diff --git a/src/graphics/host_gpu/renderer/cache/multiLevelPageTable.h b/src/graphics/host_gpu/renderer/cache/multiLevelPageTable.h index fa5bce0..a20d9cb 100644 --- a/src/graphics/host_gpu/renderer/cache/multiLevelPageTable.h +++ b/src/graphics/host_gpu/renderer/cache/multiLevelPageTable.h @@ -7,8 +7,9 @@ #include #include #include -#include +#include #include +#include #include #include @@ -103,6 +104,159 @@ private: size_t m_allocated_buckets = 0; }; +// Inline owner storage for page-table entries. Texture pages normally have only a +// handful of owners, so avoid a heap allocation on the first 16 registrations as +// shadPS4 does with boost::container::small_vector. +// +// Owners are packed into live uint64_t objects instead of being constructed in +// raw storage. This keeps sparse bucket construction cheap without relying on +// subtle implicit-lifetime or pointer-provenance rules. +template +class InlinePageOwnerList final { +public: + using value_type = OwnerT; + using size_type = size_t; + + class const_iterator final { + public: + [[nodiscard]] OwnerT operator*() const noexcept { return m_owners->At(m_index); } + const_iterator& operator++() noexcept { + ++m_index; + return *this; + } + bool operator==(const const_iterator&) const = default; + + private: + friend class InlinePageOwnerList; + const_iterator(const InlinePageOwnerList* owners, size_type index) noexcept + : m_owners(owners), m_index(index) {} + + const InlinePageOwnerList* m_owners = nullptr; + size_type m_index = 0; + }; + + static_assert(InlineCapacity > 0); + static_assert(std::is_default_constructible_v); + static_assert(std::is_trivially_copyable_v); + static_assert(std::has_unique_object_representations_v); + static_assert(sizeof(OwnerT) <= sizeof(uint64_t)); + + InlinePageOwnerList() noexcept {} + InlinePageOwnerList(const InlinePageOwnerList&) = delete; + InlinePageOwnerList& operator=(const InlinePageOwnerList&) = delete; + InlinePageOwnerList(InlinePageOwnerList&& other) noexcept + : m_overflow(std::move(other.m_overflow)), + m_inline_size(std::exchange(other.m_inline_size, 0)) { + if (m_overflow == nullptr) { + std::copy_n(other.m_inline.begin(), m_inline_size, m_inline.begin()); + } + } + InlinePageOwnerList& operator=(InlinePageOwnerList&& other) noexcept { + if (this != &other) { + m_overflow = std::move(other.m_overflow); + m_inline_size = std::exchange(other.m_inline_size, 0); + if (m_overflow == nullptr) { + std::copy_n(other.m_inline.begin(), m_inline_size, m_inline.begin()); + } + } + return *this; + } + + [[nodiscard]] const_iterator begin() const noexcept { return {this, 0}; } + [[nodiscard]] const_iterator end() const noexcept { return {this, size()}; } + + [[nodiscard]] bool empty() const noexcept { return size() == 0; } + [[nodiscard]] size_type size() const noexcept { + return m_overflow == nullptr ? m_inline_size : m_overflow->size(); + } + [[nodiscard]] OwnerT front() const noexcept { return At(0); } + [[nodiscard]] OwnerT operator[](size_type index) const noexcept { return At(index); } + + void push_back(const OwnerT& owner) { + const uint64_t packed = Pack(owner); + if (m_overflow != nullptr) { + m_overflow->push_back(packed); + return; + } + if (m_inline_size < InlineCapacity) { + m_inline[m_inline_size] = packed; + ++m_inline_size; + return; + } + auto overflow = std::make_unique>(); + overflow->reserve(InlineCapacity * 2); + overflow->assign(m_inline.begin(), m_inline.end()); + overflow->push_back(packed); + m_overflow = std::move(overflow); + } + + [[nodiscard]] bool Contains(const OwnerT& owner) const noexcept { + const uint64_t packed = Pack(owner); + if (m_overflow != nullptr) { + return std::find(m_overflow->begin(), m_overflow->end(), packed) != m_overflow->end(); + } + return std::find(m_inline.begin(), m_inline.begin() + static_cast(m_inline_size), + packed) != m_inline.begin() + static_cast(m_inline_size); + } + + [[nodiscard]] bool Erase(const OwnerT& owner) noexcept { + const uint64_t packed = Pack(owner); + if (m_overflow != nullptr) { + const auto found = std::find(m_overflow->begin(), m_overflow->end(), packed); + if (found == m_overflow->end()) { + return false; + } + m_overflow->erase(found); + if (m_overflow->size() == InlineCapacity) { + std::copy(m_overflow->begin(), m_overflow->end(), m_inline.begin()); + m_inline_size = InlineCapacity; + m_overflow.reset(); + } + return true; + } + const auto end = m_inline.begin() + static_cast(m_inline_size); + const auto found = std::find(m_inline.begin(), end, packed); + if (found == end) { + return false; + } + std::move(found + 1, end, found); + --m_inline_size; + return true; + } + + template + void ForEach(Func&& func) const { + if (m_overflow != nullptr) { + for (const uint64_t owner: *m_overflow) { + func(Unpack(owner)); + } + return; + } + for (size_type index = 0; index < m_inline_size; ++index) { + func(Unpack(m_inline[index])); + } + } + +private: + [[nodiscard]] OwnerT At(size_type index) const noexcept { + return Unpack(m_overflow == nullptr ? m_inline[index] : (*m_overflow)[index]); + } + [[nodiscard]] static uint64_t Pack(const OwnerT& owner) noexcept { + uint64_t packed = 0; + std::memcpy(&packed, &owner, sizeof(owner)); + return packed; + } + [[nodiscard]] static OwnerT Unpack(uint64_t packed) noexcept { + OwnerT owner {}; + std::memcpy(&owner, &packed, sizeof(owner)); + return owner; + } + + std::array m_inline; + std::unique_ptr> m_overflow; + size_type m_inline_size = 0; +}; + // Removes only the requested owner. Other owners in the same page entry remain intact. template [[nodiscard]] bool EraseExact(Container& owners, const Value& owner) { @@ -114,247 +268,6 @@ template return true; } -// Multi-range ownership with 1 MiB candidate buckets and precise 4 KiB -// lifetime accounting. OwnerT only needs equality. -template -class MultiRangePageOwnerIndex final { -private: - struct Registration; - using MembershipList = std::vector; - -public: - struct ByteRange final { - uint64_t address = 0; - uint64_t size = 0; - }; - - using CoarseTable = MultiLevelPageTable; - using TrackingTable = MultiLevelPageTable; - - [[nodiscard]] bool Register(const OwnerT& owner, const std::vector& ranges) { - if (FindRegistration(owner) != m_registrations.end()) { - return false; - } - auto normalized = Normalize(ranges); - if (normalized.empty()) { - return false; - } - m_registrations.push_back({owner, std::move(normalized)}); - const Registration* registration = &m_registrations.back(); - for (const size_t page: CollectPages<20>(registration->ranges)) { - m_coarse_pages[page].push_back(registration); - } - for (const size_t page: CollectPages<12>(registration->ranges)) { - m_tracking_pages[page].push_back(registration); - } - return true; - } - - // No state changes if an expected membership is absent. Final releases are - // sorted and coalesced 4 KiB spans whose final owner disappeared. - [[nodiscard]] bool Unregister(const OwnerT& owner, std::vector& final_releases) { - final_releases.clear(); - auto registration = FindRegistration(owner); - if (registration == m_registrations.end()) { - return false; - } - const auto coarse_pages = CollectPages<20>(registration->ranges); - const auto tracking_pages = CollectPages<12>(registration->ranges); - const Registration* registration_ptr = &*registration; - if (!HasAllMemberships(m_coarse_pages, coarse_pages, registration_ptr) || - !HasAllMemberships(m_tracking_pages, tracking_pages, registration_ptr)) { - return false; - } - std::vector final_pages; - for (const size_t page: coarse_pages) { - (void)EraseExact(*m_coarse_pages.Find(page), registration_ptr); - } - for (const size_t page: tracking_pages) { - auto& owners = *m_tracking_pages.Find(page); - if (owners.size() == 1) { - final_pages.push_back(page); - } - (void)EraseExact(owners, registration_ptr); - } - m_registrations.erase(registration); - final_releases = CoalesceTrackingPages(final_pages); - return true; - } - - [[nodiscard]] std::vector Query(uint64_t address, uint64_t size) const { - return Query(address, size, [](const OwnerT&) { return true; }); - } - - template - [[nodiscard]] std::vector Query(uint64_t address, uint64_t size, - Predicate&& predicate) const { - return QueryImpl(address, size, true, std::forward(predicate)); - } - - // Fault paths use page candidates: exact byte-disjoint owners sharing a - // touched 4 KiB page are intentionally retained. - [[nodiscard]] std::vector QueryCandidates(uint64_t address, uint64_t size) const { - return QueryCandidates(address, size, [](const OwnerT&) { return true; }); - } - - template - [[nodiscard]] std::vector QueryCandidates(uint64_t address, uint64_t size, - Predicate&& predicate) const { - return QueryImpl(address, size, false, std::forward(predicate)); - } - - [[nodiscard]] size_t CoarseMembershipCount(size_t page) const { - const auto* owners = m_coarse_pages.Find(page); - return owners == nullptr ? 0 : owners->size(); - } - [[nodiscard]] size_t TrackingMembershipCount(size_t page) const { - const auto* owners = m_tracking_pages.Find(page); - return owners == nullptr ? 0 : owners->size(); - } - -private: - template - [[nodiscard]] std::vector QueryImpl(uint64_t address, uint64_t size, bool strict_bytes, - Predicate&& predicate) const { - typename CoarseTable::PageRange coarse_range {}; - typename TrackingTable::PageRange tracking_range {}; - if (!CoarseTable::TryGetPageRange(address, size, coarse_range) || - (!strict_bytes && !TrackingTable::TryGetPageRange(address, size, tracking_range))) { - return {}; - } - MembershipList candidates; - for (size_t page = coarse_range.first; page < coarse_range.last_exclusive; ++page) { - if (const auto* owners = m_coarse_pages.Find(page); owners != nullptr) { - AppendUnique(candidates, *owners); - } - } - std::vector result; - for (const Registration* registration: candidates) { - if ((strict_bytes ? Overlaps(registration->ranges, address, size) - : HasTrackingMembership(registration, tracking_range)) && - predicate(registration->owner)) { - result.push_back(registration->owner); - } - } - return result; - } - - struct Registration final { - OwnerT owner; - std::vector ranges; - }; - using RegistrationIterator = typename std::list::iterator; - using ConstRegistrationIterator = typename std::list::const_iterator; - - [[nodiscard]] RegistrationIterator FindRegistration(const OwnerT& owner) { - return std::find_if(m_registrations.begin(), m_registrations.end(), - [&](const Registration& item) { return item.owner == owner; }); - } - [[nodiscard]] ConstRegistrationIterator FindRegistration(const OwnerT& owner) const { - return std::find_if(m_registrations.begin(), m_registrations.end(), - [&](const Registration& item) { return item.owner == owner; }); - } - - [[nodiscard]] static std::vector Normalize(const std::vector& ranges) { - std::vector sorted; - for (const auto& range: ranges) { - typename CoarseTable::PageRange ignored {}; - if (!CoarseTable::TryGetPageRange(range.address, range.size, ignored)) { - return {}; - } - sorted.push_back(range); - } - std::sort(sorted.begin(), sorted.end(), [](const ByteRange& lhs, const ByteRange& rhs) { - return lhs.address < rhs.address; - }); - std::vector merged; - for (const auto& range: sorted) { - if (merged.empty() || range.address > merged.back().address + merged.back().size) { - merged.push_back(range); - } else { - const uint64_t end = std::max(merged.back().address + merged.back().size, - range.address + range.size); - merged.back().size = end - merged.back().address; - } - } - return merged; - } - - template - [[nodiscard]] static std::vector CollectPages(const std::vector& ranges) { - std::vector pages; - for (const auto& range: ranges) { - const size_t first = static_cast(range.address >> Bits); - const size_t last = static_cast((range.address + range.size - 1) >> Bits); - for (size_t page = first; page <= last; ++page) { - pages.push_back(page); - } - } - std::sort(pages.begin(), pages.end()); - pages.erase(std::unique(pages.begin(), pages.end()), pages.end()); - return pages; - } - - template - [[nodiscard]] static bool HasAllMemberships(const Table& table, - const std::vector& pages, - const Registration* registration) { - for (const size_t page: pages) { - const auto* owners = table.Find(page); - if (owners == nullptr || - std::find(owners->begin(), owners->end(), registration) == owners->end()) { - return false; - } - } - return true; - } - - [[nodiscard]] bool HasTrackingMembership(const Registration* registration, - const typename TrackingTable::PageRange& range) const { - for (size_t page = range.first; page < range.last_exclusive; ++page) { - const auto* owners = m_tracking_pages.Find(page); - if (owners != nullptr && - std::find(owners->begin(), owners->end(), registration) != owners->end()) { - return true; - } - } - return false; - } - - [[nodiscard]] static bool Overlaps(const std::vector& ranges, uint64_t address, - uint64_t size) { - const uint64_t end = address + size; - return std::any_of(ranges.begin(), ranges.end(), [&](const ByteRange& range) { - return range.address < end && address < range.address + range.size; - }); - } - static void AppendUnique(MembershipList& destination, const MembershipList& source) { - for (const Registration* registration: source) { - if (std::find(destination.begin(), destination.end(), registration) == - destination.end()) { - destination.push_back(registration); - } - } - } - [[nodiscard]] static std::vector - CoalesceTrackingPages(const std::vector& pages) { - std::vector result; - for (const size_t page: pages) { - const uint64_t address = static_cast(page) << 12; - if (!result.empty() && result.back().address + result.back().size == address) { - result.back().size += uint64_t {1} << 12; - } else { - result.push_back({address, uint64_t {1} << 12}); - } - } - return result; - } - - CoarseTable m_coarse_pages; - TrackingTable m_tracking_pages; - std::list m_registrations; -}; - } // namespace Libs::Graphics #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_ diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.cpp b/src/graphics/host_gpu/renderer/cache/textureCache.cpp index 3f4c161..668661e 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/textureCache.cpp @@ -195,10 +195,12 @@ void TextureCache::RegisterImage(ImageId id) { if (image.registered || image.info.data.Empty()) { EXIT("TextureCache: invalid image registration\n"); } - std::vector ranges; - ranges.push_back({image.info.data.address, image.info.data.size}); - if (!m_image_owner_index.Register(id, ranges)) { - EXIT("TextureCache: duplicate or invalid image registration\n"); + ImagePageTable::PageRange pages {}; + if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) { + EXIT("TextureCache: image registration is outside the guest address space\n"); + } + for (size_t page = pages.first; page < pages.last_exclusive; ++page) { + m_image_page_table[page].push_back(id); } image.registered = true; image.lru_id = m_lru_cache.Insert(id, m_gc_tick); @@ -211,9 +213,15 @@ void TextureCache::UnregisterImage(ImageId id) { return; } UntrackImage(id); - std::vector releases; - if (!m_image_owner_index.Unregister(id, releases)) { - EXIT("TextureCache: image missing from owner index\n"); + ImagePageTable::PageRange pages {}; + if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) { + EXIT("TextureCache: registered image is outside the guest address space\n"); + } + for (size_t page = pages.first; page < pages.last_exclusive; ++page) { + auto* owners = m_image_page_table.Find(page); + if (owners == nullptr || !owners->Erase(id)) { + EXIT("TextureCache: image missing from page owner index\n"); + } } m_lru_cache.Free(image.lru_id); const auto accounted = image.AccountedSize(); @@ -451,10 +459,48 @@ const Image& TextureCache::GetImage(ImageId id) const { return ResolveImage(id); } -std::vector TextureCache::FindImagesInRegion(uint64_t address, uint64_t size, - bool page_overlap) const { - return page_overlap ? m_image_owner_index.QueryCandidates(address, size) - : m_image_owner_index.Query(address, size); +TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64_t size, + bool page_overlap) const { + ImagePageTable::PageRange pages {}; + if (!ImagePageTable::TryGetPageRange(address, size, pages)) { + return {}; + } + + uint32_t query_epoch = ++m_image_query_epoch; + if (query_epoch == 0) { + for (const auto& slot: m_slots) { + if (slot.image != nullptr) { + slot.image->query_epoch = 0; + } + } + query_epoch = ++m_image_query_epoch; + } + + ImageIds result; + for (size_t page = pages.first; page < pages.last_exclusive; ++page) { + const auto* owners = m_image_page_table.Find(page); + if (owners == nullptr) { + continue; + } + owners->ForEach([&](ImageId id) { + if (!id || id.index >= m_slots.size()) { + return; + } + const auto& slot = m_slots[id.index]; + if (slot.generation != id.generation || slot.image == nullptr) { + return; + } + auto& image = *slot.image; + if (image.query_epoch == query_epoch) { + return; + } + image.query_epoch = query_epoch; + if (image.Overlaps(address, size, page_overlap)) { + result.push_back(id); + } + }); + } + return result; } ImageId TextureCache::GetNullImage(const ImageDesc& desc) { @@ -1152,9 +1198,9 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { ImageId result {}; bool inserted_new = false; { - std::lock_guard transaction(m_resource_mutex); - CacheLock lock(*this, m_lock); - const std::vector candidates = + std::lock_guard transaction(m_resource_mutex); + CacheLock lock(*this, m_lock); + const auto candidates = FindImagesInRegion(desc.info.data.address, desc.info.data.size, false); for (const auto id: candidates) { diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.h b/src/graphics/host_gpu/renderer/cache/textureCache.h index 622da33..ba0e847 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.h +++ b/src/graphics/host_gpu/renderer/cache/textureCache.h @@ -99,7 +99,8 @@ private: int32_t layer = -1; }; - using ImageOwnerIndex = MultiRangePageOwnerIndex; + using ImageIds = InlinePageOwnerList; + using ImagePageTable = MultiLevelPageTable; [[nodiscard]] Image& ResolveImage(ImageId id); [[nodiscard]] const Image& ResolveImage(ImageId id) const; @@ -125,8 +126,9 @@ private: [[nodiscard]] static BindingType UploadBinding(const Image& image); [[nodiscard]] bool SafeToDownload(const Image& image); - [[nodiscard]] std::vector FindImagesInRegion(uint64_t address, uint64_t size, - bool page_overlap) const; + // Caller holds m_lock; it also serializes the per-image query epoch. + [[nodiscard]] ImageIds FindImagesInRegion(uint64_t address, uint64_t size, + bool page_overlap) const; [[nodiscard]] OverlapResult ResolveOverlap(const ImageInfo& requested, BindingType binding, ImageId cached, ImageId merged); [[nodiscard]] ImageId ResolveDepthOverlap(const ImageInfo& requested, BindingType binding, @@ -169,7 +171,7 @@ private: ResourceMutex& m_resource_mutex; std::vector m_slots; std::vector m_free_slots; - ImageOwnerIndex m_image_owner_index; + ImagePageTable m_image_page_table; std::map m_null_images; Common::LeastRecentlyUsedCache m_lru_cache; std::set m_download_images; @@ -177,9 +179,10 @@ private: uint64_t m_total_used_memory = 0; uint64_t m_trigger_gc_memory = 0; uint64_t m_pressure_gc_memory = 1536ull * 1024 * 1024; - uint64_t m_critical_gc_memory = 3ull * 1024 * 1024 * 1024; - uint64_t m_gc_tick = 0; - bool m_readback_linear_images = false; + uint64_t m_critical_gc_memory = 3ull * 1024 * 1024 * 1024; + uint64_t m_gc_tick = 0; + mutable uint32_t m_image_query_epoch = 0; + bool m_readback_linear_images = false; friend struct TextureCacheTestAccess; friend class BufferCache; diff --git a/src/graphics/host_gpu/renderer/image/image.h b/src/graphics/host_gpu/renderer/image/image.h index c092983..afea6ad 100644 --- a/src/graphics/host_gpu/renderer/image/image.h +++ b/src/graphics/host_gpu/renderer/image/image.h @@ -154,17 +154,18 @@ public: } [[nodiscard]] uint64_t HashGuestEdges() const; - ImageInfo info; - VulkanImage backing; - ImageViewCache views; - ImageUsage usage; - ImageBinding binding; - bool registered = false; - uint64_t track_addr = 0; - uint64_t track_addr_end = 0; - ImageId depth_id {}; - uint64_t tick_accessed_last = 0; - size_t lru_id = 0; + ImageInfo info; + VulkanImage backing; + ImageViewCache views; + ImageUsage usage; + ImageBinding binding; + bool registered = false; + mutable uint32_t query_epoch = 0; + uint64_t track_addr = 0; + uint64_t track_addr_end = 0; + ImageId depth_id {}; + uint64_t tick_accessed_last = 0; + size_t lru_id = 0; private: friend struct ImageTestAccess; diff --git a/tests/ImagePageTableTests.cpp b/tests/ImagePageTableTests.cpp index 6ef71c8..d399c16 100644 --- a/tests/ImagePageTableTests.cpp +++ b/tests/ImagePageTableTests.cpp @@ -3,13 +3,15 @@ #include #include #include +#include #include namespace { using Owners = std::vector; using Table = Libs::Graphics::MultiLevelPageTable; -using OwnerIndex = Libs::Graphics::MultiRangePageOwnerIndex; +using PageOwners = Libs::Graphics::InlinePageOwnerList; +using OwnerTable = Libs::Graphics::MultiLevelPageTable; void Check(bool value, const char* text) { if (!value) { @@ -77,76 +79,93 @@ void TestAddressSpaceBoundaries() { "final page supports allocating and nonallocating access"); } -void TestMultiRangeRegistrationDeduplicatesPages() { - OwnerIndex index; - // Depth and stencil-like planes overlap tracking pages and share one 1 MiB bucket. - Check(index.Register(7, {{0x101000, 0x2800}, {0x102000, 0x3000}}), - "multi-range owner registers"); - Check(index.CoarseMembershipCount(1) == 1, - "one owner is inserted once in a shared 1 MiB bucket"); - Check(index.TrackingMembershipCount(0x102) == 1, - "overlapping planes insert one 4 KiB membership"); - Check(!index.Register(7, {{0x101000, 0x1000}}), "duplicate owner registration hard-fails"); +void TestInlineOwnerStorageAndOverflow() { + PageOwners owners; + for (uint32_t owner = 1; owner <= 18; ++owner) { + owners.push_back(owner); + } + Check(owners.size() == 18 && owners.front() == 1, + "inline owner storage grows past its 16-owner capacity"); + uint32_t expected = 1; + for (const uint32_t owner: owners) { + Check(owner == expected++, "overflow preserves registration order"); + } + Check(owners.Erase(5) && owners.size() == 17 && !owners.Contains(5), + "overflow erase removes only the requested owner"); + Check(owners.Erase(18) && owners.size() == 16, + "overflow storage shrinks back to inline capacity"); + const std::vector remaining {1, 2, 3, 4, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17}; + size_t remaining_index = 0; + for (const uint32_t owner: owners) { + Check(owner == remaining[remaining_index++], + "overflow-to-inline shrink preserves every owner"); + } + Check(!owners.Erase(99), "missing inline owner is reported without mutation"); - const auto owners = index.Query(0x100000, 0x10000); - Check(owners.size() == 1 && owners.front() == 7, "multi-page query returns an owner once"); + PageOwners moved = std::move(owners); + Check(owners.empty() && moved.size() == remaining.size() && moved.front() == 1, + "moving a full inline owner list leaves the source empty"); + PageOwners partial; + partial.push_back(41); + partial.push_back(42); + PageOwners partial_moved = std::move(partial); + Check(partial.empty() && partial_moved.size() == 2 && partial_moved[1] == 42, + "moving a partially populated list copies only live owners"); + PageOwners assigned; + assigned.push_back(99); + assigned = std::move(partial_moved); + Check(partial_moved.empty() && assigned.size() == 2 && assigned.front() == 41, + "move assignment replaces an inline list without reading inactive slots"); + PageOwners empty; + PageOwners empty_moved = std::move(empty); + Check(empty.empty() && empty_moved.empty(), "moving an empty owner list is safe"); } -void TestSharedPageUnregisterLifecycle() { - OwnerIndex index; - const std::vector ranges {{0x202000, 0x2000}}; - Check(index.Register(11, ranges) && index.Register(22, ranges), - "two owners register on identical pages"); - Check(index.CoarseMembershipCount(2) == 2 && index.TrackingMembershipCount(0x202) == 2, - "coarse and tracking pages retain both owners"); - - std::vector releases; - Check(index.Unregister(11, releases), "first owner unregisters"); - Check(releases.empty(), "shared tracking pages are not released with one owner remaining"); - Check(index.Query(0x202000, 1).size() == 1 && index.Query(0x202000, 1).front() == 22, - "unregistering one owner preserves the other"); - Check(!index.Unregister(11, releases), "missing membership hard-fails without mutation"); - - Check(index.Unregister(22, releases), "final owner unregisters"); - Check(releases.size() == 1 && releases.front().address == 0x202000 && - releases.front().size == 0x2000, - "adjacent final-owner tracking pages return one contiguous release"); +void TestOneMiBRegistrationGranularity() { + static_assert(OwnerTable::kPageBits == 20); + OwnerTable table; + OwnerTable::PageRange pages {}; + constexpr uint64_t range_size = 64ull * 1024 * 1024; + Check(OwnerTable::TryGetPageRange(0, range_size, pages), "large owner range is valid"); + Check(pages.first == 0 && pages.last_exclusive == 64, + "64 MiB registration touches exactly 64 one-MiB entries"); + for (size_t page = pages.first; page < pages.last_exclusive; ++page) { + table[page].push_back(7); + } + Check(table.AllocatedBucketCount() == 1, + "large registration uses one sparse second-level bucket"); + for (size_t page = pages.first; page < pages.last_exclusive; ++page) { + auto* owners = table.Find(page); + Check(owners != nullptr && owners->size() == 1 && owners->front() == 7, + "each touched one-MiB entry retains the owner once"); + Check(owners->Erase(7), "large owner unregisters by coarse page"); + } } -void TestStrictByteFilteringAndPredicate() { - OwnerIndex index; - Check(index.Register(31, {{0x300100, 0x100}}), "first byte-disjoint owner registers"); - Check(index.Register(32, {{0x300800, 0x100}}), "second byte-disjoint owner registers"); - Check(index.Register(33, {{0x30f000, 0x100}}), "coarse-only owner registers"); - Check(index.TrackingMembershipCount(0x300) == 2, - "byte-disjoint owners share one tracking page"); - Check(index.CoarseMembershipCount(3) == 3, - "all owners share one coarse candidate bucket"); - Check(index.Query(0x300400, 0x40).empty(), "page hit without byte overlap is filtered out"); - const auto page_candidates = index.QueryCandidates(0x300400, 0x40); - Check(page_candidates.size() == 2, - "fault candidate query retains touched-page owners and rejects coarse-only owners"); - const auto first = index.Query(0x300180, 0x10); - Check(first.size() == 1 && first.front() == 31, - "strict byte overlap selects only the matching owner"); - const auto predicate_filtered = - index.Query(0x300000, 0x1000, [](uint32_t owner) { return owner == 32; }); - Check(predicate_filtered.size() == 1 && predicate_filtered.front() == 32, - "supplied predicate filters query owners"); +void TestSharedCoarsePageLifecycle() { + OwnerTable table; + auto& owners = table[2]; + owners.push_back(11); + owners.push_back(22); + Check(owners.size() == 2, "two images share one coarse page"); + Check(owners.Erase(11) && owners.size() == 1 && owners.front() == 22, + "unregistering one image preserves its coarse-page neighbor"); + Check(!owners.Erase(11), "double unregister is rejected without mutation"); + Check(owners.Erase(22) && owners.empty(), "final coarse-page owner unregisters"); } -void TestOwnerIndexAddressSpaceBoundary() { - OwnerIndex index; - constexpr uint64_t last_byte = OwnerIndex::CoarseTable::kAddressSpaceSize - 1; - Check(index.Register(41, {{last_byte, 1}}), "final guest byte registers"); - const auto exact = index.Query(last_byte, 1); - Check(exact.size() == 1 && exact.front() == 41, - "strict query finds an exact overlap at the final guest byte"); - Check(index.Query(last_byte - 1, 1).empty(), - "strict query preserves half-open overlap boundaries"); - const auto page_candidates = index.QueryCandidates(last_byte - 1, 1); - Check(page_candidates.size() == 1 && page_candidates.front() == 41, - "page candidate query retains a byte-disjoint owner on the final tracking page"); +void TestOneMiBBoundaries() { + OwnerTable::PageRange range {}; + Check(OwnerTable::TryGetPageRange(0x0fffff, 2, range), + "range crossing a one-MiB boundary is valid"); + Check(range.first == 0 && range.last_exclusive == 2, + "cross-boundary registration touches both coarse pages"); + Check(OwnerTable::TryGetPageRange(OwnerTable::kAddressSpaceSize - 1, 1, range), + "final guest byte maps to a coarse owner page"); + Check(range.first == OwnerTable::kPageCount - 1 && + range.last_exclusive == OwnerTable::kPageCount, + "final guest byte uses the final one-MiB page"); } } // namespace @@ -156,10 +175,10 @@ int main() { TestCrossBucketRange(); TestQueriesDoNotAllocate(); TestAddressSpaceBoundaries(); - TestMultiRangeRegistrationDeduplicatesPages(); - TestSharedPageUnregisterLifecycle(); - TestStrictByteFilteringAndPredicate(); - TestOwnerIndexAddressSpaceBoundary(); + TestInlineOwnerStorageAndOverflow(); + TestOneMiBRegistrationGranularity(); + TestSharedCoarsePageLifecycle(); + TestOneMiBBoundaries(); std::printf("ImagePageTableTests: all cases passed\n"); return 0; } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index f56f9a7..dbc3344 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -163,6 +163,10 @@ struct TileManagerTestAccess { }; struct TextureCacheTestAccess { + static_assert(TextureCache::ImagePageTable::kPageBits == 20); + static_assert(TextureCache::ImagePageTable::kAddressSpaceBits == 40); + static_assert(TextureCache::ImagePageTable::kFirstLevelBits == 10); + static void ConfigureGarbageCollection(TextureCache& cache, std::span oldest, uint64_t tick, uint64_t pressure) { cache.m_trigger_gc_memory = 0; @@ -197,6 +201,75 @@ struct TextureCacheTestAccess { return owner != nullptr && owner->registered; } + static std::vector FindImages(TextureCache& cache, uint64_t address, uint64_t size, + bool page_overlap) { + std::lock_guard lock(cache.m_lock); + const auto found = cache.FindImagesInRegion(address, size, page_overlap); + std::vector result; + result.reserve(found.size()); + for (const auto id: found) { + result.push_back(id); + } + return result; + } + + static size_t PageOwnerCount(TextureCache& cache, uint64_t address) { + std::lock_guard lock(cache.m_lock); + const auto* owners = cache.m_image_page_table.Find( + static_cast(address >> TextureCache::ImagePageTable::kPageBits)); + return owners == nullptr ? 0 : owners->size(); + } + + static size_t OwnedPageCount(TextureCache& cache, uint64_t address, uint64_t size, ImageId id) { + std::lock_guard lock(cache.m_lock); + TextureCache::ImagePageTable::PageRange pages {}; + if (!TextureCache::ImagePageTable::TryGetPageRange(address, size, pages)) { + return 0; + } + size_t count = 0; + for (size_t page = pages.first; page < pages.last_exclusive; ++page) { + const auto* owners = cache.m_image_page_table.Find(page); + count += owners != nullptr && owners->Contains(id) ? 1 : 0; + } + return count; + } + + static void AddPageOwner(TextureCache& cache, uint64_t address, ImageId id) { + std::lock_guard lock(cache.m_lock); + cache.m_image_page_table[static_cast( + address >> TextureCache::ImagePageTable::kPageBits)] + .push_back(id); + } + + static bool RemovePageOwner(TextureCache& cache, uint64_t address, ImageId id) { + std::lock_guard lock(cache.m_lock); + auto* owners = cache.m_image_page_table.Find( + static_cast(address >> TextureCache::ImagePageTable::kPageBits)); + return owners != nullptr && owners->Erase(id); + } + + static void SetQueryEpoch(TextureCache& cache, uint32_t epoch) { + std::lock_guard lock(cache.m_lock); + cache.m_image_query_epoch = epoch; + } + + static uint32_t QueryEpoch(TextureCache& cache) { + std::lock_guard lock(cache.m_lock); + return cache.m_image_query_epoch; + } + + static ImageId InsertImage(TextureCache& cache, const ImageInfo& info) { + std::lock_guard transaction(cache.m_resource_mutex); + std::lock_guard lock(cache.m_lock); + return cache.InsertImage(info); + } + + static void DeleteImage(TextureCache& cache, ImageId id) { + std::lock_guard transaction(cache.m_resource_mutex); + std::lock_guard lock(cache.m_lock); + cache.DeleteImage(id); + } + static std::shared_ptr Owner(const TextureCache& cache, ImageId id) { return cache.ResolveOwner(id); } @@ -2488,6 +2561,110 @@ public: vk::Format::eR8G8B8A8Srgb, "registered compatible backing did not reuse one ImageId"); + const auto IsOnlyImage = [](const std::vector& ids, ImageId expected) { + return ids.size() == 1 && ids.front() == expected; + }; + const auto exact_byte_miss = + TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, false); + const auto touched_page_hit = + TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, true); + const auto next_page_miss = + TextureCacheTestAccess::FindImages(texture_cache, base + 0x1000, 1, true); + Require(name, "exact and 4-KiB page filtering", + exact_byte_miss.empty() && IsOnlyImage(touched_page_hit, first) && + next_page_miss.empty(), + "coarse candidates did not preserve exact-byte and touched-page semantics"); + + const auto MakeOwnershipInfo = [](uint64_t address, uint64_t size) { + ImageInfo info {}; + info.data = {address, size}; + info.extent = {1, 1, 1}; + info.resources = {1, 1}; + info.samples = 1; + return info; + }; + const auto spanning_info = MakeOwnershipInfo(base + 0x1ff000, 0x2000); + const auto spanning = TextureCacheTestAccess::InsertImage(texture_cache, spanning_info); + const auto spanning_results = TextureCacheTestAccess::FindImages( + texture_cache, spanning_info.data.address, spanning_info.data.size, false); + Require(name, "one-MiB cross-page deduplication", + spanning && IsOnlyImage(spanning_results, spanning) && + TextureCacheTestAccess::PageOwnerCount(texture_cache, + spanning_info.data.address) == 1 && + TextureCacheTestAccess::PageOwnerCount( + texture_cache, spanning_info.data.End() - 1) == 1, + "an image spanning two coarse pages was missing or returned more than once"); + TextureCacheTestAccess::DeleteImage(texture_cache, spanning); + Require(name, "cross-page owner cleanup", + TextureCacheTestAccess::PageOwnerCount(texture_cache, + spanning_info.data.address) == 0 && + TextureCacheTestAccess::PageOwnerCount( + texture_cache, spanning_info.data.End() - 1) == 0 && + TextureCacheTestAccess::FindImages(texture_cache, + spanning_info.data.address, + spanning_info.data.size, false) + .empty(), + "cross-page unregister left a stale coarse-page membership"); + + const auto shared_info = MakeOwnershipInfo(base + 0x500100, 0x100); + const auto shared_first = + TextureCacheTestAccess::InsertImage(texture_cache, shared_info); + const auto shared_second = + TextureCacheTestAccess::InsertImage(texture_cache, shared_info); + const auto shared_results = TextureCacheTestAccess::FindImages( + texture_cache, shared_info.data.address, shared_info.data.size, false); + Require(name, "shared coarse-page registration", + shared_results.size() == 2 && + std::find(shared_results.begin(), shared_results.end(), shared_first) != + shared_results.end() && + std::find(shared_results.begin(), shared_results.end(), shared_second) != + shared_results.end(), + "two registered owners were not retained in one coarse page"); + TextureCacheTestAccess::DeleteImage(texture_cache, shared_first); + const auto shared_survivor = TextureCacheTestAccess::FindImages( + texture_cache, shared_info.data.address, shared_info.data.size, false); + Require(name, "shared coarse-page unregister", + IsOnlyImage(shared_survivor, shared_second) && + TextureCacheTestAccess::PageOwnerCount(texture_cache, + shared_info.data.address) == 1, + "unregistering one owner removed its coarse-page neighbor"); + TextureCacheTestAccess::DeleteImage(texture_cache, shared_second); + Require(name, "final shared coarse-page unregister", + TextureCacheTestAccess::PageOwnerCount(texture_cache, + shared_info.data.address) == 0, + "the final shared owner remained registered"); + + constexpr uint64_t large_owner_size = 64ull * 1024 * 1024; + const auto large_info = MakeOwnershipInfo(base + 0x800000, large_owner_size); + const auto large_owner = + TextureCacheTestAccess::InsertImage(texture_cache, large_info); + Require(name, "production one-MiB registration granularity", + TextureCacheTestAccess::OwnedPageCount( + texture_cache, large_info.data.address, large_info.data.size, large_owner) == 64, + "64 MiB image registration did not create exactly 64 coarse memberships"); + TextureCacheTestAccess::DeleteImage(texture_cache, large_owner); + Require(name, "production one-MiB unregister granularity", + TextureCacheTestAccess::OwnedPageCount( + texture_cache, large_info.data.address, large_info.data.size, large_owner) == 0, + "large image unregister left coarse memberships behind"); + + const ImageId stale {first.index, first.generation + 1}; + TextureCacheTestAccess::AddPageOwner(texture_cache, base, stale); + const auto stale_filtered = + TextureCacheTestAccess::FindImages(texture_cache, base, sizeof(initial), false); + Require(name, "stale page owner filtering", + IsOnlyImage(stale_filtered, first) && + TextureCacheTestAccess::RemovePageOwner(texture_cache, base, stale), + "a stale generation escaped the direct page-owner lookup"); + + TextureCacheTestAccess::SetQueryEpoch(texture_cache, UINT32_MAX); + const auto wrap_results = + TextureCacheTestAccess::FindImages(texture_cache, base, sizeof(initial), false); + Require(name, "page-owner query epoch wrap", + IsOnlyImage(wrap_results, first) && + TextureCacheTestAccess::QueryEpoch(texture_cache) == 1, + "query deduplication failed while wrapping its epoch"); + auto& image = texture_cache.GetImage(first); constexpr uint32_t final_sampled_value = 0x88776655u; Require(name, "sampled write between discovery and acquisition",