H3cJP/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-08-03 11:23:49 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

perf: reduce texture registration and lookup overhead

Browse Source
This commit is contained in:
nmzik
2026-08-03 08:18:05 +02:00
parent 2e9e2ae566
commit 26f0c7b0bf
6 changed files with 500 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/host_gpu/renderer/cache/multiLevelPageTable.h
Vendored
+154 -242
View File
@@ -7,8 +7,9 @@
#include <array> #include <array>
#include <cstddef> #include <cstddef>
#include <cstdint> #include <cstdint>
#include <list> #include <cstring>
#include <memory> #include <memory>
#include <type_traits>
#include <utility> #include <utility>
#include <vector> #include <vector>
@@ -103,6 +104,158 @@ private:
size_t m_allocated_buckets = 0; size_t m_allocated_buckets = 0;
}; };
// Inline owner storage for page-table entries. Texture pages normally have only a
// handful of owners, so avoid a heap allocation on the first 16 registrations.
//
// Owners are packed into live uint64_t objects instead of being constructed in
// raw storage. This keeps sparse bucket construction cheap without relying on
// subtle implicit-lifetime or pointer-provenance rules.
template <typename OwnerT, size_t InlineCapacity = 16>
class InlinePageOwnerList final {
public:
using value_type = OwnerT;
using size_type = size_t;
class const_iterator final {
public:
[[nodiscard]] OwnerT operator*() const noexcept { return m_owners->At(m_index); }
const_iterator& operator++() noexcept {
++m_index;
return *this;
}
bool operator==(const const_iterator&) const = default;
private:
friend class InlinePageOwnerList;
const_iterator(const InlinePageOwnerList* owners, size_type index) noexcept
: m_owners(owners), m_index(index) {}
const InlinePageOwnerList* m_owners = nullptr;
size_type m_index = 0;
};
static_assert(InlineCapacity > 0);
static_assert(std::is_default_constructible_v<OwnerT>);
static_assert(std::is_trivially_copyable_v<OwnerT>);
static_assert(std::has_unique_object_representations_v<OwnerT>);
static_assert(sizeof(OwnerT) <= sizeof(uint64_t));
InlinePageOwnerList() noexcept {}
InlinePageOwnerList(const InlinePageOwnerList&) = delete;
InlinePageOwnerList& operator=(const InlinePageOwnerList&) = delete;
InlinePageOwnerList(InlinePageOwnerList&& other) noexcept
: m_overflow(std::move(other.m_overflow)),
m_inline_size(std::exchange(other.m_inline_size, 0)) {
if (m_overflow == nullptr) {
std::copy_n(other.m_inline.begin(), m_inline_size, m_inline.begin());
}
}
InlinePageOwnerList& operator=(InlinePageOwnerList&& other) noexcept {
if (this != &other) {
m_overflow = std::move(other.m_overflow);
m_inline_size = std::exchange(other.m_inline_size, 0);
if (m_overflow == nullptr) {
std::copy_n(other.m_inline.begin(), m_inline_size, m_inline.begin());
}
}
return *this;
}
[[nodiscard]] const_iterator begin() const noexcept { return {this, 0}; }
[[nodiscard]] const_iterator end() const noexcept { return {this, size()}; }
[[nodiscard]] bool empty() const noexcept { return size() == 0; }
[[nodiscard]] size_type size() const noexcept {
return m_overflow == nullptr ? m_inline_size : m_overflow->size();
}
[[nodiscard]] OwnerT front() const noexcept { return At(0); }
[[nodiscard]] OwnerT operator[](size_type index) const noexcept { return At(index); }
void push_back(const OwnerT& owner) {
const uint64_t packed = Pack(owner);
if (m_overflow != nullptr) {
m_overflow->push_back(packed);
return;
}
if (m_inline_size < InlineCapacity) {
m_inline[m_inline_size] = packed;
++m_inline_size;
return;
}
auto overflow = std::make_unique<std::vector<uint64_t>>();
overflow->reserve(InlineCapacity * 2);
overflow->assign(m_inline.begin(), m_inline.end());
overflow->push_back(packed);
m_overflow = std::move(overflow);
}
[[nodiscard]] bool Contains(const OwnerT& owner) const noexcept {
const uint64_t packed = Pack(owner);
if (m_overflow != nullptr) {
return std::find(m_overflow->begin(), m_overflow->end(), packed) != m_overflow->end();
}
return std::find(m_inline.begin(), m_inline.begin() + static_cast<ptrdiff_t>(m_inline_size),
packed) != m_inline.begin() + static_cast<ptrdiff_t>(m_inline_size);
}
[[nodiscard]] bool Erase(const OwnerT& owner) noexcept {
const uint64_t packed = Pack(owner);
if (m_overflow != nullptr) {
const auto found = std::find(m_overflow->begin(), m_overflow->end(), packed);
if (found == m_overflow->end()) {
return false;
}
m_overflow->erase(found);
if (m_overflow->size() == InlineCapacity) {
std::copy(m_overflow->begin(), m_overflow->end(), m_inline.begin());
m_inline_size = InlineCapacity;
m_overflow.reset();
}
return true;
}
const auto end = m_inline.begin() + static_cast<ptrdiff_t>(m_inline_size);
const auto found = std::find(m_inline.begin(), end, packed);
if (found == end) {
return false;
}
std::move(found + 1, end, found);
--m_inline_size;
return true;
}
template <typename Func>
void ForEach(Func&& func) const {
if (m_overflow != nullptr) {
for (const uint64_t owner: *m_overflow) {
func(Unpack(owner));
}
return;
}
for (size_type index = 0; index < m_inline_size; ++index) {
func(Unpack(m_inline[index]));
}
}
private:
[[nodiscard]] OwnerT At(size_type index) const noexcept {
return Unpack(m_overflow == nullptr ? m_inline[index] : (*m_overflow)[index]);
}
[[nodiscard]] static uint64_t Pack(const OwnerT& owner) noexcept {
uint64_t packed = 0;
std::memcpy(&packed, &owner, sizeof(owner));
return packed;
}
[[nodiscard]] static OwnerT Unpack(uint64_t packed) noexcept {
OwnerT owner {};
std::memcpy(&owner, &packed, sizeof(owner));
return owner;
}
std::array<uint64_t, InlineCapacity> m_inline;
std::unique_ptr<std::vector<uint64_t>> m_overflow;
size_type m_inline_size = 0;
};
// Removes only the requested owner. Other owners in the same page entry remain intact. // Removes only the requested owner. Other owners in the same page entry remain intact.
template <typename Container, typename Value> template <typename Container, typename Value>
[[nodiscard]] bool EraseExact(Container& owners, const Value& owner) { [[nodiscard]] bool EraseExact(Container& owners, const Value& owner) {
@@ -114,247 +267,6 @@ template <typename Container, typename Value>
return true; return true;
} }
// Multi-range ownership with 1 MiB candidate buckets and precise 4 KiB
// lifetime accounting. OwnerT only needs equality.
template <typename OwnerT>
class MultiRangePageOwnerIndex final {
private:
struct Registration;
using MembershipList = std::vector<const Registration*>;
public:
struct ByteRange final {
uint64_t address = 0;
uint64_t size = 0;
};
using CoarseTable = MultiLevelPageTable<MembershipList, 20, 40, 10>;
using TrackingTable = MultiLevelPageTable<MembershipList, 12, 40, 18>;
[[nodiscard]] bool Register(const OwnerT& owner, const std::vector<ByteRange>& ranges) {
if (FindRegistration(owner) != m_registrations.end()) {
return false;
}
auto normalized = Normalize(ranges);
if (normalized.empty()) {
return false;
}
m_registrations.push_back({owner, std::move(normalized)});
const Registration* registration = &m_registrations.back();
for (const size_t page: CollectPages<20>(registration->ranges)) {
m_coarse_pages[page].push_back(registration);
}
for (const size_t page: CollectPages<12>(registration->ranges)) {
m_tracking_pages[page].push_back(registration);
}
return true;
}
// No state changes if an expected membership is absent. Final releases are
// sorted and coalesced 4 KiB spans whose final owner disappeared.
[[nodiscard]] bool Unregister(const OwnerT& owner, std::vector<ByteRange>& final_releases) {
final_releases.clear();
auto registration = FindRegistration(owner);
if (registration == m_registrations.end()) {
return false;
}
const auto coarse_pages = CollectPages<20>(registration->ranges);
const auto tracking_pages = CollectPages<12>(registration->ranges);
const Registration* registration_ptr = &*registration;
if (!HasAllMemberships(m_coarse_pages, coarse_pages, registration_ptr) ||
!HasAllMemberships(m_tracking_pages, tracking_pages, registration_ptr)) {
return false;
}
std::vector<size_t> final_pages;
for (const size_t page: coarse_pages) {
(void)EraseExact(*m_coarse_pages.Find(page), registration_ptr);
}
for (const size_t page: tracking_pages) {
auto& owners = *m_tracking_pages.Find(page);
if (owners.size() == 1) {
final_pages.push_back(page);
}
(void)EraseExact(owners, registration_ptr);
}
m_registrations.erase(registration);
final_releases = CoalesceTrackingPages(final_pages);
return true;
}
[[nodiscard]] std::vector<OwnerT> Query(uint64_t address, uint64_t size) const {
return Query(address, size, [](const OwnerT&) { return true; });
}
template <typename Predicate>
[[nodiscard]] std::vector<OwnerT> Query(uint64_t address, uint64_t size,
Predicate&& predicate) const {
return QueryImpl(address, size, true, std::forward<Predicate>(predicate));
}
// Fault paths use page candidates: exact byte-disjoint owners sharing a
// touched 4 KiB page are intentionally retained.
[[nodiscard]] std::vector<OwnerT> QueryCandidates(uint64_t address, uint64_t size) const {
return QueryCandidates(address, size, [](const OwnerT&) { return true; });
}
template <typename Predicate>
[[nodiscard]] std::vector<OwnerT> QueryCandidates(uint64_t address, uint64_t size,
Predicate&& predicate) const {
return QueryImpl(address, size, false, std::forward<Predicate>(predicate));
}
[[nodiscard]] size_t CoarseMembershipCount(size_t page) const {
const auto* owners = m_coarse_pages.Find(page);
return owners == nullptr ? 0 : owners->size();
}
[[nodiscard]] size_t TrackingMembershipCount(size_t page) const {
const auto* owners = m_tracking_pages.Find(page);
return owners == nullptr ? 0 : owners->size();
}
private:
template <typename Predicate>
[[nodiscard]] std::vector<OwnerT> QueryImpl(uint64_t address, uint64_t size, bool strict_bytes,
Predicate&& predicate) const {
typename CoarseTable::PageRange coarse_range {};
typename TrackingTable::PageRange tracking_range {};
if (!CoarseTable::TryGetPageRange(address, size, coarse_range) ||
(!strict_bytes && !TrackingTable::TryGetPageRange(address, size, tracking_range))) {
return {};
}
MembershipList candidates;
for (size_t page = coarse_range.first; page < coarse_range.last_exclusive; ++page) {
if (const auto* owners = m_coarse_pages.Find(page); owners != nullptr) {
AppendUnique(candidates, *owners);
}
}
std::vector<OwnerT> result;
for (const Registration* registration: candidates) {
if ((strict_bytes ? Overlaps(registration->ranges, address, size)
: HasTrackingMembership(registration, tracking_range)) &&
predicate(registration->owner)) {
result.push_back(registration->owner);
}
}
return result;
}
struct Registration final {
OwnerT owner;
std::vector<ByteRange> ranges;
};
using RegistrationIterator = typename std::list<Registration>::iterator;
using ConstRegistrationIterator = typename std::list<Registration>::const_iterator;
[[nodiscard]] RegistrationIterator FindRegistration(const OwnerT& owner) {
return std::find_if(m_registrations.begin(), m_registrations.end(),
[&](const Registration& item) { return item.owner == owner; });
}
[[nodiscard]] ConstRegistrationIterator FindRegistration(const OwnerT& owner) const {
return std::find_if(m_registrations.begin(), m_registrations.end(),
[&](const Registration& item) { return item.owner == owner; });
}
[[nodiscard]] static std::vector<ByteRange> Normalize(const std::vector<ByteRange>& ranges) {
std::vector<ByteRange> sorted;
for (const auto& range: ranges) {
typename CoarseTable::PageRange ignored {};
if (!CoarseTable::TryGetPageRange(range.address, range.size, ignored)) {
return {};
}
sorted.push_back(range);
}
std::sort(sorted.begin(), sorted.end(), [](const ByteRange& lhs, const ByteRange& rhs) {
return lhs.address < rhs.address;
});
std::vector<ByteRange> merged;
for (const auto& range: sorted) {
if (merged.empty() || range.address > merged.back().address + merged.back().size) {
merged.push_back(range);
} else {
const uint64_t end = std::max(merged.back().address + merged.back().size,
range.address + range.size);
merged.back().size = end - merged.back().address;
}
}
return merged;
}
template <size_t Bits>
[[nodiscard]] static std::vector<size_t> CollectPages(const std::vector<ByteRange>& ranges) {
std::vector<size_t> pages;
for (const auto& range: ranges) {
const size_t first = static_cast<size_t>(range.address >> Bits);
const size_t last = static_cast<size_t>((range.address + range.size - 1) >> Bits);
for (size_t page = first; page <= last; ++page) {
pages.push_back(page);
}
}
std::sort(pages.begin(), pages.end());
pages.erase(std::unique(pages.begin(), pages.end()), pages.end());
return pages;
}
template <typename Table>
[[nodiscard]] static bool HasAllMemberships(const Table& table,
const std::vector<size_t>& pages,
const Registration* registration) {
for (const size_t page: pages) {
const auto* owners = table.Find(page);
if (owners == nullptr ||
std::find(owners->begin(), owners->end(), registration) == owners->end()) {
return false;
}
}
return true;
}
[[nodiscard]] bool HasTrackingMembership(const Registration* registration,
const typename TrackingTable::PageRange& range) const {
for (size_t page = range.first; page < range.last_exclusive; ++page) {
const auto* owners = m_tracking_pages.Find(page);
if (owners != nullptr &&
std::find(owners->begin(), owners->end(), registration) != owners->end()) {
return true;
}
}
return false;
}
[[nodiscard]] static bool Overlaps(const std::vector<ByteRange>& ranges, uint64_t address,
uint64_t size) {
const uint64_t end = address + size;
return std::any_of(ranges.begin(), ranges.end(), [&](const ByteRange& range) {
return range.address < end && address < range.address + range.size;
});
}
static void AppendUnique(MembershipList& destination, const MembershipList& source) {
for (const Registration* registration: source) {
if (std::find(destination.begin(), destination.end(), registration) ==
destination.end()) {
destination.push_back(registration);
}
}
}
[[nodiscard]] static std::vector<ByteRange>
CoalesceTrackingPages(const std::vector<size_t>& pages) {
std::vector<ByteRange> result;
for (const size_t page: pages) {
const uint64_t address = static_cast<uint64_t>(page) << 12;
if (!result.empty() && result.back().address + result.back().size == address) {
result.back().size += uint64_t {1} << 12;
} else {
result.push_back({address, uint64_t {1} << 12});
}
}
return result;
}
CoarseTable m_coarse_pages;
TrackingTable m_tracking_pages;
std::list<Registration> m_registrations;
};
} // namespace Libs::Graphics } // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_ #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_
src/graphics/host_gpu/renderer/cache/textureCache.cpp
Vendored
+60 -14
View File
@@ -195,10 +195,12 @@ void TextureCache::RegisterImage(ImageId id) {
if (image.registered || image.info.data.Empty()) { if (image.registered || image.info.data.Empty()) {
EXIT("TextureCache: invalid image registration\n"); EXIT("TextureCache: invalid image registration\n");
} }
std::vector<ImageOwnerIndex::ByteRange> ranges; ImagePageTable::PageRange pages {};
ranges.push_back({image.info.data.address, image.info.data.size}); if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
if (!m_image_owner_index.Register(id, ranges)) { EXIT("TextureCache: image registration is outside the guest address space\n");
EXIT("TextureCache: duplicate or invalid image registration\n"); }
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
m_image_page_table[page].push_back(id);
} }
image.registered = true; image.registered = true;
image.lru_id = m_lru_cache.Insert(id, m_gc_tick); image.lru_id = m_lru_cache.Insert(id, m_gc_tick);
@@ -211,9 +213,15 @@ void TextureCache::UnregisterImage(ImageId id) {
return; return;
} }
UntrackImage(id); UntrackImage(id);
std::vector<ImageOwnerIndex::ByteRange> releases; ImagePageTable::PageRange pages {};
if (!m_image_owner_index.Unregister(id, releases)) { if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
EXIT("TextureCache: image missing from owner index\n"); EXIT("TextureCache: registered image is outside the guest address space\n");
}
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
auto* owners = m_image_page_table.Find(page);
if (owners == nullptr || !owners->Erase(id)) {
EXIT("TextureCache: image missing from page owner index\n");
}
} }
m_lru_cache.Free(image.lru_id); m_lru_cache.Free(image.lru_id);
const auto accounted = image.AccountedSize(); const auto accounted = image.AccountedSize();
@@ -451,10 +459,48 @@ const Image& TextureCache::GetImage(ImageId id) const {
return ResolveImage(id); return ResolveImage(id);
} }
std::vector<ImageId> TextureCache::FindImagesInRegion(uint64_t address, uint64_t size, TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64_t size,
bool page_overlap) const { bool page_overlap) const {
return page_overlap ? m_image_owner_index.QueryCandidates(address, size) ImagePageTable::PageRange pages {};
: m_image_owner_index.Query(address, size); if (!ImagePageTable::TryGetPageRange(address, size, pages)) {
return {};
}
uint32_t query_epoch = ++m_image_query_epoch;
if (query_epoch == 0) {
for (const auto& slot: m_slots) {
if (slot.image != nullptr) {
slot.image->query_epoch = 0;
}
}
query_epoch = ++m_image_query_epoch;
}
ImageIds result;
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
const auto* owners = m_image_page_table.Find(page);
if (owners == nullptr) {
continue;
}
owners->ForEach([&](ImageId id) {
if (!id || id.index >= m_slots.size()) {
return;
}
const auto& slot = m_slots[id.index];
if (slot.generation != id.generation || slot.image == nullptr) {
return;
}
auto& image = *slot.image;
if (image.query_epoch == query_epoch) {
return;
}
image.query_epoch = query_epoch;
if (image.Overlaps(address, size, page_overlap)) {
result.push_back(id);
}
});
}
return result;
} }
ImageId TextureCache::GetNullImage(const ImageDesc& desc) { ImageId TextureCache::GetNullImage(const ImageDesc& desc) {
@@ -1152,9 +1198,9 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
ImageId result {}; ImageId result {};
bool inserted_new = false; bool inserted_new = false;
{ {
std::lock_guard transaction(m_resource_mutex); std::lock_guard transaction(m_resource_mutex);
CacheLock lock(*this, m_lock); CacheLock lock(*this, m_lock);
const std::vector<ImageId> candidates = const auto candidates =
FindImagesInRegion(desc.info.data.address, desc.info.data.size, false); FindImagesInRegion(desc.info.data.address, desc.info.data.size, false);
for (const auto id: candidates) { for (const auto id: candidates) {
src/graphics/host_gpu/renderer/cache/textureCache.h
Vendored
+10 -7
View File
@@ -99,7 +99,8 @@ private:
int32_t layer = -1; int32_t layer = -1;
}; };
using ImageOwnerIndex = MultiRangePageOwnerIndex<ImageId>; using ImageIds = InlinePageOwnerList<ImageId, 16>;
using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
[[nodiscard]] Image& ResolveImage(ImageId id); [[nodiscard]] Image& ResolveImage(ImageId id);
[[nodiscard]] const Image& ResolveImage(ImageId id) const; [[nodiscard]] const Image& ResolveImage(ImageId id) const;
@@ -125,8 +126,9 @@ private:
[[nodiscard]] static BindingType UploadBinding(const Image& image); [[nodiscard]] static BindingType UploadBinding(const Image& image);
[[nodiscard]] bool SafeToDownload(const Image& image); [[nodiscard]] bool SafeToDownload(const Image& image);
[[nodiscard]] std::vector<ImageId> FindImagesInRegion(uint64_t address, uint64_t size, // Caller holds m_lock; it also serializes the per-image query epoch.
bool page_overlap) const; [[nodiscard]] ImageIds FindImagesInRegion(uint64_t address, uint64_t size,
bool page_overlap) const;
[[nodiscard]] OverlapResult ResolveOverlap(const ImageInfo& requested, BindingType binding, [[nodiscard]] OverlapResult ResolveOverlap(const ImageInfo& requested, BindingType binding,
ImageId cached, ImageId merged); ImageId cached, ImageId merged);
[[nodiscard]] ImageId ResolveDepthOverlap(const ImageInfo& requested, BindingType binding, [[nodiscard]] ImageId ResolveDepthOverlap(const ImageInfo& requested, BindingType binding,
@@ -169,7 +171,7 @@ private:
ResourceMutex& m_resource_mutex; ResourceMutex& m_resource_mutex;
std::vector<Slot> m_slots; std::vector<Slot> m_slots;
std::vector<uint32_t> m_free_slots; std::vector<uint32_t> m_free_slots;
ImageOwnerIndex m_image_owner_index; ImagePageTable m_image_page_table;
std::map<vk::Format, ImageId> m_null_images; std::map<vk::Format, ImageId> m_null_images;
Common::LeastRecentlyUsedCache<ImageId, uint64_t> m_lru_cache; Common::LeastRecentlyUsedCache<ImageId, uint64_t> m_lru_cache;
std::set<ImageId> m_download_images; std::set<ImageId> m_download_images;
@@ -177,9 +179,10 @@ private:
uint64_t m_total_used_memory = 0; uint64_t m_total_used_memory = 0;
uint64_t m_trigger_gc_memory = 0; uint64_t m_trigger_gc_memory = 0;
uint64_t m_pressure_gc_memory = 1536ull * 1024 * 1024; uint64_t m_pressure_gc_memory = 1536ull * 1024 * 1024;
uint64_t m_critical_gc_memory = 3ull * 1024 * 1024 * 1024; uint64_t m_critical_gc_memory = 3ull * 1024 * 1024 * 1024;
uint64_t m_gc_tick = 0; uint64_t m_gc_tick = 0;
bool m_readback_linear_images = false; mutable uint32_t m_image_query_epoch = 0;
bool m_readback_linear_images = false;
friend struct TextureCacheTestAccess; friend struct TextureCacheTestAccess;
friend class BufferCache; friend class BufferCache;
src/graphics/host_gpu/renderer/image/image.h
+12 -11
View File
@@ -154,17 +154,18 @@ public:
} }
[[nodiscard]] uint64_t HashGuestEdges() const; [[nodiscard]] uint64_t HashGuestEdges() const;
ImageInfo info; ImageInfo info;
VulkanImage backing; VulkanImage backing;
ImageViewCache views; ImageViewCache views;
ImageUsage usage; ImageUsage usage;
ImageBinding binding; ImageBinding binding;
bool registered = false; bool registered = false;
uint64_t track_addr = 0; mutable uint32_t query_epoch = 0;
uint64_t track_addr_end = 0; uint64_t track_addr = 0;
ImageId depth_id {}; uint64_t track_addr_end = 0;
uint64_t tick_accessed_last = 0; ImageId depth_id {};
size_t lru_id = 0; uint64_t tick_accessed_last = 0;
size_t lru_id = 0;
private: private:
friend struct ImageTestAccess; friend struct ImageTestAccess;
tests/ImagePageTableTests.cpp
+87 -68
View File
@@ -3,13 +3,15 @@
#include <cstdint> #include <cstdint>
#include <cstdio> #include <cstdio>
#include <cstdlib> #include <cstdlib>
#include <utility>
#include <vector> #include <vector>
namespace { namespace {
using Owners = std::vector<uint32_t>; using Owners = std::vector<uint32_t>;
using Table = Libs::Graphics::MultiLevelPageTable<Owners>; using Table = Libs::Graphics::MultiLevelPageTable<Owners>;
using OwnerIndex = Libs::Graphics::MultiRangePageOwnerIndex<uint32_t>; using PageOwners = Libs::Graphics::InlinePageOwnerList<uint32_t, 16>;
using OwnerTable = Libs::Graphics::MultiLevelPageTable<PageOwners, 20, 40, 10>;
void Check(bool value, const char* text) { void Check(bool value, const char* text) {
if (!value) { if (!value) {
@@ -77,76 +79,93 @@ void TestAddressSpaceBoundaries() {
"final page supports allocating and nonallocating access"); "final page supports allocating and nonallocating access");
} }
void TestMultiRangeRegistrationDeduplicatesPages() { void TestInlineOwnerStorageAndOverflow() {
OwnerIndex index; PageOwners owners;
// Depth and stencil-like planes overlap tracking pages and share one 1 MiB bucket. for (uint32_t owner = 1; owner <= 18; ++owner) {
Check(index.Register(7, {{0x101000, 0x2800}, {0x102000, 0x3000}}), owners.push_back(owner);
"multi-range owner registers"); }
Check(index.CoarseMembershipCount(1) == 1, Check(owners.size() == 18 && owners.front() == 1,
"one owner is inserted once in a shared 1 MiB bucket"); "inline owner storage grows past its 16-owner capacity");
Check(index.TrackingMembershipCount(0x102) == 1, uint32_t expected = 1;
"overlapping planes insert one 4 KiB membership"); for (const uint32_t owner: owners) {
Check(!index.Register(7, {{0x101000, 0x1000}}), "duplicate owner registration hard-fails"); Check(owner == expected++, "overflow preserves registration order");
}
Check(owners.Erase(5) && owners.size() == 17 && !owners.Contains(5),
"overflow erase removes only the requested owner");
Check(owners.Erase(18) && owners.size() == 16,
"overflow storage shrinks back to inline capacity");
const std::vector<uint32_t> remaining {1, 2, 3, 4, 6, 7, 8, 9,
10, 11, 12, 13, 14, 15, 16, 17};
size_t remaining_index = 0;
for (const uint32_t owner: owners) {
Check(owner == remaining[remaining_index++],
"overflow-to-inline shrink preserves every owner");
}
Check(!owners.Erase(99), "missing inline owner is reported without mutation");
const auto owners = index.Query(0x100000, 0x10000); PageOwners moved = std::move(owners);
Check(owners.size() == 1 && owners.front() == 7, "multi-page query returns an owner once"); Check(owners.empty() && moved.size() == remaining.size() && moved.front() == 1,
"moving a full inline owner list leaves the source empty");
PageOwners partial;
partial.push_back(41);
partial.push_back(42);
PageOwners partial_moved = std::move(partial);
Check(partial.empty() && partial_moved.size() == 2 && partial_moved[1] == 42,
"moving a partially populated list copies only live owners");
PageOwners assigned;
assigned.push_back(99);
assigned = std::move(partial_moved);
Check(partial_moved.empty() && assigned.size() == 2 && assigned.front() == 41,
"move assignment replaces an inline list without reading inactive slots");
PageOwners empty;
PageOwners empty_moved = std::move(empty);
Check(empty.empty() && empty_moved.empty(), "moving an empty owner list is safe");
} }
void TestSharedPageUnregisterLifecycle() { void TestOneMiBRegistrationGranularity() {
OwnerIndex index; static_assert(OwnerTable::kPageBits == 20);
const std::vector<OwnerIndex::ByteRange> ranges {{0x202000, 0x2000}}; OwnerTable table;
Check(index.Register(11, ranges) && index.Register(22, ranges), OwnerTable::PageRange pages {};
"two owners register on identical pages"); constexpr uint64_t range_size = 64ull * 1024 * 1024;
Check(index.CoarseMembershipCount(2) == 2 && index.TrackingMembershipCount(0x202) == 2, Check(OwnerTable::TryGetPageRange(0, range_size, pages), "large owner range is valid");
"coarse and tracking pages retain both owners"); Check(pages.first == 0 && pages.last_exclusive == 64,
"64 MiB registration touches exactly 64 one-MiB entries");
std::vector<OwnerIndex::ByteRange> releases; for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
Check(index.Unregister(11, releases), "first owner unregisters"); table[page].push_back(7);
Check(releases.empty(), "shared tracking pages are not released with one owner remaining"); }
Check(index.Query(0x202000, 1).size() == 1 && index.Query(0x202000, 1).front() == 22, Check(table.AllocatedBucketCount() == 1,
"unregistering one owner preserves the other"); "large registration uses one sparse second-level bucket");
Check(!index.Unregister(11, releases), "missing membership hard-fails without mutation"); for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
auto* owners = table.Find(page);
Check(index.Unregister(22, releases), "final owner unregisters"); Check(owners != nullptr && owners->size() == 1 && owners->front() == 7,
Check(releases.size() == 1 && releases.front().address == 0x202000 && "each touched one-MiB entry retains the owner once");
releases.front().size == 0x2000, Check(owners->Erase(7), "large owner unregisters by coarse page");
"adjacent final-owner tracking pages return one contiguous release"); }
} }
void TestStrictByteFilteringAndPredicate() { void TestSharedCoarsePageLifecycle() {
OwnerIndex index; OwnerTable table;
Check(index.Register(31, {{0x300100, 0x100}}), "first byte-disjoint owner registers"); auto& owners = table[2];
Check(index.Register(32, {{0x300800, 0x100}}), "second byte-disjoint owner registers"); owners.push_back(11);
Check(index.Register(33, {{0x30f000, 0x100}}), "coarse-only owner registers"); owners.push_back(22);
Check(index.TrackingMembershipCount(0x300) == 2, Check(owners.size() == 2, "two images share one coarse page");
"byte-disjoint owners share one tracking page"); Check(owners.Erase(11) && owners.size() == 1 && owners.front() == 22,
Check(index.CoarseMembershipCount(3) == 3, "unregistering one image preserves its coarse-page neighbor");
"all owners share one coarse candidate bucket"); Check(!owners.Erase(11), "double unregister is rejected without mutation");
Check(index.Query(0x300400, 0x40).empty(), "page hit without byte overlap is filtered out"); Check(owners.Erase(22) && owners.empty(), "final coarse-page owner unregisters");
const auto page_candidates = index.QueryCandidates(0x300400, 0x40);
Check(page_candidates.size() == 2,
"fault candidate query retains touched-page owners and rejects coarse-only owners");
const auto first = index.Query(0x300180, 0x10);
Check(first.size() == 1 && first.front() == 31,
"strict byte overlap selects only the matching owner");
const auto predicate_filtered =
index.Query(0x300000, 0x1000, [](uint32_t owner) { return owner == 32; });
Check(predicate_filtered.size() == 1 && predicate_filtered.front() == 32,
"supplied predicate filters query owners");
} }
void TestOwnerIndexAddressSpaceBoundary() { void TestOneMiBBoundaries() {
OwnerIndex index; OwnerTable::PageRange range {};
constexpr uint64_t last_byte = OwnerIndex::CoarseTable::kAddressSpaceSize - 1; Check(OwnerTable::TryGetPageRange(0x0fffff, 2, range),
Check(index.Register(41, {{last_byte, 1}}), "final guest byte registers"); "range crossing a one-MiB boundary is valid");
const auto exact = index.Query(last_byte, 1); Check(range.first == 0 && range.last_exclusive == 2,
Check(exact.size() == 1 && exact.front() == 41, "cross-boundary registration touches both coarse pages");
"strict query finds an exact overlap at the final guest byte"); Check(OwnerTable::TryGetPageRange(OwnerTable::kAddressSpaceSize - 1, 1, range),
Check(index.Query(last_byte - 1, 1).empty(), "final guest byte maps to a coarse owner page");
"strict query preserves half-open overlap boundaries"); Check(range.first == OwnerTable::kPageCount - 1 &&
const auto page_candidates = index.QueryCandidates(last_byte - 1, 1); range.last_exclusive == OwnerTable::kPageCount,
Check(page_candidates.size() == 1 && page_candidates.front() == 41, "final guest byte uses the final one-MiB page");
"page candidate query retains a byte-disjoint owner on the final tracking page");
} }
} // namespace } // namespace
@@ -156,10 +175,10 @@ int main() {
TestCrossBucketRange(); TestCrossBucketRange();
TestQueriesDoNotAllocate(); TestQueriesDoNotAllocate();
TestAddressSpaceBoundaries(); TestAddressSpaceBoundaries();
TestMultiRangeRegistrationDeduplicatesPages(); TestInlineOwnerStorageAndOverflow();
TestSharedPageUnregisterLifecycle(); TestOneMiBRegistrationGranularity();
TestStrictByteFilteringAndPredicate(); TestSharedCoarsePageLifecycle();
TestOwnerIndexAddressSpaceBoundary(); TestOneMiBBoundaries();
std::printf("ImagePageTableTests: all cases passed\n"); std::printf("ImagePageTableTests: all cases passed\n");
return 0; return 0;
} }
tests/ShaderRecompilerComputeTests.cpp
+177
View File
@@ -163,6 +163,10 @@ struct TileManagerTestAccess {
}; };
struct TextureCacheTestAccess { struct TextureCacheTestAccess {
static_assert(TextureCache::ImagePageTable::kPageBits == 20);
static_assert(TextureCache::ImagePageTable::kAddressSpaceBits == 40);
static_assert(TextureCache::ImagePageTable::kFirstLevelBits == 10);
static void ConfigureGarbageCollection(TextureCache& cache, std::span<const ImageId> oldest, static void ConfigureGarbageCollection(TextureCache& cache, std::span<const ImageId> oldest,
uint64_t tick, uint64_t pressure) { uint64_t tick, uint64_t pressure) {
cache.m_trigger_gc_memory = 0; cache.m_trigger_gc_memory = 0;
@@ -197,6 +201,75 @@ struct TextureCacheTestAccess {
return owner != nullptr && owner->registered; return owner != nullptr && owner->registered;
} }
static std::vector<ImageId> FindImages(TextureCache& cache, uint64_t address, uint64_t size,
bool page_overlap) {
std::lock_guard lock(cache.m_lock);
const auto found = cache.FindImagesInRegion(address, size, page_overlap);
std::vector<ImageId> result;
result.reserve(found.size());
for (const auto id: found) {
result.push_back(id);
}
return result;
}
static size_t PageOwnerCount(TextureCache& cache, uint64_t address) {
std::lock_guard lock(cache.m_lock);
const auto* owners = cache.m_image_page_table.Find(
static_cast<size_t>(address >> TextureCache::ImagePageTable::kPageBits));
return owners == nullptr ? 0 : owners->size();
}
static size_t OwnedPageCount(TextureCache& cache, uint64_t address, uint64_t size, ImageId id) {
std::lock_guard lock(cache.m_lock);
TextureCache::ImagePageTable::PageRange pages {};
if (!TextureCache::ImagePageTable::TryGetPageRange(address, size, pages)) {
return 0;
}
size_t count = 0;
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
const auto* owners = cache.m_image_page_table.Find(page);
count += owners != nullptr && owners->Contains(id) ? 1 : 0;
}
return count;
}
static void AddPageOwner(TextureCache& cache, uint64_t address, ImageId id) {
std::lock_guard lock(cache.m_lock);
cache.m_image_page_table[static_cast<size_t>(
address >> TextureCache::ImagePageTable::kPageBits)]
.push_back(id);
}
static bool RemovePageOwner(TextureCache& cache, uint64_t address, ImageId id) {
std::lock_guard lock(cache.m_lock);
auto* owners = cache.m_image_page_table.Find(
static_cast<size_t>(address >> TextureCache::ImagePageTable::kPageBits));
return owners != nullptr && owners->Erase(id);
}
static void SetQueryEpoch(TextureCache& cache, uint32_t epoch) {
std::lock_guard lock(cache.m_lock);
cache.m_image_query_epoch = epoch;
}
static uint32_t QueryEpoch(TextureCache& cache) {
std::lock_guard lock(cache.m_lock);
return cache.m_image_query_epoch;
}
static ImageId InsertImage(TextureCache& cache, const ImageInfo& info) {
std::lock_guard transaction(cache.m_resource_mutex);
std::lock_guard lock(cache.m_lock);
return cache.InsertImage(info);
}
static void DeleteImage(TextureCache& cache, ImageId id) {
std::lock_guard transaction(cache.m_resource_mutex);
std::lock_guard lock(cache.m_lock);
cache.DeleteImage(id);
}
static std::shared_ptr<Image> Owner(const TextureCache& cache, ImageId id) { static std::shared_ptr<Image> Owner(const TextureCache& cache, ImageId id) {
return cache.ResolveOwner(id); return cache.ResolveOwner(id);
} }
@@ -2488,6 +2561,110 @@ public:
vk::Format::eR8G8B8A8Srgb, vk::Format::eR8G8B8A8Srgb,
"registered compatible backing did not reuse one ImageId"); "registered compatible backing did not reuse one ImageId");
const auto IsOnlyImage = [](const std::vector<ImageId>& ids, ImageId expected) {
return ids.size() == 1 && ids.front() == expected;
};
const auto exact_byte_miss =
TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, false);
const auto touched_page_hit =
TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, true);
const auto next_page_miss =
TextureCacheTestAccess::FindImages(texture_cache, base + 0x1000, 1, true);
Require(name, "exact and 4-KiB page filtering",
exact_byte_miss.empty() && IsOnlyImage(touched_page_hit, first) &&
next_page_miss.empty(),
"coarse candidates did not preserve exact-byte and touched-page semantics");
const auto MakeOwnershipInfo = [](uint64_t address, uint64_t size) {
ImageInfo info {};
info.data = {address, size};
info.extent = {1, 1, 1};
info.resources = {1, 1};
info.samples = 1;
return info;
};
const auto spanning_info = MakeOwnershipInfo(base + 0x1ff000, 0x2000);
const auto spanning = TextureCacheTestAccess::InsertImage(texture_cache, spanning_info);
const auto spanning_results = TextureCacheTestAccess::FindImages(
texture_cache, spanning_info.data.address, spanning_info.data.size, false);
Require(name, "one-MiB cross-page deduplication",
spanning && IsOnlyImage(spanning_results, spanning) &&
TextureCacheTestAccess::PageOwnerCount(texture_cache,
spanning_info.data.address) == 1 &&
TextureCacheTestAccess::PageOwnerCount(
texture_cache, spanning_info.data.End() - 1) == 1,
"an image spanning two coarse pages was missing or returned more than once");
TextureCacheTestAccess::DeleteImage(texture_cache, spanning);
Require(name, "cross-page owner cleanup",
TextureCacheTestAccess::PageOwnerCount(texture_cache,
spanning_info.data.address) == 0 &&
TextureCacheTestAccess::PageOwnerCount(
texture_cache, spanning_info.data.End() - 1) == 0 &&
TextureCacheTestAccess::FindImages(texture_cache,
spanning_info.data.address,
spanning_info.data.size, false)
.empty(),
"cross-page unregister left a stale coarse-page membership");
const auto shared_info = MakeOwnershipInfo(base + 0x500100, 0x100);
const auto shared_first =
TextureCacheTestAccess::InsertImage(texture_cache, shared_info);
const auto shared_second =
TextureCacheTestAccess::InsertImage(texture_cache, shared_info);
const auto shared_results = TextureCacheTestAccess::FindImages(
texture_cache, shared_info.data.address, shared_info.data.size, false);
Require(name, "shared coarse-page registration",
shared_results.size() == 2 &&
std::find(shared_results.begin(), shared_results.end(), shared_first) !=
shared_results.end() &&
std::find(shared_results.begin(), shared_results.end(), shared_second) !=
shared_results.end(),
"two registered owners were not retained in one coarse page");
TextureCacheTestAccess::DeleteImage(texture_cache, shared_first);
const auto shared_survivor = TextureCacheTestAccess::FindImages(
texture_cache, shared_info.data.address, shared_info.data.size, false);
Require(name, "shared coarse-page unregister",
IsOnlyImage(shared_survivor, shared_second) &&
TextureCacheTestAccess::PageOwnerCount(texture_cache,
shared_info.data.address) == 1,
"unregistering one owner removed its coarse-page neighbor");
TextureCacheTestAccess::DeleteImage(texture_cache, shared_second);
Require(name, "final shared coarse-page unregister",
TextureCacheTestAccess::PageOwnerCount(texture_cache,
shared_info.data.address) == 0,
"the final shared owner remained registered");
constexpr uint64_t large_owner_size = 64ull * 1024 * 1024;
const auto large_info = MakeOwnershipInfo(base + 0x800000, large_owner_size);
const auto large_owner =
TextureCacheTestAccess::InsertImage(texture_cache, large_info);
Require(name, "production one-MiB registration granularity",
TextureCacheTestAccess::OwnedPageCount(
texture_cache, large_info.data.address, large_info.data.size, large_owner) == 64,
"64 MiB image registration did not create exactly 64 coarse memberships");
TextureCacheTestAccess::DeleteImage(texture_cache, large_owner);
Require(name, "production one-MiB unregister granularity",
TextureCacheTestAccess::OwnedPageCount(
texture_cache, large_info.data.address, large_info.data.size, large_owner) == 0,
"large image unregister left coarse memberships behind");
const ImageId stale {first.index, first.generation + 1};
TextureCacheTestAccess::AddPageOwner(texture_cache, base, stale);
const auto stale_filtered =
TextureCacheTestAccess::FindImages(texture_cache, base, sizeof(initial), false);
Require(name, "stale page owner filtering",
IsOnlyImage(stale_filtered, first) &&
TextureCacheTestAccess::RemovePageOwner(texture_cache, base, stale),
"a stale generation escaped the direct page-owner lookup");
TextureCacheTestAccess::SetQueryEpoch(texture_cache, UINT32_MAX);
const auto wrap_results =
TextureCacheTestAccess::FindImages(texture_cache, base, sizeof(initial), false);
Require(name, "page-owner query epoch wrap",
IsOnlyImage(wrap_results, first) &&
TextureCacheTestAccess::QueryEpoch(texture_cache) == 1,
"query deduplication failed while wrapping its epoch");
auto& image = texture_cache.GetImage(first); auto& image = texture_cache.GetImage(first);
constexpr uint32_t final_sampled_value = 0x88776655u; constexpr uint32_t final_sampled_value = 0x88776655u;
Require(name, "sampled write between discovery and acquisition", Require(name, "sampled write between discovery and acquisition",