H3cJP/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-08-03 11:23:49 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Optimize bulk memory invalidation (#124)
Build and Release KytyPS5 / Build KytyPS5 (Windows) (push) Canceled after 0s
Details
Build and Release KytyPS5 / Build KytyPS5 (macOS) (push) Canceled after 0s
Details
Build and Release KytyPS5 / Build KytyPS5 (Linux) (push) Canceled after 0s
Details
Build and Release KytyPS5 / Release KytyPS5 (push) Canceled after 0s
Details

Browse Source 
* Per page -> per range search (optimization)
This commit is contained in:
nmzik
2026-07-29 05:09:36 +02:00
committed by GitHub
parent 861729fc6c
commit f6e01e5403
16 changed files with 322 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/host_gpu/memoryTracker.h
+43 -2
View File
@@ -38,10 +38,51 @@ public:
bool downloaded) noexcept;
[[nodiscard]] bool InvalidateRegion(uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept;
template <typename Flush>
void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) {
static_assert(std::is_invocable_v<Flush&>);
CheckNotInUploadCallback();
ValidateRange(vaddr, size);
const auto update_cpu_state = [this, vaddr, size] {
std::lock_guard access(m_access_mutex);
std::vector<RegionManager*> managers;
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
managers.push_back(manager);
});
std::vector<std::unique_lock<TrackingSpinLock>> locks;
locks.reserve(managers.size());
for (auto* manager: managers) {
locks.emplace_back(manager->lock);
}
const bool gpu_modified = Iterate<false>(
vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
return manager->IsModified<DirtySource::Gpu>(offset, bytes);
});
if (gpu_modified) {
return true;
}
Iterate<false>(vaddr, size,
[](RegionManager* manager, uint64_t offset, uint64_t bytes) {
const auto changed = manager->ChangeState<DirtySource::Cpu, true>(
manager->GetCpuAddr() + offset, bytes);
manager->ApplyProtection(changed, false);
});
return false;
};
if (!update_cpu_state()) {
return;
}
std::forward<Flush>(on_flush)();
if (update_cpu_state()) {
EXIT("memory invalidation retained GPU-owned pages\n");
}
}
[[nodiscard]] bool InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr,
uint64_t size, PageFaultPhase phase) noexcept;
void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
const char* operation) const noexcept;
void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
const char* operation) const noexcept;
void ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
const char* operation);
src/graphics/host_gpu/pageManager.cpp
-38
View File
@@ -699,26 +699,6 @@ bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
return true;
}
bool PageManager::HasAnyMapping(uint64_t vaddr, uint64_t size) const noexcept {
if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE ||
size > ADDRESS_SIZE - vaddr) {
return false;
}
const auto end = PageEnd(vaddr, size);
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
auto* region = m_impl->FindRegion(page_vaddr);
if (region == nullptr) {
continue;
}
auto& page = m_impl->GetPage(*region, page_vaddr);
SpinGuard lock(page.lock);
if (page.mappings != 0) {
return true;
}
}
return false;
}
bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept {
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
FailFast("HasGpuAccess received an invalid GPU access mode");
@@ -1150,22 +1130,4 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex
return true;
}
bool PageManager::HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept {
if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE ||
size > ADDRESS_SIZE - vaddr) {
return false;
}
const auto end = PageEnd(vaddr, size);
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
if (!IsMapped(page_vaddr, 1)) {
continue;
}
const auto fault_vaddr = std::max(page_vaddr, vaddr);
if (!HandleFault(PageFaultAccess::Write, fault_vaddr)) {
return false;
}
}
return true;
}
} // namespace Libs::Graphics
src/graphics/host_gpu/pageManager.h
+1 -3
View File
@@ -41,7 +41,6 @@ public:
[[nodiscard]] uint64_t GetPageSize() const;
[[nodiscard]] bool IsTracked(uint64_t vaddr) const noexcept;
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
[[nodiscard]] bool HasAnyMapping(uint64_t vaddr, uint64_t size) const noexcept;
[[nodiscard]] bool HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept;
void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
@@ -50,9 +49,8 @@ public:
void OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
[[nodiscard]] bool HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept;
[[nodiscard]] std::vector<std::unique_ptr<BackingWrite>>
ReserveBackingWrites(std::span<const RangeSet::Range> ranges);
ReserveBackingWrites(std::span<const RangeSet::Range> ranges);
private:
void BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept;
src/graphics/host_gpu/rangeSet.h
+10
View File
@@ -59,6 +59,16 @@ public:
return result;
}
[[nodiscard]] bool Contains(uint64_t address, uint64_t size) const {
const auto end = End(address, size);
auto it = m_ranges.upper_bound(address);
if (it == m_ranges.begin()) {
return false;
}
--it;
return it->first <= address && it->second >= end;
}
template <typename Func>
void ForEachIntersection(uint64_t address, uint64_t size, Func&& func) const {
const auto end = End(address, size);
src/graphics/host_gpu/renderer/cache/bufferCache.cpp
Vendored
+131 -72
View File
@@ -4,10 +4,10 @@
#include "common/logging/log.h"
#include "common/profiler.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/renderer/render.h"
#include "kernel/memory.h"
#include <algorithm>
@@ -183,9 +183,8 @@ BufferCache::RecordDownloads(std::span<const DownloadCopy> copies) {
return {};
}
auto& download = m_download_buffer;
const auto [mapped, base_offset] =
download.Map(reservation_size, DOWNLOAD_ALIGNMENT);
auto& download = m_download_buffer;
const auto [mapped, base_offset] = download.Map(reservation_size, DOWNLOAD_ALIGNMENT);
if (mapped == nullptr) {
EXIT("BufferCache: download batch could not reserve the shared stream\n");
}
@@ -215,40 +214,38 @@ void BufferCache::PublishDownloads(std::span<const DownloadRange> downloads) {
}
}
void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies,
RetiredBuffer retire) {
void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire) {
if (copies.empty()) {
return;
}
auto downloads = RecordDownloads(copies);
const auto tick = m_scheduler.CurrentTick();
auto downloads = RecordDownloads(copies);
const auto tick = m_scheduler.CurrentTick();
BeginBackingPublication(retire.address, retire.size, tick);
m_scheduler.DeferOperation(
[this, downloads = std::move(downloads), retire = std::move(retire), tick]() mutable {
PublishDownloads(downloads);
{
FaultSafeCacheLock lock(this, m_mutex);
if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
m_memory_tracker.ForEachDownloadRange<true>(
retire.address, retire.size,
[&](uint64_t address, uint64_t size) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(
m_gpu_modified_ranges, address, size,
"asynchronous garbage retirement");
},
[](uint64_t, uint64_t) noexcept {});
}
for (const auto& range: downloads) {
m_gpu_modified_ranges.Subtract(range.address, range.size);
}
if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
!m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
}
m_memory_tracker.UntrackMemory(retire.address, retire.size);
}
CompleteBackingPublication(retire.address, retire.size, tick);
});
m_scheduler.DeferOperation([this, downloads = std::move(downloads), retire = std::move(retire),
tick]() mutable {
PublishDownloads(downloads);
{
FaultSafeCacheLock lock(this, m_mutex);
if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
m_memory_tracker.ForEachDownloadRange<true>(
retire.address, retire.size,
[&](uint64_t address, uint64_t size) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
"asynchronous garbage retirement");
},
[](uint64_t, uint64_t) noexcept {});
}
for (const auto& range: downloads) {
m_gpu_modified_ranges.Subtract(range.address, range.size);
}
if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
!m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
}
m_memory_tracker.UntrackMemory(retire.address, retire.size);
}
CompleteBackingPublication(retire.address, retire.size, tick);
});
}
BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
@@ -301,14 +298,13 @@ BufferCache::~BufferCache() {
bool BufferCache::SynchronizeBacking(uint64_t vaddr, uint64_t size) {
bool waited = false;
for (;;) {
uint64_t tick = 0;
uint64_t tick = 0;
const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
const auto page_end =
(vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
const auto page_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
CacheRange affected {.address = page_begin, .size = page_end - page_begin};
{
FaultSafeCacheLock lock(this, m_mutex);
bool changed = true;
bool changed = true;
while (changed) {
changed = false;
for (const auto& [address, cached]: m_buffers) {
@@ -384,6 +380,72 @@ BufferBinding BufferCache::UploadTransient(const void* data, uint64_t size, uint
return {owner, owner->Handle(), 0};
}
void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("BufferCache: invalid memory-invalidation range\n");
}
(void)SynchronizeBacking(vaddr, size);
if (!HasPageOverlap(vaddr, size)) {
return;
}
m_memory_tracker.InvalidateRegion(vaddr, size,
[this, vaddr, size] { ReadMemory(vaddr, size); });
}
void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
std::vector<DownloadCopy> copies;
{
FaultSafeCacheLock lock(this, m_mutex);
m_memory_tracker.ForEachDownloadRange<false>(
vaddr, size,
[&](uint64_t address, uint64_t bytes) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
"memory invalidation");
},
[&](uint64_t address, uint64_t bytes) noexcept {
for (const auto range: m_gpu_modified_ranges.Intersections(address, bytes)) {
for (uint64_t copied = 0; copied < range.size;) {
const auto copy_address = range.address + copied;
auto owner = m_buffers.upper_bound(copy_address);
if (owner == m_buffers.begin()) {
EXIT("BufferCache: invalidation readback has no buffer owner\n");
}
auto& cached = *std::prev(owner)->second;
if (!cached.buffer->IsInBounds(copy_address, 1)) {
EXIT(
"BufferCache: invalidation readback is outside its buffer owner\n");
}
const auto copy_size = std::min(range.size - copied,
cached.vaddr + cached.size - copy_address);
copies.push_back({cached.buffer, cached.buffer->Offset(copy_address),
copy_address, copy_size});
copied += copy_size;
}
}
});
}
if (copies.empty()) {
EXIT("BufferCache: GPU-owned invalidation has no dirty byte ranges\n");
}
auto downloads = RecordDownloads(copies);
m_scheduler.FinishCurrent();
PublishDownloads(downloads);
{
FaultSafeCacheLock lock(this, m_mutex);
m_memory_tracker.ForEachDownloadRange<true>(
vaddr, size,
[&](uint64_t address, uint64_t bytes) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
"memory invalidation completion");
},
[](uint64_t, uint64_t) noexcept {});
for (const auto& range: downloads) {
m_gpu_modified_ranges.Subtract(range.address, range.size);
}
}
}
bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept {
const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1);
@@ -544,8 +606,8 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
m_memory_tracker.ForEachDownloadRange<true>(
begin, bytes,
[&](uint64_t address, uint64_t download_size) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(
m_gpu_modified_ranges, address, download_size, "unmap retirement");
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address,
download_size, "unmap retirement");
},
[](uint64_t, uint64_t) noexcept {});
}
@@ -722,12 +784,11 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
{
FaultSafeCacheLock lock(this, m_mutex);
const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size);
const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size);
const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size);
const bool invalidated =
!m_image_invalidated_ranges.Intersections(vaddr, size).empty();
const bool requested_gpu_owned = !dirty.empty();
const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size);
const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size);
const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size);
const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
const bool requested_gpu_owned = !dirty.empty();
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
"image source");
@@ -807,8 +868,8 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
}
FaultSafeCacheLock lock(this, m_mutex);
const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size);
const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size);
const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
const bool requested_gpu_owned = !dirty.empty();
auto owner = find_owner();
if (requested_gpu_owned && owner == m_buffers.end()) {
@@ -831,9 +892,8 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
[&]() noexcept {
for (const auto& [address, upload_size]: uploads) {
cached.buffer->CopyFrom(
m_scheduler.Current(), m_staging_buffer,
stage_offset + address - stage_address, cached.buffer->Offset(address),
upload_size, vk::AccessFlagBits::eHostWrite);
m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address,
cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
}
});
DiscardGpuDirtyBytesLocked(vaddr, size, "staged image source transfer");
@@ -917,9 +977,8 @@ std::pair<std::shared_ptr<Buffer>, uint64_t> BufferCache::ObtainBufferForImageWr
[&]() noexcept {
for (const auto& [address, upload_size]: uploads) {
cached.buffer->CopyFrom(
m_scheduler.Current(), m_staging_buffer,
stage_offset + address - stage_address, cached.buffer->Offset(address),
upload_size, vk::AccessFlagBits::eHostWrite);
m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address,
cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
}
});
return {cached.buffer, cached.buffer->Offset(vaddr)};
@@ -946,12 +1005,12 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
const auto region = m_texture_cache.QueryRegion(vaddr, size);
if (!HasGpuDirtyBytes(vaddr, size) && !region.gpu_image_bytes) {
if (region.image_bytes) {
m_texture_cache.PrepareHostWrite(vaddr, size);
m_texture_cache.InvalidateMemory(vaddr, size);
}
std::array<uint32_t, 4096> values;
values.fill(value);
const std::span<const uint8_t> bytes {
reinterpret_cast<const uint8_t*>(values.data()), sizeof(values)};
const std::span<const uint8_t> bytes {reinterpret_cast<const uint8_t*>(values.data()),
sizeof(values)};
for (uint64_t offset = 0; offset < size;) {
const auto chunk = std::min<uint64_t>(size - offset, bytes.size());
WriteHostMemory(vaddr + offset, bytes.first(chunk));
@@ -992,7 +1051,7 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
if (src_memory) {
(void)SynchronizeBacking(src_vaddr, size);
}
const auto src_region =
const auto src_region =
src_memory ? m_texture_cache.QueryRegion(src_vaddr, size) : TextureCache::RegionInfo {};
const auto dst_region =
dst_memory ? m_texture_cache.QueryRegion(dst_vaddr, size) : TextureCache::RegionInfo {};
@@ -1004,7 +1063,7 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
!HasGpuDirtyBytes(dst_vaddr, size) && !src_region.gpu_image_bytes &&
!dst_region.gpu_image_bytes) {
if (dst_region.image_bytes) {
m_texture_cache.PrepareHostWrite(dst_vaddr, size);
m_texture_cache.InvalidateMemory(dst_vaddr, size);
}
std::array<uint8_t, 64 * 1024> bytes;
for (uint64_t offset = 0; offset < size;) {
@@ -1037,10 +1096,10 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
EXIT("BufferCache: resolved Vulkan copy ranges overlap\n");
}
auto& source = src.owner != nullptr ? *std::static_pointer_cast<Buffer>(src.owner)
: src_gds ? m_gds_buffer
: m_stream_buffer;
auto& destination = dst.owner != nullptr ? *std::static_pointer_cast<Buffer>(dst.owner)
: m_gds_buffer;
: src_gds ? m_gds_buffer
: m_stream_buffer;
auto& destination =
dst.owner != nullptr ? *std::static_pointer_cast<Buffer>(dst.owner) : m_gds_buffer;
if (source.Handle() != src.buffer || destination.Handle() != dst.buffer) {
EXIT("BufferCache: resolved copy owner does not match its Vulkan handle\n");
}
@@ -1107,7 +1166,7 @@ void BufferCache::BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_
void BufferCache::CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) {
std::lock_guard lock(m_publication_mutex);
const auto publication =
const auto publication =
std::ranges::find_if(m_pending_backing_publications, [&](const auto& pending) {
return pending.address == vaddr && pending.size == size && pending.tick == tick;
});
@@ -1170,7 +1229,7 @@ void BufferCache::RunGarbageCollector() {
const uint64_t age = std::min<uint64_t>(aggressive ? 80 : 160, tick);
const size_t limit = aggressive ? 64 : 32;
std::vector<RetiredBuffer> retires;
std::vector<RetiredBuffer> retires;
std::vector<std::pair<RetiredBuffer, std::vector<DownloadCopy>>> dirty_retires;
{
FaultSafeCacheLock lock(this, m_mutex);
@@ -1190,8 +1249,8 @@ void BufferCache::RunGarbageCollector() {
}
for (const auto address: candidates) {
auto& cached = *m_buffers.at(address);
m_memory_tracker.ValidateGpuDirtyOwnership(
m_gpu_modified_ranges, cached.vaddr, cached.size, "garbage collection");
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, cached.vaddr,
cached.size, "garbage collection");
retires.push_back({address, cached.size, cached.buffer});
// GC runs immediately before submission. Preserve every source referenced by commands
// already recorded in the active batch.
@@ -1205,13 +1264,13 @@ void BufferCache::RunGarbageCollector() {
m_memory_tracker.ForEachDownloadRange<false>(
retire.address, retire.size,
[&](uint64_t address, uint64_t size) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(
m_gpu_modified_ranges, address, size, "garbage collection");
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
"garbage collection");
},
[&](uint64_t address, uint64_t size) noexcept {
for (const auto range: m_gpu_modified_ranges.Intersections(address, size)) {
copies.push_back({retire.owner, range.address - retire.address, range.address,
range.size});
copies.push_back({retire.owner, range.address - retire.address,
range.address, range.size});
}
});
}
src/graphics/host_gpu/renderer/cache/bufferCache.h
Vendored
+11 -11
View File
@@ -49,6 +49,7 @@ public:
[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept;
void InvalidateMemory(uint64_t vaddr, uint64_t size);
void UnmapMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
bool is_written = false, bool is_read = true,
@@ -71,8 +72,8 @@ public:
[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
void InvalidateImageAliases(uint64_t vaddr, uint64_t size);
void BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
void BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
[[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size);
void PublishImageBuffer(uint64_t vaddr, uint64_t size);
void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const;
@@ -91,27 +92,26 @@ private:
struct RetiredBuffer;
struct FaultReadback;
struct PendingBackingPublication;
static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64;
[[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept;
[[nodiscard]] static uint64_t AlignUp(uint64_t value);
static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64;
[[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept;
[[nodiscard]] static uint64_t AlignUp(uint64_t value);
[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
}
[[nodiscard]] static bool PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right,
uint64_t right_size) noexcept;
[[nodiscard]] static std::pair<uint64_t, uint64_t>
DownloadEnvelope(const DownloadCopy& copy);
[[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
uint64_t right_size) noexcept;
[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
[[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset,
const void* source, uint64_t size);
[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
uint64_t size);
[[nodiscard]] std::vector<DownloadRange>
RecordDownloads(std::span<const DownloadCopy> copies);
[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
void PublishDownloads(std::span<const DownloadRange> downloads);
void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
void RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, uint64_t vaddr,
uint64_t size, bool upload);
void ReadMemory(uint64_t vaddr, uint64_t size);
void DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation);
void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp
Vendored
+24 -17
View File
@@ -68,47 +68,52 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
return handled;
}
void GpuResourceManager::PrepareHostWrite(uint64_t vaddr, uint64_t size) {
if (!m_page_manager.HasAnyMapping(vaddr, size)) {
return;
bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) {
if (!IsMapped(vaddr, size)) {
return false;
}
if (CommandScheduler::InDeferredOperation()) {
EXIT("unsupported host write from an asynchronous GPU completion, addr=0x%016" PRIx64
" size=0x%016" PRIx64 "\n",
EXIT("unsupported memory invalidation from an asynchronous GPU completion, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
const auto handle_range = [this, vaddr, size] {
if (!m_page_manager.HandleWriteRange(vaddr, size)) {
EXIT("failed to prepare host write, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
};
const auto resolve = [this, &handle_range](CommandProcessor& cp) {
const auto resolve = [this, vaddr, size](CommandProcessor& cp) {
cp.BeginReadbackTransaction();
{
ResourceMutex::FaultScope fault(m_resource_mutex);
handle_range();
m_buffer_cache.InvalidateMemory(vaddr, size);
m_texture_cache.InvalidateMemory(vaddr, size);
}
cp.EndReadbackTransaction();
};
if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
resolve(*cp);
return;
return true;
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported host write from a pre-owned resource transaction, addr=0x%016" PRIx64
" size=0x%016" PRIx64 "\n",
EXIT("unsupported memory invalidation from a pre-owned resource transaction, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
EXIT_IF(m_gpu == nullptr);
m_gpu->SendCommandSyncWithProcessor(resolve);
return true;
}
bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
return m_page_manager.IsMapped(vaddr, size);
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
return false;
}
std::shared_lock lock(m_mapped_ranges_mutex);
return m_mapped_ranges.Contains(vaddr, size);
}
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
{
std::lock_guard lock(m_mapped_ranges_mutex);
m_mapped_ranges.Add(vaddr, size);
}
m_page_manager.OnGpuMap(vaddr, size, access);
}
@@ -120,6 +125,8 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess ac
m_texture_cache.UnmapMemory(vaddr, size);
m_buffer_cache.UnmapMemory(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size, access);
std::lock_guard lock(m_mapped_ranges_mutex);
m_mapped_ranges.Subtract(vaddr, size);
};
if (m_gpu == nullptr) {
if (m_resource_mutex.IsOwnedByCurrentThread()) {
src/graphics/host_gpu/renderer/cache/gpuResourceManager.h
Vendored
+9 -6
View File
@@ -9,6 +9,7 @@
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include <cstdint>
#include <shared_mutex>
namespace Libs::Graphics {
@@ -26,7 +27,7 @@ public:
void SetGpu(Gpu* gpu) noexcept { m_gpu = gpu; }
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
void PrepareHostWrite(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
@@ -38,11 +39,13 @@ private:
[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept;
PageManager m_page_manager;
ResourceMutex m_resource_mutex;
BufferCache m_buffer_cache;
TextureCache m_texture_cache;
Gpu* m_gpu = nullptr;
PageManager m_page_manager;
ResourceMutex m_resource_mutex;
BufferCache m_buffer_cache;
TextureCache m_texture_cache;
mutable std::shared_mutex m_mapped_ranges_mutex;
RangeSet m_mapped_ranges;
Gpu* m_gpu = nullptr;
};
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/textureCache.cpp
Vendored
+2 -2
View File
@@ -1439,9 +1439,9 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
return true;
}
void TextureCache::PrepareHostWrite(uint64_t address, uint64_t size) {
void TextureCache::InvalidateMemory(uint64_t address, uint64_t size) {
if (!GuestRange {address, size}.Valid()) {
EXIT("TextureCache: invalid host-write range\n");
EXIT("TextureCache: invalid memory-invalidation range\n");
}
CacheLock lock(*this, m_lock);
InvalidateCpuAliases(address, size);
src/graphics/host_gpu/renderer/cache/textureCache.h
Vendored
+1 -1
View File
@@ -65,7 +65,7 @@ public:
[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
uint32_t packed_clear);
void PrepareHostWrite(uint64_t address, uint64_t size);
void InvalidateMemory(uint64_t address, uint64_t size);
[[nodiscard]] bool SynchronizeImageToBuffer(uint64_t address, uint64_t size);
[[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
bool formatted_buffer_write = false);
src/kernel/fileSystem.cpp
+14 -14
View File
@@ -238,8 +238,9 @@ static std::filesystem::path ResolvePathIgnoringCase(const std::filesystem::path
}
// Preserve unmatched components for the caller's ENOENT path.
std::filesystem::path resolved = path.has_root_path() ? path.root_path() : std::filesystem::path(".");
bool matched = true;
std::filesystem::path resolved =
path.has_root_path() ? path.root_path() : std::filesystem::path(".");
bool matched = true;
for (const auto& component: path.relative_path()) {
if (component.empty()) {
@@ -568,11 +569,11 @@ int64_t KYTY_SYSV_ABI KernelRead(int d, void* buf, size_t nbytes) {
file->mutex.Lock();
bool is_invalid = file->f.IsInvalid();
const auto pos = file->f.Tell();
const auto file_size = file->f.Size();
const auto remaining = pos < file_size ? file_size - pos : 0;
Memory::PrepareHostWrite(reinterpret_cast<uint64_t>(buf),
bool is_invalid = file->f.IsInvalid();
const auto pos = file->f.Tell();
const auto file_size = file->f.Size();
const auto remaining = pos < file_size ? file_size - pos : 0;
Memory::InvalidateMemory(reinterpret_cast<uint64_t>(buf),
std::min<uint64_t>(nbytes, remaining));
uint32_t bytes_read = 0;
file->f.Read(buf, static_cast<uint32_t>(nbytes), &bytes_read);
@@ -692,13 +693,12 @@ int64_t KYTY_SYSV_ABI KernelPread(int d, void* buf, size_t nbytes, int64_t offse
file->mutex.Lock();
bool is_invalid = file->f.IsInvalid();
auto pos = file->f.Tell();
const auto file_size = file->f.Size();
const auto remaining = static_cast<uint64_t>(offset) < file_size
? file_size - static_cast<uint64_t>(offset)
: 0;
Memory::PrepareHostWrite(reinterpret_cast<uint64_t>(buf),
bool is_invalid = file->f.IsInvalid();
auto pos = file->f.Tell();
const auto file_size = file->f.Size();
const auto remaining =
static_cast<uint64_t>(offset) < file_size ? file_size - static_cast<uint64_t>(offset) : 0;
Memory::InvalidateMemory(reinterpret_cast<uint64_t>(buf),
std::min<uint64_t>(nbytes, remaining));
uint32_t bytes_read = 0;
file->f.Seek(offset);
src/kernel/memory.cpp
+23 -23
View File
@@ -66,8 +66,8 @@ constexpr int PAGE_TABLE_POOL_ENTRIES =
static_cast<int>(PAGE_TABLE_POOL_SIZE / PAGE_TABLE_GRANULARITY);
constexpr uint64_t DEFAULT_FLEXIBLE_MEMORY_SIZE = 4ull * 1024ull * 1024ull * 1024ull;
static uint64_t g_flexible_memory_size = DEFAULT_FLEXIBLE_MEMORY_SIZE;
static Graphics::GpuResourceManager* g_gpu_resources = nullptr;
static uint64_t g_flexible_memory_size = DEFAULT_FLEXIBLE_MEMORY_SIZE;
static Graphics::GpuResourceManager* g_gpu_resources = nullptr;
static Graphics::GpuResourceManager& GetGpuResources() {
EXIT_IF(g_gpu_resources == nullptr);
@@ -892,11 +892,11 @@ void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept {
}
}
void PrepareHostWrite(uint64_t vaddr, uint64_t size) {
void InvalidateMemory(uint64_t vaddr, uint64_t size) {
if (size == 0) {
return;
}
GetGpuResources().PrepareHostWrite(vaddr, size);
(void)GetGpuResources().InvalidateMemory(vaddr, size);
}
void InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept {
@@ -1915,24 +1915,24 @@ int32_t KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t le
EXIT_NOT_IMPLEMENTED(addr_in_out == nullptr);
constexpr size_t PAGE_SIZE = 0x4000;
constexpr size_t MAXIMUM_NAME_SIZE = 32;
constexpr uint64_t DEFAULT_PS5_BASE = 0x200000000;
constexpr int GUEST_MAP_FIXED = 0x10;
constexpr int GUEST_MAP_SHARED = 0x01;
constexpr int GUEST_MAP_PRIVATE = 0x02;
constexpr int GUEST_MAP_NO_OVERWRITE = 0x80;
constexpr int GUEST_MAP_VOID = 0x100;
constexpr int GUEST_MAP_STACK = 0x400;
constexpr int GUEST_MAP_NO_SYNC = 0x800;
constexpr int GUEST_MAP_ANON = 0x1000;
constexpr int GUEST_MAP_UNKNOWN_8000 = 0x8000;
constexpr int GUEST_MAP_NO_CORE = 0x20000;
constexpr int GUEST_MAP_NO_COALESCE = 0x400000;
constexpr int SUPPORTED_MAP_BITS =
GUEST_MAP_SHARED | GUEST_MAP_PRIVATE | GUEST_MAP_FIXED | GUEST_MAP_NO_OVERWRITE |
GUEST_MAP_VOID | GUEST_MAP_STACK | GUEST_MAP_NO_SYNC | GUEST_MAP_ANON |
GUEST_MAP_UNKNOWN_8000 | GUEST_MAP_NO_CORE | GUEST_MAP_NO_COALESCE;
constexpr size_t PAGE_SIZE = 0x4000;
constexpr size_t MAXIMUM_NAME_SIZE = 32;
constexpr uint64_t DEFAULT_PS5_BASE = 0x200000000;
constexpr int GUEST_MAP_FIXED = 0x10;
constexpr int GUEST_MAP_SHARED = 0x01;
constexpr int GUEST_MAP_PRIVATE = 0x02;
constexpr int GUEST_MAP_NO_OVERWRITE = 0x80;
constexpr int GUEST_MAP_VOID = 0x100;
constexpr int GUEST_MAP_STACK = 0x400;
constexpr int GUEST_MAP_NO_SYNC = 0x800;
constexpr int GUEST_MAP_ANON = 0x1000;
constexpr int GUEST_MAP_UNKNOWN_8000 = 0x8000;
constexpr int GUEST_MAP_NO_CORE = 0x20000;
constexpr int GUEST_MAP_NO_COALESCE = 0x400000;
constexpr int SUPPORTED_MAP_BITS = GUEST_MAP_SHARED | GUEST_MAP_PRIVATE | GUEST_MAP_FIXED |
GUEST_MAP_NO_OVERWRITE | GUEST_MAP_VOID | GUEST_MAP_STACK |
GUEST_MAP_NO_SYNC | GUEST_MAP_ANON | GUEST_MAP_UNKNOWN_8000 |
GUEST_MAP_NO_CORE | GUEST_MAP_NO_COALESCE;
if (len == 0 || (len & (PAGE_SIZE - 1)) != 0) {
return KERNEL_ERROR_EINVAL;
@@ -3294,7 +3294,7 @@ int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags,
"\t alignment = 0x%016" PRIx64 "\n",
in_addr, len, flags, alignment);
constexpr size_t PAGE_SIZE = 0x4000;
constexpr size_t PAGE_SIZE = 0x4000;
constexpr int GUEST_MAP_FIXED = 0x10;
constexpr int GUEST_MAP_NO_OVERWRITE = 0x80;
src/kernel/memory.h
+7 -7
View File
@@ -99,13 +99,13 @@ struct KernelMemoryPoolBlockStats {
static_assert(sizeof(KernelMemoryPoolBlockStats) == 16,
"KernelMemoryPoolBlockStats struct size is incorrect");
void RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func);
void SetFlexibleMemorySize(uint64_t size);
bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size);
bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept;
void PrepareHostWrite(uint64_t vaddr, uint64_t size);
void InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept;
void RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func);
void SetFlexibleMemorySize(uint64_t size);
bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size);
bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept;
void InvalidateMemory(uint64_t vaddr, uint64_t size);
void InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept;
[[nodiscard]] bool HandleGpuFault(Graphics::PageFaultAccess access, uint64_t fault_vaddr) noexcept;
int KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags,
tests/MemoryTrackerTests.cpp
+43
View File
@@ -669,6 +669,8 @@ void TestRangeSet() {
ranges.Add(0x1000, 0x80);
ranges.Add(0x1080, 0x80);
ranges.Add(0x1200, 0x40);
Check(ranges.Contains(0x1010, 0xe0) && !ranges.Contains(0x1010, 0x200),
"range set containment did not require full coverage");
auto intersections = ranges.Intersections(0x1070, 0x1b0);
Check(intersections.size() == 2 && intersections[0].address == 0x1070 &&
intersections[0].size == 0x90 && intersections[1].address == 0x1200 &&
@@ -682,6 +684,46 @@ void TestRangeSet() {
"range set subtraction did not preserve both exact tails");
}
void TestRangeInvalidation() {
constexpr uintptr_t base = 0x0000000201000000ull;
TrackerHarness harness;
auto &tracker = harness.tracker;
auto &page_manager = harness.page_manager;
constexpr uint64_t size = Libs::Graphics::TRACKER_REGION_SIZE * 2;
auto *memory = static_cast<uint8_t *>(
VirtualAlloc(reinterpret_cast<void *>(base), size, MEM_RESERVE | MEM_COMMIT,
PAGE_READWRITE));
Check(memory == reinterpret_cast<void *>(base),
"range invalidation allocation failed");
const auto address = reinterpret_cast<uint64_t>(memory);
page_manager.OnGpuMap(address, size);
tracker.ForEachUploadRange(
address, size, true, [](uint64_t, uint64_t) noexcept {},
[]() noexcept {});
Check(tracker.IsRegionGpuModified(address, size) && !IsWritable(memory),
"range invalidation setup did not establish GPU ownership");
uint32_t flushes = 0;
tracker.InvalidateRegion(address + 16, size - 32, [&] {
flushes++;
tracker.ForEachDownloadRange<true>(
address + 16, size - 32, [](uint64_t, uint64_t) noexcept {});
});
Check(flushes == 1 && !tracker.IsRegionGpuModified(address, size) &&
tracker.IsRegionCpuModified(address, size) && IsWritable(memory) &&
IsWritable(memory + size - 1),
"range invalidation did not batch ownership transfer across regions");
tracker.InvalidateRegion(address + 16, size - 32, [&] { flushes++; });
Check(flushes == 1,
"clean range invalidation unnecessarily requested a GPU flush");
tracker.UntrackMemory(address, size);
page_manager.OnGpuUnmap(address, size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0,
"range invalidation VirtualFree failed");
}
void TestCpuDirtyUploadAndFault() {
constexpr uintptr_t base = 0x0000000200010000ull;
TrackerHarness harness;
@@ -1207,6 +1249,7 @@ int main(int argc, char **argv) {
TestSameSlabTrackerArbitration();
TestSharedMetadataAndImagePageFault();
TestRangeSet();
TestRangeInvalidation();
TestGpuDirtyBits();
TestCrossRegionUpload();
TestFaultDuringUploadRemainsDirty();
tests/PageManagerTests.cpp
-31
View File
@@ -301,36 +301,6 @@ void TestSharedWatcherFault() {
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
void TestMappedHostWriteRange() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize();
auto *memory = Allocate(page_size * 3);
const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size);
manager.OnGpuMap(address + page_size * 2, page_size);
manager.UpdatePageWatchers(true, address, page_size);
manager.UpdatePageWatchers(true, address + page_size * 2, page_size);
Check(manager.HasAnyMapping(address + 16, page_size * 3 - 32),
"host-write range did not find partial GPU mappings");
Check(!manager.IsMapped(address, page_size * 3),
"partial GPU mappings were reported as a full mapping");
Check(manager.HandleWriteRange(address + 16, page_size * 3 - 32),
"mapped host-write range was not handled");
Check(context.calls.load(std::memory_order_relaxed) == 2,
"host-write range did not invalidate each mapped watched page");
Check(IsWritable(memory) && IsWritable(memory + page_size * 2),
"host-write range did not restore writable protection");
manager.OnGpuUnmap(address, page_size);
manager.OnGpuUnmap(address + page_size * 2, page_size);
Check(!manager.HasAnyMapping(address, page_size * 3),
"host-write range retained stale GPU mappings");
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
void TestReadWriteWatcherFault() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
@@ -855,7 +825,6 @@ int main(int argc, char **argv) {
}
TestWatchFaultAndUnwatch();
TestSharedWatcherFault();
TestMappedHostWriteRange();
TestReadWriteWatcherFault();
TestPermittedMappedLateFaultsResume();
TestPartialMappingUnmapPreservesTokens();
tests/ShaderRecompilerComputeTests.cpp
+3 -1
View File
@@ -1679,7 +1679,9 @@ public:
Require("GpuCommandLane", "processor fault context",
Gpu::CurrentCommandProcessor() == &processor,
"processor resource test lost its command context");
resources.PrepareHostWrite(fault_base, sizeof(uint32_t));
Require("GpuCommandLane", "processor memory invalidation",
resources.InvalidateMemory(fault_base, sizeof(uint32_t)),
"processor memory invalidation did not find its mapped range");
});
resources.UnmapMemory(fault_base, fault_size, GpuAccess::ReadWrite);
Require("GpuCommandLane", "processor fault unmap",