H3cJP/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-08-19 06:52:36 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

perf: sync only GPU-dirty readbacks

Browse Source
This commit is contained in:
nmzik
2026-08-04 02:26:50 +02:00
parent e8752aca7e
commit 51d0e02543
13 changed files with 373 additions and 390 deletions
Download Patch File Download Diff File Expand all files Collapse all files
3rdparty/tracy
Vendored
+1 -1
Submodule 3rdparty/tracy updated: 00c079cec9...05cceee0df
src/graphics/guest_gpu/command_processor/commandProcessor.h
+7 -21
View File
@@ -50,23 +50,10 @@ public:
void Reset();
void BufferInit();
void BufferFlush();
void BufferFlushAndWait();
void BufferWait();
void BeginReadbackTransaction() {
if (m_readback_active) {
EXIT("nested command-processor readback transaction\n");
}
m_readback_active = true;
}
void EndReadbackTransaction() {
if (!m_readback_active) {
EXIT("command-processor readback transaction is not active\n");
}
m_readback_active = false;
}
void BufferInit();
void BufferFlush();
void BufferFlushAndWait();
void BufferWait();
HW::Context& GetCtx() { return m_ctx; }
HW::UserConfig& GetUcfg() { return m_ucfg; }
HW::Shader& GetShCtx() { return m_sh_ctx; }
@@ -173,10 +160,9 @@ private:
// Persistent draw state: indirect draws update it for subsequent draws.
uint32_t m_num_instances = 1;
uint32_t m_de_count = 0;
uint32_t m_ce_count = 0;
bool m_ce_complete = false;
bool m_readback_active = false;
uint32_t m_de_count = 0;
uint32_t m_ce_count = 0;
bool m_ce_complete = false;
uint32_t m_const_ram[0x3000] = {0};
src/graphics/guest_gpu/graphicsRun.cpp
+33 -26
View File
@@ -36,6 +36,7 @@ static thread_local CommandProcessor* g_current_processor = nullptr;
static thread_local Pm4Execution* g_current_execution = nullptr;
static thread_local bool g_gpu_mutex_owned = false;
static thread_local bool g_gpu_thread = false;
static thread_local GpuState* g_gpu_state = nullptr;
class GpuMutexLock final {
public:
@@ -98,11 +99,10 @@ public:
void SubmitFlipPreparation(uint64_t request_id);
void Done();
void Shutdown();
[[nodiscard]] bool IsStopping();
void SendCommand(Common::UniqueFunction<void>&& command);
void SendCommandSync(Common::UniqueFunction<void>&& command);
void SendCommandSyncWithProcessor(Common::UniqueFunction<void, CommandProcessor&>&& command);
int GetFrameNum();
[[nodiscard]] bool IsStopping();
void SendCommand(Common::UniqueFunction<void>&& command);
void SendCommandSync(Common::UniqueFunction<void>&& command);
int GetFrameNum();
[[nodiscard]] static bool IsGpuThread() noexcept { return g_gpu_thread; }
private:
@@ -127,6 +127,7 @@ private:
void WaitLocked();
void Enqueue(Submission submission);
void WaitForIdle();
void ProcessCommands();
bool Process(Submission& submission);
static void ThreadRun(void* data);
CommandProcessor& GetProcessor(uint32_t queue_id);
@@ -139,6 +140,7 @@ private:
Common::CondVar m_idle;
std::array<std::deque<Submission>, QueueCount> m_queues;
std::deque<Common::UniqueFunction<void>> m_commands;
std::atomic_uint32_t m_pending_commands {0};
uint32_t m_next_queue = 0;
uint32_t m_submission_count = 0;
bool m_processing = false;
@@ -153,6 +155,8 @@ private:
uint64_t m_submit_id = 0;
std::atomic_int m_done_num = 0;
std::jthread m_thread;
friend class CommandProcessor;
};
static bool GraphicsRunDebugDumpEnabled() {
@@ -195,9 +199,25 @@ void GpuState::SendCommand(Common::UniqueFunction<void>&& command) {
Common::LockGuard lock(m_queue_mutex);
EXIT_IF(!m_accepting);
m_commands.push_back(std::move(command));
m_pending_commands.fetch_add(1, std::memory_order_release);
m_work_available.Signal();
}
void GpuState::ProcessCommands() {
EXIT_IF(!IsGpuThread());
while (m_pending_commands.load(std::memory_order_acquire) != 0) {
Common::UniqueFunction<void> command;
{
Common::LockGuard lock(m_queue_mutex);
EXIT_IF(m_commands.empty());
command = std::move(m_commands.front());
m_commands.pop_front();
EXIT_IF(m_pending_commands.fetch_sub(1, std::memory_order_acq_rel) == 0);
}
command();
}
}
void GpuState::SendCommandSync(Common::UniqueFunction<void>&& command) {
EXIT_IF(!command);
if (IsGpuThread()) {
@@ -212,17 +232,6 @@ void GpuState::SendCommandSync(Common::UniqueFunction<void>&& command) {
done.acquire();
}
void GpuState::SendCommandSyncWithProcessor(
Common::UniqueFunction<void, CommandProcessor&>&& command) {
EXIT_IF(!command);
SendCommandSync([this, operation = std::move(command)]() mutable {
EXIT_IF(g_current_processor != nullptr);
g_current_processor = m_gfx_cp.get();
operation(*m_gfx_cp);
g_current_processor = nullptr;
});
}
void GpuState::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) {
GpuMutexLock lock(m_submission_mutex);
@@ -509,6 +518,7 @@ void GpuState::ThreadRun(void* data) {
EXIT_IF(gpu == nullptr);
KYTY_PROFILER_THREAD("Thread_Gpu");
g_gpu_thread = true;
g_gpu_state = gpu;
for (;;) {
Submission submission;
@@ -529,6 +539,7 @@ void GpuState::ThreadRun(void* data) {
} else if (!gpu->m_commands.empty()) {
command = std::move(gpu->m_commands.front());
gpu->m_commands.pop_front();
EXIT_IF(gpu->m_pending_commands.fetch_sub(1, std::memory_order_acq_rel) == 0);
gpu->m_processing = true;
} else {
int selected_queue = -1;
@@ -560,6 +571,7 @@ void GpuState::ThreadRun(void* data) {
}
if (should_stop) {
gpu->m_gfx_cp->BufferWait();
g_gpu_state = nullptr;
g_gpu_thread = false;
return;
}
@@ -741,6 +753,9 @@ void CommandProcessor::SuspendPm4() {
void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
while (execution.m_buffer_stack.size() > stop_depth) {
if (g_gpu_state != nullptr) {
g_gpu_state->ProcessCommands();
}
const auto buffer_index = execution.m_buffer_stack.size() - 1;
auto& cursor = execution.m_buffer_stack[buffer_index];
if (cursor.deferred_advance_dw != 0) {
@@ -1650,10 +1665,6 @@ void Gpu::SendCommandSync(Common::UniqueFunction<void>&& command) {
m_state->SendCommandSync(std::move(command));
}
void Gpu::SendCommandSyncWithProcessor(Common::UniqueFunction<void, CommandProcessor&>&& command) {
m_state->SendCommandSyncWithProcessor(std::move(command));
}
void Gpu::Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands,
uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done) {
EXIT_IF(draw_commands == nullptr || draw_size_dw == 0);
@@ -1679,12 +1690,8 @@ int Gpu::GetFrameNum() const {
return m_state->GetFrameNum();
}
bool Gpu::IsCommandProcessorThread() noexcept {
return g_current_processor != nullptr;
}
CommandProcessor* Gpu::CurrentCommandProcessor() noexcept {
return g_current_processor;
bool Gpu::IsGpuThread() noexcept {
return GpuState::IsGpuThread();
}
} // namespace Libs::Graphics
src/graphics/guest_gpu/graphicsRun.h
+1 -4
View File
@@ -9,7 +9,6 @@
namespace Libs::Graphics {
class CommandProcessor;
class GpuState;
class RenderContext;
@@ -23,7 +22,6 @@ public:
[[nodiscard]] bool IsStopping();
void SendCommand(Common::UniqueFunction<void>&& command);
void SendCommandSync(Common::UniqueFunction<void>&& command);
void SendCommandSyncWithProcessor(Common::UniqueFunction<void, CommandProcessor&>&& command);
void Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands,
uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done = false);
@@ -33,8 +31,7 @@ public:
void Done();
[[nodiscard]] int GetFrameNum() const;
[[nodiscard]] static bool IsCommandProcessorThread() noexcept;
[[nodiscard]] static CommandProcessor* CurrentCommandProcessor() noexcept;
[[nodiscard]] static bool IsGpuThread() noexcept;
private:
std::unique_ptr<GpuState> m_state;
src/graphics/host_gpu/memoryTracker.h
+17 -33
View File
@@ -29,45 +29,29 @@ public:
void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
void UntrackMemory(uint64_t vaddr, uint64_t size);
// Removes protection from a range and flushes GPU-owned data when required.
template <typename Flush>
void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) {
void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept {
static_assert(std::is_invocable_v<Flush&>);
CheckNotInUploadCallback();
ValidateRange(vaddr, size);
const auto update_cpu_state = [this, vaddr, size] {
std::lock_guard access(m_access_mutex);
std::vector<RegionManager*> managers;
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
managers.push_back(manager);
});
std::vector<std::unique_lock<TrackingSpinLock>> locks;
locks.reserve(managers.size());
for (auto* manager: managers) {
locks.emplace_back(manager->lock);
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
const bool should_flush = [&] {
// Perform both the GPU modification check and CPU state change with the lock in
// case the GPU thread is racing to mark the page modified. If a flush is needed,
// on_flush performs the CPU state change.
std::scoped_lock lock(manager->lock);
if (manager->IsModified<DirtySource::Gpu>(offset, bytes)) {
return true;
}
manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
return false;
}();
if (should_flush) {
on_flush();
}
const bool gpu_modified = Iterate<false>(
vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
return manager->IsModified<DirtySource::Gpu>(offset, bytes);
});
if (gpu_modified) {
return true;
}
Iterate<false>(vaddr, size,
[](RegionManager* manager, uint64_t offset, uint64_t bytes) {
manager->ChangeState<DirtySource::Cpu, true>(
manager->GetCpuAddr() + offset, bytes);
});
return false;
};
if (!update_cpu_state()) {
return;
}
std::forward<Flush>(on_flush)();
if (update_cpu_state()) {
EXIT("memory invalidation retained GPU-owned pages\n");
}
});
}
#if KYTY_BUILD == KYTY_BUILD_DEBUG
void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
src/graphics/host_gpu/renderer/cache/bufferCache.cpp
Vendored
+37 -5
View File
@@ -3,15 +3,18 @@
#include "common/assert.h"
#include "common/logging/log.h"
#include "common/profiler.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "kernel/memory.h"
#include <algorithm>
#include <array>
#include <cinttypes>
#include <cstring>
#include <utility>
#include <vector>
@@ -288,10 +291,29 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
return;
}
m_memory_tracker.InvalidateRegion(vaddr, size,
[this, vaddr, size] { ReadMemory(vaddr, size); });
[this, vaddr, size] { ReadMemory(vaddr, size, true); });
}
void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
if (Gpu::IsGpuThread()) {
ReadMemoryOnGpu(vaddr, size, is_write);
return;
}
if (CommandScheduler::InDeferredOperation()) {
EXIT("unsupported buffer readback from an asynchronous GPU completion, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported buffer readback from a pre-owned resource transaction, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
m_scheduler.Context().GetGpu().SendCommandSync(
[this, vaddr, size, is_write] { ReadMemoryOnGpu(vaddr, size, is_write); });
}
void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write) {
std::vector<DownloadCopy> copies;
{
FaultSafeCacheLock lock(this, m_mutex);
@@ -322,9 +344,16 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
}
}
});
}
if (copies.empty()) {
return;
if (copies.empty()) {
if (!is_write) {
return;
}
// A preceding read fault can consume the last GPU-owned copy after this write
// invalidation has already chosen to flush. Complete the CPU ownership handoff even
// though this callback no longer has bytes to download.
m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
return;
}
}
auto downloads = RecordDownloads(copies);
m_scheduler.FinishCurrent();
@@ -336,6 +365,9 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
}
// The enumeration above covered whole dirty pages and every exact interval on them.
m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size);
if (is_write) {
m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
}
}
}
src/graphics/host_gpu/renderer/cache/bufferCache.h
Vendored
+2 -1
View File
@@ -46,7 +46,7 @@ public:
KYTY_CLASS_NO_COPY(BufferCache);
void InvalidateMemory(uint64_t vaddr, uint64_t size);
void ReadMemory(uint64_t vaddr, uint64_t size);
void ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false);
void UnmapMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
bool is_written = false, bool is_read = true,
@@ -97,6 +97,7 @@ private:
void PublishDownloads(std::span<const DownloadRange> downloads);
void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write);
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp
Vendored
+8 -57
View File
@@ -1,7 +1,6 @@
#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
#include "common/assert.h"
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
namespace Libs::Graphics {
@@ -18,69 +17,21 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
if (!IsMapped(fault_vaddr, fault_size)) {
return false;
}
if (CommandScheduler::InDeferredOperation()) {
EXIT("unsupported guest-memory fault from an asynchronous GPU completion, "
"addr=0x%016" PRIx64 " access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
if (access == PageFaultAccess::Write) {
m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
} else {
m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
}
bool handled = false;
const auto resolve = [this, access, fault_vaddr, &handled](CommandProcessor& cp) {
cp.BeginReadbackTransaction();
{
ResourceMutex::FaultScope fault(m_resource_mutex);
if (access == PageFaultAccess::Write) {
m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
} else {
m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
}
handled = true;
}
cp.EndReadbackTransaction();
};
if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
resolve(*cp);
return handled;
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64
" access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
}
EXIT_IF(m_gpu == nullptr);
m_gpu->SendCommandSyncWithProcessor(resolve);
return handled;
return true;
}
bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) {
if (!IsMapped(vaddr, size)) {
return false;
}
if (CommandScheduler::InDeferredOperation()) {
EXIT("unsupported memory invalidation from an asynchronous GPU completion, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
const auto resolve = [this, vaddr, size](CommandProcessor& cp) {
cp.BeginReadbackTransaction();
{
ResourceMutex::FaultScope fault(m_resource_mutex);
m_buffer_cache.InvalidateMemory(vaddr, size);
m_texture_cache.InvalidateMemory(vaddr, size);
}
cp.EndReadbackTransaction();
};
if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
resolve(*cp);
return true;
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported memory invalidation from a pre-owned resource transaction, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
EXIT_IF(m_gpu == nullptr);
m_gpu->SendCommandSyncWithProcessor(resolve);
m_buffer_cache.InvalidateMemory(vaddr, size);
m_texture_cache.InvalidateMemory(vaddr, size);
return true;
}
src/graphics/host_gpu/renderer/cache/resourceMutex.cpp
Vendored
+2 -48
View File
@@ -4,18 +4,10 @@
namespace Libs::Graphics {
ResourceMutex::FaultScope::FaultScope(ResourceMutex& mutex): m_mutex(mutex) {
m_resource_preowned = m_mutex.BeginFault();
}
ResourceMutex::FaultScope::~FaultScope() {
m_mutex.EndFault(m_resource_preowned);
}
ResourceMutex::~ResourceMutex() {
std::lock_guard state(m_state);
if (m_resource_owner != std::thread::id {} || m_fault_owner != std::thread::id {}) {
EXIT("ResourceMutex destroyed with an active owner or fault transaction\n");
if (m_resource_owner != std::thread::id {}) {
EXIT("ResourceMutex destroyed with an active owner\n");
}
}
@@ -26,9 +18,6 @@ void ResourceMutex::lock() {
if (m_resource_owner == current) {
EXIT("recursive resource transaction\n");
}
if (m_fault_owner == current) {
EXIT("resource transaction re-entered from a page-fault callback\n");
}
}
m_resource.lock();
std::lock_guard state(m_state);
@@ -55,39 +44,4 @@ bool ResourceMutex::IsOwnedByCurrentThread() {
return m_resource_owner == std::this_thread::get_id();
}
bool ResourceMutex::BeginFault() {
const auto current = std::this_thread::get_id();
{
std::lock_guard state(m_state);
if (m_fault_owner == current) {
EXIT("nested resource page-fault transaction\n");
}
if (m_resource_owner == current) {
m_fault_owner = current;
return true;
}
}
lock();
std::lock_guard state(m_state);
if (m_resource_owner != current || m_fault_owner != std::thread::id {}) {
EXIT("resource fault transaction acquired inconsistent ownership\n");
}
m_fault_owner = current;
return false;
}
void ResourceMutex::EndFault(bool resource_preowned) {
const auto current = std::this_thread::get_id();
{
std::lock_guard state(m_state);
if (m_fault_owner != current || m_resource_owner != current) {
EXIT("resource page-fault transaction ended with inconsistent ownership\n");
}
m_fault_owner = {};
}
if (!resource_preowned) {
unlock();
}
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/resourceMutex.h
Vendored
+1 -19
View File
@@ -8,21 +8,9 @@
namespace Libs::Graphics {
// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions first;
// command-processor faults drain pending guest processors before entering this transaction.
// Owner-tracked shared buffer/image transaction.
class ResourceMutex final {
public:
class FaultScope final {
public:
explicit FaultScope(ResourceMutex& mutex);
~FaultScope();
KYTY_CLASS_NO_COPY(FaultScope);
private:
ResourceMutex& m_mutex;
bool m_resource_preowned = false;
};
ResourceMutex() = default;
~ResourceMutex();
KYTY_CLASS_NO_COPY(ResourceMutex);
@@ -32,15 +20,9 @@ public:
[[nodiscard]] bool IsOwnedByCurrentThread();
private:
friend class FaultScope;
[[nodiscard]] bool BeginFault();
void EndFault(bool resource_preowned);
std::mutex m_resource;
std::mutex m_state;
std::thread::id m_resource_owner;
std::thread::id m_fault_owner;
};
} // namespace Libs::Graphics
tests/MemoryTrackerTests.cpp
+51
View File
@@ -6,7 +6,9 @@
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <semaphore>
#include <string>
#include <thread>
#include <vector>
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
@@ -265,6 +267,7 @@ void TestRangeInvalidation() {
flushes++;
tracker.ForEachDownloadRange<true>(address + 16, size - 32,
[](uint64_t, uint64_t) noexcept {});
tracker.MarkRegionAsCpuModified(address + 16, size - 32);
});
Check(flushes == 1 && !tracker.IsRegionGpuModified(address, size) &&
tracker.IsRegionCpuModified(address, size) && IsWritable(memory) &&
@@ -277,6 +280,53 @@ void TestRangeInvalidation() {
Release(page_manager, memory, size);
}
void TestGpuReacquisitionAfterInvalidation() {
TrackerHarness harness;
auto &tracker = harness.tracker;
auto &page_manager = harness.page_manager;
const auto page_size = page_manager.GetPageSize();
auto *memory = Allocate(page_manager, 1);
const auto address = reinterpret_cast<uint64_t>(memory);
tracker.ForEachUploadRange(
address, page_size, true, [](uint64_t, uint64_t) noexcept {},
[]() noexcept {});
Check(tracker.IsRegionGpuModified(address, page_size) &&
!tracker.IsRegionCpuModified(address, page_size),
"reacquisition setup did not establish GPU ownership");
uint32_t flushes = 0;
uint32_t uploads = 0;
std::binary_semaphore reacquire{0};
std::binary_semaphore reacquired{0};
std::jthread publisher([&] {
reacquire.acquire();
tracker.ForEachUploadRange(
address + 16, 32, true,
[&](uint64_t, uint64_t) noexcept { uploads++; }, []() noexcept {});
reacquired.release();
});
tracker.InvalidateRegion(address + 16, 32, [&] {
flushes++;
tracker.ForEachDownloadRange<true>(address + 16, 32,
[](uint64_t, uint64_t) noexcept {});
tracker.MarkRegionAsCpuModified(address + 16, 32);
reacquire.release();
reacquired.acquire();
});
publisher.join();
Check(flushes == 1 && uploads == 1 &&
tracker.IsRegionGpuModified(address, page_size) &&
!tracker.IsRegionCpuModified(address, page_size) &&
!IsWritable(memory),
"invalidation rejected a new generation of GPU ownership");
tracker.UnmarkRegionAsGpuModified(address, page_size);
tracker.MarkRegionAsCpuModified(address, page_size);
tracker.UntrackMemory(address, page_size);
Release(page_manager, memory, page_size);
}
void TestGpuDirtyBits() {
TrackerHarness harness;
auto &tracker = harness.tracker;
@@ -635,6 +685,7 @@ int main(int argc, char **argv) {
TestQueriesDoNotRequireMappedOwnership();
TestCpuDirtyUpload();
TestRangeInvalidation();
TestGpuReacquisitionAfterInvalidation();
TestGpuDirtyBits();
TestExactDirtyIntervalsSharingTrackerPage();
TestGpuDownloadProtectionMirrors();
tests/ResourceMutexTests.cpp
+28 -152
View File
@@ -35,171 +35,50 @@ void YieldMany() {
}
}
void TestFaultBlocksPublisher() {
ResourceMutex mutex;
std::atomic_bool publisher_started {false};
std::atomic_bool publisher_entered {false};
std::thread publisher;
void TestOwnership() {
ResourceMutex mutex;
Check(!mutex.IsOwnedByCurrentThread(), "fresh mutex reported an owner");
{
ResourceMutex::FaultScope fault(mutex);
publisher = std::thread([&] {
publisher_started.store(true, std::memory_order_release);
std::lock_guard lock(mutex);
publisher_entered.store(true, std::memory_order_release);
});
while (!publisher_started.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
YieldMany();
Check(!publisher_entered.load(std::memory_order_acquire),
"publisher entered during active fault transaction");
std::lock_guard lock(mutex);
Check(mutex.IsOwnedByCurrentThread(), "owner tracking was not installed");
}
publisher.join();
Check(publisher_entered.load(std::memory_order_acquire),
"publisher did not resume after fault transaction");
Check(!mutex.IsOwnedByCurrentThread(), "owner tracking survived unlock");
}
void TestFaultDrainsExistingOwner() {
void TestSerializesTransactions() {
ResourceMutex mutex;
std::unique_lock owner(mutex);
std::atomic_bool fault_started {false};
std::atomic_bool fault_entered {false};
std::atomic_bool release_fault {false};
std::atomic_bool publisher_started {false};
std::atomic_bool publisher_entered {false};
std::thread fault([&] {
fault_started.store(true, std::memory_order_release);
ResourceMutex::FaultScope scope(mutex);
fault_entered.store(true, std::memory_order_release);
while (!release_fault.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
});
while (!fault_started.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
YieldMany();
Check(!fault_entered.load(std::memory_order_acquire),
"fault transaction did not wait for existing owner");
owner.unlock();
while (!fault_entered.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
std::thread publisher([&] {
publisher_started.store(true, std::memory_order_release);
std::atomic_bool contender_started {false};
std::atomic_bool contender_entered {false};
std::thread contender([&] {
contender_started.store(true, std::memory_order_release);
std::lock_guard lock(mutex);
publisher_entered.store(true, std::memory_order_release);
contender_entered.store(true, std::memory_order_release);
});
while (!publisher_started.load(std::memory_order_acquire)) {
while (!contender_started.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
YieldMany();
Check(!publisher_entered.load(std::memory_order_acquire),
"publisher entered before drained fault completed");
release_fault.store(true, std::memory_order_release);
fault.join();
publisher.join();
Check(publisher_entered.load(std::memory_order_acquire),
"publisher did not resume after drained fault");
}
void TestFaultScopesSerialize() {
ResourceMutex mutex;
std::atomic_bool first_entered {false};
std::atomic_bool second_started {false};
std::atomic_bool second_entered {false};
std::atomic_bool release_first {false};
std::atomic_bool release_second {false};
std::atomic_bool publisher_entered {false};
std::thread first([&] {
ResourceMutex::FaultScope scope(mutex);
first_entered.store(true, std::memory_order_release);
while (!release_first.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
});
while (!first_entered.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
std::thread second([&] {
second_started.store(true, std::memory_order_release);
ResourceMutex::FaultScope scope(mutex);
second_entered.store(true, std::memory_order_release);
while (!release_second.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
});
while (!second_started.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
YieldMany();
Check(!second_entered.load(std::memory_order_acquire),
"second fault transaction entered before first completed");
release_first.store(true, std::memory_order_release);
while (!second_entered.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
std::thread publisher([&] {
std::lock_guard lock(mutex);
publisher_entered.store(true, std::memory_order_release);
});
YieldMany();
Check(!publisher_entered.load(std::memory_order_acquire),
"publisher entered during second fault transaction");
release_second.store(true, std::memory_order_release);
first.join();
second.join();
publisher.join();
Check(publisher_entered.load(std::memory_order_acquire),
"publisher did not resume after serialized faults");
}
void TestPreownedFaultKeepsResourceTransaction() {
ResourceMutex mutex;
std::atomic_bool publisher_started {false};
std::atomic_bool publisher_entered {false};
std::thread publisher;
std::unique_lock owner(mutex);
{
ResourceMutex::FaultScope fault(mutex);
publisher = std::thread([&] {
publisher_started.store(true, std::memory_order_release);
std::lock_guard lock(mutex);
publisher_entered.store(true, std::memory_order_release);
});
while (!publisher_started.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
YieldMany();
Check(!publisher_entered.load(std::memory_order_acquire),
"publisher entered during preowned fault transaction");
}
YieldMany();
Check(!publisher_entered.load(std::memory_order_acquire),
"preowned fault released its outer resource transaction");
Check(!contender_entered.load(std::memory_order_acquire),
"contender entered an active resource transaction");
owner.unlock();
publisher.join();
Check(publisher_entered.load(std::memory_order_acquire),
"publisher did not resume after outer transaction");
contender.join();
Check(contender_entered.load(std::memory_order_acquire),
"contender did not resume after resource transaction");
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
[[noreturn]] void RunDeathCase(const char* name) {
[[noreturn]] void RunDeathCase() {
ResourceMutex mutex;
if (std::strcmp(name, "recursive-lock") == 0) {
std::lock_guard first(mutex);
std::lock_guard second(mutex);
} else if (std::strcmp(name, "nested-fault") == 0) {
ResourceMutex::FaultScope first(mutex);
ResourceMutex::FaultScope second(mutex);
}
std::lock_guard first(mutex);
std::lock_guard second(mutex);
std::_Exit(0x7f);
}
void CheckDeathCase(const char* name) {
void CheckDeathCase() {
char path[MAX_PATH] {};
Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, "GetModuleFileName failed");
std::string command = std::string("\"") + path + "\" --death " + name;
std::string command = std::string("\"") + path + "\" --death";
std::vector<char> mutable_command(command.begin(), command.end());
mutable_command.push_back('\0');
STARTUPINFOA startup {sizeof(startup)};
@@ -224,20 +103,17 @@ void CheckDeathCase(const char* name) {
int main(int argc, char** argv) {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (argc == 3 && std::strcmp(argv[1], "--death") == 0) {
RunDeathCase(argv[2]);
if (argc == 2 && std::strcmp(argv[1], "--death") == 0) {
RunDeathCase();
}
#else
(void)argc;
(void)argv;
#endif
TestFaultBlocksPublisher();
TestFaultDrainsExistingOwner();
TestFaultScopesSerialize();
TestPreownedFaultKeepsResourceTransaction();
TestOwnership();
TestSerializesTransactions();
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
CheckDeathCase("recursive-lock");
CheckDeathCase("nested-fault");
CheckDeathCase();
#endif
std::puts("ResourceMutexTests: all cases passed");
return 0;
tests/ShaderRecompilerComputeTests.cpp
+185 -23
View File
@@ -58,6 +58,7 @@
#include <algorithm>
#include <array>
#include <atomic>
#include <bit>
#include <chrono>
#include <cinttypes>
@@ -1372,12 +1373,14 @@ public:
void CheckGpuMappedRangeLifecycle() {
EnsureRuntimeContext();
CommandScheduler scheduler(Renderer(), m_runtime_context);
auto& context = Renderer();
CommandScheduler scheduler(context, m_runtime_context);
HW::Context registers {};
HW::UserConfig user_config {};
HW::Shader shaders {};
scheduler.Begin(registers, user_config, shaders);
Gpu gpu(Renderer());
context.InitializeGpu(nullptr);
auto& gpu = context.GetGpu();
GpuResourceManager resources(m_runtime_context, scheduler);
resources.SetGpu(&gpu);
@@ -1416,6 +1419,7 @@ public:
resources.SetGpu(nullptr);
scheduler.Finish();
context.ShutdownGpu();
std::printf("[host] %-32s ok\n", "GpuMappedRangeLifecycle");
}
@@ -1567,21 +1571,73 @@ public:
gpu_thread = std::this_thread::get_id();
Require("GpuCommandLane", "FIFO", order == 1,
"synchronous command overtook an older host command");
Require("GpuCommandLane", "generic context",
!Gpu::IsCommandProcessorThread() && Gpu::CurrentCommandProcessor() == nullptr,
"generic host command manufactured a PM4 processor context");
Require("GpuCommandLane", "GPU context", Gpu::IsGpuThread(),
"host command did not run on the GPU thread");
gpu.SendCommandSync([&nested_thread] { nested_thread = std::this_thread::get_id(); });
order = 2;
});
gpu.SendCommandSyncWithProcessor([&](CommandProcessor& processor) {
Require("GpuCommandLane", "processor context",
Gpu::IsCommandProcessorThread() && Gpu::CurrentCommandProcessor() == &processor,
"resource command did not receive its explicit processor context");
});
Require("GpuCommandLane", "dispatch thread",
order == 2 && gpu_thread != caller_thread && nested_thread == gpu_thread,
"host command did not run on the GPU thread or nested sync deadlocked");
alignas(uint32_t) uint32_t packet_marker_a = 0;
alignas(uint32_t) uint32_t packet_marker_b = 0;
const auto write_packet = [](uint32_t* packet, uint32_t* destination, uint32_t value) {
const auto destination_address = reinterpret_cast<uint64_t>(destination);
packet[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
packet[1] = 0;
packet[2] = static_cast<uint32_t>(destination_address);
packet[3] = static_cast<uint32_t>(destination_address >> 32u);
packet[4] = value;
};
std::array<uint32_t, 1024> polling_loop {};
for (size_t i = 0; i < polling_loop.size() - 4; i += 2) {
polling_loop[i] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
polling_loop[i + 1] = 0;
}
const auto loop_address = reinterpret_cast<uint64_t>(polling_loop.data());
polling_loop[1020] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
polling_loop[1021] = static_cast<uint32_t>(loop_address);
polling_loop[1022] = static_cast<uint32_t>(loop_address >> 32u);
polling_loop[1023] = 0x0f200000u | static_cast<uint32_t>(polling_loop.size());
std::array<uint32_t, 14> polling_commands {};
write_packet(polling_commands.data(), &packet_marker_a, 11);
polling_commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
polling_commands[6] = static_cast<uint32_t>(loop_address);
polling_commands[7] = static_cast<uint32_t>(loop_address >> 32u);
polling_commands[8] = 0x0f200000u | static_cast<uint32_t>(polling_loop.size());
write_packet(polling_commands.data() + 9, &packet_marker_b, 22);
std::binary_semaphore stream_gate_entered {0};
std::binary_semaphore stream_gate_release {0};
gpu.SendCommand([&] {
stream_gate_entered.release();
stream_gate_release.acquire();
});
gpu.Submit(polling_commands.data(), polling_commands.size(), nullptr, 0);
stream_gate_entered.acquire();
stream_gate_release.release();
uint32_t packet_marker_a_at_callback = UINT32_MAX;
uint32_t packet_marker_b_at_callback = UINT32_MAX;
for (uint32_t attempt = 0; attempt < 8 && packet_marker_a_at_callback != 11; attempt++) {
gpu.SendCommandSync([&] {
if (packet_marker_a == 11 && packet_marker_b == 0) {
packet_marker_a_at_callback = packet_marker_a;
packet_marker_b_at_callback = packet_marker_b;
polling_loop[1020] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
polling_loop[1021] = 0;
polling_loop[1022] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
polling_loop[1023] = 0;
}
});
std::this_thread::yield();
}
gpu.Done();
Require("GpuCommandLane", "packet-boundary command polling",
packet_marker_a_at_callback == 11 && packet_marker_b_at_callback == 0 &&
packet_marker_a == 11 && packet_marker_b == 22,
"host command waited for an entire non-suspended PM4 stream");
uint32_t label = 0;
uint32_t prefix = 0;
uint32_t suffix = 0;
@@ -1760,7 +1816,7 @@ public:
gpu.Submit(dma_commands.data(), static_cast<uint32_t>(dma_commands.size()), nullptr, 0);
gpu.Done();
constexpr uint32_t clean_fill_value = 0xdecafbad;
gpu.SendCommandSyncWithProcessor([&](CommandProcessor&) {
gpu.SendCommandSync([&] {
auto& buffer_cache = resources.GetBufferCache();
Require("GpuCommandLane", "clean cached setup",
buffer_cache.HasPageOverlap(clean_cached_fill, sizeof(source_words)) &&
@@ -1840,14 +1896,116 @@ public:
clean_copy_words[1]},
"host-memory fill/copy did not update the clean cached mirror");
gpu.SendCommandSyncWithProcessor([&](CommandProcessor& processor) {
Require("GpuCommandLane", "processor fault context",
Gpu::CurrentCommandProcessor() == &processor,
"processor resource test lost its command context");
Require("GpuCommandLane", "processor memory invalidation",
gpu.SendCommandSync([&] {
Require("GpuCommandLane", "GPU memory invalidation",
resources.InvalidateMemory(fault_base, sizeof(uint32_t)),
"processor memory invalidation did not find its mapped range");
"GPU memory invalidation did not find its mapped range");
});
auto& buffer_cache = resources.GetBufferCache();
Require("GpuCommandLane", "clean tracked fault setup",
buffer_cache.HasPageOverlap(immediate_dst, sizeof(uint32_t)) &&
!buffer_cache.HasGpuDirtyBytes(immediate_dst, sizeof(uint32_t)),
"clean write-fault test address is not backed by a clean cached Buffer");
constexpr uint64_t dirty_fault_address = fault_base + 0xd000;
constexpr uint32_t dirty_fault_value = 0x5a17c0deu;
constexpr uint32_t dirty_fault_stale = 0x0ddba11u;
std::memcpy(reinterpret_cast<void*>(dirty_fault_address), &dirty_fault_stale,
sizeof(dirty_fault_stale));
gpu.SendCommandSync([&] {
auto dirty = buffer_cache.ObtainBuffer(scheduler.Current(), dirty_fault_address,
sizeof(dirty_fault_value), true, false);
Require("GpuCommandLane", "dirty tracked fault setup", dirty.owner != nullptr,
"dirty write-fault test could not create a cached Buffer");
scheduler.Current().RetainResourceUntilFence(dirty.owner);
buffer_cache.FillBuffer(dirty_fault_address, sizeof(dirty_fault_value),
dirty_fault_value);
});
Require("GpuCommandLane", "dirty tracked fault ownership",
buffer_cache.HasGpuDirtyBytes(dirty_fault_address, sizeof(dirty_fault_value)),
"dirty write-fault test address did not acquire GPU ownership");
std::binary_semaphore block_entered {0};
std::binary_semaphore block_release {0};
std::binary_semaphore clean_fault_complete {0};
bool clean_fault_handled = false;
gpu.SendCommand([&] {
block_entered.release();
block_release.acquire();
});
block_entered.acquire();
std::jthread clean_fault_thread([&] {
clean_fault_handled = resources.HandleFault(PageFaultAccess::Write, immediate_dst);
clean_fault_complete.release();
});
const bool clean_fault_was_direct =
clean_fault_complete.try_acquire_for(std::chrono::seconds(1));
if (!clean_fault_was_direct) {
block_release.release();
clean_fault_complete.acquire();
}
clean_fault_thread.join();
Require("GpuCommandLane", "clean fault direct path",
clean_fault_was_direct && clean_fault_handled,
"a clean CPU write fault waited for the blocked GPU command lane");
std::binary_semaphore dirty_fault_complete {0};
bool dirty_fault_handled = false;
std::jthread dirty_fault_thread([&] {
dirty_fault_handled =
resources.HandleFault(PageFaultAccess::Write, dirty_fault_address);
dirty_fault_complete.release();
});
const bool dirty_fault_bypassed_lane =
dirty_fault_complete.try_acquire_for(std::chrono::milliseconds(200));
block_release.release();
if (!dirty_fault_bypassed_lane) {
dirty_fault_complete.acquire();
}
dirty_fault_thread.join();
uint32_t dirty_fault_backing = 0;
std::memcpy(&dirty_fault_backing, reinterpret_cast<const void*>(dirty_fault_address),
sizeof(dirty_fault_backing));
Require("GpuCommandLane", "dirty fault synchronization",
!dirty_fault_bypassed_lane && dirty_fault_handled &&
dirty_fault_backing == dirty_fault_value &&
!buffer_cache.HasGpuDirtyBytes(dirty_fault_address, sizeof(dirty_fault_value)),
"a GPU-dirty write fault bypassed synchronization or lost the published bytes");
constexpr uint64_t empty_copy_fault_address = fault_base + 0xe000;
constexpr uint32_t empty_copy_fault_value = 0x6b28d1efu;
gpu.SendCommandSync([&] {
auto dirty = buffer_cache.ObtainBuffer(scheduler.Current(), empty_copy_fault_address,
sizeof(empty_copy_fault_value), true, false);
Require("GpuCommandLane", "empty-copy write setup", dirty.owner != nullptr,
"empty-copy write test could not create a cached Buffer");
scheduler.Current().RetainResourceUntilFence(dirty.owner);
buffer_cache.FillBuffer(empty_copy_fault_address, sizeof(empty_copy_fault_value),
empty_copy_fault_value);
buffer_cache.ReadMemory(empty_copy_fault_address, sizeof(empty_copy_fault_value));
const bool gpu_owned = buffer_cache.IsRegionGpuModified(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
const bool cpu_owned = buffer_cache.IsRegionCpuModified(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
Require("GpuCommandLane", "empty-copy read ownership",
!gpu_owned && !cpu_owned,
"readback did not leave clean ownership for the queued write invalidation");
buffer_cache.ReadMemory(empty_copy_fault_address, sizeof(empty_copy_fault_value), true);
});
uint32_t empty_copy_fault_backing = 0;
std::memcpy(&empty_copy_fault_backing, reinterpret_cast<const void*>(empty_copy_fault_address),
sizeof(empty_copy_fault_backing));
const bool empty_copy_cpu_owned = buffer_cache.IsRegionCpuModified(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
const bool empty_copy_gpu_owned = buffer_cache.IsRegionGpuModified(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
const bool empty_copy_dirty =
buffer_cache.HasGpuDirtyBytes(empty_copy_fault_address, sizeof(empty_copy_fault_value));
Require("GpuCommandLane", "empty-copy write ownership",
empty_copy_fault_backing == empty_copy_fault_value &&
empty_copy_cpu_owned && !empty_copy_gpu_owned && !empty_copy_dirty,
"write invalidation without a remaining copy did not establish CPU ownership");
resources.UnmapMemory(fault_base, fault_size);
Require("GpuCommandLane", "processor fault unmap",
Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0,
@@ -2073,12 +2231,14 @@ public:
constexpr uint32_t ring_fault_second_value = 0x4e5f6071u;
EnsureRuntimeContext();
CommandScheduler scheduler(Renderer(), m_runtime_context);
auto& context = Renderer();
CommandScheduler scheduler(context, m_runtime_context);
HW::Context registers {};
HW::UserConfig user_config {};
HW::Shader shaders {};
scheduler.Begin(registers, user_config, shaders);
Gpu gpu(Renderer());
context.InitializeGpu(nullptr);
auto& gpu = context.GetGpu();
int64_t direct_offset = -1;
Require(name, "direct allocation",
@@ -2471,7 +2631,7 @@ public:
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
gpu.Shutdown();
context.ShutdownGpu();
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"dirty-GC direct-memory mapping release failed");
@@ -2488,12 +2648,14 @@ public:
constexpr uint64_t allocation_size = 0x2800000;
constexpr uint64_t allocation_alignment = 0x200000;
EnsureRuntimeContext();
CommandScheduler scheduler(Renderer(), m_runtime_context);
auto& context = Renderer();
CommandScheduler scheduler(context, m_runtime_context);
HW::Context registers {};
HW::UserConfig user_config {};
HW::Shader shaders {};
scheduler.Begin(registers, user_config, shaders);
Gpu gpu(Renderer());
context.InitializeGpu(nullptr);
auto& gpu = context.GetGpu();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
@@ -4949,7 +5111,7 @@ public:
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
gpu.Shutdown();
context.ShutdownGpu();
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"cache direct-memory mapping release failed");