Implement syncOnAddress ABIs (perf improvement)

CMakeLists.txt

@@ -450,6 +450,13 @@ add_executable(event_queue_lifetime_tests EXCLUDE_FROM_ALL
 target_link_libraries(event_queue_lifetime_tests common fmt::fmt)
 target_include_directories(event_queue_lifetime_tests PRIVATE ${inc_headers})
 
+add_executable(sync_on_address_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/SyncOnAddressTests.cpp"
+	"${KYTY_SOURCE_DIR}/kernel/syncOnAddress.cpp"
+)
+target_link_libraries(sync_on_address_tests common)
+target_include_directories(sync_on_address_tests PRIVATE ${inc_headers})
+
 add_executable(image_page_table_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/ImagePageTableTests.cpp"
 )
@@ -503,6 +510,7 @@ if(BUILD_TESTING)
 	add_test(NAME resource_tracking COMMAND $<TARGET_FILE:resource_tracking_tests>)
 	add_test(NAME resource_mutex COMMAND $<TARGET_FILE:resource_mutex_tests>)
 	add_test(NAME event_queue_lifetime COMMAND $<TARGET_FILE:event_queue_lifetime_tests>)
+	add_test(NAME sync_on_address COMMAND $<TARGET_FILE:sync_on_address_tests>)
 	add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
 	add_test(NAME virtual_memory_allocation
@@ -551,6 +559,7 @@ if(BUILD_TESTING)
 		resource_tracking_tests
 		resource_mutex_tests
 		event_queue_lifetime_tests
+		sync_on_address_tests
 		shader_recompiler_compute_tests
 		virtual_memory_allocation_tests
 	)

src/kernel/syncOnAddress.cpp

@@ -0,0 +1,287 @@
+#include "kernel/syncOnAddress.h"
+
+#include "common/threads.h"
+#include "libs/errno.h"
+
+#include <algorithm>
+#include <chrono>
+#include <climits>
+#include <cstdint>
+#include <list>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+#include <cerrno>
+#include <linux/futex.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#endif
+
+namespace Libs::LibKernel::SyncOnAddress {
+
+namespace {
+
+constexpr uint32_t SIGNAL_POLL_MICROS = 10000;
+
+using Clock = std::chrono::steady_clock;
+
+template <typename T>
+[[nodiscard]] bool IsValidWaitAddress(const volatile T* address) {
+	return address != nullptr && (reinterpret_cast<uintptr_t>(address) & (alignof(T) - 1u)) == 0;
+}
+
+[[nodiscard]] bool IsValidWakeAddress(const volatile void* address) {
+	return address != nullptr &&
+	       (reinterpret_cast<uintptr_t>(address) & (alignof(uint32_t) - 1u)) == 0;
+}
+
+template <typename T>
+[[nodiscard]] T ReadWord(const volatile T* address) {
+	return __atomic_load_n(address, __ATOMIC_ACQUIRE);
+}
+
+struct WaitDeadline {
+	bool              finite = false;
+	Clock::time_point end {};
+};
+
+[[nodiscard]] WaitDeadline MakeDeadline(const uint32_t* timeout_micros) {
+	if (timeout_micros == nullptr) {
+		return {};
+	}
+	return {true, Clock::now() + std::chrono::microseconds(*timeout_micros)};
+}
+
+[[nodiscard]] uint32_t GetWaitSliceMicros(const WaitDeadline& deadline, bool first_wait) {
+	if (!deadline.finite) {
+		return SIGNAL_POLL_MICROS;
+	}
+
+	const auto now = Clock::now();
+	if (now >= deadline.end) {
+		return first_wait ? 0u : UINT32_MAX;
+	}
+
+	const auto remaining =
+	    std::chrono::duration_cast<std::chrono::microseconds>(deadline.end - now).count();
+	return static_cast<uint32_t>(std::min<int64_t>(remaining, SIGNAL_POLL_MICROS));
+}
+
+void PollSignals(signal_poll_func_t signal_poll) {
+	if (signal_poll != nullptr) {
+		signal_poll();
+	}
+}
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+
+template <typename T>
+int WaitLinux(volatile T* address, T expected, const uint32_t* timeout_micros,
+              signal_poll_func_t signal_poll) {
+	const auto deadline   = MakeDeadline(timeout_micros);
+	bool       first_wait = true;
+
+	for (;;) {
+		if (ReadWord(address) != expected) {
+			return OK;
+		}
+		const auto slice_micros = GetWaitSliceMicros(deadline, first_wait);
+		if (slice_micros == UINT32_MAX) {
+			return ReadWord(address) == expected ? KERNEL_ERROR_ETIMEDOUT : OK;
+		}
+		const timespec timeout = {
+		    .tv_sec  = static_cast<time_t>(slice_micros / 1000000u),
+		    .tv_nsec = static_cast<long>(slice_micros % 1000000u) * 1000L,
+		};
+
+		long result     = 0;
+		int  wait_error = 0;
+		result          = syscall(SYS_futex, const_cast<T*>(address), FUTEX_WAIT_PRIVATE,
+		                          static_cast<uint32_t>(expected), &timeout, nullptr, 0);
+		if (result != 0) {
+			wait_error = errno;
+		}
+		if (result == 0 || wait_error == EAGAIN) {
+			return OK;
+		}
+		if (wait_error != ETIMEDOUT && wait_error != EINTR) {
+			return KERNEL_ERROR_EINVAL;
+		}
+
+		PollSignals(signal_poll);
+		if (deadline.finite && Clock::now() >= deadline.end) {
+			return ReadWord(address) == expected ? KERNEL_ERROR_ETIMEDOUT : OK;
+		}
+		first_wait = false;
+	}
+}
+
+int WakeLinux(volatile void* address, int32_t count) {
+	const auto result = syscall(SYS_futex, const_cast<void*>(address), FUTEX_WAKE_PRIVATE, count,
+	                            nullptr, nullptr, 0);
+	return result < 0 ? KERNEL_ERROR_EINVAL : OK;
+}
+
+#endif
+
+struct PortableWaiter {
+	Common::CondVar condition;
+	bool            wake_requested = false;
+};
+
+struct PortableAddressEntry {
+	Common::Mutex              mutex;
+	std::list<PortableWaiter*> waiters;
+};
+
+struct PortableAddressRegistry {
+	std::mutex                                                           mutex;
+	std::unordered_map<uintptr_t, std::shared_ptr<PortableAddressEntry>> entries;
+};
+
+PortableAddressRegistry& GetPortableRegistry() {
+	static PortableAddressRegistry registry;
+	return registry;
+}
+
+std::shared_ptr<PortableAddressEntry> RegisterPortableWaiter(volatile void*  address,
+                                                             PortableWaiter* waiter) {
+	auto&           registry = GetPortableRegistry();
+	std::lock_guard registry_lock(registry.mutex);
+	auto&           entry = registry.entries[reinterpret_cast<uintptr_t>(address)];
+	if (!entry) {
+		entry = std::make_shared<PortableAddressEntry>();
+	}
+	entry->mutex.Lock();
+	entry->waiters.push_back(waiter);
+	return entry;
+}
+
+void UnregisterPortableWaiter(volatile void*                               address,
+                              const std::shared_ptr<PortableAddressEntry>& entry,
+                              PortableWaiter*                              waiter) {
+	entry->waiters.remove(waiter);
+	const bool empty = entry->waiters.empty();
+	entry->mutex.Unlock();
+	if (!empty) {
+		return;
+	}
+
+	auto&           registry = GetPortableRegistry();
+	std::lock_guard registry_lock(registry.mutex);
+	entry->mutex.Lock();
+	const auto it = registry.entries.find(reinterpret_cast<uintptr_t>(address));
+	if (it != registry.entries.end() && it->second == entry && entry->waiters.empty()) {
+		registry.entries.erase(it);
+	}
+	entry->mutex.Unlock();
+}
+
+template <typename T>
+int WaitPortable(volatile T* address, T expected, const uint32_t* timeout_micros,
+                 signal_poll_func_t signal_poll) {
+	PortableWaiter waiter;
+	auto           entry      = RegisterPortableWaiter(address, &waiter);
+	const auto     deadline   = MakeDeadline(timeout_micros);
+	bool           first_wait = true;
+	int            result     = OK;
+
+	while (ReadWord(address) == expected && !waiter.wake_requested) {
+		const auto slice_micros = GetWaitSliceMicros(deadline, first_wait);
+		if (slice_micros == UINT32_MAX) {
+			result = KERNEL_ERROR_ETIMEDOUT;
+			break;
+		}
+		if (slice_micros == 0) {
+			result = KERNEL_ERROR_ETIMEDOUT;
+			break;
+		}
+
+		(void)waiter.condition.WaitFor(&entry->mutex, slice_micros);
+		entry->mutex.Unlock();
+		PollSignals(signal_poll);
+		entry->mutex.Lock();
+
+		if (deadline.finite && Clock::now() >= deadline.end && ReadWord(address) == expected &&
+		    !waiter.wake_requested) {
+			result = KERNEL_ERROR_ETIMEDOUT;
+			break;
+		}
+		first_wait = false;
+	}
+
+	UnregisterPortableWaiter(address, entry, &waiter);
+	return result;
+}
+
+int WakePortable(volatile void* address, int32_t count) {
+	if (count == 0) {
+		return OK;
+	}
+	auto&            registry = GetPortableRegistry();
+	std::unique_lock registry_lock(registry.mutex);
+	const auto       it = registry.entries.find(reinterpret_cast<uintptr_t>(address));
+	if (it == registry.entries.end()) {
+		return OK;
+	}
+	auto entry = it->second;
+	entry->mutex.Lock();
+	registry_lock.unlock();
+
+	int32_t remaining = count;
+	for (auto* waiter: entry->waiters) {
+		if (!waiter->wake_requested) {
+			waiter->wake_requested = true;
+			waiter->condition.Signal();
+			if (remaining != INT_MAX && --remaining == 0) {
+				break;
+			}
+		}
+	}
+	entry->mutex.Unlock();
+	return OK;
+}
+
+template <typename T>
+int WaitImpl(volatile T* address, T expected, const uint32_t* timeout_micros,
+             signal_poll_func_t signal_poll) {
+	if (!IsValidWaitAddress(address)) {
+		return KERNEL_ERROR_EINVAL;
+	}
+
+	int result = OK;
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+	result = WaitLinux(address, expected, timeout_micros, signal_poll);
+#else
+	result = WaitPortable(address, expected, timeout_micros, signal_poll);
+#endif
+	PollSignals(signal_poll);
+	return result;
+}
+
+} // namespace
+
+int Wait32(volatile uint32_t* address, uint32_t expected, const uint32_t* timeout_micros,
+           signal_poll_func_t signal_poll) {
+	return WaitImpl(address, expected, timeout_micros, signal_poll);
+}
+
+int Wait64(volatile uint64_t* address, uint64_t expected, const uint32_t* timeout_micros,
+           signal_poll_func_t signal_poll) {
+	return WaitImpl(address, expected, timeout_micros, signal_poll);
+}
+
+int Wake(volatile void* address, int32_t count) {
+	if (!IsValidWakeAddress(address) || count < 0) {
+		return KERNEL_ERROR_EINVAL;
+	}
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+	return WakeLinux(address, count);
+#endif
+	return WakePortable(address, count);
+}
+
+} // namespace Libs::LibKernel::SyncOnAddress

src/kernel/syncOnAddress.h

@@ -0,0 +1,19 @@
+#ifndef EMULATOR_INCLUDE_EMULATOR_KERNEL_SYNC_ON_ADDRESS_H_
+#define EMULATOR_INCLUDE_EMULATOR_KERNEL_SYNC_ON_ADDRESS_H_
+
+#include "common/abi.h"
+#include "common/common.h"
+
+namespace Libs::LibKernel::SyncOnAddress {
+
+using signal_poll_func_t = void (*)();
+
+int Wait32(volatile uint32_t* address, uint32_t expected, const uint32_t* timeout_micros,
+           signal_poll_func_t signal_poll = nullptr);
+int Wait64(volatile uint64_t* address, uint64_t expected, const uint32_t* timeout_micros,
+           signal_poll_func_t signal_poll = nullptr);
+int Wake(volatile void* address, int32_t count);
+
+} // namespace Libs::LibKernel::SyncOnAddress
+
+#endif /* EMULATOR_INCLUDE_EMULATOR_KERNEL_SYNC_ON_ADDRESS_H_ */

src/libs/libKernel.cpp

@@ -13,6 +13,7 @@
 #include "kernel/memory.h"
 #include "kernel/pthread.h"
 #include "kernel/semaphore.h"
+#include "kernel/syncOnAddress.h"
 #include "libs/errno.h"
 #include "libs/libs.h"
 #include "libs/network.h"
@@ -2134,26 +2135,37 @@ uint64_t KYTY_SYSV_ABI cfwBSQyr5Ys(uint64_t a1, uint64_t a2, uint64_t a3, uint64
 	return 0;
 }
 
-uint64_t KYTY_SYSV_ABI KernelSyncOnAddressV1(uint64_t op, uint64_t address, uint64_t value,
-                                             uint64_t size, uint64_t timeout, uint64_t flags) {
-	static std::atomic_uint32_t log_count = 0;
-	const auto                  index     = log_count.fetch_add(1, std::memory_order_relaxed);
-
-	if (index < 16) {
-		LOGF("\t libkernel_sync_on_address_v1: op=0x%016" PRIx64 ", address=0x%016" PRIx64
-		     ", value=0x%016" PRIx64 ", size=0x%016" PRIx64 ", timeout=0x%016" PRIx64
-		     ", flags=0x%016" PRIx64 "\n",
-		     op, address, value, size, timeout, flags);
+static void LogExperimentalSyncOnAddress(std::atomic_bool& logged, const char* function_name) {
+	if (!logged.exchange(true, std::memory_order_relaxed)) {
+		::printf("WARNING: %s is experimental\n", function_name);
+		LOGF("WARNING: %s is experimental\n", function_name);
 	}
+}
 
-	if (op != 0 && address == 0 && value == 0 && size == 0) {
-		// This unsupported form is used as a yield/wait by some Unity jobs.
-		// SleepMicro() uses a sub-millisecond busy wait on Windows, which can
-		// pin every worker thread when the guest polls this path.
-		Common::Thread::Sleep(timeout == 0 ? 1 : 2);
-	}
+int KYTY_SYSV_ABI KernelSyncOnAddressWait(volatile uint32_t* address, uint32_t expected,
+                                          const uint32_t* timeout_micros) {
+	return LibKernel::SyncOnAddress::Wait32(address, expected, timeout_micros,
+	                                        LibKernel::KernelDispatchPendingSignalForCurrentThread);
+}
 
-	return 0;
+int KYTY_SYSV_ABI KernelSyncOnAddressWait32(volatile uint32_t* address, uint32_t expected,
+                                            const uint32_t* timeout_micros) {
+	static std::atomic_bool logged {false};
+	LogExperimentalSyncOnAddress(logged, "sceKernelSyncOnAddressWait32");
+	return LibKernel::SyncOnAddress::Wait32(address, expected, timeout_micros,
+	                                        LibKernel::KernelDispatchPendingSignalForCurrentThread);
+}
+
+int KYTY_SYSV_ABI KernelSyncOnAddressWait64(volatile uint64_t* address, uint64_t expected,
+                                            const uint32_t* timeout_micros) {
+	static std::atomic_bool logged {false};
+	LogExperimentalSyncOnAddress(logged, "sceKernelSyncOnAddressWait64");
+	return LibKernel::SyncOnAddress::Wait64(address, expected, timeout_micros,
+	                                        LibKernel::KernelDispatchPendingSignalForCurrentThread);
+}
+
+int KYTY_SYSV_ABI KernelSyncOnAddressWake(volatile void* address, int32_t count) {
+	return LibKernel::SyncOnAddress::Wake(address, count);
 }
 
 LIB_DEFINE(InitLibKernel_1_Posix) {
@@ -3405,8 +3417,10 @@ LIB_DEFINE(InitLibKernel_1) {
 	LIB_FUNC("Xjoosiw+XPI", LibKernel::KernelUuidCreate);
 	LIB_FUNC("DLORcroUqbc", LibKernel::KernelGetOpenPsId);
 	LIB_FUNC("zE-wXIZjLoM", LibKernel::KernelDebugRaiseExceptionOnReleaseMode);
-	LIB_FUNC("Hc4CaR6JBL0", Posix::KernelSyncOnAddressV1);
-	LIB_FUNC("q2y-wDIVWZA", Posix::KernelSyncOnAddressV1);
+	LIB_FUNC("Hc4CaR6JBL0", Posix::KernelSyncOnAddressWait);
+	LIB_FUNC("B2n8aDorSH4", Posix::KernelSyncOnAddressWait32);
+	LIB_FUNC("PZQhiiLXRFs", Posix::KernelSyncOnAddressWait64);
+	LIB_FUNC("q2y-wDIVWZA", Posix::KernelSyncOnAddressWake);
 
 	AddLibkernelUnityFunc(s, "Qhv5ARAoOEc",
 	                      reinterpret_cast<uint64_t>(LibKernel::KernelRemoveExceptionHandler),

tests/SyncOnAddressTests.cpp

@@ -0,0 +1,269 @@
+#include "kernel/syncOnAddress.h"
+
+#include "libs/errno.h"
+
+#include <atomic>
+#include <chrono>
+#include <climits>
+#include <cstdio>
+#include <cstdlib>
+#include <thread>
+#include <vector>
+
+namespace {
+
+using Libs::LibKernel::SyncOnAddress::Wait32;
+using Libs::LibKernel::SyncOnAddress::Wait64;
+using Libs::LibKernel::SyncOnAddress::Wake;
+
+std::atomic<int> g_signal_poll_count{0};
+
+void Check(bool value, const char *text) {
+  if (!value) {
+    std::fprintf(stderr, "SyncOnAddressTests: failed: %s\n", text);
+    std::abort();
+  }
+}
+
+template <typename T> void Store(T *address, T value) {
+  std::atomic_ref<T>(*address).store(value, std::memory_order_release);
+}
+
+void CountSignalPoll() {
+  g_signal_poll_count.fetch_add(1, std::memory_order_relaxed);
+}
+
+void TestInvalidAddress() {
+  uint32_t timeout = 1;
+  Check(Wait32(nullptr, 0, &timeout) == Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wait32 rejects a null address");
+  Check(Wait64(nullptr, 0, &timeout) == Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wait64 rejects a null address");
+  Check(Wake(nullptr, 1) == Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wake rejects a null address");
+
+  alignas(uint64_t) uint8_t bytes[16] = {};
+  auto *misaligned = reinterpret_cast<uint32_t *>(bytes + 1);
+  Check(Wait32(misaligned, 0, &timeout) == Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wait32 rejects a misaligned address");
+  Check(Wait64(reinterpret_cast<uint64_t *>(bytes + 4), 0, &timeout) ==
+            Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wait64 rejects a misaligned address");
+  Check(Wake(misaligned, 1) == Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wake rejects a misaligned address");
+  uint64_t aligned = 0;
+  Check(Wake(&aligned, -1) == Libs::LibKernel::KERNEL_ERROR_EINVAL,
+        "wake rejects a negative count");
+}
+
+void TestMismatchReturnsImmediately() {
+  uint32_t word = 7;
+  uint64_t word64 = UINT64_C(0x100000000);
+  uint32_t timeout = 500000;
+  const auto start = std::chrono::steady_clock::now();
+  Check(Wait32(&word, 6, &timeout, CountSignalPoll) == OK,
+        "mismatch succeeds without parking");
+  Check(Wait64(&word64, 0, &timeout, CountSignalPoll) == OK,
+        "wait64 compares all 64 bits");
+  Check(std::chrono::steady_clock::now() - start <
+            std::chrono::milliseconds(100),
+        "mismatched value is a fast path");
+  Check(g_signal_poll_count.load(std::memory_order_relaxed) >= 2,
+        "mismatch remains a guest signal safe-point");
+}
+
+void TestTimeout() {
+  uint32_t word = 0;
+  uint32_t timeout = 20000;
+  const auto start = std::chrono::steady_clock::now();
+  Check(Wait32(&word, 0, &timeout) == Libs::LibKernel::KERNEL_ERROR_ETIMEDOUT,
+        "matching value times out");
+  const auto elapsed = std::chrono::steady_clock::now() - start;
+  Check(elapsed >= std::chrono::milliseconds(10),
+        "timeout does not return too early");
+  Check(elapsed < std::chrono::milliseconds(500), "timeout remains bounded");
+
+  timeout = 0;
+  Check(Wait32(&word, 0, &timeout) == Libs::LibKernel::KERNEL_ERROR_ETIMEDOUT,
+        "zero timeout polls a matching value");
+  word = 1;
+  Check(Wait32(&word, 0, &timeout) == OK,
+        "zero timeout succeeds for a mismatched value");
+}
+
+void TestValueChangeAndWake() {
+  uint64_t word = 0;
+  uint32_t timeout = 1000000;
+  std::atomic<bool> ready{false};
+  int result = Libs::LibKernel::KERNEL_ERROR_ETIMEDOUT;
+
+  std::thread waiter([&] {
+    ready.store(true, std::memory_order_release);
+    result = Wait64(&word, 0, &timeout);
+  });
+  while (!ready.load(std::memory_order_acquire)) {
+    std::this_thread::yield();
+  }
+  Store(&word, UINT64_C(0x100000000));
+  Check(Wake(&word, 1) == OK, "wake-one succeeds");
+  waiter.join();
+  Check(result == OK, "a value change plus wake releases the waiter");
+}
+
+void TestWakeOneThenAll() {
+  constexpr int WAITER_COUNT = 4;
+  uint32_t word = 0;
+  uint32_t timeout = 1000000;
+  std::atomic<int> ready{0};
+  std::atomic<int> returned{0};
+  int results[WAITER_COUNT] = {};
+  std::vector<std::thread> waiters;
+
+  for (int i = 0; i < WAITER_COUNT; i++) {
+    waiters.emplace_back([&, i] {
+      ready.fetch_add(1, std::memory_order_release);
+      results[i] = Wait32(&word, 0, &timeout);
+      returned.fetch_add(1, std::memory_order_release);
+    });
+  }
+  while (ready.load(std::memory_order_acquire) != WAITER_COUNT) {
+    std::this_thread::yield();
+  }
+  std::this_thread::sleep_for(std::chrono::milliseconds(50));
+  Check(Wake(&word, 1) == OK, "wake-one succeeds with multiple waiters");
+  const auto one_deadline =
+      std::chrono::steady_clock::now() + std::chrono::milliseconds(500);
+  while (returned.load(std::memory_order_acquire) == 0 &&
+         std::chrono::steady_clock::now() < one_deadline) {
+    std::this_thread::yield();
+  }
+  Check(returned.load(std::memory_order_acquire) == 1,
+        "wake-one releases exactly one waiter");
+
+  Check(Wake(&word, 2) == OK, "wake-two succeeds");
+  const auto two_deadline =
+      std::chrono::steady_clock::now() + std::chrono::milliseconds(500);
+  while (returned.load(std::memory_order_acquire) < 3 &&
+         std::chrono::steady_clock::now() < two_deadline) {
+    std::this_thread::yield();
+  }
+  Check(returned.load(std::memory_order_acquire) == 3,
+        "wake-two releases exactly two more waiters");
+
+  Check(Wake(&word, INT_MAX) == OK, "wake-all succeeds");
+  for (auto &waiter : waiters) {
+    waiter.join();
+  }
+  Check(returned.load(std::memory_order_acquire) == WAITER_COUNT,
+        "wake-all releases the remaining waiters");
+  for (int result : results) {
+    Check(result == OK, "explicitly woken waiters return success");
+  }
+}
+
+void TestAddressesAreIsolated() {
+  uint32_t first = 0;
+  uint32_t second = 0;
+  uint32_t first_timeout = 1000000;
+  uint32_t second_timeout = 1000000;
+  std::atomic<int> ready{0};
+  std::atomic<bool> first_returned{false};
+  std::atomic<bool> second_returned{false};
+  int first_result = 0;
+  int second_result = 0;
+
+  std::thread first_waiter([&] {
+    ready.fetch_add(1, std::memory_order_release);
+    first_result = Wait32(&first, 0, &first_timeout);
+    first_returned.store(true, std::memory_order_release);
+  });
+  std::thread second_waiter([&] {
+    ready.fetch_add(1, std::memory_order_release);
+    second_result = Wait32(&second, 0, &second_timeout);
+    second_returned.store(true, std::memory_order_release);
+  });
+  while (ready.load(std::memory_order_acquire) != 2) {
+    std::this_thread::yield();
+  }
+  std::this_thread::sleep_for(std::chrono::milliseconds(50));
+  Check(Wake(&first, 1) == OK, "first address wakes");
+  std::this_thread::sleep_for(std::chrono::milliseconds(50));
+  Check(first_returned.load(std::memory_order_acquire),
+        "first address waiter returned");
+  Check(!second_returned.load(std::memory_order_acquire),
+        "waking one address does not release another address");
+  Check(Wake(&second, 1) == OK, "second address wakes");
+
+  first_waiter.join();
+  second_waiter.join();
+  Check(first_result == OK && second_result == OK,
+        "isolated waiters return success");
+}
+
+void TestCompareRegisterWakeRace() {
+  for (int i = 0; i < 100; i++) {
+    uint32_t word = 0;
+    uint32_t timeout = 500000;
+    std::atomic<bool> ready{false};
+    int result = Libs::LibKernel::KERNEL_ERROR_ETIMEDOUT;
+    std::thread waiter([&] {
+      ready.store(true, std::memory_order_release);
+      result = Wait32(&word, 0, &timeout);
+    });
+    while (!ready.load(std::memory_order_acquire)) {
+      std::this_thread::yield();
+    }
+    Store(&word, uint32_t{1});
+    (void)Wake(&word, 1);
+    waiter.join();
+    Check(result == OK, "compare/register/wake race never loses progress");
+  }
+}
+
+void TestWakeZeroIsNoOp() {
+  constexpr int WAITER_COUNT = 2;
+  uint32_t word = 0;
+  uint32_t timeout = 1000000;
+  std::atomic<int> ready{0};
+  std::atomic<int> returned{0};
+  int results[WAITER_COUNT] = {};
+  std::vector<std::thread> waiters;
+
+  for (int i = 0; i < WAITER_COUNT; i++) {
+    waiters.emplace_back([&, i] {
+      ready.fetch_add(1, std::memory_order_release);
+      results[i] = Wait32(&word, 0, &timeout);
+      returned.fetch_add(1, std::memory_order_release);
+    });
+  }
+  while (ready.load(std::memory_order_acquire) != WAITER_COUNT) {
+    std::this_thread::yield();
+  }
+  std::this_thread::sleep_for(std::chrono::milliseconds(50));
+  Check(Wake(&word, 0) == OK, "zero-count wake succeeds");
+  std::this_thread::sleep_for(std::chrono::milliseconds(20));
+  Check(returned.load(std::memory_order_acquire) == 0,
+        "zero-count wake releases no waiters");
+  Check(Wake(&word, INT_MAX) == OK, "wake-all succeeds after zero-count wake");
+  for (auto &waiter : waiters) {
+    waiter.join();
+  }
+  for (int result : results) {
+    Check(result == OK, "wake-all releases waiters after zero-count no-op");
+  }
+}
+
+} // namespace
+
+int main() {
+  TestInvalidAddress();
+  TestMismatchReturnsImmediately();
+  TestTimeout();
+  TestValueChangeAndWake();
+  TestWakeOneThenAll();
+  TestAddressesAreIsolated();
+  TestCompareRegisterWakeRace();
+  TestWakeZeroIsNoOp();
+  std::printf("SyncOnAddressTests: all passed\n");
+  return 0;
+}