graphics: shader: ignore unused sampler border state

* Sampler dword 3 only matters when a clamp mode uses border color
  (values >= 4). When no border mode is active, dword 3 is unused
  but can still vary across loop iterations due to wave-lane spills.
  This makes resource tracking think the descriptor is dynamic and
  fail with "unsupported GPU selection".

* Fix by zeroing dword 3 when all clamp modes are non-border.

Signed-off-by: Claxten <claxten10@gmail.com>

src/CMakeLists.txt

@@ -312,6 +312,9 @@ function(add_kyty_full_emulator_test target source)
 		target_link_libraries(${target} onecore)
 		add_custom_command(TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $<TARGET_FILE_DIR:${target}>/libwinpthread-1.dll)
 	endif()
+	# The macOS x86_64 guest address space needs its .zerofill segments anchored
+	# by linker flags, or the kernel kills the binary on load (posix_spawn EIO).
+	configure_macos_guest_address_space(${target})
 endfunction()
 
 function(configure_macos_guest_address_space target)
@@ -431,7 +434,6 @@ target_sources(shader_recompiler_compute_tests PRIVATE
 add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp)
 target_compile_definitions(virtual_memory_allocation_tests PRIVATE
 	KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1)
-configure_macos_guest_address_space(virtual_memory_allocation_tests)
 
 # These tests use exceptions.
 if(NOT KYTY_CLANG_CL)

src/graphics/guest_gpu/pm4.cpp

@@ -110,7 +110,6 @@ void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t star
 	auto* cmd = cmd_buffer + start_dw;
 	auto  dw  = num_dw;
 	while (dw != 0) {
-		EXIT_NOT_IMPLEMENTED(dw < 2);
 		EXIT_NOT_IMPLEMENTED(dw > num_dw);
 
 		auto cmd_id = *cmd++;
@@ -120,6 +119,9 @@ void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t star
 		uint32_t len = 0;
 
 		const auto packet_type = static_cast<PacketType>(cmd_id >> 30u);
+		// Type-2 packets are header-only padding; every other packet type requires a body.
+		EXIT_NOT_IMPLEMENTED(dw < 2 && packet_type != PacketType::Type2);
+
 		switch (packet_type) {
 			case PacketType::Type3: {
 				const bool    sh_gx = (cmd_id & 0x2u) == 0;

src/graphics/shader/recompiler/ir/ScalarProvenance.cpp

@@ -45,6 +45,8 @@ namespace {
 
 constexpr uint32_t ScalarRegisters = 128;
 constexpr uint32_t VectorRegisters = 256;
+// Clamp X/Y/Z are consecutive three-bit fields; the high bit of each selects a border mode.
+constexpr uint32_t SamplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u);
 
 struct ScalarState {
 	std::array<uint32_t, ScalarRegisters> regs          = {};
@@ -647,6 +649,24 @@ private:
 		return AddDescriptor(descriptor);
 	}
 
+	uint32_t AddSamplerDescriptor(const ScalarState& state, uint32_t base) {
+		if (base >= ScalarRegisters || 4u > ScalarRegisters - base) {
+			return ScalarProvenance::Unknown;
+		}
+		DescriptorValue descriptor;
+		descriptor.dword_count = 4;
+		for (uint32_t i = 0; i < 4; i++) {
+			descriptor.dwords[i] = state.regs[base + i];
+		}
+		if (auto d0 = descriptor.dwords[0];
+		    d0 < m_graph.values.size() && m_graph.values[d0].op == ScalarValueOp::Constant &&
+		    (m_graph.values[d0].imm & SamplerBorderClampMask) == 0) {
+			// Without a border clamp, the border color and table index in dword 3 are unused.
+			descriptor.dwords[3] = Constant(0);
+		}
+		return AddDescriptor(descriptor);
+	}
+
 	uint32_t AddFlatAddressDescriptor(const Instruction& inst, const ScalarState& state) {
 		const uint32_t first = FlatStore(inst.op) ? 1u : 0u;
 		if (inst.src_count < first + 2u) {
@@ -704,7 +724,7 @@ private:
 			inst.memory.resource_source = AddDescriptor(state, inst.memory.resource * 4u, 8);
 			if (inst.op == Opcode::ImageSample || inst.op == Opcode::ImageGather4 ||
 			    inst.op == Opcode::ImageGetLod) {
-				inst.memory.sampler_source = AddDescriptor(state, inst.memory.sampler * 4u, 4);
+				inst.memory.sampler_source = AddSamplerDescriptor(state, inst.memory.sampler * 4u);
 			}
 		}
 	}

src/kernel/memory.cpp

@@ -818,6 +818,11 @@ static void                               MemoryPoolSubtractCommitted(uint64_t l
 // Keep host mappings, physical blocks, placeholders, and virtual ranges in step.
 static std::recursive_mutex g_memory_operation_mutex;
 
+// The base address the PS5 kernel hands out for hint-less user mappings. Guest code can
+// assume mappings it did not place explicitly are at or above this (Sony's libc rejects a
+// heap below it), so hint-less searches must not fall back to the low system-managed range.
+static constexpr uint64_t GUEST_DEFAULT_MAP_BASE = 0x200000000ull;
+
 static uint64_t FindGuestFreeRange(uint64_t search_addr, uint64_t size, uint64_t alignment) {
 	EXIT_IF(g_guest_address_space == nullptr || g_virtual_ranges == nullptr);
 
@@ -845,8 +850,11 @@ static uint64_t FindGuestFreeRange(uint64_t search_addr, uint64_t size, uint64_t
 	if (search_addr != 0) {
 		return find_in(search_addr, HOST_USER_MAX + 1u);
 	}
-	auto addr = find_in(HOST_SYSTEM_MANAGED_MIN, HOST_SYSTEM_MANAGED_MAX + 1u);
+	auto addr = find_in(GUEST_DEFAULT_MAP_BASE, HOST_SYSTEM_MANAGED_MAX + 1u);
-	return addr != 0 ? addr : find_in(HOST_USER_MIN, HOST_USER_MAX + 1u);
+	if (addr == 0) {
+		addr = find_in(HOST_USER_MIN, HOST_USER_MAX + 1u);
+	}
+	return addr;
 }
 
 bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size) {

src/loader/runtimeLinker.cpp

@@ -38,7 +38,10 @@
 #include <windows.h>
 #else
 #include <dlfcn.h>
-#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+#if defined(__APPLE__)
+#include <mach/mach.h>
+#include <mach/mach_vm.h>
+#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
 #include <sys/uio.h>
 #include <unistd.h>
 #endif
@@ -722,7 +725,26 @@ static bool IsReadableRange(uint64_t addr, uint64_t size) {
 		}
 		current = std::min(region_end, end);
 	}
-#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+#elif defined(__APPLE__)
+	// Walk the Mach regions covering the range and require read permission. The fatal
+	// report dumps memory behind raw register values, and a fault inside the reporter
+	// re-enters the signal handler and wedges the reporting thread.
+	uint64_t current = addr;
+	while (current < end) {
+		mach_vm_address_t              region_addr = current;
+		mach_vm_size_t                 region_size = 0;
+		vm_region_basic_info_data_64_t info {};
+		mach_msg_type_number_t         count       = VM_REGION_BASIC_INFO_COUNT_64;
+		mach_port_t                    object_name = MACH_PORT_NULL;
+		if (mach_vm_region(mach_task_self(), &region_addr, &region_size, VM_REGION_BASIC_INFO_64,
+		                   reinterpret_cast<vm_region_info_t>(&info), &count,
+		                   &object_name) != KERN_SUCCESS ||
+		    region_addr > current || (info.protection & VM_PROT_READ) == 0) {
+			return false;
+		}
+		current = region_addr + region_size;
+	}
+#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
 	const auto page_size = static_cast<uint64_t>(sysconf(_SC_PAGESIZE));
 	if (page_size == 0) {
 		return false;
@@ -752,7 +774,7 @@ static bool IsReadableRange(uint64_t addr, uint64_t size) {
 }
 
 static bool IsDumpableRange(uint64_t addr, uint64_t size) {
-#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
 	return IsReadableRange(addr, size);
 #else
 	(void)size;

tests/VirtualMemoryAllocationTests.cpp

@@ -1356,6 +1356,135 @@ void TestLargeDirectMapAliasesAcrossChunks() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestHintlessDirectMapUsesCanonicalGuestBase() {
+	// Mirrors the allocation Sony's libc.prx makes for its internal heap: 4 MiB of
+	// direct memory, 2 MiB aligned, mapped with no address hint. The PS5 kernel never
+	// places hint-less user mappings below 0x200000000 and guest code relies on that
+	// (libc fails its mspace setup for a lower heap address, and the first malloc then
+	// dereferences a null mspace). Writes through the mapping must also stick.
+	const char* test = "HintlessDirectMapUsesCanonicalGuestBase";
+
+	constexpr uint64_t Len   = 0x400000;
+	constexpr uint64_t Align = 0x200000;
+
+	int64_t phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(0, 0x260000000ull, Len, Align, 12,
+	                                                            &phys_addr),
+	        "KernelAllocateDirectMemory");
+
+	void* address = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(&address, Len, SceKernelProtCpuRw,
+	                                                            0, phys_addr, Align, "libc_heap"),
+	        "KernelMapNamedDirectMemory");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	{
+		char message[128] = {};
+		std::snprintf(message, sizeof(message),
+		              "hint-less direct map landed below the PS5 base: 0x%016" PRIx64, base);
+		Check(test, base >= 0x200000000ull, message);
+	}
+
+	auto* header = reinterpret_cast<uint64_t*>(base);
+	header[0]    = 0x4d53504143453030ull; // "MSPACE00"
+	header[7]    = 0x58585858ull;         // magic at +0x38, like the libc mspace
+	*reinterpret_cast<uint64_t*>(base + Len - 8) = 0x454e444d41524bull;
+
+	Check(test, header[0] == 0x4d53504143453030ull, "immediate readback of header[0] failed");
+	Check(test, header[7] == 0x58585858ull, "immediate readback of header[7] failed");
+	Check(test, *reinterpret_cast<const uint64_t*>(base + Len - 8) == 0x454e444d41524bull,
+	      "immediate readback of tail failed");
+
+	uint64_t backing = 0;
+	Check(test, Libs::LibKernel::Memory::TryReadBacking(base + 0x38, &backing, sizeof(backing)),
+	      "TryReadBacking(header+0x38)");
+	Check(test, backing == 0x58585858ull, "backing store does not see the guest write at +0x38");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, Len), "KernelMunmap");
+	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, Len),
+	        "KernelReleaseDirectMemory");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestDirectMemoryContentPersistsAcrossRemap() {
+	const char* test = "DirectMemoryContentPersistsAcrossRemap";
+
+	constexpr uint64_t MapSize = SceKernelPageSize * 4;
+
+	int64_t phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            SceKernelDirectMemoryStart, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
+	            MapSize, SceKernelPageSize, SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+
+	// Direct memory is physical: contents must survive unmapping and remapping, including
+	// a remap of a sub-range at a nonzero physical offset.
+	void* address = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(&address, MapSize,
+	                                                            SceKernelProtCpuRw, 0, phys_addr,
+	                                                            SceKernelPageSize, "persist_a"),
+	        "KernelMapNamedDirectMemory(first)");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	for (uint64_t offset = 0; offset < MapSize; offset += sizeof(uint64_t)) {
+		*reinterpret_cast<uint64_t*>(base + offset) = offset ^ 0x4b5954595045525aull; // "KYTYPERZ"
+	}
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, MapSize), "KernelMunmap(first)");
+
+	void* remap = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(&remap, MapSize,
+	                                                            SceKernelProtCpuRw, 0, phys_addr,
+	                                                            SceKernelPageSize, "persist_b"),
+	        "KernelMapNamedDirectMemory(remap)");
+	const auto remap_base = reinterpret_cast<uint64_t>(remap);
+	for (uint64_t offset = 0; offset < MapSize; offset += sizeof(uint64_t)) {
+		const auto expected = offset ^ 0x4b5954595045525aull;
+		const auto actual   = *reinterpret_cast<const uint64_t*>(remap_base + offset);
+		if (actual != expected) {
+			char message[160] = {};
+			std::snprintf(message, sizeof(message),
+			              "content lost across remap at offset 0x%" PRIx64 ": expected 0x%016" PRIx64
+			              ", read 0x%016" PRIx64,
+			              offset, expected, actual);
+			Fail(test, message);
+		}
+	}
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(remap_base, MapSize), "KernelMunmap(remap)");
+
+	// Sub-range remap at a nonzero physical offset: page 2 of the original allocation.
+	void* partial = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
+	            &partial, SceKernelPageSize, SceKernelProtCpuRw, 0,
+	            phys_addr + static_cast<int64_t>(SceKernelPageSize * 2), SceKernelPageSize,
+	            "persist_c"),
+	        "KernelMapNamedDirectMemory(partial)");
+	const auto partial_base = reinterpret_cast<uint64_t>(partial);
+	for (uint64_t offset = 0; offset < SceKernelPageSize; offset += sizeof(uint64_t)) {
+		const auto expected = (SceKernelPageSize * 2 + offset) ^ 0x4b5954595045525aull;
+		const auto actual   = *reinterpret_cast<const uint64_t*>(partial_base + offset);
+		if (actual != expected) {
+			char message[160] = {};
+			std::snprintf(message, sizeof(message),
+			              "content lost in partial remap at offset 0x%" PRIx64
+			              ": expected 0x%016" PRIx64 ", read 0x%016" PRIx64,
+			              offset, expected, actual);
+			Fail(test, message);
+		}
+	}
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(partial_base, SceKernelPageSize),
+	        "KernelMunmap(partial)");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, MapSize),
+	        "KernelReleaseDirectMemory");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestDirectMapUnmapReusesHostAddress() {
 	const char* test = "DirectMapUnmapReusesHostAddress";
 
@@ -2150,6 +2279,8 @@ int main() {
 	RunTest(TestDirectAlignmentStaysWithinSearchRange);
 	RunTest(TestDefaultDirectMapUsesSystemAddressRange);
 	RunTest(TestLargeDirectMapAliasesAcrossChunks);
+	RunTest(TestHintlessDirectMapUsesCanonicalGuestBase);
+	RunTest(TestDirectMemoryContentPersistsAcrossRemap);
 	RunTest(TestDirectMapUnmapReusesHostAddress);
 	RunTest(TestFixedReserveReplacesPartialDirectMapping);
 	RunTest(TestFixedReserveRollbackConsumesRestoredPlaceholder);