Implement ftruncate abi

Extracted from Stefanos Costa's pthread fix in #147.

.github/workflows/build.yml

@@ -73,7 +73,7 @@ jobs:
       - name: Configure
         shell: cmd
         run: |
-          cmake -S src -B _Build/windows ^
+          cmake -S . -B _Build/windows ^
             -G Ninja ^
             -DCMAKE_BUILD_TYPE=Release ^
             -DCMAKE_C_COMPILER=clang-cl ^
@@ -83,12 +83,12 @@ jobs:
       - name: Build
         shell: cmd
         run: |
-          cmake --build _Build/windows --target launcher virtual_memory_allocation_tests --parallel
+          cmake --build _Build/windows --target launcher audio_out2_port_tests virtual_memory_allocation_tests --parallel
 
       - name: Test
         shell: cmd
         run: |
-          ctest --test-dir _Build/windows --output-on-failure -R "^virtual_memory_allocation$"
+          ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|virtual_memory_allocation)$"
 
       - name: Install
         shell: cmd
@@ -147,7 +147,7 @@ jobs:
       - name: Configure
         shell: bash
         run: |
-          cmake -S src -B _Build/macos \
+          cmake -S . -B _Build/macos \
             -G Ninja \
             -DCMAKE_BUILD_TYPE=Release \
             -DCMAKE_OSX_ARCHITECTURES=x86_64 \
@@ -159,14 +159,14 @@ jobs:
         shell: bash
         run: |
           cmake --build _Build/macos \
-            --target launcher virtual_memory_allocation_tests \
+            --target launcher audio_out2_port_tests virtual_memory_allocation_tests \
             --parallel
 
       - name: Test
         shell: bash
         run: |
           ctest --test-dir _Build/macos --output-on-failure \
-            -R '^virtual_memory_allocation$'
+            -R '^(audio_out2_port|virtual_memory_allocation)$'
 
       - name: Install
         shell: bash
@@ -270,7 +270,7 @@ jobs:
         shell: bash
         run: |
           mkdir -p _Build
-          cmake -S src -B _Build/linux \
+          cmake -S . -B _Build/linux \
             -G Ninja \
             -DCMAKE_BUILD_TYPE=Release \
             -DCMAKE_C_COMPILER=clang \
@@ -297,14 +297,14 @@ jobs:
         run: |
           cmake --build _Build/linux \
             --target launcher page_manager_tests memory_tracker_tests \
-              virtual_memory_allocation_tests \
+              audio_out2_port_tests virtual_memory_allocation_tests \
             --parallel
 
       - name: Test
         shell: bash
         run: |
           ctest --test-dir _Build/linux --output-on-failure \
-            -R '^(page_manager|memory_tracker|virtual_memory_allocation)$'
+            -R '^(audio_out2_port|page_manager|memory_tracker|virtual_memory_allocation)$'
 
       - name: Install
         shell: bash

.gitignore

@@ -3,4 +3,5 @@
 .idea/
 build/
 _Build/vscode-clang/
-_Build/
+_Build/
+cmake-build-debug/

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-    "cmake.sourceDirectory": "${workspaceFolder}/src",
+    "cmake.sourceDirectory": "${workspaceFolder}",
     "cmake.buildDirectory": "${workspaceFolder}/_Build/vscode-clang",
     "cmake.generator": "Ninja",
     "cmake.environment": {

CMakeLists.txt

@@ -6,6 +6,10 @@ endif()
 
 project(Kyty)
 
+set(KYTY_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src")
+set(KYTY_TESTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests")
+set(KYTY_THIRD_PARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/3rdparty")
+
 if(CMAKE_SYSTEM_NAME MATCHES ".*Linux")
 	set(LINUX TRUE)
 endif()
@@ -19,13 +23,11 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_CXX_SCAN_FOR_MODULES OFF)
 
-include(utils.cmake)
+include("${KYTY_SOURCE_DIR}/utils.cmake")
 include(CTest)
 
 option(KYTY_ENABLE_CLANG_TIDY "Run clang-tidy checks during builds" OFF)
 
-set(KYTY_THIRD_PARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../3rdparty")
-
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 
 include(TestBigEndian)
@@ -97,36 +99,36 @@ endif()
 
 project(Kyty${KYTY_PROJECT_NAME}${CMAKE_BUILD_TYPE}${KYTY_COMPILER} VERSION 0.2.2)
 
-include_directories(	
+include_directories(
 	${KYTY_THIRD_PARTY_DIR}/gtest/include
 	${KYTY_THIRD_PARTY_DIR}/gtest
 	${KYTY_THIRD_PARTY_DIR}/fmt/include
 	${KYTY_THIRD_PARTY_DIR}/magic_enum/include/magic_enum
 	${PROJECT_BINARY_DIR}
-	${CMAKE_CURRENT_SOURCE_DIR}
+	${KYTY_SOURCE_DIR}
 )
 
 set(KYTY_VERSION "${PROJECT_VERSION}")
 
 configure_file(
-  ${PROJECT_SOURCE_DIR}/cmake_config.h.in
+  ${KYTY_SOURCE_DIR}/cmake_config.h.in
   ${PROJECT_BINARY_DIR}/cmake_config.h
   )
 
 find_package(Git)
-  
+
 add_custom_target( KytyGitVersion
-    COMMAND ${CMAKE_COMMAND} 
-	-D INPUT_FILE=${CMAKE_CURRENT_SOURCE_DIR}/kytyGitVersion.h.in
-    -D OUTPUT_FILE=${CMAKE_CURRENT_BINARY_DIR}/kytyGitVersion.h
+    COMMAND ${CMAKE_COMMAND}
+	-D INPUT_FILE=${KYTY_SOURCE_DIR}/kytyGitVersion.h.in
+	-D OUTPUT_FILE=${PROJECT_BINARY_DIR}/kytyGitVersion.h
 	-D GIT_EXECUTABLE=${GIT_EXECUTABLE}
 	-D GIT_WORKING_DIRECTORY=${CMAKE_CURRENT_SOURCE_DIR}
-	-P ${CMAKE_CURRENT_SOURCE_DIR}/generate_version.cmake
+	-P ${KYTY_SOURCE_DIR}/generate_version.cmake
 	COMMENT "Generate kytyGitVersion.h"
 )
 
 if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0.0)
-	list(APPEND KYTY_IWYU 
+	list(APPEND KYTY_IWYU
 		kyty_emulator
 		common
 		#launcher
@@ -139,55 +141,55 @@ if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0.0)
 endif()
 
 option(KYTY_BUILD_LAUNCHER "Build Qt launcher" ON)
-  
-config_compiler_and_linker()  
-  
+
+config_compiler_and_linker()
+
 add_subdirectory("${KYTY_THIRD_PARTY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/3rdparty")
-add_subdirectory(common)
+add_subdirectory("${KYTY_SOURCE_DIR}/common" "${CMAKE_CURRENT_BINARY_DIR}/common")
 
 file(GLOB kyty_emulator_src CONFIGURE_DEPENDS
-	libs/*.cpp
-	libs/*.h
-	graphics/*.cpp
-	graphics/*.h
-	graphics/guest_gpu/*.cpp
-	graphics/guest_gpu/*.h
-	graphics/guest_gpu/command_processor/*.cpp
-	graphics/guest_gpu/command_processor/*.h
-	graphics/host_gpu/*.cpp
-	graphics/host_gpu/*.h
-	graphics/host_gpu/renderer/*.cpp
-	graphics/host_gpu/renderer/*.h
-	graphics/host_gpu/renderer/cache/*.cpp
-	graphics/host_gpu/renderer/cache/*.h
-	graphics/host_gpu/renderer/image/*.cpp
-	graphics/host_gpu/renderer/image/*.h
-	graphics/host_gpu/renderer/pipeline/*.cpp
-	graphics/host_gpu/renderer/pipeline/*.h
-	graphics/shader/*.cpp
-	graphics/shader/*.h
-	graphics/shader/recompiler/*.cpp
-	graphics/shader/recompiler/*.h
-	graphics/shader/recompiler/cfg/*.cpp
-	graphics/shader/recompiler/cfg/*.h
-	graphics/shader/recompiler/decompiler/*.cpp
-	graphics/shader/recompiler/decompiler/*.h
-	graphics/shader/recompiler/emitter/*.cpp
-	graphics/shader/recompiler/emitter/*.h
-	graphics/shader/recompiler/ir/*.cpp
-	graphics/shader/recompiler/ir/*.h
-	graphics/presentation/*.cpp
-	graphics/presentation/*.h
-	graphics/presentation/window/*.cpp
-	graphics/presentation/window/*.h
-	kernel/*.cpp
-	kernel/*.h
-	loader/*.cpp
-	loader/*.h
+	"${KYTY_SOURCE_DIR}/libs/*.cpp"
+	"${KYTY_SOURCE_DIR}/libs/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/command_processor/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/command_processor/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/cache/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/cache/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/image/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/image/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/pipeline/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/pipeline/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/cfg/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/cfg/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/decompiler/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/decompiler/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/emitter/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/emitter/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/presentation/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/presentation/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/presentation/window/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/presentation/window/*.h"
+	"${KYTY_SOURCE_DIR}/kernel/*.cpp"
+	"${KYTY_SOURCE_DIR}/kernel/*.h"
+	"${KYTY_SOURCE_DIR}/loader/*.cpp"
+	"${KYTY_SOURCE_DIR}/loader/*.h"
 )
 
 find_program(KYTY_GLSLANG_VALIDATOR glslangValidator REQUIRED)
-set(gpu_tiler_shader_dir "${CMAKE_CURRENT_SOURCE_DIR}/graphics/host_gpu/shaders")
+set(gpu_tiler_shader_dir "${KYTY_SOURCE_DIR}/graphics/host_gpu/shaders")
 set(gpu_tiler_generated_dir "${PROJECT_BINARY_DIR}/gpu_tiler_shaders")
 set(gpu_tiler_shader_names
 	standard256
@@ -215,7 +217,7 @@ foreach(shader_name IN LISTS gpu_tiler_shader_names)
 		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
 		        "-I${gpu_tiler_shader_dir}" -o "${shader_spv}" "${shader_source}"
 		COMMAND ${CMAKE_COMMAND} -DINPUT=${shader_spv} -DOUTPUT=${shader_header}
-		        -DSYMBOL=${shader_symbol} -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+		        -DSYMBOL=${shader_symbol} -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
 		DEPENDS "${shader_source}" ${gpu_tiler_shader_includes}
 		VERBATIM
 	)
@@ -239,7 +241,7 @@ foreach(shader_source IN LISTS gpu_blit_shader_sources)
 		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
 		        "-I${gpu_tiler_shader_dir}" -o "${shader_spv}" "${shader_source}"
 		COMMAND ${CMAKE_COMMAND} -DINPUT=${shader_spv} -DOUTPUT=${shader_header}
-		        -DSYMBOL=${shader_symbol} -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+		        -DSYMBOL=${shader_symbol} -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
 		DEPENDS "${shader_source}"
 		VERBATIM
 	)
@@ -248,8 +250,8 @@ endforeach()
 list(APPEND kyty_emulator_src ${gpu_blit_shader_headers})
 
 list(APPEND kyty_emulator_src
-	emulator.h
-	emulator.cpp
+	"${KYTY_SOURCE_DIR}/emulator.h"
+	"${KYTY_SOURCE_DIR}/emulator.cpp"
 )
 list(REMOVE_DUPLICATES kyty_emulator_src)
 
@@ -280,7 +282,7 @@ if(LINUX)
 endif()
 
 set(inc_headers
-	${CMAKE_CURRENT_SOURCE_DIR}
+	${KYTY_SOURCE_DIR}
 	${KYTY_THIRD_PARTY_DIR}/SDL2/include
 	${KYTY_THIRD_PARTY_DIR}/VulkanMemoryAllocator/include
 	${KYTY_THIRD_PARTY_DIR}/SPIRV-Tools/include
@@ -292,13 +294,13 @@ set(inc_headers
 
 if (KYTY_CLANG_CL)
 	list(APPEND kyty_emulator_link_libraries winpthread)
-	list(APPEND inc_headers 
+	list(APPEND inc_headers
 		${KYTY_THIRD_PARTY_DIR}/winpthread/include
 	)
 endif()
 
 list(APPEND check_headers
-	${CMAKE_CURRENT_SOURCE_DIR}
+	${KYTY_SOURCE_DIR}
 )
 
 function(add_kyty_full_emulator_test target source)
@@ -312,100 +314,116 @@ function(add_kyty_full_emulator_test target source)
 		target_link_libraries(${target} onecore)
 		add_custom_command(TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $<TARGET_FILE_DIR:${target}>/libwinpthread-1.dll)
 	endif()
+	# The macOS x86_64 guest address space needs its .zerofill segments anchored
+	# by linker flags, or the kernel kills the binary on load (posix_spawn EIO).
+	configure_macos_guest_address_space(${target})
 endfunction()
 
 function(configure_macos_guest_address_space target)
 	if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
 			(NOT CMAKE_OSX_ARCHITECTURES AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$")))
-		target_sources(${target} PRIVATE kernel/macosGuestAddressSpace.cpp)
+		target_sources(${target} PRIVATE "${KYTY_SOURCE_DIR}/kernel/macosGuestAddressSpace.cpp")
 		target_compile_definitions(${target} PRIVATE KYTY_LINKED_GUEST_ADDRESS_SPACE=1)
 		target_link_options(${target} PRIVATE
 			-Wl,-ld_classic,-no_pie,-no_fixup_chains,-no_huge,-pagezero_size,0x40000,-segaddr,SYSTEM_MANAGED,0x40000,-segaddr,SYSTEM_RESERVED,0x7ffffc000,-segaddr,USER_AREA,0x7000000000,-image_base,0x700000000000)
 	endif()
 endfunction()
 
-add_kyty_full_emulator_test(shader_cfg_tests ../tests/shaderCfgTests.cpp)
+add_kyty_full_emulator_test(shader_cfg_tests "${KYTY_TESTS_DIR}/shaderCfgTests.cpp")
 
 add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL
-	../tests/ScalarProvenanceTests.cpp
-	graphics/host_gpu/hostMemory.cpp
-	graphics/shader/recompiler/ir/ReadLaneElimination.cpp
-	graphics/shader/recompiler/ir/ScalarProvenance.cpp
-	graphics/shader/recompiler/ir/SrtWalker.cpp
+	"${KYTY_TESTS_DIR}/ScalarProvenanceTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ReadLaneElimination.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp"
 )
 target_link_libraries(scalar_provenance_tests fmt::fmt)
 target_include_directories(scalar_provenance_tests PRIVATE ${inc_headers})
 
 add_executable(page_manager_tests EXCLUDE_FROM_ALL
-	../tests/PageManagerTests.cpp
-	graphics/host_gpu/pageManager.cpp
+	"${KYTY_TESTS_DIR}/PageManagerTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/pageManager.cpp"
 )
 target_include_directories(page_manager_tests PRIVATE ${inc_headers})
 
+add_executable(bit_array_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/BitArrayTests.cpp"
+)
+target_include_directories(bit_array_tests PRIVATE ${inc_headers})
+
 add_executable(memory_tracker_tests EXCLUDE_FROM_ALL
-	../tests/MemoryTrackerTests.cpp
-	graphics/host_gpu/pageManager.cpp
-	graphics/host_gpu/memoryTracker.cpp
+	"${KYTY_TESTS_DIR}/MemoryTrackerTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/pageManager.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/memoryTracker.cpp"
 )
 target_link_libraries(memory_tracker_tests fmt::fmt common)
 target_include_directories(memory_tracker_tests PRIVATE ${inc_headers})
 
 add_executable(shader_vertex_metadata_tests EXCLUDE_FROM_ALL
-	../tests/ShaderVertexMetadataTests.cpp
-	graphics/host_gpu/hostMemory.cpp
-	graphics/shader/shaderVertexMetadata.cpp
+	"${KYTY_TESTS_DIR}/ShaderVertexMetadataTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/shaderVertexMetadata.cpp"
 )
 target_include_directories(shader_vertex_metadata_tests PRIVATE ${inc_headers})
 
 add_executable(shader_stage_runtime_tests EXCLUDE_FROM_ALL
-	../tests/ShaderStageRuntimeTests.cpp
-	graphics/guest_gpu/gpu_format.cpp
-	graphics/host_gpu/hostMemory.cpp
-	graphics/shader/shaderStageRuntime.cpp
-	graphics/shader/recompiler/ir/ResourceMaterialization.cpp
-	graphics/shader/recompiler/ir/ScalarProvenance.cpp
-	graphics/shader/recompiler/ir/SrtWalker.cpp
+	"${KYTY_TESTS_DIR}/ShaderStageRuntimeTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/shaderStageRuntime.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceMaterialization.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp"
 )
 target_link_libraries(shader_stage_runtime_tests fmt::fmt)
 target_include_directories(shader_stage_runtime_tests PRIVATE ${inc_headers})
 
 add_executable(resource_tracking_tests EXCLUDE_FROM_ALL
-	../tests/ResourceTrackingTests.cpp
-	graphics/guest_gpu/gpu_format.cpp
-	graphics/host_gpu/hostMemory.cpp
-	graphics/shader/recompiler/ir/ScalarProvenance.cpp
-	graphics/shader/recompiler/ir/SrtWalker.cpp
-	graphics/shader/recompiler/ir/SrtPatcher.cpp
-	graphics/shader/recompiler/ir/ResourceTracking.cpp
-	graphics/shader/recompiler/ir/ResourceMaterialization.cpp
-	graphics/shader/recompiler/ir/ShaderInfoCollection.cpp
-	graphics/shader/recompiler/ir/BindingLayout.cpp
+	"${KYTY_TESTS_DIR}/ResourceTrackingTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtPatcher.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceTracking.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceMaterialization.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ShaderInfoCollection.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/BindingLayout.cpp"
 )
 target_link_libraries(resource_tracking_tests fmt::fmt)
 target_include_directories(resource_tracking_tests PRIVATE ${inc_headers})
 
 add_executable(resource_mutex_tests EXCLUDE_FROM_ALL
-	../tests/ResourceMutexTests.cpp
-	graphics/host_gpu/renderer/cache/resourceMutex.cpp
+	"${KYTY_TESTS_DIR}/ResourceMutexTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/cache/resourceMutex.cpp"
 )
 target_link_libraries(resource_mutex_tests common)
 target_include_directories(resource_mutex_tests PRIVATE ${inc_headers})
 
+add_executable(audio_out2_port_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/AudioOut2PortTests.cpp"
+	"${KYTY_SOURCE_DIR}/libs/libAudio2.cpp"
+	"${KYTY_SOURCE_DIR}/loader/timer.cpp"
+)
+target_link_libraries(audio_out2_port_tests common fmt::fmt)
+target_include_directories(audio_out2_port_tests PRIVATE ${inc_headers})
+
 add_executable(event_queue_lifetime_tests EXCLUDE_FROM_ALL
-	../tests/EventQueueLifetimeTests.cpp
-	kernel/eventQueue.cpp
-	loader/timer.cpp
+	"${KYTY_TESTS_DIR}/EventQueueLifetimeTests.cpp"
+	"${KYTY_SOURCE_DIR}/kernel/eventQueue.cpp"
+	"${KYTY_SOURCE_DIR}/loader/timer.cpp"
 )
 target_link_libraries(event_queue_lifetime_tests common fmt::fmt)
 target_include_directories(event_queue_lifetime_tests PRIVATE ${inc_headers})
 
 add_executable(image_page_table_tests EXCLUDE_FROM_ALL
-	../tests/ImagePageTableTests.cpp
+	"${KYTY_TESTS_DIR}/ImagePageTableTests.cpp"
 )
 target_link_libraries(image_page_table_tests common fmt::fmt)
 target_include_directories(image_page_table_tests PRIVATE ${inc_headers})
 
-add_kyty_full_emulator_test(shader_recompiler_compute_tests ../tests/ShaderRecompilerComputeTests.cpp)
+add_kyty_full_emulator_test(shader_recompiler_compute_tests "${KYTY_TESTS_DIR}/ShaderRecompilerComputeTests.cpp")
 
 set(gpu_test_generated_dir "${PROJECT_BINARY_DIR}/gpu_test_shaders")
 set(gpu_test_ms_depth_source
@@ -421,17 +439,16 @@ add_custom_command(
 	        -o "${gpu_test_ms_depth_spv}" "${gpu_test_ms_depth_source}"
 	COMMAND ${CMAKE_COMMAND} -DINPUT=${gpu_test_ms_depth_spv}
 	        -DOUTPUT=${gpu_test_ms_depth_header} -DSYMBOL=GPU_TEST_MS_DEPTH_SPV
-	        -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+	        -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
 	DEPENDS "${gpu_test_ms_depth_source}"
 	VERBATIM
 )
 target_sources(shader_recompiler_compute_tests PRIVATE
 	"${gpu_test_ms_depth_header}")
 
-add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp)
+add_kyty_full_emulator_test(virtual_memory_allocation_tests "${KYTY_TESTS_DIR}/VirtualMemoryAllocationTests.cpp")
 target_compile_definitions(virtual_memory_allocation_tests PRIVATE
 	KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1)
-configure_macos_guest_address_space(virtual_memory_allocation_tests)
 
 # These tests use exceptions.
 if(NOT KYTY_CLANG_CL)
@@ -442,12 +459,18 @@ if(NOT KYTY_CLANG_CL)
 endif()
 
 if(BUILD_TESTING)
+	add_test(NAME shader_cfg COMMAND $<TARGET_FILE:shader_cfg_tests>)
 	add_test(NAME scalar_provenance COMMAND $<TARGET_FILE:scalar_provenance_tests>)
 	add_test(NAME image_page_table COMMAND $<TARGET_FILE:image_page_table_tests>)
 	add_test(NAME memory_tracker COMMAND $<TARGET_FILE:memory_tracker_tests>)
 	add_test(NAME page_manager COMMAND $<TARGET_FILE:page_manager_tests>)
+	add_test(NAME bit_array COMMAND $<TARGET_FILE:bit_array_tests>)
+	add_test(NAME shader_vertex_metadata COMMAND $<TARGET_FILE:shader_vertex_metadata_tests>)
+	add_test(NAME shader_stage_runtime COMMAND $<TARGET_FILE:shader_stage_runtime_tests>)
+	add_test(NAME resource_tracking COMMAND $<TARGET_FILE:resource_tracking_tests>)
 	add_test(NAME resource_mutex COMMAND $<TARGET_FILE:resource_mutex_tests>)
 	add_test(NAME event_queue_lifetime COMMAND $<TARGET_FILE:event_queue_lifetime_tests>)
+	add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
 	add_test(NAME virtual_memory_allocation
 		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
@@ -480,10 +503,26 @@ if(BUILD_TESTING)
 		add_test(NAME buffer_cache_ranges
 			COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-range-only)
 	endif()
+
+	add_custom_target(kyty_tests DEPENDS
+		shader_cfg_tests
+		scalar_provenance_tests
+		image_page_table_tests
+		memory_tracker_tests
+		page_manager_tests
+		bit_array_tests
+		shader_vertex_metadata_tests
+		shader_stage_runtime_tests
+		resource_tracking_tests
+		resource_mutex_tests
+		event_queue_lifetime_tests
+		shader_recompiler_compute_tests
+		virtual_memory_allocation_tests
+	)
 endif()
 
 
-add_executable(kyty_emulator main.cpp ${kyty_emulator_src})
+add_executable(kyty_emulator "${KYTY_SOURCE_DIR}/main.cpp" ${kyty_emulator_src})
 configure_macos_guest_address_space(kyty_emulator)
 
 target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries})
@@ -513,7 +552,12 @@ set(KYTY_EMULATOR_MAP_LINK_PATH "${CMAKE_CURRENT_BINARY_DIR}/${KYTY_EMULATOR_MAP
 set(KYTY_EMULATOR_PDB_LINK_PATH "${CMAKE_CURRENT_BINARY_DIR}/kyty_emulator.pdb")
 
 if(KYTY_CLANG_CL)
-	set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "/DYNAMICBASE:NO /DEBUG:FULL /PDB:${KYTY_EMULATOR_PDB_LINK_PATH} /lldmap:${KYTY_EMULATOR_MAP_LINK_PATH}")
+	target_link_options(kyty_emulator PRIVATE
+		"/DYNAMICBASE:NO"
+		"/DEBUG:FULL"
+		"/PDB:${KYTY_EMULATOR_PDB_LINK_PATH}"
+		"/lldmap:${KYTY_EMULATOR_MAP_LINK_PATH}"
+	)
 	add_custom_command(TARGET kyty_emulator POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $<TARGET_FILE_DIR:kyty_emulator>/libwinpthread-1.dll)
 elseif(WIN32 OR LINUX)
 	set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "${KYTY_LD_OPTIONS} -Wl,-Map=${KYTY_EMULATOR_MAP_LINK_PATH}")
@@ -528,14 +572,14 @@ if(APPLE)
 	# reverts to the hardened defaults and aborts when it first executes written code).
 	add_custom_command(TARGET kyty_emulator POST_BUILD
 		COMMAND codesign -s - --force --options runtime
-			--entitlements "${CMAKE_CURRENT_SOURCE_DIR}/macos_jit.entitlements"
+			--entitlements "${KYTY_SOURCE_DIR}/macos_jit.entitlements"
 			$<TARGET_FILE:kyty_emulator>
 		COMMENT "Codesign kyty_emulator with JIT entitlements (macOS)")
 endif()
 
 install(TARGETS kyty_emulator DESTINATION .)
 if(KYTY_BUILD_LAUNCHER)
-	add_subdirectory(launcher)
+	add_subdirectory("${KYTY_SOURCE_DIR}/launcher" "${CMAKE_CURRENT_BINARY_DIR}/launcher")
 endif()
 if(KYTY_CLANG_CL)
 	install(FILES "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" DESTINATION .)

README.md

@@ -141,7 +141,7 @@ git submodule update --init --recursive
 Configure the project. Replace the Qt path with the version installed on your system:
 
 ```powershell
-cmake -S src -B _Build/windows -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_PREFIX_PATH="C:/Qt/6.x.x/msvc2022_64"
+cmake -S . -B _Build/windows -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_PREFIX_PATH="C:/Qt/6.x.x/msvc2022_64"
 ```
 
 Build the launcher and stage a runnable installation:
@@ -174,7 +174,7 @@ Qt 6 (Concurrent, Network, Widgets) is also required — either the distribution
 ```bash
 git submodule update --init --recursive
 
-cmake -S src -B _Build/linux -G Ninja -DCMAKE_BUILD_TYPE=Release \
+cmake -S . -B _Build/linux -G Ninja -DCMAKE_BUILD_TYPE=Release \
   -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
   -DCMAKE_PREFIX_PATH="$Qt6_DIR"
 
@@ -188,7 +188,7 @@ The install step copies the Qt libraries and plugins next to the binaries, so
 As on Windows, the MSVC compiler is not used; Clang is required. `cl.exe` is rejected at configure
 time.
 
-Note that the CMake source root is `src`, not the repository root.
+The CMake source root is the repository root.
 
 ### Building on macOS
 
@@ -207,7 +207,7 @@ Requirements:
 ```bash
 git submodule update --init --recursive
 
-cmake -S src -B _Build/macos -G Ninja -DCMAKE_BUILD_TYPE=Release \
+cmake -S . -B _Build/macos -G Ninja -DCMAKE_BUILD_TYPE=Release \
   -DCMAKE_OSX_ARCHITECTURES=x86_64 \
   -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
   -DCMAKE_PREFIX_PATH="$Qt6_DIR"
@@ -229,6 +229,17 @@ codesign --force --sign - _Build/macos/install/libMoltenVK.dylib
 
 Release archives already include a signed `libMoltenVK.dylib`.
 
+### Regression tests
+
+Build every regression executable and run the registered tests with:
+
+```powershell
+cmake --build _Build/windows --target kyty_tests
+ctest --test-dir _Build/windows --output-on-failure
+```
+
+Use `_Build/linux` instead of `_Build/windows` for a Linux build.
+
 ### Visual Studio Code
 
 A ready-made Visual Studio Code setup is included in [`.vscode`](.vscode). It configures CMake

src/common/CMakeLists.txt

@@ -13,7 +13,7 @@ endif()
 add_library(common STATIC ${common_src} ${common_headers})
 
 target_include_directories(common
-	PUBLIC ${PROJECT_SOURCE_DIR}
+	PUBLIC ${KYTY_SOURCE_DIR}
 	PRIVATE
 		${KYTY_THIRD_PARTY_DIR}/cpuinfo/include
 )

src/common/bitArray.h

@@ -0,0 +1,260 @@
+#ifndef EMULATOR_SRC_COMMON_BITARRAY_H_
+#define EMULATOR_SRC_COMMON_BITARRAY_H_
+
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+#include <utility>
+
+namespace Common {
+
+template <size_t N>
+class BitArray final {
+	static_assert(N != 0, "BitArray size must be nonzero");
+	static_assert(N % 64 == 0, "BitArray size must be a multiple of 64 bits");
+
+	static constexpr size_t BITS_PER_WORD = 64;
+	static constexpr size_t WORD_COUNT    = N / BITS_PER_WORD;
+
+public:
+	using Range = std::pair<size_t, size_t>;
+
+	class Iterator final {
+	public:
+		using iterator_category = std::forward_iterator_tag;
+		using value_type        = Range;
+		using difference_type   = std::ptrdiff_t;
+		using pointer           = const Range*;
+		using reference         = const Range&;
+
+		Iterator(const BitArray& bits, size_t start)
+		    : m_bits(bits), m_range(bits.FirstRangeFrom(start)) {}
+
+		Iterator& operator++() {
+			m_range = m_bits.FirstRangeFrom(m_range.second);
+			return *this;
+		}
+
+		[[nodiscard]] bool operator==(const Iterator& other) const {
+			return &m_bits == &other.m_bits && m_range == other.m_range;
+		}
+
+		[[nodiscard]] bool operator!=(const Iterator& other) const { return !(*this == other); }
+
+		[[nodiscard]] reference operator*() const { return m_range; }
+		[[nodiscard]] pointer   operator->() const { return &m_range; }
+
+	private:
+		const BitArray& m_bits;
+		Range           m_range;
+	};
+
+	using const_iterator = Iterator;
+
+	constexpr BitArray() = default;
+
+	constexpr BitArray(const BitArray& other, size_t start, size_t end) {
+		if (start >= end || end > N) {
+			return;
+		}
+
+		const auto first_word = start / BITS_PER_WORD;
+		const auto last_word  = (end - 1) / BITS_PER_WORD;
+		const auto start_bit  = start % BITS_PER_WORD;
+		const auto end_bit    = (end - 1) % BITS_PER_WORD;
+		const auto start_mask = ~uint64_t {0} << start_bit;
+		const auto end_mask =
+		    end_bit == BITS_PER_WORD - 1 ? ~uint64_t {0} : (uint64_t {1} << (end_bit + 1)) - 1;
+
+		if (first_word == last_word) {
+			m_data[first_word] = other.m_data[first_word] & start_mask & end_mask;
+			return;
+		}
+
+		m_data[first_word] = other.m_data[first_word] & start_mask;
+		for (auto word = first_word + 1; word < last_word; word++) {
+			m_data[word] = other.m_data[word];
+		}
+		m_data[last_word] = other.m_data[last_word] & end_mask;
+	}
+
+	[[nodiscard]] constexpr bool Get(size_t index) const {
+		return (m_data[index / BITS_PER_WORD] & (uint64_t {1} << (index % BITS_PER_WORD))) != 0;
+	}
+
+	constexpr void Set(size_t index) {
+		m_data[index / BITS_PER_WORD] |= uint64_t {1} << (index % BITS_PER_WORD);
+	}
+
+	constexpr void Unset(size_t index) {
+		m_data[index / BITS_PER_WORD] &= ~(uint64_t {1} << (index % BITS_PER_WORD));
+	}
+
+	constexpr void SetRange(size_t start, size_t end) {
+		if (start >= end || end > N) {
+			return;
+		}
+
+		const auto first_word = start / BITS_PER_WORD;
+		const auto last_word  = (end - 1) / BITS_PER_WORD;
+		const auto start_bit  = start % BITS_PER_WORD;
+		const auto end_bit    = (end - 1) % BITS_PER_WORD;
+		const auto start_mask = ~uint64_t {0} << start_bit;
+		const auto end_mask =
+		    end_bit == BITS_PER_WORD - 1 ? ~uint64_t {0} : (uint64_t {1} << (end_bit + 1)) - 1;
+
+		if (first_word == last_word) {
+			m_data[first_word] |= start_mask & end_mask;
+			return;
+		}
+
+		m_data[first_word] |= start_mask;
+		for (auto word = first_word + 1; word < last_word; word++) {
+			m_data[word] = ~uint64_t {0};
+		}
+		m_data[last_word] |= end_mask;
+	}
+
+	constexpr void UnsetRange(size_t start, size_t end) {
+		if (start >= end || end > N) {
+			return;
+		}
+
+		const auto first_word = start / BITS_PER_WORD;
+		const auto last_word  = (end - 1) / BITS_PER_WORD;
+		const auto start_bit  = start % BITS_PER_WORD;
+		const auto end_bit    = (end - 1) % BITS_PER_WORD;
+		const auto start_mask = (uint64_t {1} << start_bit) - 1;
+		const auto end_mask =
+		    end_bit == BITS_PER_WORD - 1 ? uint64_t {0} : ~((uint64_t {1} << (end_bit + 1)) - 1);
+
+		if (first_word == last_word) {
+			m_data[first_word] &= start_mask | end_mask;
+			return;
+		}
+
+		m_data[first_word] &= start_mask;
+		for (auto word = first_word + 1; word < last_word; word++) {
+			m_data[word] = 0;
+		}
+		m_data[last_word] &= end_mask;
+	}
+
+	constexpr void Clear() { m_data.fill(0); }
+	constexpr void Fill() { m_data.fill(~uint64_t {0}); }
+
+	[[nodiscard]] constexpr bool None() const {
+		uint64_t combined = 0;
+		for (const auto word: m_data) {
+			combined |= word;
+		}
+		return combined == 0;
+	}
+
+	[[nodiscard]] constexpr bool Any() const { return !None(); }
+
+	[[nodiscard]] constexpr Range FirstRangeFrom(size_t start) const {
+		if (start >= N) {
+			return {N, N};
+		}
+
+		auto word_index = start / BITS_PER_WORD;
+		auto word       = m_data[word_index] & (~uint64_t {0} << (start % BITS_PER_WORD));
+		while (word == 0) {
+			word_index++;
+			if (word_index == WORD_COUNT) {
+				return {N, N};
+			}
+			word = m_data[word_index];
+		}
+
+		const auto first     = word_index * BITS_PER_WORD + std::countr_zero(word);
+		const auto first_bit = first % BITS_PER_WORD;
+		const auto first_ones =
+		    static_cast<size_t>(std::countr_one(m_data[word_index] >> first_bit));
+		if (first_bit + first_ones < BITS_PER_WORD) {
+			return {first, first + first_ones};
+		}
+
+		for (word_index++; word_index < WORD_COUNT; word_index++) {
+			word = m_data[word_index];
+			if (word != ~uint64_t {0}) {
+				return {first, word_index * BITS_PER_WORD + std::countr_one(word)};
+			}
+		}
+		return {first, N};
+	}
+
+	[[nodiscard]] constexpr Range FirstRange() const { return FirstRangeFrom(0); }
+
+	[[nodiscard]] constexpr Range LastRangeFrom(size_t end) const {
+		if (end == 0) {
+			return {0, 0};
+		}
+		if (end > N) {
+			end = N;
+		}
+
+		auto       word_index = (end - 1) / BITS_PER_WORD;
+		const auto end_bit    = (end - 1) % BITS_PER_WORD;
+		const auto end_mask =
+		    end_bit == BITS_PER_WORD - 1 ? ~uint64_t {0} : (uint64_t {1} << (end_bit + 1)) - 1;
+		auto word = m_data[word_index] & end_mask;
+		while (word == 0) {
+			if (word_index == 0) {
+				return {0, 0};
+			}
+			word = m_data[--word_index];
+		}
+
+		const auto empty_bits = static_cast<size_t>(std::countl_zero(word));
+		const auto ones       = static_cast<size_t>(std::countl_one(word << empty_bits));
+		const auto last       = (word_index + 1) * BITS_PER_WORD - empty_bits;
+		if (empty_bits + ones < BITS_PER_WORD) {
+			return {last - ones, last};
+		}
+
+		while (word_index != 0) {
+			word = m_data[--word_index];
+			if (word != ~uint64_t {0}) {
+				return {(word_index + 1) * BITS_PER_WORD - std::countl_one(word), last};
+			}
+		}
+		return {0, last};
+	}
+
+	[[nodiscard]] constexpr Range LastRange() const { return LastRangeFrom(N); }
+
+	[[nodiscard]] const_iterator begin() const { return Iterator(*this, 0); }
+	[[nodiscard]] const_iterator end() const { return Iterator(*this, N); }
+
+	constexpr BitArray& operator^=(const BitArray& other) {
+		for (size_t word = 0; word < WORD_COUNT; word++) {
+			m_data[word] ^= other.m_data[word];
+		}
+		return *this;
+	}
+
+	[[nodiscard]] constexpr BitArray operator^(const BitArray& other) const {
+		auto result = *this;
+		result ^= other;
+		return result;
+	}
+
+	[[nodiscard]] constexpr BitArray operator~() const {
+		auto result = *this;
+		for (auto& word: result.m_data) {
+			word = ~word;
+		}
+		return result;
+	}
+
+private:
+	std::array<uint64_t, WORD_COUNT> m_data {};
+};
+
+} // namespace Common
+
+#endif // EMULATOR_SRC_COMMON_BITARRAY_H_

src/common/emulatorConfig.cpp

@@ -37,6 +37,10 @@ uint32_t GetVblankFrequency() {
 	return std::clamp(g_config->vblank_frequency, 30u, 360u);
 }
 
+uint32_t GetConsoleLanguage() {
+	return g_config->console_language;
+}
+
 bool VulkanValidationEnabled() {
 	return g_config->vulkan_validation_enabled;
 }
@@ -89,10 +93,6 @@ bool RenderDocEnabled() {
 	return g_config->renderdoc_enabled;
 }
 
-bool NggRectlistDrawEnabled() {
-	return g_config->ngg_rectlist_draw_enabled;
-}
-
 bool ReadbackLinearImagesEnabled() {
 	return g_config->readback_linear_images;
 }

src/common/emulatorConfig.h

@@ -18,10 +18,14 @@ enum class ProfilerDirection { None, Network };
 
 enum class OutputDirection { Silent, Console, File };
 
+constexpr uint32_t DEFAULT_CONSOLE_LANGUAGE = 1;
+constexpr uint32_t MAX_CONSOLE_LANGUAGE     = 29;
+
 struct ConfigOptions {
 	uint32_t               screen_width                = 1280;
 	uint32_t               screen_height               = 720;
 	uint32_t               vblank_frequency            = 60;
+	uint32_t               console_language            = DEFAULT_CONSOLE_LANGUAGE;
 	bool                   vulkan_validation_enabled   = false;
 	bool                   shader_validation_enabled   = false;
 	ShaderOptimizationType shader_optimization_type    = ShaderOptimizationType::None;
@@ -35,7 +39,6 @@ struct ConfigOptions {
 	ProfilerDirection      profiler_direction          = ProfilerDirection::None;
 	bool                   spirv_debug_printf_enabled  = false;
 	bool                   renderdoc_enabled           = false;
-	bool                   ngg_rectlist_draw_enabled   = true;
 	bool                   readback_linear_images      = false;
 };
 
@@ -44,6 +47,7 @@ void Load(const ConfigOptions& cfg);
 uint32_t GetScreenWidth();
 uint32_t GetScreenHeight();
 uint32_t GetVblankFrequency();
+uint32_t GetConsoleLanguage();
 bool     VulkanValidationEnabled();
 
 bool                   ShaderValidationEnabled();
@@ -64,7 +68,6 @@ ProfilerDirection GetProfilerDirection();
 bool SpirvDebugPrintfEnabled();
 
 bool RenderDocEnabled();
-bool NggRectlistDrawEnabled();
 bool ReadbackLinearImagesEnabled();
 
 } // namespace Config

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -83,8 +83,7 @@ public:
 	               uint32_t first_instance = 0);
 	void DrawIndexOffset(uint32_t index_offset, uint32_t index_count, uint32_t flags);
 	void DrawIndexAuto(uint32_t index_count, uint32_t flags,
-	                   uint32_t render_target_slice_offset = 0, uint32_t instance_count = 1,
-	                   uint32_t first_vertex = 0, uint32_t first_instance = 0);
+	                   uint32_t render_target_slice_offset = 0);
 	void DrawIndirect(uint32_t data_offset, uint32_t draw_initiator, bool indexed);
 	void DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count,
 	                       const volatile uint32_t* count_addr, uint32_t stride_in_bytes,
@@ -152,6 +151,9 @@ private:
 	                      uint32_t interrupt_context_id);
 	void ProcessPm4(Pm4Execution& execution, size_t stop_depth);
 	void SuspendPm4();
+	void SubmitNonIndexedDraw(uint32_t vertex_count, uint32_t flags,
+	                          uint32_t render_target_slice_offset, uint32_t first_vertex,
+	                          uint32_t first_instance);
 
 	CommandScheduler&    GetScheduler() const { return m_renderer.GetCommandScheduler(); }
 	RenderCommandBuffer& CurrentBuffer() { return GetScheduler().Current(); }
@@ -168,7 +170,8 @@ private:
 	uint64_t         m_index_base_addr                  = 0;
 	uint64_t         m_draw_indirect_args_base_addr     = 0;
 	uint64_t         m_dispatch_indirect_args_base_addr = 0;
-	uint32_t         m_num_instances                    = 1;
+	// Persistent draw state: indirect draws update it for subsequent draws.
+	uint32_t m_num_instances = 1;
 
 	uint32_t m_de_count        = 0;
 	uint32_t m_ce_count        = 0;

src/graphics/guest_gpu/graphicsRun.cpp

@@ -32,11 +32,10 @@
 
 namespace Libs::Graphics {
 
-static thread_local CommandProcessor* g_current_processor      = nullptr;
-static thread_local Pm4Execution*     g_current_execution      = nullptr;
-static thread_local uint32_t          g_submission_pause_depth = 0;
-static thread_local bool              g_gpu_mutex_owned        = false;
-static thread_local bool              g_gpu_thread             = false;
+static thread_local CommandProcessor* g_current_processor = nullptr;
+static thread_local Pm4Execution*     g_current_execution = nullptr;
+static thread_local bool              g_gpu_mutex_owned   = false;
+static thread_local bool              g_gpu_thread        = false;
 
 class GpuMutexLock final {
 public:
@@ -98,8 +97,6 @@ public:
 	                   bool trigger_agc_interrupt_on_done);
 	void SubmitFlipPreparation(uint64_t request_id);
 	void Done();
-	void PauseSubmissions();
-	void ResumeSubmissions();
 	void Shutdown();
 	[[nodiscard]] bool IsStopping();
 	void               SendCommand(Common::UniqueFunction<void>&& command);
@@ -407,7 +404,7 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw
                                  uint32_t write_control) {
 	const uint32_t dst_sel      = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu);
 	const uint32_t cache_policy = (write_control >> 25u) & 0x3u;
-	const uint32_t increment     = (write_control >> 16u) & 0x1u;
+	const uint32_t increment    = (write_control >> 16u) & 0x1u;
 	const uint32_t write_confirm = (write_control >> 20u) & 0x1u;
 
 	switch (dst_sel) {
@@ -696,26 +693,6 @@ bool GpuState::Process(Submission& submission) {
 	return complete;
 }
 
-void GpuState::PauseSubmissions() {
-	if (g_gpu_mutex_owned) {
-		EXIT("GPU submissions are already paused by this thread\n");
-	}
-	g_gpu_mutex_owned = true;
-	m_submission_mutex.Lock();
-	if (!IsGpuThread()) {
-		WaitLocked();
-	}
-	m_renderer.GetCommandScheduler().DrainPriorityOperations();
-}
-
-void GpuState::ResumeSubmissions() {
-	if (!g_gpu_mutex_owned) {
-		EXIT("GPU submissions resumed without an active pause\n");
-	}
-	m_submission_mutex.Unlock();
-	g_gpu_mutex_owned = false;
-}
-
 Pm4ProcessResult CommandProcessor::Process(Pm4Execution& execution, uint32_t* buffer,
                                            uint32_t size_dw) {
 	KYTY_PROFILER_BLOCK("CommandProcessor::Process");
@@ -1006,8 +983,9 @@ void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiato
 				     args.start_vertex_location, args.start_instance_location);
 			}
 		}
-		DrawIndexAuto(args.vertex_count_per_instance, 0, 0, args.instance_count,
-		              args.start_vertex_location, args.start_instance_location);
+		m_num_instances = args.instance_count;
+		SubmitNonIndexedDraw(args.vertex_count_per_instance, 0, 0, args.start_vertex_location,
+		                     args.start_instance_location);
 		return;
 	}
 
@@ -1047,6 +1025,7 @@ void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiato
 		}
 	}
 
+	m_num_instances = args.instance_count;
 	DrawIndex(index_count, index_addr, 0, 1, args.instance_count, nullptr, 0,
 	          static_cast<int32_t>(args.base_vertex_location), args.start_instance_location);
 }
@@ -1104,8 +1083,9 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 					     args->start_vertex_location, args->start_instance_location);
 				}
 			}
-			DrawIndexAuto(args->vertex_count_per_instance, 0, 0, args->instance_count,
-			              args->start_vertex_location, args->start_instance_location);
+			m_num_instances = args->instance_count;
+			SubmitNonIndexedDraw(args->vertex_count_per_instance, 0, 0, args->start_vertex_location,
+			                     args->start_instance_location);
 			continue;
 		}
 
@@ -1146,6 +1126,7 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 			}
 		}
 
+		m_num_instances = args->instance_count;
 		DrawIndex(index_count, index_addr, 0, 1, args->instance_count, nullptr, 0,
 		          static_cast<int32_t>(args->base_vertex_location), args->start_instance_location);
 	}
@@ -1237,12 +1218,17 @@ void CommandProcessor::DispatchIndirect(uint32_t data_offset, uint32_t mode) {
 }
 
 void CommandProcessor::DrawIndexAuto(uint32_t index_count, uint32_t flags,
-                                     uint32_t render_target_slice_offset, uint32_t instance_count,
-                                     uint32_t first_vertex, uint32_t first_instance) {
+                                     uint32_t render_target_slice_offset) {
+	SubmitNonIndexedDraw(index_count, flags, render_target_slice_offset, 0, 0);
+}
+
+void CommandProcessor::SubmitNonIndexedDraw(uint32_t vertex_count, uint32_t flags,
+                                            uint32_t render_target_slice_offset,
+                                            uint32_t first_vertex, uint32_t first_instance) {
 	CheckBuffer();
 
-	m_renderer.GetRenderExecutor().DrawAuto(m_submit_id, CurrentBuffer(), index_count, flags,
-	                                        render_target_slice_offset, instance_count,
+	m_renderer.GetRenderExecutor().DrawAuto(m_submit_id, CurrentBuffer(), vertex_count, flags,
+	                                        render_target_slice_offset, m_num_instances,
 	                                        first_vertex, first_instance);
 }
 
@@ -1693,32 +1679,6 @@ int Gpu::GetFrameNum() const {
 	return m_state->GetFrameNum();
 }
 
-void Gpu::PauseSubmissions() {
-	m_state->PauseSubmissions();
-}
-
-void Gpu::ResumeSubmissions() {
-	m_state->ResumeSubmissions();
-}
-
-Gpu::SubmissionLock::SubmissionLock(Gpu& gpu): m_gpu(gpu) {
-	if (g_current_processor != nullptr || g_submission_pause_depth == UINT32_MAX) {
-		EXIT("cannot acquire GPU submission lock in the current state\n");
-	}
-	if (g_submission_pause_depth++ == 0) {
-		m_gpu.PauseSubmissions();
-	}
-}
-
-Gpu::SubmissionLock::~SubmissionLock() {
-	if (g_submission_pause_depth == 0) {
-		EXIT("GPU submission lock released without ownership\n");
-	}
-	if (--g_submission_pause_depth == 0) {
-		m_gpu.ResumeSubmissions();
-	}
-}
-
 bool Gpu::IsCommandProcessorThread() noexcept {
 	return g_current_processor != nullptr;
 }
@@ -1727,12 +1687,4 @@ CommandProcessor* Gpu::CurrentCommandProcessor() noexcept {
 	return g_current_processor;
 }
 
-bool Gpu::SubmissionLockHeld() noexcept {
-	return g_submission_pause_depth != 0;
-}
-
-bool Gpu::MutexHeld() noexcept {
-	return g_gpu_mutex_owned;
-}
-
 } // namespace Libs::Graphics

src/graphics/guest_gpu/graphicsRun.h

@@ -35,25 +35,8 @@ public:
 
 	[[nodiscard]] static bool              IsCommandProcessorThread() noexcept;
 	[[nodiscard]] static CommandProcessor* CurrentCommandProcessor() noexcept;
-	[[nodiscard]] static bool              SubmissionLockHeld() noexcept;
-	[[nodiscard]] static bool              MutexHeld() noexcept;
-
-	class SubmissionLock final {
-	public:
-		explicit SubmissionLock(Gpu& gpu);
-		~SubmissionLock();
-		KYTY_CLASS_NO_COPY(SubmissionLock);
-
-	private:
-		Gpu& m_gpu;
-	};
 
 private:
-	friend class SubmissionLock;
-
-	void PauseSubmissions();
-	void ResumeSubmissions();
-
 	std::unique_ptr<GpuState> m_state;
 };
 } // namespace Libs::Graphics

src/graphics/host_gpu/memoryTracker.cpp

@@ -6,13 +6,7 @@ namespace Libs::Graphics {
 
 static_assert(std::atomic<void*>::is_always_lock_free);
 
-MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode)
-    : m_page_manager(page_manager), m_gpu_watch_mode(gpu_watch_mode) {
-	switch (m_gpu_watch_mode) {
-		case PageWatchMode::Write:
-		case PageWatchMode::ReadWrite: break;
-		default: EXIT("unsupported memory tracker GPU page-watch mode\n");
-	}
+MemoryTracker::MemoryTracker(PageManager& page_manager): m_page_manager(page_manager) {
 	m_regions = std::make_unique<std::atomic<RegionManager*>[]>(REGION_COUNT);
 	for (size_t i = 0; i < REGION_COUNT; i++) {
 		m_regions[i].store(nullptr, std::memory_order_relaxed);
@@ -21,6 +15,7 @@ MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_
 
 MemoryTracker::~MemoryTracker() = default;
 
+#if KYTY_BUILD == KYTY_BUILD_DEBUG
 void MemoryTracker::ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
                                           const char* operation) const noexcept {
 	if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr ||
@@ -58,6 +53,7 @@ void MemoryTracker::ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t va
 		}
 	}
 }
+#endif
 
 void MemoryTracker::ValidateRange(uint64_t vaddr, uint64_t size) {
 	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
@@ -104,34 +100,25 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 	std::lock_guard access(m_access_mutex);
 	Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
-		const auto       changed =
-		    manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
-		manager->ApplyProtection(changed, false);
+		manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
 	});
 }
 
 void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+	Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
-		const auto       changed =
-		    manager->ChangeState<DirtySource::Gpu, true>(manager->GetCpuAddr() + offset, bytes);
-		manager->ApplyGpuProtection(changed, true, m_gpu_watch_mode);
+		manager->ChangeState<DirtySource::Gpu, true>(manager->GetCpuAddr() + offset, bytes);
 	});
 }
 
 void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+	Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
-		if (!manager->IsFullyModified<DirtySource::Gpu>(offset, bytes)) {
-			EXIT("cannot clear partially GPU-dirty tracking range\n");
-		}
-		const auto changed =
-		    manager->ChangeState<DirtySource::Gpu, false>(manager->GetCpuAddr() + offset, bytes);
-		manager->ApplyGpuProtection(changed, false, m_gpu_watch_mode);
+		manager->ChangeState<DirtySource::Gpu, false>(manager->GetCpuAddr() + offset, bytes);
 	});
 }
 
@@ -154,9 +141,7 @@ void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
 		EXIT("cannot untrack GPU-dirty memory\n");
 	}
 	Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-		const auto changed =
-		    manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
-		manager->ApplyProtection(changed, false);
+		manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
 	});
 	locks.clear();
 }

src/graphics/host_gpu/memoryTracker.h

@@ -18,8 +18,7 @@ namespace Libs::Graphics {
 
 class MemoryTracker final {
 public:
-	explicit MemoryTracker(PageManager&  page_manager,
-	                       PageWatchMode gpu_watch_mode = PageWatchMode::ReadWrite);
+	explicit MemoryTracker(PageManager& page_manager);
 	~MemoryTracker();
 
 	KYTY_CLASS_NO_COPY(MemoryTracker);
@@ -56,9 +55,8 @@ public:
 			}
 			Iterate<false>(vaddr, size,
 			               [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-				               const auto changed = manager->ChangeState<DirtySource::Cpu, true>(
+				               manager->ChangeState<DirtySource::Cpu, true>(
 				                   manager->GetCpuAddr() + offset, bytes);
-				               manager->ApplyProtection(changed, false);
 			               });
 			return false;
 		};
@@ -71,10 +69,15 @@ public:
 			EXIT("memory invalidation retained GPU-owned pages\n");
 		}
 	}
+#if KYTY_BUILD == KYTY_BUILD_DEBUG
 	void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
 	                           const char* operation) const noexcept;
 	void ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
 	                               const char* operation);
+#else
+	void ValidateGpuDirtyPages(const RangeSet&, uint64_t, uint64_t, const char*) const noexcept {}
+	void ValidateGpuDirtyOwnership(const RangeSet&, uint64_t, uint64_t, const char*) {}
+#endif
 
 	template <bool clear, typename Preflight, typename Func>
 	void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Preflight&& preflight, Func&& func) {
@@ -104,10 +107,8 @@ public:
 			Iterate<false>(vaddr, size,
 			               [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 				               const auto address = manager->GetCpuAddr() + offset;
-				               const auto changed =
-				                   manager->template ForEachModifiedRange<DirtySource::Gpu, true>(
-				                       address, bytes, [](uint64_t, uint64_t) noexcept {});
-				               manager->ApplyGpuProtection(changed, false, m_gpu_watch_mode);
+				               manager->template ForEachModifiedRange<DirtySource::Gpu, true>(
+				                   address, bytes, [](uint64_t, uint64_t) noexcept {});
 			               });
 		}
 	}
@@ -137,13 +138,12 @@ public:
 		});
 		upload_func();
 		if (is_written) {
-			Iterate<false>(
-			    vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-				    const auto changed = manager->template ChangeState<DirtySource::Gpu, true>(
-				        manager->GetCpuAddr() + offset, bytes);
-				    manager->ApplyGpuProtection(changed, true, m_gpu_watch_mode);
-				    manager->lock.unlock();
-			    });
+			Iterate<false>(vaddr, size,
+			               [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+				               manager->template ChangeState<DirtySource::Gpu, true>(
+				                   manager->GetCpuAddr() + offset, bytes);
+				               manager->lock.unlock();
+			               });
 		}
 		s_upload_owner = previous_upload_owner;
 	}
@@ -197,7 +197,6 @@ private:
 	std::mutex                                     m_region_mutex;
 	std::mutex                                     m_access_mutex;
 	PageManager&                                   m_page_manager;
-	PageWatchMode                                  m_gpu_watch_mode = PageWatchMode::ReadWrite;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/pageManager.cpp

@@ -9,7 +9,6 @@
 #include <cstdarg>
 #include <cstdio>
 #include <cstdlib>
-#include <limits>
 #include <memory>
 #include <mutex>
 #include <vector>
@@ -22,13 +21,9 @@
 #undef min
 #undef max
 #elif defined(__APPLE__)
-#include <pthread.h>
-#include <sys/mman.h>
 #include <unistd.h>
 #else
 #include <execinfo.h>
-#include <sys/mman.h>
-#include <sys/syscall.h>
 #include <unistd.h>
 #endif
 
@@ -41,9 +36,8 @@ constexpr uint64_t ADDRESS_SIZE = TRACKER_ADDRESS_SIZE;
 constexpr uint64_t REGION_COUNT = ADDRESS_SIZE / REGION_SIZE;
 
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
-// The tracker reuses Win32 memory-protection tags as internal page-state values (on
-// Windows they come from <windows.h> and are what VirtualQuery returns). Mirror the
-// canonical Win32 numeric values so the shared state-machine logic is identical.
+// The tracker reuses Win32 memory-protection tags as internal page-state values.
+// Mirror their canonical numeric values so the shared state-machine logic is identical.
 constexpr uint32_t PAGE_NOACCESS  = 0x01;
 constexpr uint32_t PAGE_READONLY  = 0x02;
 constexpr uint32_t PAGE_READWRITE = 0x04;
@@ -53,8 +47,6 @@ constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE;
 constexpr uint32_t NO_ACCESS_PROTECTION  = PAGE_NOACCESS;
 constexpr uint32_t READ_ONLY_PROTECTION  = PAGE_READONLY;
 constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
-// Zero is the unknown protection sentinel.
-constexpr uint32_t UNKNOWN_PROTECTION = 0;
 
 [[noreturn]] void FailFast(const char* reason = nullptr) noexcept {
 	std::fputs("PageManager fail-fast: ", stderr);
@@ -102,25 +94,6 @@ Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
 	}
 }
 
-uint32_t CurrentThread() noexcept {
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	return GetCurrentThreadId();
-#elif defined(__APPLE__)
-	return static_cast<uint32_t>(pthread_mach_thread_np(pthread_self()));
-#elif defined(__linux__)
-	static thread_local const uint32_t tid = [] {
-		const auto raw = static_cast<uint32_t>(::syscall(SYS_gettid));
-		if (raw == 0) {
-			FailFast("gettid returned the reserved zero owner token");
-		}
-		return raw;
-	}();
-	return tid;
-#else
-	FailFast("page tracking thread identity is unsupported on this platform");
-#endif
-}
-
 class SpinGuard final {
 public:
 	explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) {
@@ -154,38 +127,58 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
 
 struct PageManager::Impl {
 	struct PageState {
-		std::atomic_flag lock                = ATOMIC_FLAG_INIT;
-		uint32_t         write_watchers      = 0;
-		uint32_t         access_watchers     = 0;
-		uint32_t         original_protection = 0;
-		uint32_t         backing_writer      = 0;
-		// Shadow the protection applied through Protect().
-		uint32_t current_protection = UNKNOWN_PROTECTION;
-		bool     resolving          = false;
-	};
+		uint8_t write_watchers  : 7 = 0;
+		uint8_t access_watchers : 1 = 0;
 
-	struct Region {
-		std::array<PageState, REGION_PAGES> pages;
-	};
+		[[nodiscard]] uint32_t Perms() const noexcept {
+			if (access_watchers != 0) {
+				return NO_ACCESS_PROTECTION;
+			}
+			if (write_watchers != 0) {
+				return READ_ONLY_PROTECTION;
+			}
+			return READ_WRITE_PROTECTION;
+		}
 
-	class PageRangeGuard final {
-	public:
-		explicit PageRangeGuard(std::span<PageState*> pages): m_pages(pages) {
-			for (auto* page: m_pages) {
-				while (page->lock.test_and_set(std::memory_order_acquire)) {
-					std::atomic_signal_fence(std::memory_order_seq_cst);
+		template <int delta, bool is_read>
+		uint32_t AddDelta(uint64_t address) {
+			static_assert(delta >= -1 && delta <= 1);
+			if constexpr (is_read) {
+				if constexpr (delta == 1) {
+					if (access_watchers != 0) {
+						Fatal("read-watcher overflow at 0x%016" PRIx64, address);
+					}
+					return ++access_watchers;
+				} else if constexpr (delta == -1) {
+					if (access_watchers == 0) {
+						Fatal("read-watcher underflow at 0x%016" PRIx64, address);
+					}
+					return --access_watchers;
+				} else {
+					return access_watchers;
+				}
+			} else {
+				if constexpr (delta == 1) {
+					if (write_watchers == 0x7f) {
+						Fatal("write-watcher overflow at 0x%016" PRIx64, address);
+					}
+					return ++write_watchers;
+				} else if constexpr (delta == -1) {
+					if (write_watchers == 0) {
+						Fatal("write-watcher underflow at 0x%016" PRIx64, address);
+					}
+					return --write_watchers;
+				} else {
+					return write_watchers;
 				}
 			}
 		}
-		~PageRangeGuard() {
-			for (auto it = m_pages.rbegin(); it != m_pages.rend(); ++it) {
-				(*it)->lock.clear(std::memory_order_release);
-			}
-		}
-		KYTY_CLASS_NO_COPY(PageRangeGuard);
+	};
+	static_assert(sizeof(PageState) == 1);
 
-	private:
-		std::span<PageState*> m_pages;
+	struct Region {
+		std::atomic_flag                    lock = ATOMIC_FLAG_INIT;
+		std::array<PageState, REGION_PAGES> pages;
 	};
 
 	Impl() {
@@ -215,10 +208,9 @@ struct PageManager::Impl {
 
 	~Impl() {
 		for (const auto& region: region_storage) {
+			SpinGuard lock(region->lock);
 			for (auto& page: region->pages) {
-				SpinGuard lock(page.lock);
-				if (page.write_watchers != 0 || page.access_watchers != 0 ||
-				    page.backing_writer != 0 || page.resolving) {
+				if (page.write_watchers != 0 || page.access_watchers != 0) {
 					FailFast("PageManager destroyed with live page state");
 				}
 			}
@@ -246,56 +238,81 @@ struct PageManager::Impl {
 		return ptr;
 	}
 
-	PageState& GetPage(Region& region, uint64_t vaddr) const {
-		return region.pages[(vaddr % REGION_SIZE) / PAGE_SIZE];
-	}
-
-	static uint32_t WatcherProtection(const PageState& page) {
-		if (page.access_watchers != 0) {
-			return NO_ACCESS_PROTECTION;
-		}
-		if (page.write_watchers != 0) {
-			return READ_ONLY_PROTECTION;
-		}
-		return page.original_protection;
-	}
-
-	static void InitializeProtection(std::span<PageState*> pages) {
-		for (auto* page: pages) {
-			page->original_protection = READ_WRITE_PROTECTION;
-			page->current_protection  = READ_WRITE_PROTECTION;
-		}
-	}
-
-	void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
-	                  std::span<const uint32_t> expected_old) noexcept {
-		const auto size = pages.size() * PAGE_SIZE;
-		if (pages.size() != expected_old.size()) {
-			FailFast("protection range state size mismatch");
-		}
-		for (size_t i = 0; i < pages.size(); i++) {
-			const auto actual = pages[i]->current_protection;
-			if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) {
-				Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
-				      ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
-				      vaddr + i * PAGE_SIZE, actual, expected_old[i], protection);
-			}
-		}
+	void Protect(uint64_t vaddr, uint64_t size, uint32_t protection) noexcept {
 		if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
 		                                                     ToMemoryMode(protection))) {
 			Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
 			      protection);
 		}
-		for (auto* page: pages) {
-			page->current_protection = protection;
-		}
 	}
 
-	void Protect(PageState& page, uint64_t vaddr, uint32_t protection,
-	             uint32_t expected_old) noexcept {
-		PageState* pages[]    = {&page};
-		uint32_t   expected[] = {expected_old};
-		ProtectRange(pages, vaddr, protection, expected);
+	template <bool track, bool is_read, bool masked>
+	void UpdateRegionWatchers(Region& region, uint64_t base_addr, size_t first, size_t last,
+	                          const RegionBits* mask = nullptr) {
+		SpinGuard lock(region.lock);
+		auto      perms                 = region.pages[first].Perms();
+		uint64_t  range_begin           = 0;
+		uint64_t  range_bytes           = 0;
+		uint64_t  potential_range_bytes = 0;
+
+		const auto release_pending = [&] {
+			if (range_bytes != 0) {
+				Protect(base_addr + range_begin * PAGE_SIZE, range_bytes, perms);
+				range_bytes           = 0;
+				potential_range_bytes = 0;
+			}
+		};
+
+		for (size_t page_index = first; page_index < last; page_index++) {
+			auto&      page    = region.pages[page_index];
+			const auto address = base_addr + page_index * PAGE_SIZE;
+			const bool update  = !masked || mask->Get(page_index);
+
+			const auto old_perms = page.Perms();
+			const auto new_count = update ? page.AddDelta<track ? 1 : -1, is_read>(address)
+			                              : page.AddDelta<0, is_read>(address);
+			const auto new_perms = page.Perms();
+
+			if (new_perms != perms) [[unlikely]] {
+				release_pending();
+				perms = new_perms;
+			} else if (range_bytes != 0) {
+				potential_range_bytes += PAGE_SIZE;
+			}
+
+			if (!update) {
+				continue;
+			}
+
+			const bool watcher_edge = (track && new_count == 1) || (!track && new_count == 0);
+			if (watcher_edge && old_perms != new_perms) {
+				if (range_bytes == 0) {
+					range_begin           = page_index;
+					potential_range_bytes = PAGE_SIZE;
+				}
+				range_bytes = potential_range_bytes;
+			}
+		}
+
+		release_pending();
+	}
+
+	template <bool track, bool is_read>
+	void UpdatePageWatchers(uint64_t vaddr, uint64_t size) {
+		const auto begin = PageStart(vaddr);
+		const auto end   = PageEnd(vaddr, size);
+		for (auto chunk_begin = begin; chunk_begin < end;) {
+			const auto chunk_end   = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE);
+			const auto region_base = chunk_begin / REGION_SIZE * REGION_SIZE;
+			auto*      region = track ? GetOrCreateRegion(chunk_begin) : FindRegion(chunk_begin);
+			if (region == nullptr) {
+				Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin);
+			}
+			const auto first = static_cast<size_t>((chunk_begin - region_base) / PAGE_SIZE);
+			const auto last  = static_cast<size_t>((chunk_end - region_base) / PAGE_SIZE);
+			UpdateRegionWatchers<track, is_read, false>(*region, region_base, first, last);
+			chunk_begin = chunk_end;
+		}
 	}
 
 	std::unique_ptr<std::atomic<Region*>[]> regions;
@@ -313,212 +330,48 @@ uint64_t PageManager::GetPageSize() const {
 	return PAGE_SIZE;
 }
 
-void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
-                                     PageWatchMode mode) {
-	if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
-		Fatal("invalid watcher mode");
-	}
-	const auto begin = PageStart(vaddr);
-	const auto end   = PageEnd(vaddr, size);
-	for (auto chunk_begin = begin; chunk_begin < end;) {
-		const auto chunk_end = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE);
-		auto*      region =
-		    track ? m_impl->GetOrCreateRegion(chunk_begin) : m_impl->FindRegion(chunk_begin);
-		if (region == nullptr) {
-			Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin);
-		}
-
-		const auto page_count = static_cast<size_t>((chunk_end - chunk_begin) / PAGE_SIZE);
-		std::vector<Impl::PageState*> pages;
-		pages.reserve(page_count);
-		for (auto address = chunk_begin; address < chunk_end; address += PAGE_SIZE) {
-			pages.push_back(&m_impl->GetPage(*region, address));
-		}
-		Impl::PageRangeGuard lock(pages);
-
-		std::vector<uint8_t> first_watchers(page_count);
-		for (size_t i = 0; i < page_count; i++) {
-			auto&      page    = *pages[i];
-			const auto address = chunk_begin + i * PAGE_SIZE;
-			if (page.resolving && track) {
-				FailFast("new page watcher raced active fault resolution");
-			}
-			auto& watchers =
-			    (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
-			if (track) {
-				if (watchers == std::numeric_limits<uint32_t>::max()) {
-					Fatal("watcher overflow at 0x%016" PRIx64, address);
-				}
-				first_watchers[i] = page.write_watchers == 0 && page.access_watchers == 0;
-			} else {
-				if (watchers == 0) {
-					Fatal("watcher underflow at 0x%016" PRIx64, address);
-				}
-				if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) {
-					Fatal("backing write ownership changed at 0x%016" PRIx64, address);
-				}
-			}
-		}
-
-		if (track) {
-			for (size_t first = 0; first < page_count;) {
-				while (first < page_count && first_watchers[first] == 0) {
-					first++;
-				}
-				auto last = first;
-				while (last < page_count && first_watchers[last] != 0) {
-					last++;
-				}
-				if (first != last) {
-					Impl::InitializeProtection(std::span {pages}.subspan(first, last - first));
-				}
-				first = last;
-			}
-		}
-
-		std::vector<uint32_t> old_protections(page_count);
-		std::vector<uint32_t> new_protections(page_count);
-		std::vector<uint8_t>  transitions(page_count);
-		for (size_t i = 0; i < page_count; i++) {
-			auto& page = *pages[i];
-			auto& watchers =
-			    (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
-			const auto old_protection = Impl::WatcherProtection(page);
-			if (track) {
-				watchers++;
-			} else {
-				watchers--;
-			}
-			const auto new_protection = Impl::WatcherProtection(page);
-			old_protections[i]        = old_protection;
-			new_protections[i]        = new_protection;
-			if (new_protection != old_protection && (track || page.backing_writer == 0)) {
-				transitions[i] = 1;
-			}
-		}
-
-		for (size_t first = 0; first < page_count;) {
-			while (first < page_count && transitions[first] == 0) {
-				first++;
-			}
-			if (first == page_count) {
-				break;
-			}
-			const auto protection = new_protections[first];
-			auto       current    = first + 1;
-			auto       last       = current;
-			for (; current < page_count && new_protections[current] == protection; current++) {
-				if (old_protections[current] != new_protections[current] &&
-				    transitions[current] == 0) {
-					break;
-				}
-				if (transitions[current] != 0) {
-					last = current + 1;
-				}
-			}
-			m_impl->ProtectRange(std::span {pages}.subspan(first, last - first),
-			                     chunk_begin + first * PAGE_SIZE, protection,
-			                     std::span {old_protections}.subspan(first, last - first));
-			first = current;
-		}
-
-		for (auto* page: pages) {
-			if (!track && page->backing_writer == 0 && page->write_watchers == 0 &&
-			    page->access_watchers == 0) {
-				page->original_protection = 0;
-			}
-		}
-		chunk_begin = chunk_end;
-	}
+template <bool track>
+void PageManager::UpdatePageWatchers(uint64_t vaddr, uint64_t size) {
+	m_impl->UpdatePageWatchers<track, false>(vaddr, size);
 }
 
+template void PageManager::UpdatePageWatchers<true>(uint64_t, uint64_t);
+template void PageManager::UpdatePageWatchers<false>(uint64_t, uint64_t);
+
+template <bool track, bool is_read>
+void PageManager::UpdatePageWatchersForRegion(uint64_t base_addr, RegionBits& mask) {
+	if (base_addr % REGION_SIZE != 0 || base_addr >= ADDRESS_SIZE ||
+	    REGION_SIZE > ADDRESS_SIZE - base_addr) {
+		Fatal("invalid tracking region base 0x%016" PRIx64, base_addr);
+	}
+
+	const auto start_range = mask.FirstRange();
+	const auto end_range   = mask.LastRange();
+	if (start_range.first == REGION_PAGES) {
+		FailFast("empty region watcher mask");
+	}
+	const auto first = start_range.first;
+	const auto last  = end_range.second;
+	if (start_range.second == end_range.second) {
+		m_impl->UpdatePageWatchers<track, is_read>(base_addr + first * PAGE_SIZE,
+		                                           (last - first) * PAGE_SIZE);
+		return;
+	}
+
+	auto* region = track ? m_impl->GetOrCreateRegion(base_addr) : m_impl->FindRegion(base_addr);
+	if (region == nullptr) {
+		Fatal("untracking unknown region 0x%016" PRIx64, base_addr);
+	}
+	m_impl->UpdateRegionWatchers<track, is_read, true>(*region, base_addr, first, last, &mask);
+}
+
+template void PageManager::UpdatePageWatchersForRegion<true, true>(uint64_t, RegionBits&);
+template void PageManager::UpdatePageWatchersForRegion<true, false>(uint64_t, RegionBits&);
+template void PageManager::UpdatePageWatchersForRegion<false, true>(uint64_t, RegionBits&);
+template void PageManager::UpdatePageWatchersForRegion<false, false>(uint64_t, RegionBits&);
+
 void PageManager::OnGpuMap(uint64_t, uint64_t) {}
 
 void PageManager::OnGpuUnmap(uint64_t, uint64_t) {}
 
-PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr,
-                                        uint64_t size) noexcept
-    : m_manager(manager), m_vaddr(vaddr), m_size(size) {
-	m_manager.BeginBackingWrite(vaddr, size);
-}
-
-PageManager::BackingWrite::~BackingWrite() {
-	m_manager.EndBackingWrite(m_vaddr, m_size);
-}
-
-std::vector<std::unique_ptr<PageManager::BackingWrite>>
-PageManager::ReserveBackingWrites(std::span<const RangeSet::Range> ranges) {
-	if (ranges.empty()) {
-		Fatal("cannot reserve empty backing-write ranges");
-	}
-	std::vector<std::unique_ptr<BackingWrite>> writes;
-	writes.reserve(ranges.size());
-	uint64_t begin = 0;
-	uint64_t end   = 0;
-	for (const auto& range: ranges) {
-		if (range.address == 0 || range.size == 0 || range.size > UINT64_MAX - range.address ||
-		    range.address + range.size > UINT64_MAX - (PAGE_SIZE - 1)) {
-			Fatal("invalid backing-write range");
-		}
-		const auto page_begin = PageStart(range.address);
-		const auto page_end   = PageStart(range.address + range.size + PAGE_SIZE - 1);
-		if (begin != 0 && page_begin > end) {
-			writes.push_back(std::make_unique<BackingWrite>(*this, begin, end - begin));
-			begin = 0;
-		}
-		if (begin == 0) {
-			begin = page_begin;
-			end   = page_end;
-		} else {
-			end = std::max(end, page_end);
-		}
-	}
-	writes.push_back(std::make_unique<BackingWrite>(*this, begin, end - begin));
-	return writes;
-}
-
-void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
-	const auto end    = PageEnd(vaddr, size);
-	const auto writer = CurrentThread();
-	for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(address);
-		if (region == nullptr) {
-			Fatal("backing write reserves an unknown page at 0x%016" PRIx64, address);
-		}
-		auto&     page = m_impl->GetPage(*region, address);
-		SpinGuard lock(page.lock);
-		if (page.resolving || page.backing_writer != 0 || page.access_watchers == 0) {
-			Fatal("backing write races page resolution at 0x%016" PRIx64, address);
-		}
-		page.resolving      = true;
-		page.backing_writer = writer;
-	}
-}
-
-void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
-	const auto end    = PageEnd(vaddr, size);
-	const auto writer = CurrentThread();
-	for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(address);
-		if (region == nullptr) {
-			FailFast("backing write ended for an unknown page");
-		}
-		auto&     page = m_impl->GetPage(*region, address);
-		SpinGuard lock(page.lock);
-		if (!page.resolving || page.backing_writer != writer) {
-			FailFast("backing write ended without matching owner and resolving state");
-		}
-		const auto old_protection = NO_ACCESS_PROTECTION;
-		const auto new_protection = Impl::WatcherProtection(page);
-		if (new_protection != old_protection) {
-			m_impl->Protect(page, address, new_protection, old_protection);
-		}
-		if (page.write_watchers == 0 && page.access_watchers == 0) {
-			page.original_protection = 0;
-		}
-		page.backing_writer = 0;
-		page.resolving      = false;
-	}
-}
-
 } // namespace Libs::Graphics

src/graphics/host_gpu/pageManager.h

@@ -2,31 +2,16 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_PAGEMANAGER_H_
 
 #include "common/common.h"
-#include "graphics/host_gpu/rangeSet.h"
+#include "graphics/host_gpu/regionDefinitions.h"
 
 #include <memory>
-#include <span>
-#include <vector>
 
 namespace Libs::Graphics {
 
 enum class PageFaultAccess { Read, Write, Execute, Unknown };
-enum class PageWatchMode { Write, ReadWrite };
 
 class PageManager final {
 public:
-	class BackingWrite final {
-	public:
-		BackingWrite(PageManager& manager, uint64_t vaddr, uint64_t size) noexcept;
-		~BackingWrite();
-		KYTY_CLASS_NO_COPY(BackingWrite);
-
-	private:
-		PageManager& m_manager;
-		uint64_t     m_vaddr = 0;
-		uint64_t     m_size  = 0;
-	};
-
 	PageManager();
 	// The owner must stop all PageManager callers before destruction.
 	~PageManager();
@@ -35,18 +20,14 @@ public:
 
 	[[nodiscard]] uint64_t GetPageSize() const;
 
-	void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
-	                        PageWatchMode mode = PageWatchMode::Write);
+	template <bool track>
+	void UpdatePageWatchers(uint64_t vaddr, uint64_t size);
+	template <bool track, bool is_read = false>
+	void UpdatePageWatchersForRegion(uint64_t base_addr, RegionBits& mask);
 	void OnGpuMap(uint64_t vaddr, uint64_t size);
 	void OnGpuUnmap(uint64_t vaddr, uint64_t size);
 
-	[[nodiscard]] std::vector<std::unique_ptr<BackingWrite>>
-	ReserveBackingWrites(std::span<const RangeSet::Range> ranges);
-
 private:
-	void BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept;
-	void EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept;
-
 	struct Impl;
 	std::unique_ptr<Impl> m_impl;
 };

src/graphics/host_gpu/regionDefinitions.h

@@ -1,10 +1,9 @@
 #ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONDEFINITIONS_H_
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONDEFINITIONS_H_
 
+#include "common/bitArray.h"
 #include "common/common.h"
 
-#include <bitset>
-
 namespace Libs::Graphics {
 
 constexpr uint64_t TRACKER_PAGE_SIZE    = 4ull * 1024ull;
@@ -13,7 +12,8 @@ constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 40u;
 constexpr size_t   TRACKER_REGION_PAGES = TRACKER_REGION_SIZE / TRACKER_PAGE_SIZE;
 
 enum class DirtySource { Cpu, Gpu };
-using RegionBits = std::bitset<TRACKER_REGION_PAGES>;
+using RegionBits = Common::BitArray<TRACKER_REGION_PAGES>;
+static_assert(sizeof(RegionBits) == TRACKER_REGION_PAGES / 8);
 
 } // namespace Libs::Graphics
 

src/graphics/host_gpu/regionManager.h

@@ -76,8 +76,9 @@ public:
 		if (m_cpu_addr % TRACKER_REGION_SIZE != 0) {
 			EXIT("invalid region tracking manager construction\n");
 		}
-		m_cpu_dirty.set();
-		m_writable.set();
+		m_cpu_dirty.Fill();
+		m_writable.Fill();
+		m_readable.Fill();
 	}
 
 	KYTY_CLASS_NO_COPY(RegionManager);
@@ -87,106 +88,81 @@ public:
 	[[nodiscard]] bool IsModified(uint64_t offset, uint64_t size) const {
 		const auto [start, end] = GetPageRange(m_cpu_addr + offset, size);
 		const auto& bits        = GetBits<source>();
-		for (auto page = start; page < end; page++) {
-			if (bits.test(page)) {
-				return true;
-			}
-		}
-		return false;
-	}
-
-	template <DirtySource source>
-	[[nodiscard]] bool IsFullyModified(uint64_t offset, uint64_t size) const {
-		const auto [start, end] = GetPageRange(m_cpu_addr + offset, size);
-		const auto& bits        = GetBits<source>();
-		for (auto page = start; page < end; page++) {
-			if (!bits.test(page)) {
-				return false;
-			}
-		}
-		return true;
+		return RegionBits(bits, start, end).Any();
 	}
 
 	template <DirtySource source, bool enable>
-	RegionBits ChangeState(uint64_t vaddr, uint64_t size) {
+	void ChangeState(uint64_t vaddr, uint64_t size) {
 		const auto [start, end] = GetPageRange(vaddr, size);
 		if constexpr (source == DirtySource::Cpu && enable) {
-			for (auto page = start; page < end; page++) {
-				if (m_gpu_dirty.test(page)) {
-					EXIT("CPU dirty state conflicts with GPU dirty state\n");
-				}
+			if (RegionBits(m_gpu_dirty, start, end).Any()) {
+				EXIT("CPU dirty state conflicts with GPU dirty state\n");
 			}
 		}
 		if constexpr (source == DirtySource::Gpu && enable) {
-			for (auto page = start; page < end; page++) {
-				if (m_cpu_dirty.test(page)) {
-					EXIT("GPU dirty state conflicts with CPU dirty state\n");
-				}
+			if (RegionBits(m_cpu_dirty, start, end).Any()) {
+				EXIT("GPU dirty state conflicts with CPU dirty state\n");
 			}
 		}
-		auto& bits    = GetBits<source>();
-		auto  changed = bits;
-		for (auto page = start; page < end; page++) {
-			bits.set(page, enable);
+		auto& bits = GetBits<source>();
+		if constexpr (enable) {
+			bits.SetRange(start, end);
+		} else {
+			bits.UnsetRange(start, end);
 		}
-		changed ^= bits;
 		if constexpr (source == DirtySource::Cpu) {
-			changed    = m_cpu_dirty ^ m_writable;
-			m_writable = m_cpu_dirty;
+			UpdateCpuProtection<!enable>();
+		} else {
+			UpdateGpuProtection<enable>();
 		}
-		return changed;
 	}
 
 	template <DirtySource source, bool clear, typename Func>
-	RegionBits ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) {
+	void ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) {
 		const auto [start, end] = GetPageRange(vaddr, size);
-		auto mask               = GetBits<source>();
-		for (auto page = 0u; page < start; page++) {
-			mask.reset(page);
-		}
-		for (auto page = end; page < TRACKER_REGION_PAGES; page++) {
-			mask.reset(page);
-		}
+		RegionBits mask(GetBits<source>(), start, end);
 		if constexpr (clear) {
-			auto& bits = GetBits<source>();
-			for (auto page = start; page < end; page++) {
-				if (mask.test(page)) {
-					bits.reset(page);
-				}
-			}
+			GetBits<source>().UnsetRange(start, end);
 		}
 		if constexpr (source == DirtySource::Cpu && clear) {
-			auto changed = m_cpu_dirty ^ m_writable;
-			m_writable   = m_cpu_dirty;
-			ApplyProtection(changed, true);
+			UpdateCpuProtection<true>();
 			ForEachRange(mask, std::forward<Func>(func));
-			return changed;
+			return;
+		}
+		if constexpr (source == DirtySource::Gpu && clear) {
+			UpdateGpuProtection<false>();
 		}
 		ForEachRange(mask, std::forward<Func>(func));
-		if constexpr (clear) {
-			return mask;
-		}
-		return {};
-	}
-
-	void ApplyProtection(const RegionBits& changed, bool track) {
-		ForEachRange(changed, [this, track](uint64_t vaddr, uint64_t size) {
-			m_page_manager.UpdatePageWatchers(track, vaddr, size);
-		});
-	}
-
-	void ApplyGpuProtection(const RegionBits& changed, bool track, PageWatchMode mode) {
-		if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
-			EXIT("unsupported GPU page-watch mode\n");
-		}
-		ForEachRange(changed, [this, track, mode](uint64_t vaddr, uint64_t size) {
-			m_page_manager.UpdatePageWatchers(track, vaddr, size, mode);
-		});
 	}
 
 	TrackingSpinLock lock;
 
 private:
+	template <bool track>
+	void UpdateCpuProtection() {
+		auto mask  = m_cpu_dirty ^ m_writable;
+		m_writable = m_cpu_dirty;
+		if (mask.None()) {
+			return;
+		}
+		m_page_manager.UpdatePageWatchersForRegion<track>(m_cpu_addr, mask);
+	}
+
+	template <bool track>
+	void UpdateGpuProtection() {
+		auto readable = ~m_gpu_dirty;
+		auto mask     = readable ^ m_readable;
+		m_readable    = readable;
+		if (mask.None()) {
+			return;
+		}
+		if constexpr (track) {
+			m_page_manager.UpdatePageWatchersForRegion<true, true>(m_cpu_addr, mask);
+		} else {
+			m_page_manager.UpdatePageWatchersForRegion<false, true>(m_cpu_addr, mask);
+		}
+	}
+
 	template <DirtySource source>
 	RegionBits& GetBits() {
 		if constexpr (source == DirtySource::Cpu) {
@@ -217,18 +193,8 @@ private:
 
 	template <typename Func>
 	void ForEachRange(const RegionBits& bits, Func&& func) const {
-		size_t page = 0;
-		while (page < TRACKER_REGION_PAGES) {
-			while (page < TRACKER_REGION_PAGES && !bits.test(page)) {
-				page++;
-			}
-			const auto start = page;
-			while (page < TRACKER_REGION_PAGES && bits.test(page)) {
-				page++;
-			}
-			if (start != page) {
-				func(m_cpu_addr + start * TRACKER_PAGE_SIZE, (page - start) * TRACKER_PAGE_SIZE);
-			}
+		for (const auto [start, end]: bits) {
+			func(m_cpu_addr + start * TRACKER_PAGE_SIZE, (end - start) * TRACKER_PAGE_SIZE);
 		}
 	}
 
@@ -237,6 +203,7 @@ private:
 	RegionBits   m_cpu_dirty;
 	RegionBits   m_gpu_dirty;
 	RegionBits   m_writable;
+	RegionBits   m_readable;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -123,12 +123,6 @@ struct BufferCache::RetiredBuffer {
 	std::shared_ptr<Buffer> owner;
 };
 
-struct BufferCache::PendingBackingPublication {
-	uint64_t address = 0;
-	uint64_t size    = 0;
-	uint64_t tick    = 0;
-};
-
 std::pair<uint64_t, uint64_t> BufferCache::DownloadEnvelope(const DownloadCopy& copy) {
 	if (copy.owner == nullptr || copy.size == 0 || copy.source_offset > copy.owner->Size() ||
 	    copy.size > copy.owner->Size() - copy.source_offset) {
@@ -200,34 +194,26 @@ void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies, Ret
 	if (copies.empty()) {
 		return;
 	}
-	auto       downloads = RecordDownloads(copies);
-	const auto tick      = m_scheduler.CurrentTick();
-	BeginBackingPublication(retire.address, retire.size, tick);
-	m_scheduler.DeferOperation([this, downloads = std::move(downloads), retire = std::move(retire),
-	                            tick]() mutable {
-		PublishDownloads(downloads);
-		{
-			FaultSafeCacheLock lock(this, m_mutex);
-			if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
-				m_memory_tracker.ForEachDownloadRange<true>(
-				    retire.address, retire.size,
-				    [&](uint64_t address, uint64_t size) noexcept {
-					    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
-					                                           "asynchronous garbage retirement");
-				    },
-				    [](uint64_t, uint64_t) noexcept {});
-			}
-			for (const auto& range: downloads) {
-				m_gpu_modified_ranges.Subtract(range.address, range.size);
-			}
-			if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
-			    !m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
-				EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
-			}
-			m_memory_tracker.UntrackMemory(retire.address, retire.size);
-		}
-		CompleteBackingPublication(retire.address, retire.size, tick);
-	});
+	auto downloads = RecordDownloads(copies);
+	m_scheduler.DeferOperation(
+	    [this, downloads = std::move(downloads), retire = std::move(retire)]() mutable {
+		    PublishDownloads(downloads);
+		    {
+			    FaultSafeCacheLock lock(this, m_mutex);
+			    for (const auto& range: downloads) {
+				    m_gpu_modified_ranges.Subtract(range.address, range.size);
+			    }
+			    // ForEachDownloadRange reports full tracker pages, and every exact GPU-owned
+			    // interval on those pages was downloaded and removed. Clearing the original
+			    // query therefore cannot orphan a dirty sibling on an edge page.
+			    m_memory_tracker.UnmarkRegionAsGpuModified(retire.address, retire.size);
+			    if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
+			        !m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
+				    EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
+			    }
+			    m_memory_tracker.UntrackMemory(retire.address, retire.size);
+		    }
+	    });
 }
 
 BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
@@ -240,8 +226,7 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
       m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
       m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
       m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB),
-      m_page_manager(page_manager), m_texture_cache(texture_cache),
-      m_resource_mutex(resource_mutex) {
+      m_texture_cache(texture_cache), m_resource_mutex(resource_mutex) {
 	std::memset(m_gds_buffer.Mapped().data(), 0, static_cast<size_t>(m_gds_buffer.Size()));
 	m_gds_buffer.Flush(0, m_gds_buffer.Size());
 	if (!m_graphics.CanReportMemoryUsage()) {
@@ -262,9 +247,6 @@ BufferCache::~BufferCache() {
 	if (!m_gpu_modified_ranges.Empty()) {
 		EXIT("BufferCache: destroyed with pending GPU-modified ranges\n");
 	}
-	if (!m_pending_backing_publications.empty()) {
-		EXIT("BufferCache: destroyed with pending backing publications\n");
-	}
 	for (const auto& [vaddr, cached]: m_buffers) {
 		(void)vaddr;
 		if (m_memory_tracker.IsRegionGpuModified(cached->vaddr, cached->size)) {
@@ -274,68 +256,6 @@ BufferCache::~BufferCache() {
 	m_buffers.clear();
 }
 
-bool BufferCache::SynchronizeBacking(uint64_t vaddr, uint64_t size) {
-	bool waited = false;
-	for (;;) {
-		uint64_t   tick       = 0;
-		const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-		const auto page_end   = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
-		CacheRange affected {.address = page_begin, .size = page_end - page_begin};
-		{
-			FaultSafeCacheLock lock(this, m_mutex);
-			bool               changed = true;
-			while (changed) {
-				changed = false;
-				for (const auto& [address, cached]: m_buffers) {
-					const CacheRange previous = affected;
-					if (ResolveOverlap(affected, {address, cached->size}) &&
-					    (previous.address != affected.address || previous.size != affected.size)) {
-						changed = true;
-					}
-				}
-			}
-		}
-		{
-			std::lock_guard lock(m_publication_mutex);
-			for (const auto& publication: m_pending_backing_publications) {
-				if (publication.address < affected.address + affected.size &&
-				    affected.address < publication.address + publication.size) {
-					tick = std::max(tick, publication.tick);
-				}
-			}
-		}
-		if (tick == 0) {
-			return waited;
-		}
-		waited = true;
-		m_scheduler.Wait(tick);
-		m_scheduler.WaitPriorityOperations(tick);
-	}
-}
-
-void BufferCache::RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached,
-                                           uint64_t vaddr, uint64_t size, bool upload) {
-	const auto invalidated = m_image_invalidated_ranges.Intersections(vaddr, size);
-	if (upload) {
-		std::array<uint8_t, 64 * 1024> bytes;
-		for (const auto& range: invalidated) {
-			for (uint64_t copied = 0; copied < range.size;) {
-				const auto chunk = std::min<uint64_t>(range.size - copied, bytes.size());
-				if (!Libs::LibKernel::Memory::TryReadBacking(range.address + copied, bytes.data(),
-				                                             chunk)) {
-					EXIT("BufferCache: failed to refresh an invalidated image alias\n");
-				}
-				Upload(command, *cached.buffer, cached.buffer->Offset(range.address + copied),
-				       bytes.data(), chunk);
-				copied += chunk;
-			}
-		}
-	}
-	if (!invalidated.empty()) {
-		m_image_invalidated_ranges.Subtract(vaddr, size);
-	}
-}
-
 StreamBuffer& BufferCache::GetUtilityBuffer(MemoryUsage usage) noexcept {
 	switch (usage) {
 		case MemoryUsage::Upload: return m_staging_buffer;
@@ -364,7 +284,6 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
 	    size > TRACKER_ADDRESS_SIZE - vaddr) {
 		EXIT("BufferCache: invalid memory-invalidation range\n");
 	}
-	(void)SynchronizeBacking(vaddr, size);
 	if (!HasPageOverlap(vaddr, size)) {
 		return;
 	}
@@ -373,7 +292,6 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
 }
 
 void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
-	(void)SynchronizeBacking(vaddr, size);
 	std::vector<DownloadCopy> copies;
 	{
 		FaultSafeCacheLock lock(this, m_mutex);
@@ -413,16 +331,11 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
 	PublishDownloads(downloads);
 	{
 		FaultSafeCacheLock lock(this, m_mutex);
-		m_memory_tracker.ForEachDownloadRange<true>(
-		    vaddr, size,
-		    [&](uint64_t address, uint64_t bytes) noexcept {
-			    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
-			                                           "memory invalidation completion");
-		    },
-		    [](uint64_t, uint64_t) noexcept {});
 		for (const auto& range: downloads) {
 			m_gpu_modified_ranges.Subtract(range.address, range.size);
 		}
+		// The enumeration above covered whole dirty pages and every exact interval on them.
+		m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size);
 	}
 }
 
@@ -430,13 +343,9 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 	if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
 		EXIT("BufferCache: invalid unmap range\n");
 	}
-	(void)SynchronizeBacking(vaddr, size);
-
-	std::vector<DownloadCopy>                               copies;
-	std::vector<RangeSet::Range>                            dirty_ranges;
-	std::vector<std::pair<uint64_t, uint64_t>>              modified_buffers;
-	std::vector<std::unique_ptr<PageManager::BackingWrite>> backing_writes;
-	std::vector<std::pair<uint64_t, uint64_t>>              retired_buffers;
+	std::vector<DownloadCopy>                  copies;
+	std::vector<std::pair<uint64_t, uint64_t>> modified_buffers;
+	std::vector<std::pair<uint64_t, uint64_t>> retired_buffers;
 	{
 		FaultSafeCacheLock lock(this, m_mutex);
 		for (const auto& [begin, cached]: m_buffers) {
@@ -453,12 +362,8 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 			if (dirty.empty()) {
 				EXIT("BufferCache: GPU-modified buffer has no dirty ranges\n");
 			}
-			dirty_ranges.insert(dirty_ranges.end(), dirty.begin(), dirty.end());
 			modified_buffers.emplace_back(begin, cached->size);
 		}
-		if (!dirty_ranges.empty()) {
-			backing_writes = m_page_manager.ReserveBackingWrites(dirty_ranges);
-		}
 		for (const auto& [begin, bytes]: modified_buffers) {
 			auto owner = m_buffers.find(begin);
 			if (owner == m_buffers.end() || owner->second->size != bytes) {
@@ -488,24 +393,11 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 		// command stream before removing such backing.
 		m_scheduler.FinishCurrent();
 	}
-	backing_writes.clear();
-
 	{
 		FaultSafeCacheLock lock(this, m_mutex);
-		for (const auto& [begin, bytes]: modified_buffers) {
-			if (!m_memory_tracker.IsRegionGpuModified(begin, bytes)) {
-				continue;
-			}
-			m_memory_tracker.ForEachDownloadRange<true>(
-			    begin, bytes,
-			    [&](uint64_t address, uint64_t download_size) noexcept {
-				    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address,
-				                                           download_size, "unmap retirement");
-			    },
-			    [](uint64_t, uint64_t) noexcept {});
-		}
 		for (const auto& [begin, bytes]: modified_buffers) {
 			m_gpu_modified_ranges.Subtract(begin, bytes);
+			m_memory_tracker.UnmarkRegionAsGpuModified(begin, bytes);
 		}
 		for (const auto& [begin, bytes]: retired_buffers) {
 			m_memory_tracker.MarkRegionAsCpuModified(begin, bytes);
@@ -513,7 +405,6 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 		if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) {
 			EXIT("BufferCache: unmap retained dirty byte ranges\n");
 		}
-		m_image_invalidated_ranges.Subtract(vaddr, size);
 		m_memory_tracker.UntrackMemory(vaddr, size);
 		for (auto it = m_buffers.begin(); it != m_buffers.end();) {
 			if (vaddr < it->first + it->second->size && it->first < vaddr + size) {
@@ -607,20 +498,29 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
 		EXIT("BufferCache: buffer request requires a recording command buffer\n");
 	}
 	std::lock_guard transaction(m_resource_mutex);
-	(void)SynchronizeBacking(vaddr, size);
 
 	if (is_read && !is_written && size <= CACHING_PAGE_SIZE &&
 	    !m_memory_tracker.IsRegionGpuModified(vaddr, size) &&
 	    m_memory_tracker.IsRegionCpuModified(vaddr, size)) {
-		std::vector<uint8_t> data(size);
-		if (Libs::LibKernel::Memory::TryReadBacking(vaddr, data.data(), size)) {
-			return UploadTransient(data.data(), size, 16);
+		const auto alignment = std::max<uint64_t>(
+		    m_graphics.physical_device_properties.limits.minUniformBufferOffsetAlignment, 1);
+		if (auto [mapped, offset] = m_stream_buffer.Map(size, alignment, false);
+		    mapped != nullptr) {
+			if (Libs::LibKernel::Memory::TryReadBacking(vaddr, mapped, size)) {
+				m_stream_buffer.Commit();
+				return {{}, m_stream_buffer.Handle(), offset};
+			}
+		} else {
+			auto owner = std::make_shared<Buffer>(m_graphics, m_scheduler, MemoryUsage::Upload, 0,
+			                                      AllFlags, size);
+			if (Libs::LibKernel::Memory::TryReadBacking(vaddr, owner->Mapped().data(), size)) {
+				owner->Flush(0, size);
+				return {owner, owner->Handle(), 0};
+			}
 		}
 	}
 
-	if (is_formatted && is_read && !is_written) {
-		(void)m_texture_cache.SynchronizeImageToBuffer(vaddr, size);
-	} else if (is_formatted && is_written) {
+	if (is_formatted && is_written) {
 		(void)m_texture_cache.InvalidateMemoryFromGPU(vaddr, size, true);
 	}
 
@@ -636,10 +536,12 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
 			           reinterpret_cast<const void*>(address), bytes);
 		    }
 	    });
-	RefreshInvalidatedRanges(command, cached, vaddr, size, is_read);
 	if (is_written) {
 		m_gpu_modified_ranges.Add(vaddr, size);
 	}
+	if (is_formatted && is_read && !is_written) {
+		(void)SynchronizeBufferFromImage(*cached.buffer, vaddr, size);
+	}
 	return {cached.buffer, cached.buffer->Handle(), cached.buffer->Offset(vaddr)};
 }
 
@@ -664,7 +566,6 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 	    size > TRACKER_ADDRESS_SIZE - vaddr) {
 		EXIT("BufferCache: invalid image source\n");
 	}
-	(void)SynchronizeBacking(vaddr, size);
 	auto find_owner = [&]() {
 		auto owner = m_buffers.upper_bound(vaddr);
 		if (owner == m_buffers.begin()) {
@@ -679,13 +580,12 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 		const bool         cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size);
 		const bool         gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size);
 		const auto         dirty        = m_gpu_modified_ranges.Intersections(vaddr, size);
-		const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
-		const bool requested_gpu_owned = !dirty.empty();
+		const bool         has_dirty_buffer_source = !dirty.empty();
 		m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
 		                                           "image source");
 
 		auto owner = find_owner();
-		if (requested_gpu_owned && owner == m_buffers.end()) {
+		if (has_dirty_buffer_source && owner == m_buffers.end()) {
 			CacheRange merged {.address = AlignDown(vaddr),
 			                   .size    = AlignUp(vaddr + size) - AlignDown(vaddr)};
 			using Iterator = decltype(m_buffers.begin());
@@ -735,42 +635,32 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 				EXIT("BufferCache: merged image source does not contain the requested range\n");
 			}
 		}
-		if (owner != m_buffers.end() && !cpu_modified && !invalidated &&
-		    (!gpu_modified || requested_gpu_owned)) {
-			DiscardGpuDirtyBytesLocked(vaddr, size, "image source transfer");
+		if (owner != m_buffers.end() && !cpu_modified &&
+		    (!gpu_modified || has_dirty_buffer_source)) {
 			owner->second->tick_accessed_last = m_gc_tick;
-			return {owner->second->buffer.get(), owner->second->buffer->Offset(vaddr),
-			        requested_gpu_owned};
+			return {owner->second->buffer.get(), owner->second->buffer->Offset(vaddr)};
 		}
-		if (requested_gpu_owned && owner == m_buffers.end()) {
+		if (has_dirty_buffer_source && owner == m_buffers.end()) {
 			EXIT("BufferCache: GPU-dirty image source could not resolve its native owner\n");
 		}
 	}
 
-	// Direct-memory backing remains readable while PageManager protects the guest mapping. The
-	// fallback exists for plain host mappings used by standalone renderer tests and is deliberately
-	// performed outside the cache lock so a page fault cannot recurse into BufferCache.
-	const auto stage_address = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-	const auto stage_end     = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
-	const auto stage_size    = stage_end - stage_address;
-	(void)SynchronizeBacking(stage_address, stage_size);
-	std::vector<uint8_t> bytes(stage_size);
-	if (!Libs::LibKernel::Memory::TryReadBacking(stage_address, bytes.data(), stage_size)) {
+	auto [staging, stage_offset] = m_staging_buffer.Map(size, 16);
+	if (staging == nullptr || !Libs::LibKernel::Memory::TryReadBacking(vaddr, staging, size)) {
 		EXIT("BufferCache: failed to read mapped guest image backing\n");
 	}
+	m_staging_buffer.Commit();
 
 	FaultSafeCacheLock lock(this, m_mutex);
-	const auto         dirty       = m_gpu_modified_ranges.Intersections(vaddr, size);
-	const bool         invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
-	const bool         requested_gpu_owned = !dirty.empty();
-	auto               owner               = find_owner();
-	if (requested_gpu_owned && owner == m_buffers.end()) {
+	const auto         dirty                   = m_gpu_modified_ranges.Intersections(vaddr, size);
+	const bool         has_dirty_buffer_source = !dirty.empty();
+	auto               owner                   = find_owner();
+	if (has_dirty_buffer_source && owner == m_buffers.end()) {
 		EXIT("BufferCache: GPU-dirty image source lost its native owner\n");
 	}
-	const auto stage_offset = m_staging_buffer.Copy(bytes.data(), stage_size, 16);
-	if (owner == m_buffers.end() || invalidated ||
-	    (m_memory_tracker.IsRegionGpuModified(vaddr, size) && !requested_gpu_owned)) {
-		return {&m_staging_buffer, stage_offset + vaddr - stage_address, false};
+	if (owner == m_buffers.end() ||
+	    (m_memory_tracker.IsRegionGpuModified(vaddr, size) && !has_dirty_buffer_source)) {
+		return {&m_staging_buffer, stage_offset};
 	}
 
 	auto& cached              = *owner->second;
@@ -784,42 +674,17 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 	    [&]() noexcept {
 		    for (const auto& [address, upload_size]: uploads) {
 			    cached.buffer->CopyFrom(
-			        m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address,
+			        m_scheduler.Current(), m_staging_buffer, stage_offset + address - vaddr,
 			        cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
 		    }
 	    });
-	DiscardGpuDirtyBytesLocked(vaddr, size, "staged image source transfer");
-	return {cached.buffer.get(), cached.buffer->Offset(vaddr), requested_gpu_owned};
-}
-
-void BufferCache::DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation) {
-	m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, operation);
-	m_gpu_modified_ranges.Subtract(vaddr, size);
-	const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-	const auto page_end   = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
-	for (auto page = page_begin; page < page_end; page += TRACKER_PAGE_SIZE) {
-		if (m_gpu_modified_ranges.Intersections(page, TRACKER_PAGE_SIZE).empty() &&
-		    m_memory_tracker.IsRegionGpuModified(page, TRACKER_PAGE_SIZE)) {
-			m_memory_tracker.UnmarkRegionAsGpuModified(page, TRACKER_PAGE_SIZE);
-		}
-	}
-	m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, operation);
-}
-
-void BufferCache::DiscardGpuDirtyBytes(uint64_t vaddr, uint64_t size) {
-	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
-	    size > TRACKER_ADDRESS_SIZE - vaddr) {
-		EXIT("BufferCache: invalid dirty-byte discard range\n");
-	}
-	FaultSafeCacheLock lock(this, m_mutex);
-	DiscardGpuDirtyBytesLocked(vaddr, size, "image output supersession");
+	return {cached.buffer.get(), cached.buffer->Offset(vaddr)};
 }
 
 void BufferCache::WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data) {
 	if (vaddr == 0 || data.empty() || data.size() > UINT64_MAX - vaddr) {
 		EXIT("BufferCache: invalid host DMA write\n");
 	}
-	(void)SynchronizeBacking(vaddr, data.size());
 	Libs::LibKernel::Memory::WriteBacking(vaddr, data.data(), data.size());
 
 	FaultSafeCacheLock lock(this, m_mutex);
@@ -835,45 +700,6 @@ void BufferCache::WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data)
 		       data.data() + begin - vaddr, range_end - begin);
 		cached->tick_accessed_last = m_gc_tick;
 	}
-	m_image_invalidated_ranges.Subtract(vaddr, data.size());
-}
-
-std::pair<std::shared_ptr<Buffer>, uint64_t> BufferCache::ObtainBufferForImageWrite(uint64_t vaddr,
-                                                                                    uint64_t size) {
-	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
-	    size > TRACKER_ADDRESS_SIZE - vaddr) {
-		EXIT("BufferCache: invalid image destination\n");
-	}
-	const auto stage_address = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-	const auto stage_end     = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
-	const auto stage_size    = stage_end - stage_address;
-	(void)SynchronizeBacking(stage_address, stage_size);
-	std::vector<uint8_t> bytes(stage_size);
-	if (!Libs::LibKernel::Memory::TryReadBacking(stage_address, bytes.data(), stage_size)) {
-		EXIT("BufferCache: failed to preserve guest bytes around an image mirror\n");
-	}
-	FaultSafeCacheLock lock(this, m_mutex);
-	auto&              cached = GetOrCreateBuffer(m_scheduler.Current(), vaddr, size);
-	m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
-	                                           "image destination");
-	if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) {
-		EXIT("BufferCache: image destination aliases GPU-owned buffer bytes\n");
-	}
-	const auto stage_offset = m_staging_buffer.Copy(bytes.data(), stage_size, 16);
-	std::vector<std::pair<uint64_t, uint64_t>> uploads;
-	m_memory_tracker.ForEachUploadRange(
-	    vaddr, size, false,
-	    [&](uint64_t address, uint64_t upload_size) noexcept {
-		    uploads.emplace_back(address, upload_size);
-	    },
-	    [&]() noexcept {
-		    for (const auto& [address, upload_size]: uploads) {
-			    cached.buffer->CopyFrom(
-			        m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address,
-			        cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
-		    }
-	    });
-	return {cached.buffer, cached.buffer->Offset(vaddr)};
 }
 
 void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds) {
@@ -933,17 +759,10 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	}
 	if (src_memory || dst_memory) {
 		std::lock_guard transaction(m_resource_mutex);
-		if (src_memory) {
-			(void)SynchronizeBacking(src_vaddr, size);
-		}
-		const auto src_region =
+		const auto      src_region =
 		    src_memory ? m_texture_cache.QueryRegion(src_vaddr, size) : TextureCache::RegionInfo {};
 		const auto dst_region =
 		    dst_memory ? m_texture_cache.QueryRegion(dst_vaddr, size) : TextureCache::RegionInfo {};
-		if (src_memory && src_region.gpu_image_bytes &&
-		    !m_texture_cache.SynchronizeImageToBuffer(src_vaddr, size)) {
-			EXIT("BufferCache: GPU copy source image could not be synchronized\n");
-		}
 		if (src_memory && dst_memory && !HasGpuDirtyBytes(src_vaddr, size) &&
 		    !HasGpuDirtyBytes(dst_vaddr, size) && !src_region.gpu_image_bytes &&
 		    !dst_region.gpu_image_bytes) {
@@ -965,7 +784,7 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	}
 
 	auto& command = m_scheduler.Current();
-	auto  src = src_memory ? ObtainBuffer(command, src_vaddr, size, false, true)
+	auto  src = src_memory ? ObtainBuffer(command, src_vaddr, size, false, true, true)
 	                       : BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = src_vaddr};
 	auto  dst = dst_memory ? ObtainBuffer(command, dst_vaddr, size, true, false, true)
 	                       : BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = dst_vaddr};
@@ -1018,75 +837,6 @@ bool BufferCache::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 	return m_memory_tracker.IsRegionCpuModified(vaddr, size);
 }
 
-void BufferCache::InvalidateImageAliases(uint64_t vaddr, uint64_t size) {
-	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
-	    size > TRACKER_ADDRESS_SIZE - vaddr) {
-		EXIT("BufferCache: invalid image-alias invalidation\n");
-	}
-	FaultSafeCacheLock lock(this, m_mutex);
-	const auto         end = vaddr + size;
-	for (const auto& [address, cached]: m_buffers) {
-		const auto cached_end = address + cached->size;
-		const auto begin      = std::max(vaddr, address);
-		const auto range_end  = std::min(end, cached_end);
-		if (begin >= range_end) {
-			continue;
-		}
-		const auto bytes = range_end - begin;
-		if (!m_gpu_modified_ranges.Intersections(begin, bytes).empty()) {
-			EXIT("BufferCache: image ownership overlaps exact dirty buffer bytes\n");
-		}
-		m_image_invalidated_ranges.Add(begin, bytes);
-	}
-}
-
-void BufferCache::BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) {
-	if (vaddr == 0 || size == 0 || tick == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
-	    size > TRACKER_ADDRESS_SIZE - vaddr) {
-		EXIT("BufferCache: invalid pending backing publication\n");
-	}
-	std::lock_guard lock(m_publication_mutex);
-	m_pending_backing_publications.push_back({vaddr, size, tick});
-}
-
-void BufferCache::CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) {
-	std::lock_guard lock(m_publication_mutex);
-	const auto      publication =
-	    std::ranges::find_if(m_pending_backing_publications, [&](const auto& pending) {
-		    return pending.address == vaddr && pending.size == size && pending.tick == tick;
-	    });
-	if (publication == m_pending_backing_publications.end()) {
-		EXIT("BufferCache: completed an unknown backing publication\n");
-	}
-	m_pending_backing_publications.erase(publication);
-}
-
-void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) {
-	FaultSafeCacheLock lock(this, m_mutex);
-	auto               owner = m_buffers.end();
-	for (auto it = m_buffers.begin(); it != m_buffers.end(); ++it) {
-		if (!PageOverlaps(vaddr, size, it->second->vaddr, it->second->size)) {
-			continue;
-		}
-		if (owner != m_buffers.end() || !it->second->buffer->IsInBounds(vaddr, size)) {
-			EXIT("BufferCache: image destination aliases a non-containing cached buffer\n");
-		}
-		owner = it;
-	}
-	m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
-	                                           "image destination publication");
-	if (owner == m_buffers.end() || m_memory_tracker.IsRegionCpuModified(vaddr, size) ||
-	    !m_gpu_modified_ranges.Intersections(vaddr, size).empty()) {
-		EXIT("BufferCache: image destination requires clean buffer ownership\n");
-	}
-	m_memory_tracker.MarkRegionAsGpuModified(vaddr, size);
-	m_gpu_modified_ranges.Add(vaddr, size);
-	m_image_invalidated_ranges.Subtract(vaddr, size);
-	m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
-	                                           "published image destination");
-	owner->second->tick_accessed_last = m_gc_tick;
-}
-
 void BufferCache::RunGarbageCollector() {
 	std::lock_guard transaction(m_resource_mutex);
 	const auto      tick = m_gc_tick++;
@@ -1162,7 +912,6 @@ void BufferCache::RunGarbageCollector() {
 		if (!m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
 			m_memory_tracker.UntrackMemory(retire.address, retire.size);
 		}
-		m_image_invalidated_ranges.Subtract(retire.address, retire.size);
 		if (retire.size > m_total_used_memory) {
 			EXIT("BufferCache: allocation accounting underflow\n");
 		}

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -10,7 +10,6 @@
 
 #include <map>
 #include <memory>
-#include <mutex>
 #include <span>
 #include <utility>
 #include <vector>
@@ -30,9 +29,8 @@ struct BufferBinding {
 };
 
 struct ImageBufferSource {
-	Buffer*  buffer    = nullptr;
-	uint64_t offset    = 0;
-	bool     gpu_owned = false;
+	Buffer*  buffer = nullptr;
+	uint64_t offset = 0;
 };
 
 class BufferCache {
@@ -60,9 +58,6 @@ public:
 	                                            uint64_t alignment);
 	[[nodiscard]] std::shared_ptr<Buffer> ObtainNullBuffer();
 	[[nodiscard]] ImageBufferSource       ObtainBufferForImage(uint64_t vaddr, uint64_t size);
-	[[nodiscard]] std::pair<std::shared_ptr<Buffer>, uint64_t>
-	     ObtainBufferForImageWrite(uint64_t vaddr, uint64_t size);
-	void DiscardGpuDirtyBytes(uint64_t vaddr, uint64_t size);
 	void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds = false);
 	void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds = false,
 	                bool src_gds = false);
@@ -70,11 +65,6 @@ public:
 	[[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
-	void               InvalidateImageAliases(uint64_t vaddr, uint64_t size);
-	void               BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
-	void               CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
-	[[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size);
-	void               PublishImageBuffer(uint64_t vaddr, uint64_t size);
 	void               RunGarbageCollector();
 
 private:
@@ -88,7 +78,6 @@ private:
 	struct DownloadCopy;
 	struct DownloadRange;
 	struct RetiredBuffer;
-	struct PendingBackingPublication;
 	static constexpr uint64_t               DOWNLOAD_ALIGNMENT = 64;
 	[[nodiscard]] static uint64_t           AlignDown(uint64_t value) noexcept;
 	[[nodiscard]] static uint64_t           AlignUp(uint64_t value);
@@ -103,12 +92,10 @@ private:
 	            const void* source, uint64_t size);
 	[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
 	                                              uint64_t size);
+	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
 	[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
 	void PublishDownloads(std::span<const DownloadRange> downloads);
 	void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
-	void RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, uint64_t vaddr,
-	                              uint64_t size, bool upload);
-	void DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation);
 	void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
 
 	GraphicContext&                                   m_graphics;
@@ -118,15 +105,11 @@ private:
 	std::shared_ptr<Buffer>                           m_null_buffer;
 	std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
 	RangeSet                                          m_gpu_modified_ranges;
-	RangeSet                                          m_image_invalidated_ranges;
-	std::mutex                                        m_publication_mutex;
-	std::vector<PendingBackingPublication>            m_pending_backing_publications;
 	MemoryTracker                                     m_memory_tracker;
 	StreamBuffer                                      m_staging_buffer;
 	StreamBuffer                                      m_stream_buffer;
 	StreamBuffer                                      m_download_buffer;
 	StreamBuffer                                      m_device_buffer;
-	PageManager&                                      m_page_manager;
 	TextureCache&                                     m_texture_cache;
 	ResourceMutex&                                    m_resource_mutex;
 	uint64_t                                          m_total_used_memory = 0;

src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp

@@ -7,7 +7,8 @@
 namespace Libs::Graphics {
 
 GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
-    : m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex),
+    : m_scheduler(scheduler),
+      m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex),
       m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache, m_resource_mutex) {}
 
 GpuResourceManager::~GpuResourceManager() = default;
@@ -101,7 +102,22 @@ void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
 }
 
 void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
+	if (CommandScheduler::InDeferredOperation()) {
+		EXIT("unsupported memory unmap from an asynchronous GPU completion, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+		     vaddr, size);
+	}
+	if (m_resource_mutex.IsOwnedByCurrentThread()) {
+		EXIT("unsupported memory unmap from a pre-owned resource transaction, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+		     vaddr, size);
+	}
 	const auto unmap = [this, vaddr, size] {
+		if (m_scheduler.Active()) {
+			const auto tick = m_scheduler.CurrentTick();
+			m_scheduler.FinishCurrent();
+			m_scheduler.WaitPriorityOperations(tick);
+		}
 		m_buffer_cache.UnmapMemory(vaddr, size);
 		m_texture_cache.UnmapMemory(vaddr, size);
 		m_page_manager.OnGpuUnmap(vaddr, size);
@@ -109,13 +125,9 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
 		m_mapped_ranges.Subtract(vaddr, size);
 	};
 	if (m_gpu == nullptr) {
-		if (m_resource_mutex.IsOwnedByCurrentThread()) {
-			EXIT("cannot synchronously unmap from a resource transaction\n");
-		}
 		unmap();
 		return;
 	}
-	Gpu::SubmissionLock submissions(*m_gpu);
 	m_gpu->SendCommandSync(unmap);
 }
 

src/graphics/host_gpu/renderer/cache/gpuResourceManager.h

@@ -36,6 +36,7 @@ public:
 private:
 	PageManager               m_page_manager;
 	ResourceMutex             m_resource_mutex;
+	CommandScheduler&         m_scheduler;
 	BufferCache               m_buffer_cache;
 	TextureCache              m_texture_cache;
 	mutable std::shared_mutex m_mapped_ranges_mutex;

src/graphics/host_gpu/renderer/cache/multiLevelPageTable.h

@@ -7,8 +7,9 @@
 #include <array>
 #include <cstddef>
 #include <cstdint>
-#include <list>
+#include <cstring>
 #include <memory>
+#include <type_traits>
 #include <utility>
 #include <vector>
 
@@ -103,6 +104,158 @@ private:
 	size_t                               m_allocated_buckets = 0;
 };
 
+// Inline owner storage for page-table entries. Texture pages normally have only a
+// handful of owners, so avoid a heap allocation on the first 16 registrations.
+//
+// Owners are packed into live uint64_t objects instead of being constructed in
+// raw storage. This keeps sparse bucket construction cheap without relying on
+// subtle implicit-lifetime or pointer-provenance rules.
+template <typename OwnerT, size_t InlineCapacity = 16>
+class InlinePageOwnerList final {
+public:
+	using value_type = OwnerT;
+	using size_type  = size_t;
+
+	class const_iterator final {
+	public:
+		[[nodiscard]] OwnerT operator*() const noexcept { return m_owners->At(m_index); }
+		const_iterator&      operator++() noexcept {
+			++m_index;
+			return *this;
+		}
+		bool operator==(const const_iterator&) const = default;
+
+	private:
+		friend class InlinePageOwnerList;
+		const_iterator(const InlinePageOwnerList* owners, size_type index) noexcept
+		    : m_owners(owners), m_index(index) {}
+
+		const InlinePageOwnerList* m_owners = nullptr;
+		size_type                  m_index  = 0;
+	};
+
+	static_assert(InlineCapacity > 0);
+	static_assert(std::is_default_constructible_v<OwnerT>);
+	static_assert(std::is_trivially_copyable_v<OwnerT>);
+	static_assert(std::has_unique_object_representations_v<OwnerT>);
+	static_assert(sizeof(OwnerT) <= sizeof(uint64_t));
+
+	InlinePageOwnerList() noexcept {}
+	InlinePageOwnerList(const InlinePageOwnerList&)            = delete;
+	InlinePageOwnerList& operator=(const InlinePageOwnerList&) = delete;
+	InlinePageOwnerList(InlinePageOwnerList&& other) noexcept
+	    : m_overflow(std::move(other.m_overflow)),
+	      m_inline_size(std::exchange(other.m_inline_size, 0)) {
+		if (m_overflow == nullptr) {
+			std::copy_n(other.m_inline.begin(), m_inline_size, m_inline.begin());
+		}
+	}
+	InlinePageOwnerList& operator=(InlinePageOwnerList&& other) noexcept {
+		if (this != &other) {
+			m_overflow    = std::move(other.m_overflow);
+			m_inline_size = std::exchange(other.m_inline_size, 0);
+			if (m_overflow == nullptr) {
+				std::copy_n(other.m_inline.begin(), m_inline_size, m_inline.begin());
+			}
+		}
+		return *this;
+	}
+
+	[[nodiscard]] const_iterator begin() const noexcept { return {this, 0}; }
+	[[nodiscard]] const_iterator end() const noexcept { return {this, size()}; }
+
+	[[nodiscard]] bool      empty() const noexcept { return size() == 0; }
+	[[nodiscard]] size_type size() const noexcept {
+		return m_overflow == nullptr ? m_inline_size : m_overflow->size();
+	}
+	[[nodiscard]] OwnerT front() const noexcept { return At(0); }
+	[[nodiscard]] OwnerT operator[](size_type index) const noexcept { return At(index); }
+
+	void push_back(const OwnerT& owner) {
+		const uint64_t packed = Pack(owner);
+		if (m_overflow != nullptr) {
+			m_overflow->push_back(packed);
+			return;
+		}
+		if (m_inline_size < InlineCapacity) {
+			m_inline[m_inline_size] = packed;
+			++m_inline_size;
+			return;
+		}
+		auto overflow = std::make_unique<std::vector<uint64_t>>();
+		overflow->reserve(InlineCapacity * 2);
+		overflow->assign(m_inline.begin(), m_inline.end());
+		overflow->push_back(packed);
+		m_overflow = std::move(overflow);
+	}
+
+	[[nodiscard]] bool Contains(const OwnerT& owner) const noexcept {
+		const uint64_t packed = Pack(owner);
+		if (m_overflow != nullptr) {
+			return std::find(m_overflow->begin(), m_overflow->end(), packed) != m_overflow->end();
+		}
+		return std::find(m_inline.begin(), m_inline.begin() + static_cast<ptrdiff_t>(m_inline_size),
+		                 packed) != m_inline.begin() + static_cast<ptrdiff_t>(m_inline_size);
+	}
+
+	[[nodiscard]] bool Erase(const OwnerT& owner) noexcept {
+		const uint64_t packed = Pack(owner);
+		if (m_overflow != nullptr) {
+			const auto found = std::find(m_overflow->begin(), m_overflow->end(), packed);
+			if (found == m_overflow->end()) {
+				return false;
+			}
+			m_overflow->erase(found);
+			if (m_overflow->size() == InlineCapacity) {
+				std::copy(m_overflow->begin(), m_overflow->end(), m_inline.begin());
+				m_inline_size = InlineCapacity;
+				m_overflow.reset();
+			}
+			return true;
+		}
+		const auto end   = m_inline.begin() + static_cast<ptrdiff_t>(m_inline_size);
+		const auto found = std::find(m_inline.begin(), end, packed);
+		if (found == end) {
+			return false;
+		}
+		std::move(found + 1, end, found);
+		--m_inline_size;
+		return true;
+	}
+
+	template <typename Func>
+	void ForEach(Func&& func) const {
+		if (m_overflow != nullptr) {
+			for (const uint64_t owner: *m_overflow) {
+				func(Unpack(owner));
+			}
+			return;
+		}
+		for (size_type index = 0; index < m_inline_size; ++index) {
+			func(Unpack(m_inline[index]));
+		}
+	}
+
+private:
+	[[nodiscard]] OwnerT At(size_type index) const noexcept {
+		return Unpack(m_overflow == nullptr ? m_inline[index] : (*m_overflow)[index]);
+	}
+	[[nodiscard]] static uint64_t Pack(const OwnerT& owner) noexcept {
+		uint64_t packed = 0;
+		std::memcpy(&packed, &owner, sizeof(owner));
+		return packed;
+	}
+	[[nodiscard]] static OwnerT Unpack(uint64_t packed) noexcept {
+		OwnerT owner {};
+		std::memcpy(&owner, &packed, sizeof(owner));
+		return owner;
+	}
+
+	std::array<uint64_t, InlineCapacity>   m_inline;
+	std::unique_ptr<std::vector<uint64_t>> m_overflow;
+	size_type                              m_inline_size = 0;
+};
+
 // Removes only the requested owner. Other owners in the same page entry remain intact.
 template <typename Container, typename Value>
 [[nodiscard]] bool EraseExact(Container& owners, const Value& owner) {
@@ -114,247 +267,6 @@ template <typename Container, typename Value>
 	return true;
 }
 
-// Multi-range ownership with 1 MiB candidate buckets and precise 4 KiB
-// lifetime accounting. OwnerT only needs equality.
-template <typename OwnerT>
-class MultiRangePageOwnerIndex final {
-private:
-	struct Registration;
-	using MembershipList = std::vector<const Registration*>;
-
-public:
-	struct ByteRange final {
-		uint64_t address = 0;
-		uint64_t size    = 0;
-	};
-
-	using CoarseTable   = MultiLevelPageTable<MembershipList, 20, 40, 10>;
-	using TrackingTable = MultiLevelPageTable<MembershipList, 12, 40, 18>;
-
-	[[nodiscard]] bool Register(const OwnerT& owner, const std::vector<ByteRange>& ranges) {
-		if (FindRegistration(owner) != m_registrations.end()) {
-			return false;
-		}
-		auto normalized = Normalize(ranges);
-		if (normalized.empty()) {
-			return false;
-		}
-		m_registrations.push_back({owner, std::move(normalized)});
-		const Registration* registration = &m_registrations.back();
-		for (const size_t page: CollectPages<20>(registration->ranges)) {
-			m_coarse_pages[page].push_back(registration);
-		}
-		for (const size_t page: CollectPages<12>(registration->ranges)) {
-			m_tracking_pages[page].push_back(registration);
-		}
-		return true;
-	}
-
-	// No state changes if an expected membership is absent. Final releases are
-	// sorted and coalesced 4 KiB spans whose final owner disappeared.
-	[[nodiscard]] bool Unregister(const OwnerT& owner, std::vector<ByteRange>& final_releases) {
-		final_releases.clear();
-		auto registration = FindRegistration(owner);
-		if (registration == m_registrations.end()) {
-			return false;
-		}
-		const auto          coarse_pages     = CollectPages<20>(registration->ranges);
-		const auto          tracking_pages   = CollectPages<12>(registration->ranges);
-		const Registration* registration_ptr = &*registration;
-		if (!HasAllMemberships(m_coarse_pages, coarse_pages, registration_ptr) ||
-		    !HasAllMemberships(m_tracking_pages, tracking_pages, registration_ptr)) {
-			return false;
-		}
-		std::vector<size_t> final_pages;
-		for (const size_t page: coarse_pages) {
-			(void)EraseExact(*m_coarse_pages.Find(page), registration_ptr);
-		}
-		for (const size_t page: tracking_pages) {
-			auto& owners = *m_tracking_pages.Find(page);
-			if (owners.size() == 1) {
-				final_pages.push_back(page);
-			}
-			(void)EraseExact(owners, registration_ptr);
-		}
-		m_registrations.erase(registration);
-		final_releases = CoalesceTrackingPages(final_pages);
-		return true;
-	}
-
-	[[nodiscard]] std::vector<OwnerT> Query(uint64_t address, uint64_t size) const {
-		return Query(address, size, [](const OwnerT&) { return true; });
-	}
-
-	template <typename Predicate>
-	[[nodiscard]] std::vector<OwnerT> Query(uint64_t address, uint64_t size,
-	                                        Predicate&& predicate) const {
-		return QueryImpl(address, size, true, std::forward<Predicate>(predicate));
-	}
-
-	// Fault paths use page candidates: exact byte-disjoint owners sharing a
-	// touched 4 KiB page are intentionally retained.
-	[[nodiscard]] std::vector<OwnerT> QueryCandidates(uint64_t address, uint64_t size) const {
-		return QueryCandidates(address, size, [](const OwnerT&) { return true; });
-	}
-
-	template <typename Predicate>
-	[[nodiscard]] std::vector<OwnerT> QueryCandidates(uint64_t address, uint64_t size,
-	                                                  Predicate&& predicate) const {
-		return QueryImpl(address, size, false, std::forward<Predicate>(predicate));
-	}
-
-	[[nodiscard]] size_t CoarseMembershipCount(size_t page) const {
-		const auto* owners = m_coarse_pages.Find(page);
-		return owners == nullptr ? 0 : owners->size();
-	}
-	[[nodiscard]] size_t TrackingMembershipCount(size_t page) const {
-		const auto* owners = m_tracking_pages.Find(page);
-		return owners == nullptr ? 0 : owners->size();
-	}
-
-private:
-	template <typename Predicate>
-	[[nodiscard]] std::vector<OwnerT> QueryImpl(uint64_t address, uint64_t size, bool strict_bytes,
-	                                            Predicate&& predicate) const {
-		typename CoarseTable::PageRange   coarse_range {};
-		typename TrackingTable::PageRange tracking_range {};
-		if (!CoarseTable::TryGetPageRange(address, size, coarse_range) ||
-		    !TrackingTable::TryGetPageRange(address, size, tracking_range)) {
-			return {};
-		}
-		MembershipList candidates;
-		for (size_t page = coarse_range.first; page < coarse_range.last_exclusive; ++page) {
-			if (const auto* owners = m_coarse_pages.Find(page); owners != nullptr) {
-				AppendUnique(candidates, *owners);
-			}
-		}
-		std::vector<OwnerT> result;
-		for (const Registration* registration: candidates) {
-			if ((!strict_bytes || Overlaps(registration->ranges, address, size)) &&
-			    HasTrackingMembership(registration, tracking_range) &&
-			    predicate(registration->owner)) {
-				result.push_back(registration->owner);
-			}
-		}
-		return result;
-	}
-
-	struct Registration final {
-		OwnerT                 owner;
-		std::vector<ByteRange> ranges;
-	};
-	using RegistrationIterator      = typename std::list<Registration>::iterator;
-	using ConstRegistrationIterator = typename std::list<Registration>::const_iterator;
-
-	[[nodiscard]] RegistrationIterator FindRegistration(const OwnerT& owner) {
-		return std::find_if(m_registrations.begin(), m_registrations.end(),
-		                    [&](const Registration& item) { return item.owner == owner; });
-	}
-	[[nodiscard]] ConstRegistrationIterator FindRegistration(const OwnerT& owner) const {
-		return std::find_if(m_registrations.begin(), m_registrations.end(),
-		                    [&](const Registration& item) { return item.owner == owner; });
-	}
-
-	[[nodiscard]] static std::vector<ByteRange> Normalize(const std::vector<ByteRange>& ranges) {
-		std::vector<ByteRange> sorted;
-		for (const auto& range: ranges) {
-			typename CoarseTable::PageRange ignored {};
-			if (!CoarseTable::TryGetPageRange(range.address, range.size, ignored)) {
-				return {};
-			}
-			sorted.push_back(range);
-		}
-		std::sort(sorted.begin(), sorted.end(), [](const ByteRange& lhs, const ByteRange& rhs) {
-			return lhs.address < rhs.address;
-		});
-		std::vector<ByteRange> merged;
-		for (const auto& range: sorted) {
-			if (merged.empty() || range.address > merged.back().address + merged.back().size) {
-				merged.push_back(range);
-			} else {
-				const uint64_t end = std::max(merged.back().address + merged.back().size,
-				                              range.address + range.size);
-				merged.back().size = end - merged.back().address;
-			}
-		}
-		return merged;
-	}
-
-	template <size_t Bits>
-	[[nodiscard]] static std::vector<size_t> CollectPages(const std::vector<ByteRange>& ranges) {
-		std::vector<size_t> pages;
-		for (const auto& range: ranges) {
-			const size_t first = static_cast<size_t>(range.address >> Bits);
-			const size_t last  = static_cast<size_t>((range.address + range.size - 1) >> Bits);
-			for (size_t page = first; page <= last; ++page) {
-				pages.push_back(page);
-			}
-		}
-		std::sort(pages.begin(), pages.end());
-		pages.erase(std::unique(pages.begin(), pages.end()), pages.end());
-		return pages;
-	}
-
-	template <typename Table>
-	[[nodiscard]] static bool HasAllMemberships(const Table&               table,
-	                                            const std::vector<size_t>& pages,
-	                                            const Registration*        registration) {
-		for (const size_t page: pages) {
-			const auto* owners = table.Find(page);
-			if (owners == nullptr ||
-			    std::find(owners->begin(), owners->end(), registration) == owners->end()) {
-				return false;
-			}
-		}
-		return true;
-	}
-
-	[[nodiscard]] bool HasTrackingMembership(const Registration*                      registration,
-	                                         const typename TrackingTable::PageRange& range) const {
-		for (size_t page = range.first; page < range.last_exclusive; ++page) {
-			const auto* owners = m_tracking_pages.Find(page);
-			if (owners != nullptr &&
-			    std::find(owners->begin(), owners->end(), registration) != owners->end()) {
-				return true;
-			}
-		}
-		return false;
-	}
-
-	[[nodiscard]] static bool Overlaps(const std::vector<ByteRange>& ranges, uint64_t address,
-	                                   uint64_t size) {
-		const uint64_t end = address + size;
-		return std::any_of(ranges.begin(), ranges.end(), [&](const ByteRange& range) {
-			return range.address < end && address < range.address + range.size;
-		});
-	}
-	static void AppendUnique(MembershipList& destination, const MembershipList& source) {
-		for (const Registration* registration: source) {
-			if (std::find(destination.begin(), destination.end(), registration) ==
-			    destination.end()) {
-				destination.push_back(registration);
-			}
-		}
-	}
-	[[nodiscard]] static std::vector<ByteRange>
-	CoalesceTrackingPages(const std::vector<size_t>& pages) {
-		std::vector<ByteRange> result;
-		for (const size_t page: pages) {
-			const uint64_t address = static_cast<uint64_t>(page) << 12;
-			if (!result.empty() && result.back().address + result.back().size == address) {
-				result.back().size += uint64_t {1} << 12;
-			} else {
-				result.push_back({address, uint64_t {1} << 12});
-			}
-		}
-		return result;
-	}
-
-	CoarseTable             m_coarse_pages;
-	TrackingTable           m_tracking_pages;
-	std::list<Registration> m_registrations;
-};
-
 } // namespace Libs::Graphics
 
 #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -195,10 +195,12 @@ void TextureCache::RegisterImage(ImageId id) {
 	if (image.registered || image.info.data.Empty()) {
 		EXIT("TextureCache: invalid image registration\n");
 	}
-	std::vector<ImageOwnerIndex::ByteRange> ranges;
-	ranges.push_back({image.info.data.address, image.info.data.size});
-	if (!m_image_owner_index.Register(id, ranges)) {
-		EXIT("TextureCache: duplicate or invalid image registration\n");
+	ImagePageTable::PageRange pages {};
+	if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
+		EXIT("TextureCache: image registration is outside the guest address space\n");
+	}
+	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+		m_image_page_table[page].push_back(id);
 	}
 	image.registered = true;
 	image.lru_id     = m_lru_cache.Insert(id, m_gc_tick);
@@ -211,9 +213,15 @@ void TextureCache::UnregisterImage(ImageId id) {
 		return;
 	}
 	UntrackImage(id);
-	std::vector<ImageOwnerIndex::ByteRange> releases;
-	if (!m_image_owner_index.Unregister(id, releases)) {
-		EXIT("TextureCache: image missing from owner index\n");
+	ImagePageTable::PageRange pages {};
+	if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
+		EXIT("TextureCache: registered image is outside the guest address space\n");
+	}
+	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+		auto* owners = m_image_page_table.Find(page);
+		if (owners == nullptr || !owners->Erase(id)) {
+			EXIT("TextureCache: image missing from page owner index\n");
+		}
 	}
 	m_lru_cache.Free(image.lru_id);
 	const auto accounted = image.AccountedSize();
@@ -323,7 +331,7 @@ void TextureCache::TrackImage(ImageId id) {
 	if (!image.IsTracked()) {
 		image.track_addr     = image_begin;
 		image.track_addr_end = image_end;
-		m_page_manager.UpdatePageWatchers(true, image_begin, image.info.data.size);
+		m_page_manager.UpdatePageWatchers<true>(image_begin, image.info.data.size);
 		return;
 	}
 	if (image_begin < image.track_addr) {
@@ -348,7 +356,7 @@ void TextureCache::TrackImageHead(ImageId id) {
 	}
 	const auto size  = image.track_addr - image_begin;
 	image.track_addr = image_begin;
-	m_page_manager.UpdatePageWatchers(true, image_begin, size);
+	m_page_manager.UpdatePageWatchers<true>(image_begin, size);
 }
 
 void TextureCache::TrackImageTail(ImageId id) {
@@ -366,7 +374,7 @@ void TextureCache::TrackImageTail(ImageId id) {
 	const auto address   = image.track_addr_end;
 	const auto size      = image_end - address;
 	image.track_addr_end = image_end;
-	m_page_manager.UpdatePageWatchers(true, address, size);
+	m_page_manager.UpdatePageWatchers<true>(address, size);
 }
 
 void TextureCache::UntrackImage(ImageId id) {
@@ -379,7 +387,7 @@ void TextureCache::UntrackImage(ImageId id) {
 	image.track_addr     = 0;
 	image.track_addr_end = 0;
 	if (size != 0) {
-		m_page_manager.UpdatePageWatchers(false, address, size);
+		m_page_manager.UpdatePageWatchers<false>(address, size);
 	}
 }
 
@@ -400,7 +408,7 @@ void TextureCache::UntrackImageHead(ImageId id) {
 		UntrackImage(id);
 	}
 	if (size != 0) {
-		m_page_manager.UpdatePageWatchers(false, begin, size);
+		m_page_manager.UpdatePageWatchers<false>(begin, size);
 	}
 }
 
@@ -421,7 +429,7 @@ void TextureCache::UntrackImageTail(ImageId id) {
 		UntrackImage(id);
 	}
 	if (size != 0) {
-		m_page_manager.UpdatePageWatchers(false, address, size);
+		m_page_manager.UpdatePageWatchers<false>(address, size);
 	}
 }
 
@@ -451,10 +459,48 @@ const Image& TextureCache::GetImage(ImageId id) const {
 	return ResolveImage(id);
 }
 
-std::vector<ImageId> TextureCache::FindImagesInRegion(uint64_t address, uint64_t size,
-                                                      bool page_overlap) const {
-	return page_overlap ? m_image_owner_index.QueryCandidates(address, size)
-	                    : m_image_owner_index.Query(address, size);
+TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64_t size,
+                                                        bool page_overlap) const {
+	ImagePageTable::PageRange pages {};
+	if (!ImagePageTable::TryGetPageRange(address, size, pages)) {
+		return {};
+	}
+
+	uint32_t query_epoch = ++m_image_query_epoch;
+	if (query_epoch == 0) {
+		for (const auto& slot: m_slots) {
+			if (slot.image != nullptr) {
+				slot.image->query_epoch = 0;
+			}
+		}
+		query_epoch = ++m_image_query_epoch;
+	}
+
+	ImageIds result;
+	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+		const auto* owners = m_image_page_table.Find(page);
+		if (owners == nullptr) {
+			continue;
+		}
+		owners->ForEach([&](ImageId id) {
+			if (!id || id.index >= m_slots.size()) {
+				return;
+			}
+			const auto& slot = m_slots[id.index];
+			if (slot.generation != id.generation || slot.image == nullptr) {
+				return;
+			}
+			auto& image = *slot.image;
+			if (image.query_epoch == query_epoch) {
+				return;
+			}
+			image.query_epoch = query_epoch;
+			if (image.Overlaps(address, size, page_overlap)) {
+				result.push_back(id);
+			}
+		});
+	}
+	return result;
 }
 
 ImageId TextureCache::GetNullImage(const ImageDesc& desc) {
@@ -838,11 +884,20 @@ struct TextureCache::ColorTransferPlan {
 	TextureUploadLayout              layout;
 	std::vector<vk::BufferImageCopy> regions;
 	std::vector<GpuTileInfo>         tiles;
+	uint64_t                         linear_size = 0;
 	bool                             tiled       = false;
 	bool                             swap_bgra16 = false;
 	bool                             valid       = false;
 };
 
+static uint64_t GetLinearSize(std::span<const GpuTileInfo> tiles) {
+	uint64_t size = 0;
+	for (const auto& tile: tiles) {
+		size = std::max(size, tile.linear_offset + tile.linear_size);
+	}
+	return size;
+}
+
 struct TextureCache::DownloadPlan {
 	ColorTransferPlan color;
 	bool              depth = false;
@@ -867,22 +922,28 @@ TextureCache::BuildColorTransfer(const Image& image, BindingType binding,
 			case BindingType::Texture: break;
 			case BindingType::Storage: owner = "StorageTextureCache"; break;
 			case BindingType::RenderTarget:
+				if (info.resources.layers == 0 || info.data.size % info.resources.layers != 0 ||
+				    info.samples != 1 || image.backing.samples != 1) {
+					EXIT("TextureCache: invalid color-attachment upload\n");
+				}
+				format           = ImageOps::RenderTargetTransferFormat(info.bytes_per_block);
+				allow_depth_tile = false;
+				plan.swap_bgra16 = info.bgra16;
+				owner            = "RenderTarget";
+				break;
 			case BindingType::VideoOut:
 				if (info.resources.layers == 0 || info.data.size % info.resources.layers != 0 ||
 				    info.samples != 1 || image.backing.samples != 1 ||
-				    (binding == BindingType::VideoOut &&
-				     info.metadata.compression != VideoOutCompression::Uncompressed)) {
+				    info.metadata.compression != VideoOutCompression::Uncompressed) {
 					EXIT("TextureCache: invalid color-attachment upload\n");
 				}
-				format           = binding == BindingType::RenderTarget
-				                       ? ImageOps::RenderTargetTransferFormat(info.bytes_per_block)
-				                       : info.guest_format;
+				format           = info.guest_format;
 				layers           = info.resources.layers;
 				volume           = false;
 				layered          = layers > 1;
 				allow_depth_tile = false;
 				plan.swap_bgra16 = info.bgra16;
-				owner = binding == BindingType::RenderTarget ? "RenderTarget" : "VideoOut";
+				owner            = "VideoOut";
 				break;
 			case BindingType::DepthTarget: return plan;
 		}
@@ -913,6 +974,7 @@ TextureCache::BuildColorTransfer(const Image& image, BindingType binding,
 		                              info.resources.levels, plan.tiles)) {
 			return plan;
 		}
+		plan.linear_size = GetLinearSize(plan.tiles);
 	}
 	plan.valid = true;
 	return plan;
@@ -950,11 +1012,19 @@ void TextureCache::UploadImage(Image& image, const ImageDesc& desc, Buffer& sour
 
 	if (desc.type != BindingType::DepthTarget) {
 		auto plan = BuildColorTransfer(image, desc.type, TransferDirection::Upload);
-		EXIT_NOT_IMPLEMENTED(!plan.valid);
+		if (!plan.valid) {
+			EXIT("TextureCache: invalid color upload: binding=%u addr=0x%016" PRIx64
+			     " size=0x%016" PRIx64 " format=%u tile=%u family=%u extent=%ux%ux%u "
+			     "pitch=%u levels=%u layers=%u samples=%u\n",
+			     static_cast<uint32_t>(desc.type), info.data.address, info.data.size,
+			     info.guest_format, info.tile_mode, static_cast<uint32_t>(plan.layout.tile_family),
+			     info.extent.width, info.extent.height, info.extent.depth, info.pitch,
+			     info.resources.levels, info.resources.layers, info.samples);
+		}
 		TileManager::Result linear {source.Handle(), source_offset, info.data.size};
 		if (plan.tiled) {
-			linear = m_tiler->Detile(source.Handle(), source_offset, info.data.size, info.data.size,
-			                         plan.tiles);
+			linear = m_tiler->Detile(source.Handle(), source_offset, info.data.size,
+			                         plan.linear_size, plan.tiles);
 		}
 		if (plan.swap_bgra16) {
 			linear = m_tiler->SwapBgra16(linear);
@@ -1038,25 +1108,20 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) {
 	if (image.info.samples > 1) {
 		return;
 	}
-	bool       data_gpu_owned = false;
-	bool       data_imported  = false;
-	const bool upload         = image.IsBufferModified() || image.IsCpuDirty();
+	bool       data_imported = false;
+	const bool upload        = image.IsBufferModified() || image.IsCpuDirty();
 	if (upload) {
 		const auto source =
 		    m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size);
 		if (source.buffer == nullptr) {
 			EXIT("TextureCache: failed to obtain image upload source\n");
 		}
-		data_gpu_owned |= source.gpu_owned;
 		data_imported = true;
 		UploadImage(image, desc, *source.buffer, source.offset);
 	}
 	if (data_imported) {
 		image.ClearBufferModified();
 	}
-	if (data_gpu_owned) {
-		image.MarkGpuModified();
-	}
 	if (image.IsCpuDirty()) {
 		image.RefreshComplete();
 	}
@@ -1131,12 +1196,11 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 	}
 
 	ImageId result {};
-	bool    replacement_buffer = false;
-	bool    inserted_new       = false;
+	bool    inserted_new = false;
 	{
-		std::lock_guard            transaction(m_resource_mutex);
-		CacheLock                  lock(*this, m_lock);
-		const std::vector<ImageId> candidates =
+		std::lock_guard transaction(m_resource_mutex);
+		CacheLock       lock(*this, m_lock);
+		const auto      candidates =
 		    FindImagesInRegion(desc.info.data.address, desc.info.data.size, false);
 
 		for (const auto id: candidates) {
@@ -1159,8 +1223,8 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 				if (owner == nullptr) {
 					continue;
 				}
-				const auto merged_info = result ? ResolveImage(result).info : desc.info;
-				const auto overlap     = ResolveOverlap(merged_info, desc.type, candidate, result);
+				const auto& merged_info = result ? ResolveImage(result).info : desc.info;
+				const auto  overlap     = ResolveOverlap(merged_info, desc.type, candidate, result);
 				if (overlap.image) {
 					result     = overlap.image;
 					view_mip   = overlap.mip;
@@ -1174,23 +1238,15 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 			if (exact_format && resolved.info.pixel_format != desc.info.pixel_format) {
 				result = {};
 			} else if (resolved.info.resources < desc.info.resources) {
-				ImageDesc refresh {
-				    .info = resolved.info, .view_info = {}, .type = UploadBinding(resolved)};
-				RefreshImage(result, refresh);
-				if (resolved.IsGpuModified() && !SynchronizeImageToBuffer(result)) {
-					EXIT("TextureCache: cannot preserve an unsupported replacement image\n");
-				}
-				replacement_buffer = resolved.IsBufferModified();
-				DeleteImage(result);
-				result = {};
+				result = ExpandImage(desc.info, result);
 			}
 		}
 		if (!result) {
 			result         = InsertImage(desc.info);
 			inserted_new   = true;
 			auto& inserted = ResolveImage(result);
-			if (replacement_buffer || m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address,
-			                                                          inserted.info.data.size)) {
+			if (m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address,
+			                                    inserted.info.data.size)) {
 				inserted.MarkBufferModified();
 			}
 		}
@@ -1360,11 +1416,6 @@ void TextureCache::CommitGpuWrite(Image& image) {
 	if (image.depth_id || image.backing.image == nullptr) {
 		EXIT("TextureCache: stencil association cannot own image contents\n");
 	}
-	const auto range = image.info.data;
-	if (m_buffer_cache.HasGpuDirtyBytes(range.address, range.size)) {
-		m_buffer_cache.DiscardGpuDirtyBytes(range.address, range.size);
-	}
-	m_buffer_cache.InvalidateImageAliases(range.address, range.size);
 	image.ClearBufferModified();
 	if (image.IsCpuDirty()) {
 		image.RefreshComplete();
@@ -1429,9 +1480,6 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 			return false;
 		}
 	}
-	if (m_buffer_cache.HasGpuDirtyBytes(address, size)) {
-		m_buffer_cache.DiscardGpuDirtyBytes(address, size);
-	}
 	if (image.IsBufferModified() || image.IsCpuDirty()) {
 		ImageDesc refresh {.info = image.info, .view_info = {}, .type = UploadBinding(image)};
 		InitializeImage(selected, refresh);
@@ -1551,11 +1599,14 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des
 }
 
 void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset,
-                                     DownloadPlan plan) {
+                                     uint64_t destination_size, DownloadPlan plan) {
 	if (!plan.valid) {
 		EXIT("TextureCache: invalid image download plan\n");
 	}
 	if (plan.depth) {
+		if (destination_size != image.info.data.size) {
+			EXIT("TextureCache: partial depth image download is unsupported\n");
+		}
 		DownloadDepth(image, destination, destination_offset);
 		return;
 	}
@@ -1565,22 +1616,119 @@ void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t
 	                                         : TileManager::ColorTransform::None;
 	if (!color.tiled) {
 		if (transform == TileManager::ColorTransform::SwapBgra16) {
-			auto linear = m_tiler->GetScratchBuffer(image.info.data.size);
+			auto linear = m_tiler->GetScratchBuffer(destination_size);
 			image.Download(color.regions, linear.buffer, 0, linear.size);
 			m_tiler->SwapBgra16(linear,
-			                    {destination.Handle(), destination_offset, image.info.data.size});
+			                    {destination.Handle(), destination_offset, destination_size});
 			return;
 		}
 		for (auto& copy: color.regions) {
 			copy.bufferOffset += destination_offset;
 		}
-		image.Download(color.regions, destination.Handle(), destination_offset,
-		               image.info.data.size);
+		image.Download(color.regions, destination.Handle(), destination_offset, destination_size);
 		return;
 	}
 
 	m_tiler->TileImage(image, color.regions, destination.Handle(), destination_offset,
-	                   image.info.data.size, image.info.data.size, color.tiles, transform);
+	                   destination_size, color.linear_size, color.tiles, transform);
+}
+
+bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size) {
+	CacheLock            lock(m_texture_cache, m_texture_cache.m_lock);
+	std::vector<ImageId> matches;
+	for (const auto id: m_texture_cache.FindImagesInRegion(vaddr, size, false)) {
+		auto owner = m_texture_cache.ResolveOwner(id);
+		if (owner == nullptr || owner->info.data.address != vaddr) {
+			continue;
+		}
+		if (owner->depth_id) {
+			owner = m_texture_cache.ResolveOwner(owner->depth_id);
+		}
+		if (owner != nullptr && owner->SafeToDownload()) {
+			matches.push_back(id);
+		}
+	}
+
+	ImageId selected {};
+	if (matches.size() == 1) {
+		selected = matches.front();
+	} else {
+		for (const auto id: matches) {
+			const auto& image = m_texture_cache.ResolveImage(id);
+			if (image.info.data.size == size) {
+				selected = id;
+				break;
+			}
+		}
+	}
+	if (!selected) {
+		return false;
+	}
+	if (const auto owner = m_texture_cache.ResolveOwner(selected);
+	    owner != nullptr && owner->depth_id) {
+		selected = owner->depth_id;
+	}
+
+	auto& image = m_texture_cache.ResolveImage(selected);
+	if (!buffer.IsInBounds(image.info.data.address, 1)) {
+		return false;
+	}
+	const auto buf_offset = buffer.Offset(image.info.data.address);
+	const auto available  = buffer.Size() - buf_offset;
+	uint32_t   levels     = 0;
+	uint64_t   copy_size  = 0;
+	if (image.info.IsVolume()) {
+		// Volume mips contain strided block slices, so a mip's linear span cannot prove that
+		// every retained slice fits. Keep volume synchronization whole-image only.
+		if (!buffer.IsInBounds(image.info.data.address, image.info.data.size)) {
+			return false;
+		}
+		levels    = image.info.resources.levels;
+		copy_size = image.info.data.size;
+	} else {
+		for (; levels < image.info.resources.levels; ++levels) {
+			const auto& mip = image.info.mip_layout[levels];
+			if (mip.size == 0 || mip.offset > available || mip.size > available - mip.offset) {
+				break;
+			}
+			copy_size = std::max(copy_size, mip.offset + mip.size);
+		}
+	}
+	if (copy_size == 0) {
+		return false;
+	}
+	auto plan = m_texture_cache.BuildDownload(image);
+	if (!plan.valid) {
+		return false;
+	}
+	if (plan.depth && copy_size != image.info.data.size) {
+		return false;
+	}
+	if (!plan.depth && levels < image.info.resources.levels) {
+		auto& color = plan.color;
+		std::erase_if(color.regions, [levels](const vk::BufferImageCopy& region) {
+			return region.imageSubresource.mipLevel >= levels;
+		});
+		if (color.regions.empty()) {
+			return false;
+		}
+		if (color.tiled) {
+			const auto binding = m_texture_cache.UploadBinding(image);
+			const auto format =
+			    binding == TextureCache::BindingType::RenderTarget
+			        ? ImageOps::RenderTargetTransferFormat(image.info.bytes_per_block)
+			        : image.info.guest_format;
+			color.tiles.clear();
+			if (!TextureBuildGpuTileInfos(copy_size, color.regions, color.layout, format,
+			                              image.info.TransferLayers(), levels, color.tiles)) {
+				return false;
+			}
+			color.linear_size = GetLinearSize(color.tiles);
+		}
+	}
+	m_texture_cache.DownloadImageData(image, buffer, buf_offset, copy_size, std::move(plan));
+	m_texture_cache.RetainImage(m_scheduler.Current(), selected);
+	return true;
 }
 
 std::pair<uint8_t*, uint64_t> TextureCache::MapDownload(uint64_t size, uint64_t alignment) {
@@ -1612,12 +1760,9 @@ void TextureCache::QueueDownload(GuestRange range, StreamBuffer& download, uint8
 	m_scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
 	                                               vk::PipelineStageFlagBits::eHost, {}, 0, nullptr,
 	                                               1, &barrier, 0, nullptr);
-	const auto tick = m_scheduler.CurrentTick();
-	m_buffer_cache.BeginBackingPublication(range.address, range.size, tick);
-	m_scheduler.DeferPriorityOperation([this, &download, range, mapped, offset, tick] {
+	m_scheduler.DeferPriorityOperation([&download, range, mapped, offset] {
 		download.Invalidate(offset, range.size);
 		LibKernel::Memory::WriteBacking(range.address, mapped, range.size);
-		m_buffer_cache.CompleteBackingPublication(range.address, range.size, tick);
 	});
 }
 
@@ -1638,7 +1783,7 @@ bool TextureCache::TryDownloadImage(ImageId id) {
 	}
 	download.Flush(offset, range.size);
 
-	DownloadImageData(image, download, offset, std::move(plan));
+	DownloadImageData(image, download, offset, range.size, std::move(plan));
 
 	QueueDownload(range, download, mapped, offset);
 	return true;
@@ -1652,62 +1797,6 @@ void TextureCache::DownloadImage(ImageId id) {
 	m_scheduler.DrainPriorityOperations();
 }
 
-bool TextureCache::SynchronizeImageToBuffer(ImageId id) {
-	auto& image = ResolveImage(id);
-	if (image.depth_id) {
-		return true;
-	}
-	auto plan = BuildDownload(image);
-	if (!plan.valid) {
-		return false;
-	}
-	const auto range = image.info.data;
-	if (image.IsCpuDirty()) {
-		RefreshImage(id,
-		             ImageDesc {.info = image.info, .view_info = {}, .type = UploadBinding(image)});
-	}
-	if (!image.IsGpuModified()) {
-		return true;
-	}
-	if (image.IsDefinitelyCpuDirty() || image.IsBufferModified()) {
-		EXIT("TextureCache: image mirror source is not native-current\n");
-	}
-	auto [destination, offset] =
-	    m_buffer_cache.ObtainBufferForImageWrite(range.address, range.size);
-	if (destination == nullptr) {
-		EXIT("TextureCache: failed to allocate image mirror\n");
-	}
-	DownloadImageData(image, *destination, offset, std::move(plan));
-	m_scheduler.Current().RetainResourceUntilFence(destination);
-	m_buffer_cache.PublishImageBuffer(range.address, range.size);
-	image.MarkBufferModified();
-	RetainImage(m_scheduler.Current(), id);
-	ClearGpuModified(id);
-	return true;
-}
-
-bool TextureCache::SynchronizeImageToBuffer(uint64_t address, uint64_t size) {
-	if (!GuestRange {address, size}.Valid()) {
-		return false;
-	}
-	CacheLock lock(*this, m_lock);
-	ImageId   selected {};
-	for (const auto id: FindImagesInRegion(address, size, true)) {
-		auto owner = ResolveOwner(id);
-		if (owner == nullptr || !owner->GpuOverlaps(address, size)) {
-			continue;
-		}
-		if (selected) {
-			EXIT("TextureCache: ambiguous image-to-buffer synchronization\n");
-		}
-		selected = id;
-	}
-	if (!selected) {
-		return false;
-	}
-	return SynchronizeImageToBuffer(selected);
-}
-
 bool TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
                                            bool formatted_buffer_write) {
 	if (!GuestRange {address, size}.Valid()) {

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -66,7 +66,6 @@ public:
 	[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
 	                                        uint32_t packed_clear);
 	void               InvalidateMemory(uint64_t address, uint64_t size);
-	[[nodiscard]] bool SynchronizeImageToBuffer(uint64_t address, uint64_t size);
 	[[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
 	                                           bool formatted_buffer_write = false);
 	[[nodiscard]] RegionInfo QueryRegion(uint64_t address, uint64_t size);
@@ -100,7 +99,8 @@ private:
 		int32_t layer = -1;
 	};
 
-	using ImageOwnerIndex = MultiRangePageOwnerIndex<ImageId>;
+	using ImageIds       = InlinePageOwnerList<ImageId, 16>;
+	using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
 
 	[[nodiscard]] Image&                 ResolveImage(ImageId id);
 	[[nodiscard]] const Image&           ResolveImage(ImageId id) const;
@@ -126,8 +126,9 @@ private:
 	[[nodiscard]] static BindingType UploadBinding(const Image& image);
 	[[nodiscard]] bool               SafeToDownload(const Image& image);
 
-	[[nodiscard]] std::vector<ImageId> FindImagesInRegion(uint64_t address, uint64_t size,
-	                                                      bool page_overlap) const;
+	// Caller holds m_lock; it also serializes the per-image query epoch.
+	[[nodiscard]] ImageIds      FindImagesInRegion(uint64_t address, uint64_t size,
+	                                               bool page_overlap) const;
 	[[nodiscard]] OverlapResult ResolveOverlap(const ImageInfo& requested, BindingType binding,
 	                                           ImageId cached, ImageId merged);
 	[[nodiscard]] ImageId       ResolveDepthOverlap(const ImageInfo& requested, BindingType binding,
@@ -140,7 +141,7 @@ private:
 	[[nodiscard]] DownloadPlan      BuildDownload(const Image& image) const;
 	void UploadImage(Image& image, const ImageDesc& desc, Buffer& source, uint64_t source_offset);
 	void DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset,
-	                       DownloadPlan plan);
+	                       uint64_t destination_size, DownloadPlan plan);
 	void DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset);
 	void CommitGpuWrite(Image& image);
 	void PrepareImageCopy(Image& image);
@@ -155,7 +156,6 @@ private:
 	void InvalidateCpuAliases(uint64_t address, uint64_t size);
 	void ClearGpuModified(ImageId id);
 
-	[[nodiscard]] bool                          SynchronizeImageToBuffer(ImageId id);
 	void                                        DownloadImage(ImageId id);
 	[[nodiscard]] bool                          TryDownloadImage(ImageId id);
 	[[nodiscard]] std::pair<uint8_t*, uint64_t> MapDownload(uint64_t size, uint64_t alignment);
@@ -171,7 +171,7 @@ private:
 	ResourceMutex&                                    m_resource_mutex;
 	std::vector<Slot>                                 m_slots;
 	std::vector<uint32_t>                             m_free_slots;
-	ImageOwnerIndex                                   m_image_owner_index;
+	ImagePageTable                                    m_image_page_table;
 	std::map<vk::Format, ImageId>                     m_null_images;
 	Common::LeastRecentlyUsedCache<ImageId, uint64_t> m_lru_cache;
 	std::set<ImageId>                                 m_download_images;
@@ -179,11 +179,13 @@ private:
 	uint64_t                                          m_total_used_memory  = 0;
 	uint64_t                                          m_trigger_gc_memory  = 0;
 	uint64_t                                          m_pressure_gc_memory = 1536ull * 1024 * 1024;
-	uint64_t m_critical_gc_memory     = 3ull * 1024 * 1024 * 1024;
-	uint64_t m_gc_tick                = 0;
-	bool     m_readback_linear_images = false;
+	uint64_t         m_critical_gc_memory     = 3ull * 1024 * 1024 * 1024;
+	uint64_t         m_gc_tick                = 0;
+	mutable uint32_t m_image_query_epoch      = 0;
+	bool             m_readback_linear_images = false;
 
 	friend struct TextureCacheTestAccess;
+	friend class BufferCache;
 	friend class RenderExecutor;
 };
 

src/graphics/host_gpu/renderer/colorRenderTarget.cpp

@@ -119,7 +119,18 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	uint32_t   pitch  = 0;
 	uint64_t   size   = 0;
 	bool       tile   = false;
-	const bool standard64 =
+	const bool volume = rt.attrib3.dimension == 2;
+	if (rt.attrib3.dimension != 1 && !volume) {
+		EXIT("unsupported render-target dimension: %u\n", rt.attrib3.dimension);
+	}
+	if (!volume && rt.attrib3.depth != 0) {
+		EXIT("2D render target has nonzero depth: %u\n", rt.attrib3.depth);
+	}
+	if (volume && samples != 1) {
+		EXIT("multisampled 3D render targets are unsupported\n");
+	}
+	const uint32_t depth = volume ? rt.attrib3.depth + 1u : 1u;
+	const bool     standard64 =
 	    rt.attrib3.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB);
 
 	switch (rt.attrib3.tile_mode) {
@@ -145,6 +156,7 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	if (bytes_per_element == 0) {
 		EXIT("render-target format has no valid element size\n");
 	}
+	const auto transfer_format = ImageOps::RenderTargetTransferFormat(bytes_per_element);
 	if (standard64 &&
 	    (rt.attrib3.dimension != 1 || rt.attrib3.depth != 0 || levels != 1 ||
 	     rt.view.current_mip_level != 0 || view.base_layer != 0 || view.image_layers != 1 ||
@@ -165,10 +177,14 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	if (rt.pitch.pitch_div8_minus1 != 0) {
 		pitch = (rt.pitch.pitch_div8_minus1 + 1u) << 3u;
 	} else if (tile) {
-		pitch = standard64
-		            ? TileGetTexturePitch(Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float),
-		                                  width, levels, rt.attrib3.tile_mode)
-		            : TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments);
+		if (volume) {
+			pitch = TileGetTexturePitch(transfer_format, width, levels, rt.attrib3.tile_mode);
+		} else if (standard64) {
+			pitch = TileGetTexturePitch(Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float),
+			                            width, levels, rt.attrib3.tile_mode);
+		} else {
+			pitch = TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments);
+		}
 		if (pitch == 0) {
 			EXIT("unsupported render-target pitch: width=%u bytes=%u\n", width, bytes_per_element);
 		}
@@ -176,9 +192,19 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		pitch = width;
 	}
 
-	TileSizeOffset mip_sizes[16] {};
-	TilePaddedSize mip_padded[16] {};
-	if (tile) {
+	TileSizeOffset   mip_sizes[16] {};
+	TilePaddedSize   mip_padded[16] {};
+	TileVolumeLayout volume_layout {};
+	uint64_t         backing_size = 0;
+	if (volume) {
+		if (!tile || !TileGetTextureVolumeLayout(transfer_format, width, height, depth, levels,
+		                                         rt.attrib3.tile_mode, volume_layout)) {
+			EXIT("unsupported 3D render-target layout: %ux%ux%u levels=%u tile=%u\n", width, height,
+			     depth, levels, rt.attrib3.tile_mode);
+		}
+		size         = volume_layout.block_slice_size;
+		backing_size = volume_layout.total_size;
+	} else if (tile) {
 		TileSizeAlign layout {};
 		bool          valid_layout = false;
 		if (standard64) {
@@ -203,12 +229,6 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 			mip_sizes[0]  = {static_cast<uint32_t>(size), 0, 0, 0, 0, 0};
 			mip_padded[0] = {pitch, height};
 		}
-		if (rt.slice.slice_div64_minus1 != 0 &&
-		    (static_cast<uint64_t>(rt.slice.slice_div64_minus1) + 1u) * 64u != size) {
-			EXIT("render-target slice span mismatch: encoded=0x%016" PRIx64 " derived=0x%016" PRIx64
-			     "\n",
-			     (static_cast<uint64_t>(rt.slice.slice_div64_minus1) + 1u) * 64u, size);
-		}
 	} else {
 		size = static_cast<uint64_t>(pitch) * height * bytes_per_element * samples;
 		if (size > UINT32_MAX) {
@@ -217,40 +237,66 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		mip_sizes[0]  = {static_cast<uint32_t>(size), 0, 0, 0, 0, 0};
 		mip_padded[0] = {pitch, height};
 	}
-	if (size == 0 || size > UINT64_MAX / view.image_layers) {
+	if (rt.slice.slice_div64_minus1 != 0 &&
+	    (static_cast<uint64_t>(rt.slice.slice_div64_minus1) + 1u) * 64u != size) {
+		EXIT("render-target slice span mismatch: encoded=0x%016" PRIx64 " derived=0x%016" PRIx64
+		     "\n",
+		     (static_cast<uint64_t>(rt.slice.slice_div64_minus1) + 1u) * 64u, size);
+	}
+	if (size == 0 || (!volume && size > UINT64_MAX / view.image_layers)) {
 		EXIT("render-target memory footprint is invalid\n");
 	}
-	const auto backing_size = size * view.image_layers;
+	if (!volume) {
+		backing_size = size * view.image_layers;
+	}
+	if (backing_size == 0) {
+		EXIT("render-target backing is empty\n");
+	}
 	if (backing_size > TRACKER_ADDRESS_SIZE - rt.base.addr) {
 		EXIT("render-target backing range is invalid\n");
 	}
 
 	const vk::Extent2D view_extent = {std::max(width >> rt.view.current_mip_level, 1u),
 	                                  std::max(height >> rt.view.current_mip_level, 1u)};
+	const uint32_t     view_depth  = std::max(depth >> rt.view.current_mip_level, 1u);
+	if (volume &&
+	    (view.base_layer >= view_depth || view.layer_count > view_depth - view.base_layer)) {
+		EXIT("3D render-target view exceeds mip depth: base=%u count=%u depth=%u mip=%u\n",
+		     view.base_layer, view.layer_count, view_depth, rt.view.current_mip_level);
+	}
 
 	auto decision_log_id = g_render_color_log_count.fetch_add(1);
 	if (decision_log_id < 128) {
 		LOGF("RenderColorTarget: slot=%" PRIu32 " addr=0x%010" PRIx64 " size=0x%016" PRIx64
-		     " extent=%ux%u view_mip=%u view_extent=%ux%u levels=%u pitch=%u"
+		     " extent=%ux%ux%u view_mip=%u view_extent=%ux%u levels=%u pitch=%u"
 		     " fmt=0x%08" PRIx32 " nfmt=0x%08" PRIx32 " order=0x%08" PRIx32 " samples=%u tile=%s\n",
-		     rt_slot, rt.base.addr, backing_size, width, height, rt.view.current_mip_level,
+		     rt_slot, rt.base.addr, backing_size, width, height, depth, rt.view.current_mip_level,
 		     view_extent.width, view_extent.height, levels, pitch, rt.info.format,
 		     rt.info.channel_type, rt.info.channel_order, samples, tile ? "tiled" : "linear");
 	}
 
 	TextureCache::ImageDesc desc {};
-	desc.type                 = TextureCache::BindingType::RenderTarget;
-	desc.info.data            = {rt.base.addr, backing_size};
-	desc.info.pixel_format    = target_format.format;
-	desc.info.guest_format    = ImageOps::RenderTargetTransferFormat(bytes_per_element);
-	desc.info.type            = Prospero::ImageType::kColor2D;
-	desc.info.extent          = {width, height, 1};
-	desc.info.resources       = {levels, view.image_layers};
-	desc.info.pitch           = pitch;
+	desc.type              = TextureCache::BindingType::RenderTarget;
+	desc.info.data         = {rt.base.addr, backing_size};
+	desc.info.pixel_format = target_format.format;
+	desc.info.guest_format = transfer_format;
+	desc.info.type         = volume ? Prospero::ImageType::kColor3D : Prospero::ImageType::kColor2D;
+	desc.info.extent       = {width, height, depth};
+	desc.info.resources    = {levels, volume ? 1u : view.image_layers};
+	desc.info.pitch        = pitch;
 	desc.info.bytes_per_block = bytes_per_element;
 	desc.info.samples         = samples;
 	desc.info.tile_mode       = rt.attrib3.tile_mode;
 	for (uint32_t level = 0; level < levels; level++) {
+		if (volume) {
+			desc.info.mip_layout[level] = {
+			    volume_layout.level_offsets[level],
+			    volume_layout.level_sizes[level],
+			    volume_layout.level_widths[level],
+			    volume_layout.level_heights[level],
+			};
+			continue;
+		}
 		const auto level_offset =
 		    mip_sizes[level].src_size != 0 ? mip_sizes[level].src_offset : mip_sizes[level].offset;
 		const auto level_size =

src/graphics/host_gpu/renderer/debug.cpp

@@ -359,30 +359,12 @@ static void RtCheck(const HW::RenderTarget& rt) {
 				logged = true;
 			}
 		}
-		if (rt.attrib3.depth != 0x00000000) {
-			static bool logged = false;
-			if (!logged) {
-				LOGF("RenderTarget: temporary: ignoring PS5 color target depth_minus1=0x%08" PRIx32
-				     "\n",
-				     rt.attrib3.depth);
-				logged = true;
-			}
-		}
 		if (!RenderIsColorTileMode(rt.attrib3.tile_mode)) {
 			EXIT("unknown PS5 render-target tile mode: 0x%08" PRIx32 "\n", rt.attrib3.tile_mode);
 		}
 		if (!RenderIsColorDimension(rt.attrib3.dimension)) {
 			EXIT("unknown PS5 render-target dimension: 0x%08" PRIx32 "\n", rt.attrib3.dimension);
 		}
-		if (rt.attrib3.dimension != 0x00000001) {
-			static bool logged = false;
-			if (!logged) {
-				LOGF("RenderTarget: temporary: using 2D fallback for PS5 color "
-				     "dimension=0x%08" PRIx32 "\n",
-				     rt.attrib3.dimension);
-				logged = true;
-			}
-		}
 		if (!rt.attrib3.cmask_pipe_aligned) {
 			static bool logged = false;
 			if (!logged) {

src/graphics/host_gpu/renderer/depthRenderTarget.cpp

@@ -96,9 +96,11 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 	const auto& dc = hw.GetDepthControl();
 	const auto& sc = hw.GetStencilControl();
 	const auto& sm = hw.GetStencilMask();
-	const bool  depth_active =
+	const bool  has_stencil =
+	    z.stencil_info.format != Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid);
+	const bool depth_active =
 	    dc.z_enable || dc.z_write_enable || dc.depth_bounds_enable || rc.depth_clear_enable;
-	const bool stencil_active = dc.stencil_enable || rc.stencil_clear_enable;
+	const bool stencil_active = has_stencil && (dc.stencil_enable || rc.stencil_clear_enable);
 	if (!depth_active && !stencil_active) {
 		return;
 	}
@@ -139,8 +141,6 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 		}
 		return;
 	}
-	const bool has_stencil =
-	    z.stencil_info.format != Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid);
 	const bool has_htile = z.z_info.htile_acceleration;
 	const auto samples   = render_sample_count(z.z_info.num_samples);
 	if (samples == 0) {
@@ -156,8 +156,8 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 			DepthFatal("invalid depth view: base=%u last=%u", z.depth_view.slice_start,
 			           z.depth_view.slice_max);
 	}
-	if ((stencil_active && !has_stencil) || rc.resummarize_enable || rc.copy_centroid ||
-	    rc.copy_sample != 0 || z.z_info.expclear_enabled || z.stencil_info.expclear_enabled ||
+	if (rc.resummarize_enable || rc.copy_centroid || rc.copy_sample != 0 ||
+	    z.z_info.expclear_enabled || z.stencil_info.expclear_enabled ||
 	    z.z_info.partially_resident || z.stencil_info.partially_resident ||
 	    z.z_info.max_mip_level != 0 || z.depth_view.current_mip_level != 0 ||
 	    z.depth_info.addr5_swizzle_mask != 0 || z.depth_info.array_mode != 0 ||
@@ -279,10 +279,10 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 	r.depth_min_bounds         = hw.GetDepthBoundsMin();
 	r.depth_max_bounds         = hw.GetDepthBoundsMax();
 
-	r.stencil_clear_enable = rc.stencil_clear_enable;
+	r.stencil_clear_enable = has_stencil && rc.stencil_clear_enable;
 	r.stencil_clear_value  = hw.GetStencilClearValue();
-	r.stencil_test_enable  = dc.stencil_enable;
-	if (dc.stencil_enable) {
+	r.stencil_test_enable  = has_stencil && dc.stencil_enable;
+	if (r.stencil_test_enable) {
 		if (dc.stencilfunc > static_cast<uint8_t>(vk::CompareOp::eAlways) ||
 		    (dc.backface_enable &&
 		     dc.stencilfunc_bf > static_cast<uint8_t>(vk::CompareOp::eAlways)) ||

src/graphics/host_gpu/renderer/image/image.cpp

@@ -105,6 +105,10 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout                      destinat
                                    std::optional<ImageSubresourceRange> range) {
 	auto& state              = backing.state;
 	auto& subresource_states = backing.subresource_states;
+	if (range && info.IsVolume()) {
+		range->base_layer  = 0;
+		range->layer_count = 1;
+	}
 
 	const bool partial =
 	    range && (range->base_level != 0 || range->level_count != info.resources.levels ||

src/graphics/host_gpu/renderer/image/image.h

@@ -154,17 +154,18 @@ public:
 	}
 	[[nodiscard]] uint64_t HashGuestEdges() const;
 
-	ImageInfo      info;
-	VulkanImage    backing;
-	ImageViewCache views;
-	ImageUsage     usage;
-	ImageBinding   binding;
-	bool           registered     = false;
-	uint64_t       track_addr     = 0;
-	uint64_t       track_addr_end = 0;
-	ImageId        depth_id {};
-	uint64_t       tick_accessed_last = 0;
-	size_t         lru_id             = 0;
+	ImageInfo        info;
+	VulkanImage      backing;
+	ImageViewCache   views;
+	ImageUsage       usage;
+	ImageBinding     binding;
+	bool             registered     = false;
+	mutable uint32_t query_epoch    = 0;
+	uint64_t         track_addr     = 0;
+	uint64_t         track_addr_end = 0;
+	ImageId          depth_id {};
+	uint64_t         tick_accessed_last = 0;
+	size_t           lru_id             = 0;
 
 private:
 	friend struct ImageTestAccess;

src/graphics/host_gpu/renderer/image/textureCommon.cpp

@@ -482,10 +482,10 @@ static bool SetGpuTileSize(uint64_t offset, uint64_t length, uint64_t capacity,
 	return true;
 }
 
-bool TextureBuildGpuTileInfos(uint64_t size, const std::vector<vk::BufferImageCopy>& regions,
+bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferImageCopy>& regions,
                               const TextureUploadLayout& layout, uint32_t fmt, uint32_t depth,
                               uint64_t levels, std::vector<GpuTileInfo>& out_infos) {
-	if (size == 0 || levels == 0 || levels > 16 || depth == 0 ||
+	if (tiled_size == 0 || levels == 0 || levels > 16 || depth == 0 ||
 	    regions.size() != GetTextureRegionCount(depth, levels, layout.volume_texture) ||
 	    Prospero::IsFmaskTextureFormat(fmt)) {
 		return false;
@@ -538,8 +538,9 @@ bool TextureBuildGpuTileInfos(uint64_t size, const std::vector<vk::BufferImageCo
 				const uint64_t linear_span =
 				    static_cast<uint64_t>(copy_depth - 1u) * linear_stride +
 				    layout.level_sizes[level].size;
-				if (!SetGpuTileSize(info.linear_offset, linear_span, size, info.linear_size) ||
-				    !SetGpuTileSize(info.tiled_offset, volume.level_sizes[level], size,
+				if (!SetGpuTileSize(info.linear_offset, linear_span, UINT64_MAX,
+				                    info.linear_size) ||
+				    !SetGpuTileSize(info.tiled_offset, volume.level_sizes[level], tiled_size,
 				                    info.tiled_size)) {
 					return false;
 				}
@@ -588,8 +589,9 @@ bool TextureBuildGpuTileInfos(uint64_t size, const std::vector<vk::BufferImageCo
 				info.bytes_per_element = block.bytes_per_element;
 				info.linear_offset     = region.bufferOffset;
 				info.tiled_offset      = TextureUploadSliceSourceOffset(layout, level, z);
-				if (!SetGpuTileSize(info.linear_offset, level_size.size, size, info.linear_size) ||
-				    !SetGpuTileSize(info.tiled_offset, GetLevelSrcSize(level_size), size,
+				if (!SetGpuTileSize(info.linear_offset, level_size.size, UINT64_MAX,
+				                    info.linear_size) ||
+				    !SetGpuTileSize(info.tiled_offset, GetLevelSrcSize(level_size), tiled_size,
 				                    info.tiled_size)) {
 					return false;
 				}

src/graphics/host_gpu/renderer/image/textureCommon.h

@@ -44,7 +44,7 @@ std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayo
                                                          uint32_t width, uint32_t height,
                                                          uint32_t depth, uint64_t levels,
                                                          bool array_texture, bool volume_texture);
-bool TextureBuildGpuTileInfos(uint64_t size, const std::vector<vk::BufferImageCopy>& regions,
+bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferImageCopy>& regions,
                               const TextureUploadLayout& layout, uint32_t fmt, uint32_t depth,
                               uint64_t levels, std::vector<GpuTileInfo>& infos);
 

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -391,9 +391,8 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	const bool supported_swizzle =
 	    IsValidImageSwizzle(swizzle) &&
 	    (swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
-	const bool supported_mip_view = descriptor.BaseLevel() == 0 || is_1d || is_2d;
 	return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
-	       supported_mip_view && descriptor.BaseLevel() == descriptor.LastLevel() &&
+	       descriptor.BaseLevel() == descriptor.LastLevel() &&
 	       descriptor.LastLevel() <= descriptor.MaxMip() && descriptor.MinLod() == 0 &&
 	       supported_swizzle && descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth();
 }
@@ -618,14 +617,15 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	const bool multisampled = IsMultisampledTexture(type);
 	const auto levels       = multisampled ? 1u : static_cast<uint32_t>(descriptor.MaxMip()) + 1u;
 	const auto tile         = descriptor.TileMode();
-	const bool depth_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kDepth);
+	const bool depth_tile   = tile == Prospero::GpuEnumValue(Prospero::TileMode::kDepth);
 	const bool msaa_tile =
 	    depth_tile || tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget);
 	const bool msaa_array = type == Prospero::ImageType::kColor2DMsaaArray;
 	if ((!multisampled && (base_level > last_level || last_level >= levels)) ||
 	    (multisampled &&
 	     (base_level != 0 || last_level == 0 || last_level > 3 ||
-	      descriptor.MaxMip() != last_level || !msaa_tile || (descriptor.MsaaDepth() && !depth_tile) ||
+	      descriptor.MaxMip() != last_level || !msaa_tile ||
+	      (descriptor.MsaaDepth() && !depth_tile) ||
 	      (!msaa_array && (descriptor.Depth() != 0 || descriptor.BaseArray5() != 0))))) {
 		EXIT("unsupported texture mip view: base=%u last=%u levels=%u max=%u type=%u tile=%u "
 		     "kind=%u dimension=%u mip_mode=%u read=%d written=%d "
@@ -634,7 +634,8 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 		     static_cast<uint32_t>(resource.kind), static_cast<uint32_t>(resource.dimension),
 		     static_cast<uint32_t>(resource.mip_mode), resource.read, resource.written,
 		     descriptor.fields[0], descriptor.fields[1], descriptor.fields[2], descriptor.fields[3],
-		     descriptor.fields[4], descriptor.fields[5], descriptor.fields[6], descriptor.fields[7]);
+		     descriptor.fields[4], descriptor.fields[5], descriptor.fields[6],
+		     descriptor.fields[7]);
 	}
 	const auto samples = multisampled ? 1u << last_level : 1u;
 	const auto view_levels =
@@ -661,8 +662,8 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	TileSizeAlign size {};
 	if (multisampled) {
 		const auto bytes = Prospero::NumBytesPerElement(format);
-		pitch = depth_tile ? TileGetDepthPitch(width, bytes, last_level)
-		                   : TileGetRenderTargetPitch(width, bytes, last_level);
+		pitch            = depth_tile ? TileGetDepthPitch(width, bytes, last_level)
+		                              : TileGetRenderTargetPitch(width, bytes, last_level);
 		if (pitch == 0 || !TileGetRenderTargetSize(width, height, pitch, bytes, size, last_level) ||
 		    size.size > UINT32_MAX / image_layers) {
 			EXIT("unsupported multisample texture layout\n");
@@ -679,8 +680,8 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 		ValidateStorageTexture(resource, descriptor, size.size);
 	}
 
-	const auto              pixel_format = TextureGetFormat(format);
-	const auto              storage_view_format =
+	const auto pixel_format = TextureGetFormat(format);
+	const auto storage_view_format =
 	    storage && format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32SInt)
 	        ? vk::Format::eR32Uint
 	        : SrgbStorageViewFormat(pixel_format);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -154,8 +154,9 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 	for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) {
 		static_params.color_mask[i] = color_mask[i];
 	}
-	static_params.cull_back  = mc.cull_back;
-	static_params.cull_front = mc.cull_front;
+	const bool rect_list     = topology == vk::PrimitiveTopology::ePatchList;
+	static_params.cull_back  = !rect_list && mc.cull_back;
+	static_params.cull_front = !rect_list && mc.cull_front;
 	static_params.face       = mc.face;
 
 	for (uint32_t i = 0; i < color_count; i++) {

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -14,6 +14,7 @@
 #include "graphics/host_gpu/renderer/renderTarget.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/rectListShader.h"
 #include "graphics/shader/shader.h"
 
 #include <algorithm>
@@ -463,8 +464,12 @@ void CreatePipelineInternal(
     uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active) {
 	EXIT_IF(ps_active && ps_input_info == nullptr);
 
-	vk::ShaderModule vert_shader_module = nullptr;
-	vk::ShaderModule frag_shader_module = nullptr;
+	const bool rect_list = static_params.topology == vk::PrimitiveTopology::ePatchList;
+
+	vk::ShaderModule vert_shader_module         = nullptr;
+	vk::ShaderModule tess_control_shader_module = nullptr;
+	vk::ShaderModule tess_eval_shader_module    = nullptr;
+	vk::ShaderModule frag_shader_module         = nullptr;
 
 	vk::ShaderModuleCreateInfo create_info {};
 
@@ -491,8 +496,33 @@ void CreatePipelineInternal(
 		}
 		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 	}
+	if (rect_list) {
+		const auto shaders =
+		    BuildRectListShaders(vs_input_info, ps_active ? ps_input_info : nullptr);
+		create_info.codeSize = shaders.control.size() * 4;
+		create_info.pCode    = shaders.control.data();
+		result =
+		    graphics.device.createShaderModule(&create_info, nullptr, &tess_control_shader_module);
+		if (graphics_debug_dump_enabled()) {
+			LOGF("PipelineTrace: vkCreateShaderModule RectList TCS done result=%s module=%p\n",
+			     VulkanToString(result).c_str(), static_cast<void*>(tess_control_shader_module));
+		}
+		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
+
+		create_info.codeSize = shaders.evaluation.size() * 4;
+		create_info.pCode    = shaders.evaluation.data();
+		result =
+		    graphics.device.createShaderModule(&create_info, nullptr, &tess_eval_shader_module);
+		if (graphics_debug_dump_enabled()) {
+			LOGF("PipelineTrace: vkCreateShaderModule RectList TES done result=%s module=%p\n",
+			     VulkanToString(result).c_str(), static_cast<void*>(tess_eval_shader_module));
+		}
+		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
+	}
 
 	EXIT_NOT_IMPLEMENTED(vert_shader_module == nullptr);
+	EXIT_NOT_IMPLEMENTED(
+	    rect_list && (tess_control_shader_module == nullptr || tess_eval_shader_module == nullptr));
 	EXIT_NOT_IMPLEMENTED(ps_active && frag_shader_module == nullptr);
 
 	vk::PipelineShaderStageCreateInfo                     vert_shader_stage_info {};
@@ -525,9 +555,28 @@ void CreatePipelineInternal(
 		                      frag_shader_stage_info);
 	}
 
-	vk::PipelineShaderStageCreateInfo shader_stages[]    = {vert_shader_stage_info,
-	                                                        frag_shader_stage_info};
-	const uint32_t                    shader_stage_count = ps_active ? 2u : 1u;
+	vk::PipelineShaderStageCreateInfo tess_control_shader_stage_info {};
+	tess_control_shader_stage_info.sType  = vk::StructureType::ePipelineShaderStageCreateInfo;
+	tess_control_shader_stage_info.stage  = vk::ShaderStageFlagBits::eTessellationControl;
+	tess_control_shader_stage_info.module = tess_control_shader_module;
+	tess_control_shader_stage_info.pName  = "main";
+
+	vk::PipelineShaderStageCreateInfo tess_eval_shader_stage_info {};
+	tess_eval_shader_stage_info.sType  = vk::StructureType::ePipelineShaderStageCreateInfo;
+	tess_eval_shader_stage_info.stage  = vk::ShaderStageFlagBits::eTessellationEvaluation;
+	tess_eval_shader_stage_info.module = tess_eval_shader_module;
+	tess_eval_shader_stage_info.pName  = "main";
+
+	vk::PipelineShaderStageCreateInfo shader_stages[4]   = {};
+	uint32_t                          shader_stage_count = 0;
+	shader_stages[shader_stage_count++]                  = vert_shader_stage_info;
+	if (rect_list) {
+		shader_stages[shader_stage_count++] = tess_control_shader_stage_info;
+		shader_stages[shader_stage_count++] = tess_eval_shader_stage_info;
+	}
+	if (ps_active) {
+		shader_stages[shader_stage_count++] = frag_shader_stage_info;
+	}
 
 	vk::VertexInputAttributeDescription input_attr[ShaderVertexInputInfo::RES_MAX];
 	vk::VertexInputBindingDescription   input_desc[ShaderVertexInputInfo::RES_MAX];
@@ -929,10 +978,13 @@ void CreatePipelineInternal(
 	pipeline_info.pStages                  = shader_stages;
 	pipeline_info.pVertexInputState        = &vertex_input_info;
 	pipeline_info.pInputAssemblyState      = &input_assembly;
-	pipeline_info.pTessellationState       = nullptr;
-	pipeline_info.pViewportState           = &viewport_state;
-	pipeline_info.pRasterizationState      = &rasterizer;
-	pipeline_info.pMultisampleState        = &multisampling;
+	vk::PipelineTessellationStateCreateInfo tessellation_state {};
+	tessellation_state.sType              = vk::StructureType::ePipelineTessellationStateCreateInfo;
+	tessellation_state.patchControlPoints = 3;
+	pipeline_info.pTessellationState      = (rect_list ? &tessellation_state : nullptr);
+	pipeline_info.pViewportState          = &viewport_state;
+	pipeline_info.pRasterizationState     = &rasterizer;
+	pipeline_info.pMultisampleState       = &multisampling;
 	pipeline_info.pDepthStencilState = (static_params.with_depth ? &depth_stencil_info : nullptr);
 	pipeline_info.pColorBlendState   = &color_blending;
 	pipeline_info.pDynamicState      = &dynamic_state;
@@ -968,6 +1020,12 @@ void CreatePipelineInternal(
 	if (frag_shader_module != nullptr) {
 		graphics.device.destroyShaderModule(frag_shader_module, nullptr);
 	}
+	if (tess_control_shader_module != nullptr) {
+		graphics.device.destroyShaderModule(tess_control_shader_module, nullptr);
+	}
+	if (tess_eval_shader_module != nullptr) {
+		graphics.device.destroyShaderModule(tess_eval_shader_module, nullptr);
+	}
 	graphics.device.destroyShaderModule(vert_shader_module, nullptr);
 }
 

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -2,7 +2,6 @@
 
 #include "common/assert.h"
 #include "common/common.h"
-#include "common/emulatorConfig.h"
 #include "common/file.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
@@ -650,11 +649,9 @@ static bool ConsumeMetadataColorOperation(const RenderCommandBuffer& buffer) {
 }
 
 struct DrawEmitInfo {
-	bool     indexed           = false;
-	bool     draw_prim7_as_ngg = false;
-	uint32_t draw_vertex_count = 0;
-	int32_t  vertex_offset     = 0;
-	uint32_t first_vertex      = 0;
+	bool     indexed       = false;
+	int32_t  vertex_offset = 0;
+	uint32_t first_vertex  = 0;
 };
 
 struct DrawIndexBufferSource {
@@ -767,7 +764,7 @@ static void SetDrawDebugPhase(RenderCommandBuffer& buffer, uint64_t submit_id,
 	                    draw.flags, draw.instance_count, draw.first_instance);
 }
 
-static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw, bool use_ngg_rectlist_draw,
+static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw,
                             vk::PrimitiveTopology& topology) {
 
 	topology = vk::PrimitiveTopology::ePointList;
@@ -791,8 +788,7 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw, bool use
 			topology = vk::PrimitiveTopology::eTriangleStrip;
 			break;
 		case Prospero::PrimitiveType::kRectList:
-			topology = (auto_draw && use_ngg_rectlist_draw ? vk::PrimitiveTopology::eTriangleStrip
-			                                               : vk::PrimitiveTopology::eTriangleList);
+			topology = vk::PrimitiveTopology::ePatchList;
 			break;
 		case Prospero::PrimitiveType::kRectListLegacy:
 			if (!auto_draw) {
@@ -991,20 +987,6 @@ static void LogDrawStateIfNeeded(const RenderCommandBuffer& buffer, const DrawCa
 	// LogDrawTextureState(draw.name, state.color_info[0], state.ps_input_info);
 }
 
-static bool IsHostExpandedRectListDrawSupported(const ShaderVertexInputInfo& vs_input_info,
-                                                const DrawCallInfo&          draw,
-                                                const DrawEmitInfo&          emit) {
-	if (!emit.draw_prim7_as_ngg) {
-		return true;
-	}
-
-	if (vs_input_info.buffers_num != 0) {
-		return false;
-	}
-
-	return draw.index_count == 3 || draw.index_count == emit.draw_vertex_count;
-}
-
 static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_buffer,
                                const ShaderVertexInputInfo& vs_input_info, const DrawCallInfo& draw,
                                const DrawEmitInfo& emit) {
@@ -1017,22 +999,12 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 		case Prospero::PrimitiveType::kTriList:
 		case Prospero::PrimitiveType::kTriFan:
 		case Prospero::PrimitiveType::kTriStrip:
-			if (emit.indexed) {
-				vk_buffer.drawIndexed(draw.index_count, draw.instance_count, 0, emit.vertex_offset,
-				                      draw.first_instance);
-			} else {
-				vk_buffer.draw(draw.index_count, draw.instance_count, emit.first_vertex,
-				               draw.first_instance);
-			}
-			break;
 		case Prospero::PrimitiveType::kRectList:
 			if (emit.indexed) {
 				vk_buffer.drawIndexed(draw.index_count, draw.instance_count, 0, emit.vertex_offset,
 				                      draw.first_instance);
 			} else {
-				EXIT_NOT_IMPLEMENTED(
-				    !IsHostExpandedRectListDrawSupported(vs_input_info, draw, emit));
-				vk_buffer.draw(emit.draw_vertex_count, draw.instance_count, emit.first_vertex,
+				vk_buffer.draw(draw.index_count, draw.instance_count, emit.first_vertex,
 				               draw.first_instance);
 			}
 			break;
@@ -1159,7 +1131,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 	                    reinterpret_cast<uint64_t>(index_addr));
 
 	Common::LockGuard lock(m_context.GetMutex());
-	if (index_count == 0) {
+	if (index_count == 0 || instance_count == 0) {
 		return;
 	}
 
@@ -1201,7 +1173,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 	hw_check(buffer);
 
 	vk::PrimitiveTopology topology = vk::PrimitiveTopology::ePointList;
-	if (!GetDrawTopology(ucfg, false, false, topology)) {
+	if (!GetDrawTopology(ucfg, false, topology)) {
 		return;
 	}
 
@@ -1229,10 +1201,6 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 
 	EXIT_NOT_IMPLEMENTED(flags != 0);
 	EXIT_NOT_IMPLEMENTED(type != 1);
-	if (instance_count == 0) {
-		instance_count = 1;
-	}
-
 	const DrawCallInfo    draw {"DrawIndex",    CommandBufferDebugOp::DrawIndex,
 	                            index_count,    flags,
 	                            instance_count, first_instance};
@@ -1292,7 +1260,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, u
 	                    index_count, flags, first_vertex, instance_count, first_instance);
 
 	Common::LockGuard lock(m_context.GetMutex());
-	if (index_count == 0) {
+	if (index_count == 0 || instance_count == 0) {
 		return;
 	}
 
@@ -1330,10 +1298,6 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, u
 	hw_check(buffer);
 
 	EXIT_NOT_IMPLEMENTED(flags != 0);
-	if (instance_count == 0) {
-		instance_count = 1;
-	}
-
 	const DrawCallInfo draw {"DrawIndexAuto", CommandBufferDebugOp::DrawIndexAuto,
 	                         index_count,     flags,
 	                         instance_count,  first_instance};
@@ -1345,20 +1309,15 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, u
 		return;
 	}
 
-	vk::PrimitiveTopology topology              = vk::PrimitiveTopology::ePointList;
-	const bool            use_ngg_rectlist_draw = Config::NggRectlistDrawEnabled();
-
-	if (!GetDrawTopology(ucfg, true, use_ngg_rectlist_draw, topology)) {
+	vk::PrimitiveTopology topology = vk::PrimitiveTopology::ePointList;
+	if (!GetDrawTopology(ucfg, true, topology)) {
 		ResetBindings();
 		return;
 	}
-	const bool draw_prim7_as_ngg =
-	    (use_ngg_rectlist_draw &&
-	     ucfg.GetPrimType() == Prospero::GpuEnumValue(Prospero::PrimitiveType::kRectList));
-
 	RefreshShaders(buffer, draw, false, state);
 
-	if (draw_prim7_as_ngg && state.vs_input_info.buffers_num == 0 &&
+	const bool rect_list = topology == vk::PrimitiveTopology::ePatchList;
+	if (rect_list && state.vs_input_info.buffers_num == 0 &&
 	    state.vs_input_info.param_export_mask == 0 && state.ps_input_info.input_num != 0) {
 		if (graphics_debug_dump_enabled()) {
 			LOGF("DrawIndexAuto: skipping rect-list draw with no VS param exports and PS inputs: "
@@ -1375,13 +1334,10 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, u
 	                         Prospero::GpuEnumValue(Prospero::PrimitiveType::kRectListLegacy),
 	                     0, nullptr);
 
-	const uint32_t draw_vertex_count = (draw_prim7_as_ngg ? 4u : index_count);
-	const auto     vertex_offset = ResolveVertexOffset(ucfg.GetIndexOffset(), state.vs_input_info) +
-	                               static_cast<int32_t>(first_vertex);
-	DrawEmitInfo   emit {};
-	emit.draw_prim7_as_ngg = draw_prim7_as_ngg;
-	emit.draw_vertex_count = draw_vertex_count;
-	emit.first_vertex      = static_cast<uint32_t>(vertex_offset);
+	const auto   vertex_offset = ResolveVertexOffset(ucfg.GetIndexOffset(), state.vs_input_info) +
+	                             static_cast<int32_t>(first_vertex);
+	DrawEmitInfo emit {};
+	emit.first_vertex = static_cast<uint32_t>(vertex_offset);
 
 	DrawIndexBufferSource index_source {};
 	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, false, false,

src/graphics/shader/recompiler/cfg/ShaderCFG.cpp

@@ -871,19 +871,19 @@ std::vector<uint32_t> DominatedBlocks(const Graph& graph, uint32_t header,
 	return blocks;
 }
 
-uint32_t AppendSyntheticMergeBlock(Graph& graph, uint32_t old_merge) {
-	const auto* merge = graph.FindBlock(old_merge);
+uint32_t AppendSyntheticBranchBlock(Graph& graph, uint32_t target) {
+	const auto* target_block = graph.FindBlock(target);
 
 	BasicBlock block;
 	block.id                    = static_cast<uint32_t>(graph.blocks.size());
-	block.start_pc              = merge != nullptr ? merge->start_pc : 0u;
+	block.start_pc              = target_block != nullptr ? target_block->start_pc : 0u;
 	block.end_pc                = block.start_pc;
-	block.inst_begin            = merge != nullptr ? merge->inst_begin : 0u;
+	block.inst_begin            = target_block != nullptr ? target_block->inst_begin : 0u;
 	block.inst_end              = block.inst_begin;
-	block.successors            = {old_merge};
+	block.successors            = {target};
 	block.terminator.kind       = TerminatorKind::Branch;
 	block.terminator.condition  = BranchCondition::Always;
-	block.terminator.true_block = old_merge;
+	block.terminator.true_block = target;
 	graph.blocks.push_back(std::move(block));
 	return graph.blocks.back().id;
 }
@@ -897,15 +897,50 @@ bool IsSyntheticMergeForwarder(const Graph& graph, uint32_t block_id, uint32_t m
 	       block->terminator.true_block == merge;
 }
 
-bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) {
-	return block_id != UINT32_MAX && block_id != loop.merge && block_id != loop.continue_block &&
-	       graph.Dominates(loop.header, block_id) &&
-	       (loop.merge == UINT32_MAX || !graph.Dominates(loop.merge, block_id));
+const NaturalLoop* FindInnermostContainingLoop(const Graph& graph, uint32_t block_id) {
+	const NaturalLoop* innermost = nullptr;
+	for (const auto& loop: graph.natural_loops) {
+		if (Contains(loop.body_blocks, block_id) &&
+		    (innermost == nullptr || loop.body_blocks.size() < innermost->body_blocks.size())) {
+			innermost = &loop;
+		}
+	}
+	return innermost;
 }
 
-bool SelectionMergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t merge) {
+bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) {
+	return block_id != UINT32_MAX && block_id != loop.merge && block_id != loop.continue_block &&
+	       graph.Dominates(loop.header, block_id) && !graph.Dominates(loop.merge, block_id);
+}
+
+bool IsInnermostLoopControlConditional(const Graph& graph, const BasicBlock& block) {
+	if (block.terminator.kind != TerminatorKind::ConditionalBranch) {
+		return false;
+	}
+	const auto* loop = FindInnermostContainingLoop(graph, block.id);
+	if (loop == nullptr || loop->merge == UINT32_MAX || loop->continue_block == UINT32_MAX) {
+		return false;
+	}
+	const auto true_target  = block.terminator.true_block;
+	const auto false_target = block.terminator.false_block;
+	if (block.id == loop->continue_block) {
+		const auto is_repeat_target = [&](uint32_t target) {
+			return target == loop->header || target == loop->merge;
+		};
+		return is_repeat_target(true_target) && is_repeat_target(false_target);
+	}
+	const auto is_control_target = [&](uint32_t target) {
+		return target == loop->merge || target == loop->continue_block;
+	};
+	return (is_control_target(true_target) &&
+	        (is_control_target(false_target) ||
+	         IsInsideLoopConstruct(graph, *loop, false_target))) ||
+	       (is_control_target(false_target) && IsInsideLoopConstruct(graph, *loop, true_target));
+}
+
+bool MergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t merge) {
 	for (const auto& loop: graph.natural_loops) {
-		if (IsInsideLoopConstruct(graph, loop, header) &&
+		if (loop.header != header && IsInsideLoopConstruct(graph, loop, header) &&
 		    !IsInsideLoopConstruct(graph, loop, merge)) {
 			return true;
 		}
@@ -913,6 +948,80 @@ bool SelectionMergeLeavesContainingLoop(const Graph& graph, uint32_t header, uin
 	return false;
 }
 
+bool CanonicalizeNaturalLoops(Graph& graph, std::string* error) {
+	const auto rewrite_budget = graph.blocks.size() * 2u + 16u;
+	for (size_t rewrite = 0; rewrite < rewrite_budget; rewrite++) {
+		bool changed = false;
+		for (const auto& loop: graph.natural_loops) {
+			std::vector<uint32_t> latches;
+			for (const auto& edge: graph.back_edges) {
+				if (edge.to == loop.header) {
+					AddUnique(latches, edge.from);
+				}
+			}
+			if (latches.size() <= 1u) {
+				continue;
+			}
+
+			const auto continue_block = AppendSyntheticBranchBlock(graph, loop.header);
+			for (auto latch: latches) {
+				auto* block = graph.FindBlock(latch);
+				if (block != nullptr) {
+					ReplaceValue(block->successors, loop.header, continue_block);
+					ReplaceTerminatorTarget(block->terminator, loop.header, continue_block);
+				}
+			}
+			RebuildPredecessors(graph);
+			RecomputeAnalyses(graph);
+			changed = true;
+			break;
+		}
+		if (changed) {
+			continue;
+		}
+
+		for (const auto& loop: graph.natural_loops) {
+			const auto* header                 = graph.FindBlock(loop.header);
+			const auto  is_loop_control_target = [&](uint32_t target) {
+				return target == loop.merge || target == loop.continue_block;
+			};
+			if (header == nullptr || header->terminator.kind != TerminatorKind::ConditionalBranch ||
+			    is_loop_control_target(header->terminator.true_block) ||
+			    is_loop_control_target(header->terminator.false_block) ||
+			    !Contains(loop.body_blocks, header->terminator.true_block) ||
+			    !Contains(loop.body_blocks, header->terminator.false_block)) {
+				continue;
+			}
+
+			const auto old_header   = loop.header;
+			const auto predecessors = header->predecessors;
+			const auto new_header   = AppendSyntheticBranchBlock(graph, old_header);
+			for (auto pred: predecessors) {
+				auto* block = graph.FindBlock(pred);
+				if (block != nullptr) {
+					ReplaceValue(block->successors, old_header, new_header);
+					ReplaceTerminatorTarget(block->terminator, old_header, new_header);
+				}
+			}
+			if (graph.entry_block == old_header) {
+				graph.entry_block = new_header;
+			}
+			MoveBlockBefore(graph, new_header, old_header);
+			RebuildPredecessors(graph);
+			RecomputeAnalyses(graph);
+			changed = true;
+			break;
+		}
+		if (!changed) {
+			return true;
+		}
+	}
+
+	SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block,
+	           "CFG loop canonicalization exceeded rewrite budget", error);
+	return false;
+}
+
 bool SplitSharedMergeBlock(Graph& graph, uint32_t merge,
                            const std::vector<uint32_t>& construct_blocks,
                            bool                         force_split = false) {
@@ -948,7 +1057,7 @@ bool SplitSharedMergeBlock(Graph& graph, uint32_t merge,
 		return false;
 	}
 
-	const auto synthetic_merge = AppendSyntheticMergeBlock(graph, merge);
+	const auto synthetic_merge = AppendSyntheticBranchBlock(graph, merge);
 	auto*      synthetic_block = graph.FindBlock(synthetic_merge);
 	if (synthetic_block != nullptr) {
 		synthetic_block->predecessors = predecessors_to_split;
@@ -980,14 +1089,111 @@ bool SplitSharedMergeBlock(Graph& graph, uint32_t merge,
 bool SplitOneLoopMerge(Graph& graph) {
 	const auto& loops = graph.natural_loops;
 	for (const auto& loop: loops) {
-		if (SplitSharedMergeBlock(graph, loop.merge, loop.body_blocks)) {
+		const auto construct_blocks = DominatedBlocks(graph, loop.header, loop.merge);
+		const auto force_split      = MergeLeavesContainingLoop(graph, loop.header, loop.merge);
+		if (SplitSharedMergeBlock(graph, loop.merge, construct_blocks, force_split)) {
 			return true;
 		}
 	}
 	return false;
 }
 
-bool SplitOneSelectionMerge(Graph& graph) {
+std::vector<uint32_t> SelectionRegion(const Graph& graph, const BasicBlock& header,
+                                      uint32_t merge) {
+	std::vector<uint32_t> region;
+	std::vector<uint32_t> pending = {header.terminator.true_block,
+	                                 header.terminator.false_block};
+	while (!pending.empty()) {
+		const auto block_id = pending.back();
+		pending.pop_back();
+		if (block_id == merge || Contains(region, block_id)) {
+			continue;
+		}
+		const auto* block = graph.FindBlock(block_id);
+		if (block == nullptr) {
+			continue;
+		}
+		AddUnique(region, block_id);
+		pending.insert(pending.end(), block->successors.begin(), block->successors.end());
+	}
+	SortUnique(region);
+	return region;
+}
+
+bool DuplicateSelectionRegion(Graph& graph, uint32_t header_id, uint32_t merge,
+                              const std::vector<uint32_t>& region, uint32_t block_budget) {
+	std::vector<uint32_t> cloned_blocks;
+	for (auto block_id: region) {
+		if (!graph.Dominates(header_id, block_id)) {
+			cloned_blocks.push_back(block_id);
+		}
+	}
+	if (cloned_blocks.empty() || graph.FindBlock(header_id) == nullptr || header_id >= merge ||
+	    graph.blocks.size() + cloned_blocks.size() + 1u > block_budget) {
+		return false;
+	}
+
+	const auto first_clone = static_cast<uint32_t>(graph.blocks.size());
+	std::map<uint32_t, uint32_t> clones;
+	for (uint32_t i = 0; i < cloned_blocks.size(); i++) {
+		clones.emplace(cloned_blocks[i], first_clone + i);
+	}
+
+	for (auto block_id: cloned_blocks) {
+		BasicBlock clone = *graph.FindBlock(block_id);
+		clone.id         = clones.at(block_id);
+		clone.predecessors.clear();
+		clone.dominators.clear();
+		clone.post_dominators.clear();
+		graph.blocks.push_back(std::move(clone));
+	}
+
+	const auto remap_block = [&](BasicBlock& block) {
+		const auto remap_target = [&](uint32_t& target) {
+			if (const auto it = clones.find(target); it != clones.end()) {
+				target = it->second;
+			}
+		};
+		for (auto& successor: block.successors) {
+			remap_target(successor);
+		}
+		remap_target(block.terminator.true_block);
+		remap_target(block.terminator.false_block);
+		remap_target(block.terminator.merge_block);
+		remap_target(block.terminator.continue_block);
+		for (auto& target: block.terminator.indirect_targets) {
+			remap_target(target);
+		}
+	};
+	for (auto block_id: region) {
+		const auto owned_id = clones.contains(block_id) ? clones.at(block_id) : block_id;
+		remap_block(*graph.FindBlock(owned_id));
+	}
+
+	const auto private_merge = AppendSyntheticBranchBlock(graph, merge);
+	auto&      header        = *graph.FindBlock(header_id);
+	remap_block(header);
+
+	for (auto block_id: region) {
+		const auto owned_id = clones.contains(block_id) ? clones.at(block_id) : block_id;
+		auto*      block    = graph.FindBlock(owned_id);
+		if (block != nullptr) {
+			ReplaceValue(block->successors, merge, private_merge);
+			ReplaceTerminatorTarget(block->terminator, merge, private_merge);
+		}
+	}
+	ReplaceValue(header.successors, merge, private_merge);
+	ReplaceTerminatorTarget(header.terminator, merge, private_merge);
+
+	for (uint32_t i = 0; i <= cloned_blocks.size(); i++) {
+		MoveBlockBefore(graph, first_clone + i, merge + i);
+	}
+	RebuildPredecessors(graph);
+	RecomputeAnalyses(graph);
+	return true;
+}
+
+bool SplitOneSelectionMerge(Graph& graph, uint32_t block_budget) {
 	std::vector<uint32_t> loop_headers;
 	loop_headers.reserve(graph.natural_loops.size());
 	for (const auto& loop: graph.natural_loops) {
@@ -1001,11 +1207,26 @@ bool SplitOneSelectionMerge(Graph& graph) {
 		    Contains(loop_headers, block_id)) {
 			continue;
 		}
+		if (IsInnermostLoopControlConditional(graph, *block)) {
+			continue;
+		}
 
 		const auto merge = graph.FindNearestCommonPostDominator(block->terminator.true_block,
 		                                                        block->terminator.false_block);
+		if (merge == UINT32_MAX || graph.FindBlock(merge) == nullptr) {
+			continue;
+		}
+		const auto region = SelectionRegion(graph, *block, merge);
+		if (std::any_of(region.begin(), region.end(),
+		                [&](uint32_t member) { return !graph.Dominates(block_id, member); })) {
+			if (graph.natural_loops.empty() &&
+			    DuplicateSelectionRegion(graph, block_id, merge, region, block_budget)) {
+				return true;
+			}
+			continue;
+		}
 		const auto construct_blocks = DominatedBlocks(graph, block_id, merge);
-		const auto force_split      = SelectionMergeLeavesContainingLoop(graph, block_id, merge);
+		const auto force_split      = MergeLeavesContainingLoop(graph, block_id, merge);
 		if (SplitSharedMergeBlock(graph, merge, construct_blocks, force_split)) {
 			return true;
 		}
@@ -1015,10 +1236,12 @@ bool SplitOneSelectionMerge(Graph& graph) {
 
 bool SplitSharedMergeBlocks(Graph& graph, std::string* error) {
 	const auto original_block_count = static_cast<uint32_t>(graph.blocks.size());
-	const auto split_budget =
-	    std::max<uint32_t>(16u, std::min<uint32_t>(128u, original_block_count));
+	const auto split_budget = std::max<uint32_t>(
+	    16u, std::min<uint32_t>(128u, original_block_count * 4u));
+	const auto block_budget = std::max<uint32_t>(
+	    32u, std::min<uint32_t>(512u, original_block_count * 8u));
 	for (uint32_t splits = 0; splits < split_budget; splits++) {
-		if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph)) {
+		if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph, block_budget)) {
 			return true;
 		}
 		RebuildPredecessors(graph);
@@ -1353,6 +1576,9 @@ bool Structurize(Graph& graph, std::string* error) {
 		return false;
 	}
 
+	if (!CanonicalizeNaturalLoops(graph, error)) {
+		return false;
+	}
 	if (!SplitSharedMergeBlocks(graph, error)) {
 		return false;
 	}
@@ -1395,6 +1621,9 @@ bool Structurize(Graph& graph, std::string* error) {
 		    block.terminator.loop_header) {
 			continue;
 		}
+		if (IsInnermostLoopControlConditional(graph, block)) {
+			continue;
+		}
 
 		const auto merge = graph.FindNearestCommonPostDominator(block.terminator.true_block,
 		                                                        block.terminator.false_block);

src/graphics/shader/recompiler/emitter/SpirvBuilder.cpp

@@ -19,7 +19,7 @@ static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_
 	section.insert(section.end(), words + 1, words + words_num);
 }
 
-Builder::Builder() {
+Builder::Builder(uint32_t version): m_version(version) {
 	m_debug.reserve(InitialSpirvSectionReserve);
 	m_annotations.reserve(InitialSpirvSectionReserve);
 	m_types.reserve(InitialSpirvSectionReserve);
@@ -138,7 +138,7 @@ std::vector<uint32_t> Builder::Build() const {
 	               m_debug.size() + m_annotations.size() + m_types.size() + m_functions.size());
 
 	module.push_back(0x07230203u);
-	module.push_back(0x00010300u);
+	module.push_back(m_version);
 	module.push_back(0u);
 	module.push_back(m_next_id);
 	module.push_back(0u);

src/graphics/shader/recompiler/emitter/SpirvBuilder.h

@@ -10,7 +10,7 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
 class Builder {
 public:
-	Builder();
+	explicit Builder(uint32_t version = 0x00010300u);
 	~Builder() = default;
 	KYTY_CLASS_DEFAULT_COPY(Builder);
 
@@ -39,6 +39,7 @@ private:
 	static void AppendString(std::vector<uint32_t>& words, const char* text);
 
 	uint32_t              m_next_id = 1;
+	uint32_t              m_version = 0;
 	std::vector<uint32_t> m_capabilities;
 	std::vector<uint32_t> m_extensions;
 	std::vector<uint32_t> m_ext_inst_imports;

src/graphics/shader/recompiler/emitter/spirvEmitterAnalysis.cpp

@@ -7,51 +7,30 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
 uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr) {
 	const auto* ps = state.pixel_input_info;
-	if (state.stage != ShaderType::Pixel || ps == nullptr || attr >= ps->input_num) {
+	if (state.stage != ShaderType::Pixel || ps == nullptr) {
 		return attr;
 	}
-	// VINTRP ATTR selects the PS input slot. SPI_PS_INPUT_CNTL maps that slot to a
-	// VS parameter export, which is the SPIR-V location we must link against.
-	return ps->interpolator_settings[attr] & PsInputOffsetMask;
+	return ShaderPixelParameterMappedLocation(*ps, attr);
 }
 
 uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
-	bool used_locations[32] = {};
-
+	std::array<uint32_t, 32> active_inputs {};
+	uint32_t                 active_count = 0;
 	for (const auto& input: state.inputs) {
-		if (input.kind != IR::StageInputKind::Parameter) {
-			continue;
-		}
-
-		auto location = PixelParameterMappedLocation(state, input.location);
-		if (location < std::size(used_locations) && used_locations[location]) {
-			auto fallback_location = input.location;
-			while (fallback_location < std::size(used_locations) &&
-			       used_locations[fallback_location]) {
-				fallback_location++;
-			}
-			EXIT_NOT_IMPLEMENTED(fallback_location >= std::size(used_locations));
-			location = fallback_location;
-		}
-
-		if (input.location == attr) {
-			return location;
-		}
-
-		if (location < std::size(used_locations)) {
-			used_locations[location] = true;
+		if (input.kind == IR::StageInputKind::Parameter) {
+			active_inputs[active_count++] = input.location;
 		}
 	}
-
-	return PixelParameterMappedLocation(state, attr);
+	return state.stage == ShaderType::Pixel && state.pixel_input_info != nullptr
+	           ? ShaderPixelParameterLocation(*state.pixel_input_info,
+	                                          {active_inputs.data(), active_count}, attr)
+	           : attr;
 }
 
 bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
 	const auto* ps = state.pixel_input_info;
-	if (state.stage != ShaderType::Pixel || ps == nullptr || attr >= ps->input_num) {
-		return false;
-	}
-	return (ps->interpolator_settings[attr] & PsInputFlatShade) != 0;
+	return state.stage == ShaderType::Pixel && ps != nullptr &&
+	       ShaderPixelParameterIsFlat(*ps, attr);
 }
 
 void SetError(std::string* error, const char* message) {

src/graphics/shader/recompiler/emitter/spirvEmitterInternal.h

@@ -425,9 +425,6 @@ struct EmitterState {
 	std::map<uint32_t, uint32_t>                     float_constants;
 };
 
-constexpr uint32_t PsInputOffsetMask = 0x0000001fu;
-constexpr uint32_t PsInputFlatShade  = 0x00000400u;
-
 enum class VertexInputScalarKind { Float, Sint, Uint };
 
 constexpr uint32_t NoImageComponent = 0xffffffffu;

src/graphics/shader/recompiler/ir/ResourceMaterialization.cpp

@@ -200,12 +200,12 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
 		if (dimension == Decoder::ImageDimension::Unknown || dimension != image.dimension ||
 		    DescriptorIsCube(descriptor) != image.cube) {
 			if (error != nullptr) {
-				*error = fmt::format(
-				    "image descriptor {} no longer matches specialized dimension: "
-				    "{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x}",
-				    i, descriptor.dwords[0], descriptor.dwords[1], descriptor.dwords[2],
-				    descriptor.dwords[3], descriptor.dwords[4], descriptor.dwords[5],
-				    descriptor.dwords[6], descriptor.dwords[7]);
+				*error =
+				    fmt::format("image descriptor {} no longer matches specialized dimension: "
+				                "{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x}",
+				                i, descriptor.dwords[0], descriptor.dwords[1], descriptor.dwords[2],
+				                descriptor.dwords[3], descriptor.dwords[4], descriptor.dwords[5],
+				                descriptor.dwords[6], descriptor.dwords[7]);
 			}
 			return false;
 		}
@@ -224,14 +224,17 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
 			const bool raw_sint_storage =
 			    storage && format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32SInt) &&
 			    !image.read && !image.atomic;
-			const bool uint_descriptor =
-			    Prospero::IsUintTextureFormat(format) || raw_sint_storage;
-			const auto uint_program = image.kind == ResourceKind::ImageUint ||
-			                          image.kind == ResourceKind::StorageImageUint;
+			const bool uint_descriptor = Prospero::IsUintTextureFormat(format) || raw_sint_storage;
+			const auto uint_program    = image.kind == ResourceKind::ImageUint ||
+			                             image.kind == ResourceKind::StorageImageUint;
 			if (uint_descriptor != uint_program && !(image.atomic && uint_program)) {
 				if (error != nullptr) {
-					*error =
-					    fmt::format("image descriptor {} no longer matches specialized format", i);
+					*error = fmt::format(
+					    "image descriptor {} no longer matches specialized format: "
+					    "{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x}",
+					    i, descriptor.dwords[0], descriptor.dwords[1], descriptor.dwords[2],
+					    descriptor.dwords[3], descriptor.dwords[4], descriptor.dwords[5],
+					    descriptor.dwords[6], descriptor.dwords[7]);
 				}
 				return false;
 			}

src/graphics/shader/recompiler/ir/ScalarProvenance.cpp

@@ -45,6 +45,8 @@ namespace {
 
 constexpr uint32_t ScalarRegisters = 128;
 constexpr uint32_t VectorRegisters = 256;
+// Clamp X/Y/Z are consecutive three-bit fields; the high bit of each selects a border mode.
+constexpr uint32_t SamplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u);
 
 struct ScalarState {
 	std::array<uint32_t, ScalarRegisters> regs          = {};
@@ -647,6 +649,24 @@ private:
 		return AddDescriptor(descriptor);
 	}
 
+	uint32_t AddSamplerDescriptor(const ScalarState& state, uint32_t base) {
+		if (base >= ScalarRegisters || 4u > ScalarRegisters - base) {
+			return ScalarProvenance::Unknown;
+		}
+		DescriptorValue descriptor;
+		descriptor.dword_count = 4;
+		for (uint32_t i = 0; i < 4; i++) {
+			descriptor.dwords[i] = state.regs[base + i];
+		}
+		if (auto d0 = descriptor.dwords[0];
+		    d0 < m_graph.values.size() && m_graph.values[d0].op == ScalarValueOp::Constant &&
+		    (m_graph.values[d0].imm & SamplerBorderClampMask) == 0) {
+			// Without a border clamp, the border color and table index in dword 3 are unused.
+			descriptor.dwords[3] = Constant(0);
+		}
+		return AddDescriptor(descriptor);
+	}
+
 	uint32_t AddFlatAddressDescriptor(const Instruction& inst, const ScalarState& state) {
 		const uint32_t first = FlatStore(inst.op) ? 1u : 0u;
 		if (inst.src_count < first + 2u) {
@@ -704,7 +724,7 @@ private:
 			inst.memory.resource_source = AddDescriptor(state, inst.memory.resource * 4u, 8);
 			if (inst.op == Opcode::ImageSample || inst.op == Opcode::ImageGather4 ||
 			    inst.op == Opcode::ImageGetLod) {
-				inst.memory.sampler_source = AddDescriptor(state, inst.memory.sampler * 4u, 4);
+				inst.memory.sampler_source = AddSamplerDescriptor(state, inst.memory.sampler * 4u);
 			}
 		}
 	}

src/graphics/shader/rectListShader.cpp

@@ -0,0 +1,397 @@
+#include "graphics/shader/rectListShader.h"
+
+#include "common/assert.h"
+#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/shader.h"
+#include "spirv/unified1/spirv.hpp11"
+
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <utility>
+
+namespace Libs::Graphics {
+
+namespace {
+
+using ShaderRecompiler::Spirv::Builder;
+
+constexpr uint32_t SpirvVersion15 = 0x00010500u;
+
+template <typename T>
+constexpr uint32_t Word(T value) {
+	return static_cast<uint32_t>(value);
+}
+
+struct Parameter {
+	uint32_t input_location  = 0;
+	uint32_t output_location = 0;
+	bool     flat            = false;
+};
+
+std::vector<Parameter> GetParameters(const ShaderVertexInputInfo& vertex_info,
+                                     const ShaderPixelInputInfo*  pixel_info) {
+	if (pixel_info == nullptr) {
+		return {};
+	}
+	EXIT_IF(pixel_info->input_num > ShaderVertexInputInfo::RES_MAX);
+	EXIT_IF(pixel_info->stage.program == nullptr);
+
+	std::vector<uint32_t> active_inputs;
+	for (const auto& input: pixel_info->stage.program->info.inputs) {
+		if (input.kind == ShaderRecompiler::IR::StageInputKind::Parameter) {
+			active_inputs.push_back(input.location);
+		}
+	}
+
+	std::vector<Parameter> parameters;
+	for (const auto input: active_inputs) {
+		const auto input_location = ShaderPixelParameterMappedLocation(*pixel_info, input);
+		if ((vertex_info.param_export_mask & (1u << input_location)) != 0) {
+			parameters.push_back({input_location,
+			                      ShaderPixelParameterLocation(*pixel_info, active_inputs, input),
+			                      ShaderPixelParameterIsFlat(*pixel_info, input)});
+		}
+	}
+	return parameters;
+}
+
+class RectListEmitter {
+public:
+	RectListEmitter(const std::vector<Parameter>& parameters_, spv::ExecutionModel model)
+	    : parameters(parameters_) {
+		builder.AddMemoryModel(
+		    {Word(spv::AddressingModel::Logical), Word(spv::MemoryModel::GLSL450)});
+
+		void_type       = Type(spv::Op::OpTypeVoid);
+		uint_type       = Type(spv::Op::OpTypeInt, 32u, 0u);
+		int_type        = Type(spv::Op::OpTypeInt, 32u, 1u);
+		float_type      = Type(spv::Op::OpTypeFloat, 32u);
+		vec4_float_type = Type(spv::Op::OpTypeVector, float_type, 4u);
+		function_type   = Type(spv::Op::OpTypeFunction, void_type);
+
+		per_vertex_type = Type(spv::Op::OpTypeStruct, vec4_float_type);
+		builder.AddAnnotation({Word(spv::Op::OpMemberDecorate), per_vertex_type, 0u,
+		                       Word(spv::Decoration::BuiltIn), Word(spv::BuiltIn::Position)});
+		builder.AddAnnotation(
+		    {Word(spv::Op::OpDecorate), per_vertex_type, Word(spv::Decoration::Block)});
+
+		ptr_input_vec4_float  = Pointer(spv::StorageClass::Input, vec4_float_type);
+		ptr_output_vec4_float = Pointer(spv::StorageClass::Output, vec4_float_type);
+		if (model == spv::ExecutionModel::TessellationControl) {
+			bool_type        = Type(spv::Op::OpTypeBool);
+			vec2_bool_type   = Type(spv::Op::OpTypeVector, bool_type, 2u);
+			vec2_float_type  = Type(spv::Op::OpTypeVector, float_type, 2u);
+			ptr_output_float = Pointer(spv::StorageClass::Output, float_type);
+		} else {
+			vec3_float_type = Type(spv::Op::OpTypeVector, float_type, 3u);
+			ptr_input_float = Pointer(spv::StorageClass::Input, float_type);
+		}
+	}
+
+	std::vector<uint32_t> EmitControl() {
+		DefineEntry(spv::ExecutionModel::TessellationControl);
+		const auto float_one = Constant(float_type, std::bit_cast<uint32_t>(1.0f));
+
+		for (uint32_t i = 0; i < 4; i++) {
+			Store(Access(ptr_output_float, tess_outer, Int(i)), float_one);
+		}
+		for (uint32_t i = 0; i < 2; i++) {
+			Store(Access(ptr_output_float, tess_inner, Int(i)), float_one);
+		}
+
+		std::array<uint32_t, 3> positions {};
+		for (uint32_t i = 0; i < positions.size(); i++) {
+			positions[i] =
+			    Load(vec4_float_type, Access(ptr_input_vec4_float, gl_in, Int(i), Int(0)));
+		}
+
+		std::array<uint32_t, 3> coordinate_equal {};
+		for (uint32_t i = 0; i < coordinate_equal.size(); i++) {
+			const auto left  = Result(spv::Op::OpVectorShuffle, vec2_float_type, positions[i],
+			                          positions[i], 0u, 1u);
+			const auto right = Result(spv::Op::OpVectorShuffle, vec2_float_type,
+			                          positions[(i + 1u) % 3u], positions[(i + 1u) % 3u], 0u, 1u);
+			coordinate_equal[i] = Result(spv::Op::OpFOrdEqual, vec2_bool_type, left, right);
+		}
+
+		std::array<uint32_t, 3> barycentric {};
+		std::array<uint32_t, 3> edge_vertex {};
+		const auto float_minus_one = Constant(float_type, std::bit_cast<uint32_t>(-1.0f));
+		for (uint32_t i = 0; i < edge_vertex.size(); i++) {
+			const auto previous = (i + 2u) % 3u;
+			const auto xy       = Result(
+			    spv::Op::OpLogicalAnd, bool_type,
+			    Result(spv::Op::OpCompositeExtract, bool_type, coordinate_equal[i], 0u),
+			    Result(spv::Op::OpCompositeExtract, bool_type, coordinate_equal[previous], 1u));
+			const auto yx = Result(
+			    spv::Op::OpLogicalAnd, bool_type,
+			    Result(spv::Op::OpCompositeExtract, bool_type, coordinate_equal[i], 1u),
+			    Result(spv::Op::OpCompositeExtract, bool_type, coordinate_equal[previous], 0u));
+			edge_vertex[i] = Result(spv::Op::OpLogicalOr, bool_type, xy, yx);
+			barycentric[i] =
+			    Result(spv::Op::OpSelect, float_type, edge_vertex[i], float_minus_one, float_one);
+		}
+
+		auto vertex_index = Result(spv::Op::OpSelect, int_type, edge_vertex[2], Int(2), Int(0));
+		vertex_index = Result(spv::Op::OpSelect, int_type, edge_vertex[1], Int(1), vertex_index);
+		const auto invocation = Load(int_type, invocation_id);
+		const auto is_fourth  = Result(spv::Op::OpIEqual, bool_type, invocation, Int(3));
+		const auto index =
+		    Result(spv::Op::OpSMod, int_type,
+		           Result(spv::Op::OpIAdd, int_type, vertex_index, invocation), Int(3));
+
+		const auto position3 = Interpolate(positions[0], positions[1], positions[2], barycentric);
+		const auto position =
+		    Result(spv::Op::OpSelect, vec4_float_type, is_fourth, position3,
+		           Load(vec4_float_type, Access(ptr_input_vec4_float, gl_in, index, Int(0))));
+		Store(Access(ptr_output_vec4_float, gl_out, invocation, Int(0)), position);
+
+		for (uint32_t i = 0; i < parameters.size(); i++) {
+			const auto input0 =
+			    Load(vec4_float_type, Access(ptr_input_vec4_float, inputs[i], Int(0)));
+			if (parameters[i].flat) {
+				Store(Access(ptr_output_vec4_float, outputs[i], invocation), input0);
+				continue;
+			}
+			const auto input1 =
+			    Load(vec4_float_type, Access(ptr_input_vec4_float, inputs[i], Int(1)));
+			const auto input2 =
+			    Load(vec4_float_type, Access(ptr_input_vec4_float, inputs[i], Int(2)));
+			const auto input3 = Interpolate(input0, input1, input2, barycentric);
+			const auto value =
+			    Result(spv::Op::OpSelect, vec4_float_type, is_fourth, input3,
+			           Load(vec4_float_type, Access(ptr_input_vec4_float, inputs[i], index)));
+			Store(Access(ptr_output_vec4_float, outputs[i], invocation), value);
+		}
+
+		Emit(spv::Op::OpReturn);
+		Emit(spv::Op::OpFunctionEnd);
+		return builder.Build();
+	}
+
+	std::vector<uint32_t> EmitEvaluation() {
+		DefineEntry(spv::ExecutionModel::TessellationEvaluation);
+
+		const auto x     = Load(float_type, Access(ptr_input_float, tess_coord, Int(0)));
+		const auto y     = Load(float_type, Access(ptr_input_float, tess_coord, Int(1)));
+		const auto index = Result(
+		    spv::Op::OpIAdd, int_type,
+		    Result(spv::Op::OpIMul, int_type, Result(spv::Op::OpConvertFToS, int_type, y), Int(2)),
+		    Result(spv::Op::OpConvertFToS, int_type, x));
+
+		const auto position =
+		    Load(vec4_float_type, Access(ptr_input_vec4_float, gl_in, index, Int(0)));
+		Store(Access(ptr_output_vec4_float, gl_out, Int(0)), position);
+		for (uint32_t i = 0; i < parameters.size(); i++) {
+			Store(outputs[i],
+			      Load(vec4_float_type, Access(ptr_input_vec4_float, inputs[i], index)));
+		}
+
+		Emit(spv::Op::OpReturn);
+		Emit(spv::Op::OpFunctionEnd);
+		return builder.Build();
+	}
+
+private:
+	template <typename... Args>
+	uint32_t Type(spv::Op opcode, Args... operands) {
+		const auto id = builder.AllocateId();
+		builder.AddType({Word(opcode), id, Word(operands)...});
+		return id;
+	}
+
+	uint32_t Constant(uint32_t type, uint32_t value) {
+		const auto id = builder.AllocateId();
+		builder.AddType({Word(spv::Op::OpConstant), type, id, value});
+		return id;
+	}
+
+	uint32_t Pointer(spv::StorageClass storage, uint32_t type) {
+		return Type(spv::Op::OpTypePointer, storage, type);
+	}
+
+	uint32_t Array(uint32_t type, uint32_t size) {
+		return Type(spv::Op::OpTypeArray, type, Uint(size));
+	}
+
+	template <typename... Args>
+	uint32_t Result(spv::Op opcode, uint32_t type, Args... operands) {
+		const auto id = builder.AllocateId();
+		builder.AddFunction({Word(opcode), type, id, Word(operands)...});
+		return id;
+	}
+
+	template <typename... Args>
+	uint32_t ResultWithoutType(spv::Op opcode, Args... operands) {
+		const auto id = builder.AllocateId();
+		builder.AddFunction({Word(opcode), id, Word(operands)...});
+		return id;
+	}
+
+	template <typename... Args>
+	void Emit(spv::Op opcode, Args... operands) {
+		builder.AddFunction({Word(opcode), Word(operands)...});
+	}
+
+	template <typename... Args>
+	uint32_t Access(uint32_t pointer_type, uint32_t base, Args... indices) {
+		return Result(spv::Op::OpAccessChain, pointer_type, base, Word(indices)...);
+	}
+
+	uint32_t Load(uint32_t type, uint32_t pointer) {
+		return Result(spv::Op::OpLoad, type, pointer);
+	}
+
+	void Store(uint32_t pointer, uint32_t value) { Emit(spv::Op::OpStore, pointer, value); }
+
+	uint32_t Int(uint32_t value) {
+		auto& id = int_constants[value];
+		if (id == 0) {
+			id = Constant(int_type, value);
+		}
+		return id;
+	}
+
+	uint32_t Uint(uint32_t value) {
+		auto& id = uint_constants[value];
+		if (id == 0) {
+			id = Constant(uint_type, value);
+		}
+		return id;
+	}
+
+	uint32_t AddInterface(spv::StorageClass storage, uint32_t type) {
+		const auto variable = builder.AllocateId();
+		builder.AddType(
+		    {Word(spv::Op::OpVariable), Pointer(storage, type), variable, Word(storage)});
+		interfaces.push_back(variable);
+		return variable;
+	}
+
+	void Decorate(uint32_t target, spv::Decoration decoration, uint32_t value) {
+		builder.AddAnnotation({Word(spv::Op::OpDecorate), target, Word(decoration), value});
+	}
+
+	void DefineEntry(spv::ExecutionModel model) {
+		builder.AddCapability({Word(spv::Capability::Shader)});
+		builder.AddCapability({Word(spv::Capability::Tessellation)});
+		main = Result(spv::Op::OpFunction, void_type, spv::FunctionControlMask::MaskNone,
+		              function_type);
+		if (model == spv::ExecutionModel::TessellationControl) {
+			builder.AddExecutionMode({main, Word(spv::ExecutionMode::OutputVertices), 4u});
+		} else {
+			builder.AddExecutionMode({main, Word(spv::ExecutionMode::Quads)});
+			builder.AddExecutionMode({main, Word(spv::ExecutionMode::SpacingEqual)});
+			builder.AddExecutionMode({main, Word(spv::ExecutionMode::VertexOrderCw)});
+		}
+		DefineInputs(model);
+		DefineOutputs(model);
+		builder.AddEntryPoint(Word(model), main, "main", interfaces);
+		ResultWithoutType(spv::Op::OpLabel);
+	}
+
+	void DefineInputs(spv::ExecutionModel model) {
+		const auto tess_control = model == spv::ExecutionModel::TessellationControl;
+		if (tess_control) {
+			invocation_id = AddInterface(spv::StorageClass::Input, int_type);
+			Decorate(invocation_id, spv::Decoration::BuiltIn, Word(spv::BuiltIn::InvocationId));
+		} else {
+			tess_coord = AddInterface(spv::StorageClass::Input, vec3_float_type);
+			Decorate(tess_coord, spv::Decoration::BuiltIn, Word(spv::BuiltIn::TessCoord));
+		}
+		gl_in =
+		    AddInterface(spv::StorageClass::Input, Array(per_vertex_type, tess_control ? 3u : 4u));
+
+		inputs.resize(parameters.size());
+		std::array<uint32_t, ShaderVertexInputInfo::RES_MAX> locations {};
+		for (uint32_t i = 0; i < parameters.size(); i++) {
+			const auto location =
+			    tess_control ? parameters[i].input_location : parameters[i].output_location;
+			if (tess_control && locations[location] != 0) {
+				inputs[i] = locations[location];
+				continue;
+			}
+			inputs[i] = AddInterface(spv::StorageClass::Input,
+			                         Array(vec4_float_type, tess_control ? 3u : 4u));
+			Decorate(inputs[i], spv::Decoration::Location, location);
+			locations[location] = inputs[i];
+		}
+	}
+
+	void DefineOutputs(spv::ExecutionModel model) {
+		const auto tess_control = model == spv::ExecutionModel::TessellationControl;
+		if (tess_control) {
+			gl_out     = AddInterface(spv::StorageClass::Output, Array(per_vertex_type, 4u));
+			tess_inner = AddInterface(spv::StorageClass::Output, Array(float_type, 2u));
+			Decorate(tess_inner, spv::Decoration::BuiltIn, Word(spv::BuiltIn::TessLevelInner));
+			builder.AddAnnotation(
+			    {Word(spv::Op::OpDecorate), tess_inner, Word(spv::Decoration::Patch)});
+			tess_outer = AddInterface(spv::StorageClass::Output, Array(float_type, 4u));
+			Decorate(tess_outer, spv::Decoration::BuiltIn, Word(spv::BuiltIn::TessLevelOuter));
+			builder.AddAnnotation(
+			    {Word(spv::Op::OpDecorate), tess_outer, Word(spv::Decoration::Patch)});
+		} else {
+			gl_out = AddInterface(spv::StorageClass::Output, per_vertex_type);
+		}
+
+		outputs.resize(parameters.size());
+		for (uint32_t i = 0; i < parameters.size(); i++) {
+			outputs[i] = AddInterface(spv::StorageClass::Output,
+			                          tess_control ? Array(vec4_float_type, 4u) : vec4_float_type);
+			Decorate(outputs[i], spv::Decoration::Location, parameters[i].output_location);
+		}
+	}
+
+	uint32_t Interpolate(uint32_t v0, uint32_t v1, uint32_t v2,
+	                     const std::array<uint32_t, 3>& barycentric) {
+		const auto p0 = Result(spv::Op::OpVectorTimesScalar, vec4_float_type, v0, barycentric[0]);
+		const auto p1 = Result(spv::Op::OpVectorTimesScalar, vec4_float_type, v1, barycentric[1]);
+		const auto p2 = Result(spv::Op::OpVectorTimesScalar, vec4_float_type, v2, barycentric[2]);
+		return Result(spv::Op::OpFAdd, vec4_float_type, p0,
+		              Result(spv::Op::OpFAdd, vec4_float_type, p1, p2));
+	}
+
+	Builder                       builder {SpirvVersion15};
+	const std::vector<Parameter>& parameters;
+	std::vector<uint32_t>         interfaces;
+	std::vector<uint32_t>         inputs;
+	std::vector<uint32_t>         outputs;
+	std::array<uint32_t, 5>       int_constants {};
+	std::array<uint32_t, 5>       uint_constants {};
+	uint32_t                      main                  = 0;
+	uint32_t                      void_type             = 0;
+	uint32_t                      bool_type             = 0;
+	uint32_t                      uint_type             = 0;
+	uint32_t                      int_type              = 0;
+	uint32_t                      float_type            = 0;
+	uint32_t                      vec2_bool_type        = 0;
+	uint32_t                      vec2_float_type       = 0;
+	uint32_t                      vec3_float_type       = 0;
+	uint32_t                      vec4_float_type       = 0;
+	uint32_t                      function_type         = 0;
+	uint32_t                      per_vertex_type       = 0;
+	uint32_t                      ptr_input_float       = 0;
+	uint32_t                      ptr_input_vec4_float  = 0;
+	uint32_t                      ptr_output_float      = 0;
+	uint32_t                      ptr_output_vec4_float = 0;
+	uint32_t                      gl_in                 = 0;
+	uint32_t                      gl_out                = 0;
+	uint32_t                      tess_inner            = 0;
+	uint32_t                      tess_outer            = 0;
+	uint32_t                      tess_coord            = 0;
+	uint32_t                      invocation_id         = 0;
+};
+
+} // namespace
+
+RectListShaders BuildRectListShaders(const ShaderVertexInputInfo& vertex_info,
+                                     const ShaderPixelInputInfo*  pixel_info) {
+	const auto      parameters = GetParameters(vertex_info, pixel_info);
+	RectListEmitter control(parameters, spv::ExecutionModel::TessellationControl);
+	RectListEmitter evaluation(parameters, spv::ExecutionModel::TessellationEvaluation);
+	return {control.EmitControl(), evaluation.EmitEvaluation()};
+}
+
+} // namespace Libs::Graphics

src/graphics/shader/rectListShader.h

@@ -0,0 +1,22 @@
+#ifndef EMULATOR_SRC_GRAPHICS_SHADER_RECTLISTSHADER_H_
+#define EMULATOR_SRC_GRAPHICS_SHADER_RECTLISTSHADER_H_
+
+#include <cstdint>
+#include <vector>
+
+namespace Libs::Graphics {
+
+struct ShaderPixelInputInfo;
+struct ShaderVertexInputInfo;
+
+struct RectListShaders {
+	std::vector<uint32_t> control;
+	std::vector<uint32_t> evaluation;
+};
+
+RectListShaders BuildRectListShaders(const ShaderVertexInputInfo& vertex_info,
+                                     const ShaderPixelInputInfo*  pixel_info);
+
+} // namespace Libs::Graphics
+
+#endif // EMULATOR_SRC_GRAPHICS_SHADER_RECTLISTSHADER_H_

src/graphics/shader/shader.cpp

@@ -45,6 +45,42 @@
 
 namespace Libs::Graphics {
 
+namespace {
+
+constexpr uint32_t PsInputOffsetMask = 0x0000001fu;
+constexpr uint32_t PsInputFlatShade  = 0x00000400u;
+
+} // namespace
+
+uint32_t ShaderPixelParameterMappedLocation(const ShaderPixelInputInfo& info, uint32_t input) {
+	return input < info.input_num ? info.interpolator_settings[input] & PsInputOffsetMask : input;
+}
+
+uint32_t ShaderPixelParameterLocation(const ShaderPixelInputInfo& info,
+                                      std::span<const uint32_t> active_inputs, uint32_t input) {
+	std::array<bool, 32> used_locations {};
+	for (const auto active_input: active_inputs) {
+		auto location = ShaderPixelParameterMappedLocation(info, active_input);
+		if (location < used_locations.size() && used_locations[location]) {
+			location = active_input;
+			while (location < used_locations.size() && used_locations[location]) {
+				location++;
+			}
+			EXIT_NOT_IMPLEMENTED(location >= used_locations.size());
+		}
+
+		if (active_input == input) {
+			return location;
+		}
+		used_locations[location] = true;
+	}
+	return ShaderPixelParameterMappedLocation(info, input);
+}
+
+bool ShaderPixelParameterIsFlat(const ShaderPixelInputInfo& info, uint32_t input) {
+	return input < info.input_num && (info.interpolator_settings[input] & PsInputFlatShade) != 0;
+}
+
 struct ShaderBinaryInfo {
 	uint8_t  signature[7];
 	uint8_t  version;
@@ -1606,6 +1642,7 @@ ShaderId ShaderGetIdPS(const HW::PixelShaderInfo& regs, const ShaderPixelInputIn
 	ret.ids.push_back(static_cast<uint32_t>(input_info.ps_pos_z));
 	ret.ids.push_back(static_cast<uint32_t>(input_info.ps_pos_w));
 	ret.ids.push_back(static_cast<uint32_t>(input_info.ps_front_face));
+	ret.ids.push_back(static_cast<uint32_t>(input_info.ps_no_perspective));
 	ret.ids.push_back(static_cast<uint32_t>(input_info.ps_pixel_kill_enable));
 	ret.ids.push_back(static_cast<uint32_t>(input_info.ps_sample_mask_export_enable));
 	ret.ids.push_back(static_cast<uint32_t>(input_info.ps_early_z));

src/graphics/shader/shader.h

@@ -122,6 +122,11 @@ struct ShaderPixelInputInfo {
 	bool HasPositionInput() const { return ps_pos_x || ps_pos_y || ps_pos_z || ps_pos_w; }
 };
 
+uint32_t ShaderPixelParameterMappedLocation(const ShaderPixelInputInfo& info, uint32_t input);
+uint32_t ShaderPixelParameterLocation(const ShaderPixelInputInfo& info,
+                                      std::span<const uint32_t> active_inputs, uint32_t input);
+bool     ShaderPixelParameterIsFlat(const ShaderPixelInputInfo& info, uint32_t input);
+
 struct ShaderSharp {
 	uint16_t offset_dw : 15;
 	uint16_t size      : 1;

src/kernel/fileSystem.cpp

@@ -64,6 +64,7 @@ struct File {
 	std::filesystem::path               real_name;
 	std::atomic_bool                    opened;
 	std::atomic_bool                    directory;
+	std::atomic_bool                    writable;
 	std::atomic_bool                    append;
 	std::atomic_bool                    sync_writes;
 	SpecialFile                         special;
@@ -129,6 +130,7 @@ int FileDescriptors::CreateDescriptor() {
 	auto* file        = new File {};
 	file->opened      = false;
 	file->directory   = false;
+	file->writable    = false;
 	file->append      = false;
 	file->sync_writes = false;
 	file->special     = SpecialFile::None;
@@ -408,6 +410,7 @@ int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode) {
 	EXIT_IF(file == nullptr || file->opened || file->directory);
 
 	file->name        = path;
+	file->writable    = rw_mode != Common::File::Mode::Read;
 	file->append      = append;
 	file->sync_writes = fsync || sync || dsync;
 
@@ -953,6 +956,47 @@ int KYTY_SYSV_ABI KernelFstat(int d, FileStat* sb) {
 	return OK;
 }
 
+int KYTY_SYSV_ABI KernelFtruncate(int d, int64_t length) {
+	PRINT_NAME();
+
+	if (d < DESCRIPTOR_MIN) {
+		return KERNEL_ERROR_EBADF;
+	}
+
+	if (length < 0) {
+		return KERNEL_ERROR_EINVAL;
+	}
+
+	if (::Libs::Network::Net::IsSocket(d)) {
+		return KERNEL_ERROR_EINVAL;
+	}
+
+	auto* file = g_files->GetFile(d);
+
+	if (file == nullptr || !file->opened) {
+		return KERNEL_ERROR_EBADF;
+	}
+
+	if (!file->writable) {
+		return KERNEL_ERROR_EBADF;
+	}
+
+	if (file->directory || file->special != SpecialFile::None) {
+		return KERNEL_ERROR_EINVAL;
+	}
+
+	Common::LockGuard lock(file->mutex);
+
+	if (file->f.IsInvalid() || !file->f.Truncate(static_cast<uint64_t>(length))) {
+		return KERNEL_ERROR_EIO;
+	}
+
+	LOGF("\tFtruncate (size = %" PRId64 ") file: %s\n", length,
+	     Common::PathToString(file->real_name).c_str());
+
+	return OK;
+}
+
 int KYTY_SYSV_ABI KernelUnlink(const char* path) {
 	PRINT_NAME();
 

src/kernel/fileSystem.h

@@ -48,6 +48,7 @@ int64_t KYTY_SYSV_ABI KernelPwrite(int d, const void* buf, size_t nbytes, int64_
 int64_t KYTY_SYSV_ABI KernelLseek(int d, int64_t offset, int whence);
 int KYTY_SYSV_ABI     KernelStat(const char* path, FileStat* sb);
 int KYTY_SYSV_ABI     KernelFstat(int d, FileStat* sb);
+int KYTY_SYSV_ABI     KernelFtruncate(int d, int64_t length);
 int KYTY_SYSV_ABI     KernelUnlink(const char* path);
 int KYTY_SYSV_ABI     KernelRename(const char* from, const char* to);
 int KYTY_SYSV_ABI     KernelGetdirentries(int fd, char* buf, int nbytes, int64_t* basep);

src/kernel/memoryAddressSpace.inc

@@ -1058,6 +1058,76 @@ private:
 		    {HOST_SYSTEM_RESERVED_MIN, HOST_SYSTEM_RESERVED_MAX + 1u},
 		    {HOST_USER_MIN, HOST_USER_MAX + 1u},
 		}};
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+		std::vector<std::pair<uint64_t, uint64_t>> occupied;
+		FILE*                                      maps = fopen("/proc/self/maps", "r");
+		EXIT_IF(maps == nullptr);
+
+		char line[512];
+		while (fgets(line, sizeof(line), maps) != nullptr) {
+			unsigned long long mapping_start = 0;
+			unsigned long long mapping_end   = 0;
+			if (sscanf(line, "%llx-%llx", &mapping_start, &mapping_end) == 2) {
+				occupied.emplace_back(static_cast<uint64_t>(mapping_start),
+				                      static_cast<uint64_t>(mapping_end));
+			}
+		}
+		fclose(maps);
+
+		auto reserve_range = [this](uint64_t start, uint64_t end) {
+			start = AlignUp(start, PageSize());
+			end   = AlignDown(end, PageSize());
+			if (start == 0 || end <= start) {
+				return;
+			}
+
+			const auto size = end - start;
+			int        flags = MAP_PRIVATE | MAP_ANON | MAP_NORESERVE;
+#if defined(KYTY_LINKED_GUEST_ADDRESS_SPACE)
+			flags |= MAP_FIXED;
+#elif defined(MAP_FIXED_NOREPLACE)
+			flags |= MAP_FIXED_NOREPLACE;
+#endif
+
+			void* ptr = mmap(reinterpret_cast<void*>(start), size, PROT_NONE, flags, -1, 0);
+			if (ptr == MAP_FAILED || reinterpret_cast<uint64_t>(ptr) != start) {
+				if (ptr != MAP_FAILED) {
+					munmap(ptr, size);
+				}
+				return;
+			}
+
+			AddFreeUnlocked(start, size);
+			m_owned.emplace_back(start, size);
+		};
+
+		for (const auto& [region_start, region_end]: regions) {
+			auto current = region_start;
+
+			for (const auto& [mapping_start, mapping_end]: occupied) {
+				if (mapping_end <= current) {
+					continue;
+				}
+				if (mapping_start >= region_end) {
+					break;
+				}
+
+				if (mapping_start > current) {
+					reserve_range(current, std::min(mapping_start, region_end));
+				}
+
+				current = std::max(current, mapping_end);
+				if (current >= region_end) {
+					break;
+				}
+			}
+
+			if (current < region_end) {
+				reserve_range(current, region_end);
+			}
+		}
+#else
 		for (const auto& [start, end]: regions) {
 			int flags = MAP_PRIVATE | MAP_ANON | MAP_NORESERVE;
 #if defined(KYTY_LINKED_GUEST_ADDRESS_SPACE)
@@ -1070,13 +1140,14 @@ private:
 				if (ptr != MAP_FAILED) {
 					munmap(ptr, end - start);
 				}
-				EXIT("failed to reserve guest address space at 0x%016" PRIx64 ", size 0x%016" PRIx64
-				     "\n",
+				EXIT("failed to reserve guest address space at 0x%016" PRIx64
+				     ", size 0x%016" PRIx64 "\n",
 				     start, end - start);
 			}
 			AddFreeUnlocked(start, end - start);
 			m_owned.emplace_back(start, end - start);
 		}
+#endif
 #endif
 	}
 

src/kernel/pthread.cpp

@@ -65,6 +65,10 @@
 
 namespace Libs {
 
+namespace LibcInternalExt {
+void RunThreadAtexitDestructors();
+} // namespace LibcInternalExt
+
 namespace LibKernel {
 
 LIB_NAME("libkernel", "libkernel");
@@ -361,6 +365,7 @@ struct PthreadAttrPrivate {
 	uint64_t       stack_map_addr;
 	size_t         stack_map_size;
 	int            policy;
+	int            guest_priority;
 	int            inherit_sched;
 	int            solosched;
 	bool           detached;
@@ -871,9 +876,11 @@ bool TestGuestStackOwnerLifecycle(uint64_t* first_address, uint64_t* second_addr
 
 static KYTY_SYSV_ABI void* RunOnGuestStack(void* arg, pthread_entry_func_t func, void* stack_top) {
 #if defined(__x86_64__) || defined(_M_X64)
-	void*      ret       = nullptr;
-	const auto guest_rsp = reinterpret_cast<uintptr_t>(stack_top) & ~static_cast<uintptr_t>(0x0f);
-	const auto guest_rbp = guest_rsp - 4u * sizeof(uint64_t);
+	void*      ret = nullptr;
+	const auto aligned_stack_top =
+	    reinterpret_cast<uintptr_t>(stack_top) & ~static_cast<uintptr_t>(0x0f);
+	const auto guest_rsp = aligned_stack_top - 2u * sizeof(uintptr_t);
+	const auto guest_rbp = guest_rsp;
 
 	auto* guest_root_frame = reinterpret_cast<uintptr_t*>(guest_rbp);
 	guest_root_frame[0]    = 0;
@@ -2124,13 +2131,8 @@ int KYTY_SYSV_ABI PthreadAttrGetschedparam(const PthreadAttr* attr, KernelSchedP
 
 	int result = pthread_attr_getschedparam(&(*attr)->p, param);
 
-	if (param->sched_priority <= -2) {
-		param->sched_priority = 767;
-	} else if (param->sched_priority >= +2) {
-		param->sched_priority = 256;
-	} else {
-		param->sched_priority = 700;
-	}
+	// Host priority mapping is lossy; return the exact guest value.
+	param->sched_priority = (*attr)->guest_priority;
 
 	if (result == 0) {
 		return OK;
@@ -2294,6 +2296,7 @@ int KYTY_SYSV_ABI PthreadAttrSetschedparam(PthreadAttr* attr, const KernelSchedP
 		return KERNEL_ERROR_EINVAL;
 	}
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	KernelSchedParam pparam {};
 	if (param->sched_priority <= 478) {
 		pparam.sched_priority = +2;
@@ -2303,12 +2306,13 @@ int KYTY_SYSV_ABI PthreadAttrSetschedparam(PthreadAttr* attr, const KernelSchedP
 		pparam.sched_priority = 0;
 	}
 
-	int result = pthread_attr_setschedparam(&attr_value->p, &pparam);
-
-	if (result == 0) {
-		return OK;
+	if (pthread_attr_setschedparam(&attr_value->p, &pparam) != 0) {
+		return KERNEL_ERROR_EINVAL;
 	}
-	return KERNEL_ERROR_EINVAL;
+#endif
+
+	attr_value->guest_priority = param->sched_priority;
+	return OK;
 }
 
 int KYTY_SYSV_ABI PthreadAttrSetschedpolicy(PthreadAttr* attr, int policy) {
@@ -3347,6 +3351,8 @@ int PthreadGetCurrentPriorityForKernel() {
 static void CleanupThread(void* arg) {
 	auto* thread = static_cast<Pthread>(arg);
 
+	LibcInternalExt::RunThreadAtexitDestructors();
+
 	auto thread_dtors = g_pthread_context->GetThreadDtors();
 
 	if (thread_dtors != nullptr) {
@@ -3633,26 +3639,15 @@ int KYTY_SYSV_ABI PthreadGetprio(Pthread thread, int* prio) {
 
 	EXIT_NOT_IMPLEMENTED(prio == nullptr);
 
-	sched_param param {};
-	int         pol = 0;
-
-	int result = pthread_getschedparam(thread->p, &pol, &param);
-
-	if (result == 0) {
-		if (param.sched_priority <= -2) {
-			*prio = 767;
-		} else if (param.sched_priority >= +2) {
-			*prio = 256;
-		} else {
-			*prio = 700;
-		}
-
-		LOGF("\t PthreadGetprio: %d, %d\n", thread->unique_id, *prio);
-
-		return OK;
+	sched_param native_param {};
+	int         native_policy = 0;
+	if (pthread_getschedparam(thread->p, &native_policy, &native_param) != 0) {
+		return KERNEL_ERROR_EINVAL;
 	}
 
-	return KERNEL_ERROR_EINVAL;
+	*prio = thread->attr->guest_priority;
+	LOGF("\t PthreadGetprio: %d, %d\n", thread->unique_id, *prio);
+	return OK;
 }
 
 int KYTY_SYSV_ABI PthreadSetprio(Pthread thread, int prio) {
@@ -3667,25 +3662,27 @@ int KYTY_SYSV_ABI PthreadSetprio(Pthread thread, int prio) {
 
 	int result = pthread_getschedparam(thread->p, &pol, &param);
 
-	if (result == 0) {
-		if (prio <= 478) {
-			param.sched_priority = +2;
-		} else if (prio >= 733) {
-			param.sched_priority = -2;
-		} else {
-			param.sched_priority = 0;
-		}
-
-		result = pthread_setschedparam(thread->p, pol, &param);
-
-		if (result == 0) {
-			LOGF("\t PthreadSetprio: %d, %d\n", thread->unique_id, prio);
-
-			return OK;
-		}
+	if (result != 0) {
+		return KERNEL_ERROR_EINVAL;
 	}
 
-	return KERNEL_ERROR_EINVAL;
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	if (prio <= 478) {
+		param.sched_priority = +2;
+	} else if (prio >= 733) {
+		param.sched_priority = -2;
+	} else {
+		param.sched_priority = 0;
+	}
+
+	if (pthread_setschedparam(thread->p, pol, &param) != 0) {
+		return KERNEL_ERROR_EINVAL;
+	}
+#endif
+
+	thread->attr->guest_priority = prio;
+	LOGF("\t PthreadSetprio: %d, %d\n", thread->unique_id, prio);
+	return OK;
 }
 
 void KYTY_SYSV_ABI PthreadTestcancel() {
@@ -3880,8 +3877,12 @@ int KYTY_SYSV_ABI KernelGettimeofday(KernelTimeval* tp) {
 	tp->tv_sec  = static_cast<int64_t>(ticks / 1000000);
 	tp->tv_usec = static_cast<int64_t>(ticks % 1000000);
 #else
-	auto dt = Common::DateTime::FromSystemUTC();
-	sec_to_timeval(tp, dt.ToUnix());
+	struct timespec ts {};
+	result = ::clock_gettime(CLOCK_REALTIME, &ts);
+	if (result == 0) {
+		tp->tv_sec  = static_cast<int64_t>(ts.tv_sec);
+		tp->tv_usec = static_cast<int64_t>(ts.tv_nsec / 1000);
+	}
 #endif
 
 	if (result == 0) {

src/launcher/CMakeLists.txt

@@ -141,14 +141,14 @@ elseif(LINUX)
 endif()
 
 
-set(tidy_dirs "${CMAKE_SOURCE_DIR}/launcher/include") 
-set(iwyu_maps "${CMAKE_SOURCE_DIR}/launcher/utils/qt6_16.imp")
+set(tidy_dirs "${CMAKE_CURRENT_SOURCE_DIR}/include")
+set(iwyu_maps "${CMAKE_CURRENT_SOURCE_DIR}/utils/qt6_16.imp")
 
 get_property(inc_headers TARGET launcher PROPERTY INCLUDE_DIRECTORIES)
 
 list(APPEND inc_headers
-	${CMAKE_SOURCE_DIR}/launcher
-	${CMAKE_BINARY_DIR}/launcher/launcher_autogen/include
+	${CMAKE_CURRENT_SOURCE_DIR}
+	${CMAKE_CURRENT_BINARY_DIR}/launcher_autogen/include
 	${Qt6Widgets_INCLUDE_DIRS}
 	${Qt6Core_INCLUDE_DIRS}
 	${Qt6Gui_INCLUDE_DIRS}

src/launcher/forms/configuration_edit_dialog.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>560</width>
-    <height>420</height>
+    <height>450</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -78,19 +78,6 @@
            </property>
           </widget>
          </item>
-         <item row="2" column="0" colspan="2">
-          <widget class="QCheckBox" name="checkBox_ngg_rectlist_draw">
-           <property name="toolTip">
-            <string>Use the NGG 4-vertex path for rect-list DrawIndexAuto primitive 7</string>
-           </property>
-           <property name="text">
-            <string>Use NGG rect-list draw</string>
-           </property>
-           <property name="checked">
-            <bool>true</bool>
-           </property>
-          </widget>
-         </item>
         </layout>
        </widget>
       </item>
@@ -132,41 +119,55 @@
        </widget>
       </item>
       <item row="3" column="0">
+       <widget class="QLabel" name="label_console_language">
+        <property name="text">
+         <string>Console language:</string>
+        </property>
+       </widget>
+      </item>
+      <item row="3" column="1">
+       <widget class="QComboBox" name="comboBox_console_language">
+        <property name="toolTip">
+         <string>Language</string>
+        </property>
+       </widget>
+      </item>
+      <item row="4" column="0">
        <widget class="QLabel" name="label_20">
         <property name="text">
          <string>Shader optimization type:</string>
         </property>
        </widget>
       </item>
-      <item row="3" column="1">
+      <item row="4" column="1">
        <widget class="QComboBox" name="comboBox_shader_optimization_type">
         <property name="toolTip">
          <string>Optimize shaders for code size or performance</string>
         </property>
        </widget>
       </item>
-      <item row="4" column="0">
+      <item row="5" column="0">
        <widget class="QLabel" name="label_21">
         <property name="text">
          <string>Shader log direction:</string>
         </property>
        </widget>
       </item>
-      <item row="4" column="1">
+      <item row="5" column="1">
        <widget class="QComboBox" name="comboBox_shader_log_direction">
         <property name="toolTip">
          <string>Dump shaders to file or console window. If enabled may decrease emulator performance</string>
         </property>
        </widget>
       </item>
-      <item row="5" column="0">
+      <item row="6" column="0">
        <widget class="QLabel" name="label_22">
         <property name="text">
          <string>Shader log folder:</string>
         </property>
        </widget>
       </item>
-      <item row="5" column="1">
+      <item row="6" column="1">
        <widget class="MandatoryLineEdit" name="lineEdit_shader_log_folder">
         <property name="toolTip">
          <string>Specify directory to dump shaders</string>
@@ -176,14 +177,14 @@
         </property>
        </widget>
       </item>
-      <item row="6" column="0">
+      <item row="7" column="0">
        <widget class="QLabel" name="label_24">
         <property name="text">
          <string>Command buffer dump folder:</string>
         </property>
        </widget>
       </item>
-      <item row="6" column="1">
+      <item row="7" column="1">
        <widget class="MandatoryLineEdit" name="lineEdit_cmd_dump_folder">
         <property name="toolTip">
          <string>Specify directory to dump command buffers</string>
@@ -193,28 +194,28 @@
         </property>
        </widget>
       </item>
-      <item row="7" column="0">
+      <item row="8" column="0">
        <widget class="QLabel" name="label_25">
         <property name="text">
          <string>Printf direction:</string>
         </property>
        </widget>
       </item>
-      <item row="7" column="1">
+      <item row="8" column="1">
        <widget class="QComboBox" name="comboBox_printf_direction">
         <property name="toolTip">
          <string>Print logs to file or console window. If enabled may decrease emulator performance</string>
         </property>
        </widget>
       </item>
-      <item row="8" column="0">
+      <item row="9" column="0">
        <widget class="QLabel" name="label_26">
         <property name="text">
          <string>Printf output file:</string>
         </property>
        </widget>
       </item>
-      <item row="8" column="1">
+      <item row="9" column="1">
        <widget class="MandatoryLineEdit" name="lineEdit_printf_file">
         <property name="toolTip">
          <string>Specify file to dump logs</string>
@@ -224,14 +225,14 @@
         </property>
        </widget>
       </item>
-      <item row="9" column="0">
+      <item row="10" column="0">
        <widget class="QLabel" name="label_27">
         <property name="text">
          <string>Profiler direction:</string>
         </property>
        </widget>
       </item>
-      <item row="9" column="1">
+      <item row="10" column="1">
        <widget class="QComboBox" name="comboBox_profiler_direction">
         <property name="toolTip">
          <string>Enable or disable profiler. If enabled may decrease emulator performance</string>

src/launcher/include/configuration.h

@@ -48,6 +48,9 @@ class Configuration: public QObject {
 	Q_OBJECT
 
 public:
+	static constexpr int DEFAULT_CONSOLE_LANGUAGE = 1;
+	static constexpr int MAX_CONSOLE_LANGUAGE     = 29;
+
 	enum class Resolution {
 		R1280X720,
 		R1920X1080,
@@ -83,6 +86,7 @@ public:
 
 	Resolution             screen_resolution           = Resolution::R1280X720;
 	int                    vblank_frequency            = 60;
+	int                    console_language            = DEFAULT_CONSOLE_LANGUAGE;
 	bool                   vulkan_validation_enabled   = true;
 	bool                   shader_validation_enabled   = true;
 	ShaderOptimizationType shader_optimization_type    = ShaderOptimizationType::Performance;
@@ -94,13 +98,13 @@ public:
 	QString                printf_output_file          = "_kyty.txt";
 	ProfilerDirection      profiler_direction          = ProfilerDirection::None;
 	bool                   renderdoc_enabled           = false;
-	bool                   ngg_rectlist_draw_enabled   = true;
 
 	QString elf = QStringLiteral("eboot.bin");
 
 	void CopyEmulatorSettingsFrom(const Configuration& other) {
 		screen_resolution           = other.screen_resolution;
 		vblank_frequency            = other.vblank_frequency;
+		console_language            = other.console_language;
 		vulkan_validation_enabled   = other.vulkan_validation_enabled;
 		shader_validation_enabled   = other.shader_validation_enabled;
 		shader_optimization_type    = other.shader_optimization_type;
@@ -112,7 +116,6 @@ public:
 		printf_output_file          = other.printf_output_file;
 		profiler_direction          = other.profiler_direction;
 		renderdoc_enabled           = other.renderdoc_enabled;
-		ngg_rectlist_draw_enabled   = other.ngg_rectlist_draw_enabled;
 	}
 
 	void CopyFrom(const Configuration& other) {
@@ -136,6 +139,7 @@ public:
 		KYTY_CFG_SET(custom_settings);
 		KYTY_CFG_SET(screen_resolution);
 		KYTY_CFG_SET(vblank_frequency);
+		KYTY_CFG_SET(console_language);
 		KYTY_CFG_SET(vulkan_validation_enabled);
 		KYTY_CFG_SET(shader_validation_enabled);
 		KYTY_CFG_SET(shader_optimization_type);
@@ -147,7 +151,6 @@ public:
 		KYTY_CFG_SET(printf_output_file);
 		KYTY_CFG_SET(profiler_direction);
 		KYTY_CFG_SET(renderdoc_enabled);
-		KYTY_CFG_SET(ngg_rectlist_draw_enabled);
 		KYTY_CFG_SET(elf);
 	}
 
@@ -158,6 +161,10 @@ public:
 		KYTY_CFG_GET(custom_settings);
 		KYTY_CFG_GET(screen_resolution);
 		vblank_frequency = s->value("vblank_frequency", vblank_frequency).toInt();
+		console_language = s->value("console_language", console_language).toInt();
+		if (console_language < 0 || console_language > MAX_CONSOLE_LANGUAGE) {
+			console_language = DEFAULT_CONSOLE_LANGUAGE;
+		}
 		KYTY_CFG_GET(vulkan_validation_enabled);
 		KYTY_CFG_GET(shader_validation_enabled);
 		KYTY_CFG_GET(shader_optimization_type);
@@ -169,8 +176,6 @@ public:
 		KYTY_CFG_GET(printf_output_file);
 		KYTY_CFG_GET(profiler_direction);
 		KYTY_CFG_GET(renderdoc_enabled);
-		ngg_rectlist_draw_enabled =
-		    s->value("ngg_rectlist_draw_enabled", ngg_rectlist_draw_enabled).toBool();
 		elf = s->value("elf", elf).toString();
 	}
 };

src/launcher/src/configurationEditDialog.cpp

@@ -30,6 +30,39 @@ constexpr char SETTINGS_CFG_DIALOG[]               = "ConfigurationEditDialog";
 constexpr char SETTINGS_CFG_LAST_GEOMETRY[]        = "geometry";
 constexpr int  GLOBAL_SETTINGS_GAME_DIRS_MIN_WIDTH = 560;
 
+const QStringList CONSOLE_LANGUAGE_NAMES = {
+    "Japanese",
+    "English (United States)",
+    "French (France)",
+    "Spanish (Spain)",
+    "German",
+    "Italian",
+    "Dutch",
+    "Portuguese (Portugal)",
+    "Russian",
+    "Korean",
+    "Chinese (Traditional)",
+    "Chinese (Simplified)",
+    "Finnish",
+    "Swedish",
+    "Danish",
+    "Norwegian",
+    "Polish",
+    "Portuguese (Brazil)",
+    "English (United Kingdom)",
+    "Turkish",
+    "Spanish (Latin America)",
+    "Arabic",
+    "French (Canada)",
+    "Czech",
+    "Hungarian",
+    "Greek",
+    "Romanian",
+    "Thai",
+    "Vietnamese",
+    "Indonesian",
+};
+
 static QString NormalizeGameDirectory(const QString& dir) {
 	const auto trimmed = dir.trimmed();
 	if (trimmed.isEmpty()) {
@@ -121,10 +154,15 @@ static void ListInit(QComboBox* combo, T value) {
 void ConfigurationEditDialog::Init(const Configuration& info) {
 	ListInit(m_ui->comboBox_screen_resolution, info.screen_resolution);
 	m_ui->spinBox_vblank_frequency->setValue(info.vblank_frequency);
+	m_ui->comboBox_console_language->clear();
+	m_ui->comboBox_console_language->addItems(CONSOLE_LANGUAGE_NAMES);
+	m_ui->comboBox_console_language->setCurrentIndex(
+	    info.console_language >= 0 && info.console_language < CONSOLE_LANGUAGE_NAMES.size()
+	        ? info.console_language
+	        : Configuration::DEFAULT_CONSOLE_LANGUAGE);
 	m_ui->checkBox_shader_validation->setChecked(info.shader_validation_enabled);
 	m_ui->checkBox_vulkan_validation->setChecked(info.vulkan_validation_enabled);
 	m_ui->checkBox_renderdoc_capture->setChecked(info.renderdoc_enabled);
-	m_ui->checkBox_ngg_rectlist_draw->setChecked(info.ngg_rectlist_draw_enabled);
 	ListInit(m_ui->comboBox_shader_optimization_type, info.shader_optimization_type);
 	ListInit(m_ui->comboBox_shader_log_direction, info.shader_log_direction);
 	m_ui->lineEdit_shader_log_folder->setText(info.shader_log_folder);
@@ -242,10 +280,10 @@ static void UpdateInfo(Configuration& info, Ui::ConfigurationEditDialog& ui) {
 	info.screen_resolution =
 	    TextToEnum<Configuration::Resolution>(ui.comboBox_screen_resolution->currentText());
 	info.vblank_frequency          = ui.spinBox_vblank_frequency->value();
+	info.console_language          = ui.comboBox_console_language->currentIndex();
 	info.vulkan_validation_enabled = ui.checkBox_vulkan_validation->isChecked();
 	info.shader_validation_enabled = ui.checkBox_shader_validation->isChecked();
 	info.renderdoc_enabled         = ui.checkBox_renderdoc_capture->isChecked();
-	info.ngg_rectlist_draw_enabled = ui.checkBox_ngg_rectlist_draw->isChecked();
 	info.shader_optimization_type  = TextToEnum<Configuration::ShaderOptimizationType>(
 	    ui.comboBox_shader_optimization_type->currentText());
 	info.shader_log_direction = TextToEnum<Configuration::ShaderLogDirection>(

src/launcher/src/mainDialog.cpp

@@ -205,6 +205,7 @@ static QStringList CreateEmulatorArgs(const Configuration& info) {
 	args << "--screen-width" << r.at(0);
 	args << "--screen-height" << r.at(1);
 	args << "--vblank-frequency" << QString::number(info.vblank_frequency);
+	args << "--console-language" << QString::number(info.console_language);
 	args << "--vulkan-validation" << BoolArg(info.vulkan_validation_enabled);
 	args << "--shader-validation" << BoolArg(info.shader_validation_enabled);
 	args << "--shader-optimization-type" << EnumToText(info.shader_optimization_type);
@@ -216,7 +217,6 @@ static QStringList CreateEmulatorArgs(const Configuration& info) {
 	args << "--printf-output-file" << info.printf_output_file;
 	args << "--profiler-direction" << EnumToText(info.profiler_direction);
 	args << "--spirv-debug-printf" << "false";
-	args << "--ngg-rectlist-draw" << BoolArg(info.ngg_rectlist_draw_enabled);
 	if (info.renderdoc_enabled) {
 		args << "--rd";
 	}

src/libs/agc.cpp

@@ -221,57 +221,6 @@ static RegisterDefaults* get_internal_register_defaults(uint32_t ver) {
 	return get_register_defaults(g_agc_internal_reg_defaults_by_version[index], &storage[index]);
 }
 
-struct PendingGraphicsSegment {
-	uint32_t* start     = nullptr;
-	uint32_t* end       = nullptr;
-	uint32_t* range_end = nullptr;
-};
-
-static std::mutex             g_pending_graphics_segment_mutex;
-static PendingGraphicsSegment g_pending_graphics_segment;
-
-static void track_pending_graphics_segment_after_submit(uint32_t* dcb, uint32_t size_in_dwords) {
-	if (dcb == nullptr || size_in_dwords == 0) {
-		return;
-	}
-
-	auto* segment_start = dcb + size_in_dwords;
-	auto* range_end     = segment_start + 0xfffffu;
-
-	std::lock_guard lock(g_pending_graphics_segment_mutex);
-	g_pending_graphics_segment.start     = segment_start;
-	g_pending_graphics_segment.end       = segment_start;
-	g_pending_graphics_segment.range_end = range_end;
-}
-
-static void track_pending_graphics_allocation(uint32_t* cmd, uint32_t size_dw) {
-	if (cmd == nullptr || size_dw == 0) {
-		return;
-	}
-
-	std::lock_guard lock(g_pending_graphics_segment_mutex);
-	auto*           range_start = g_pending_graphics_segment.start;
-	auto*           range_end   = g_pending_graphics_segment.range_end;
-	if (range_start == nullptr || range_end == nullptr || cmd < range_start || cmd >= range_end) {
-		return;
-	}
-
-	auto* cmd_end = cmd + size_dw;
-	if (cmd > g_pending_graphics_segment.end) {
-		static std::atomic<uint32_t> log_count {0};
-		if (log_count.fetch_add(1) < 64) {
-			LOGF("\t pending graphics segment: ignoring non-contiguous allocation cmd = "
-			     "0x%016" PRIx64 ", tracked_end = 0x%016" PRIx64 "\n",
-			     reinterpret_cast<uint64_t>(cmd),
-			     reinterpret_cast<uint64_t>(g_pending_graphics_segment.end));
-		}
-		return;
-	}
-	if (cmd_end > g_pending_graphics_segment.end && cmd_end <= range_end) {
-		g_pending_graphics_segment.end = cmd_end;
-	}
-}
-
 struct CommandBuffer {
 	using Callback = KYTY_SYSV_ABI bool (*)(CommandBuffer*, uint32_t, void*);
 
@@ -370,7 +319,6 @@ struct CommandBuffer {
 		}
 		auto* ret_ptr = cursor_up;
 		cursor_up += size_dw;
-		track_pending_graphics_allocation(ret_ptr, size_dw);
 		return ret_ptr;
 	}
 };
@@ -834,6 +782,104 @@ int KYTY_SYSV_ABI GraphicsUnknownFuseShaderHalves(Shader* fused_result, const Sh
 	return OK;
 }
 
+static void merge_shader_register_max_field(ShaderRegister* dst, const ShaderRegister* src,
+                                            uint32_t shift, uint32_t mask) {
+	const auto dst_field = (dst->value >> shift) & mask;
+	const auto src_field = (src->value >> shift) & mask;
+	const auto field     = std::max(dst_field, src_field);
+
+	dst->value &= ~(mask << shift);
+	dst->value |= field << shift;
+}
+
+int KYTY_SYSV_ABI GraphicsUnknownNApJjpKNBl4(Shader* fused_result, const Shader* front,
+                                             const Shader* back, void* scratch_mem) {
+	PRINT_NAME();
+
+	LOGF("\t fused_result = 0x%016" PRIx64 "\n"
+	     "\t front        = 0x%016" PRIx64 "\n"
+	     "\t back         = 0x%016" PRIx64 "\n"
+	     "\t scratch_mem  = 0x%016" PRIx64 "\n",
+	     reinterpret_cast<uint64_t>(fused_result), reinterpret_cast<uint64_t>(front),
+	     reinterpret_cast<uint64_t>(back), reinterpret_cast<uint64_t>(scratch_mem));
+
+	const auto front_type = static_cast<Prospero::ShaderBinaryType>(front->type);
+	const auto is_gs      = front_type == Prospero::ShaderBinaryType::kGsFront;
+	const auto is_hs      = front_type == Prospero::ShaderBinaryType::kHsFront;
+	if ((!is_gs && !is_hs) ||
+	    (is_gs && back->type != static_cast<uint8_t>(Prospero::ShaderBinaryType::kGsBack)) ||
+	    (is_hs && back->type != static_cast<uint8_t>(Prospero::ShaderBinaryType::kHsBack))) {
+		return GRAPHICS5_ERROR_INVALID_SHADER_HALVES;
+	}
+
+	*fused_result      = *back;
+	fused_result->type = static_cast<uint8_t>(is_gs ? Prospero::ShaderBinaryType::kGs
+	                                                : Prospero::ShaderBinaryType::kHs);
+
+	const auto back_stages  = back->specials->vgt_shader_stages_en.value;
+	const auto front_stages = front->specials->vgt_shader_stages_en.value;
+	const auto mismatch_bit = is_gs ? (1u << 22u) : (1u << 21u);
+	if (((front_stages ^ back_stages) & mismatch_bit) != 0) {
+		return GRAPHICS5_ERROR_INVALID_SHADER_HALVES;
+	}
+
+	if (scratch_mem != nullptr) {
+		auto* sh_registers = static_cast<ShaderRegister*>(scratch_mem);
+		memcpy(sh_registers, back->sh_registers,
+		       static_cast<size_t>(back->num_sh_registers) * sizeof(ShaderRegister));
+		fused_result->sh_registers = sh_registers;
+	}
+
+	auto*      fused_regs      = fused_result->sh_registers;
+	const auto fused_reg_count = static_cast<uint32_t>(fused_result->num_sh_registers);
+	const auto front_reg_count = static_cast<uint32_t>(front->num_sh_registers);
+	const auto checksum_offset =
+	    is_gs ? Pm4::SPI_SHADER_PGM_CHKSUM_GS : Pm4::SPI_SHADER_PGM_CHKSUM_HS;
+	const auto* front_checksum0 =
+	    find_shader_register(front->sh_registers, front_reg_count, checksum_offset, 0);
+	const auto* front_checksum1 =
+	    find_shader_register(front->sh_registers, front_reg_count, checksum_offset, 1);
+	auto* fused_checksum0  = find_shader_register(fused_regs, fused_reg_count, checksum_offset, 0);
+	auto* fused_checksum1  = find_shader_register(fused_regs, fused_reg_count, checksum_offset, 1);
+	fused_checksum0->value = front_checksum0->value;
+	fused_checksum1->value = front_checksum1->value;
+
+	const auto  rsrc1_offset = is_gs ? Pm4::SPI_SHADER_PGM_RSRC1_GS : Pm4::SPI_SHADER_PGM_RSRC1_HS;
+	const auto  rsrc2_offset = is_gs ? Pm4::SPI_SHADER_PGM_RSRC2_GS : Pm4::SPI_SHADER_PGM_RSRC2_HS;
+	const auto* front_rsrc1 =
+	    find_shader_register(front->sh_registers, front_reg_count, rsrc1_offset);
+	const auto* front_rsrc2 =
+	    find_shader_register(front->sh_registers, front_reg_count, rsrc2_offset);
+	auto* fused_rsrc1 = find_shader_register(fused_regs, fused_reg_count, rsrc1_offset);
+	auto* fused_rsrc2 = find_shader_register(fused_regs, fused_reg_count, rsrc2_offset);
+
+	merge_shader_register_max_field(fused_rsrc1, front_rsrc1, 0, 0x3fu);
+	merge_shader_register_max_field(fused_rsrc2, front_rsrc2, 28, 0x0fu);
+	if (is_gs) {
+		merge_shader_register_max_field(fused_rsrc1, front_rsrc1, 29, 0x03u);
+		merge_shader_register_max_field(fused_rsrc2, front_rsrc2, 16, 0x03u);
+		fused_rsrc2->value =
+		    (fused_rsrc2->value & 0xf7ffffc1u) | (front_rsrc2->value & 0x0800003eu);
+		fused_rsrc2->value =
+		    (fused_rsrc2->value & 0xfffbffffu) | (front_rsrc2->value & 0x00040000u);
+	} else {
+		merge_shader_register_max_field(fused_rsrc1, front_rsrc1, 28, 0x03u);
+		fused_rsrc2->value =
+		    (fused_rsrc2->value & 0xf7ffffc1u) | (front_rsrc2->value & 0x0800003eu);
+	}
+
+	const auto program_lo_offset = is_gs ? Pm4::SPI_SHADER_PGM_LO_ES : Pm4::SPI_SHADER_PGM_LO_LS;
+	auto*      program_lo = find_shader_register(fused_regs, fused_reg_count, program_lo_offset);
+	const auto address    = reinterpret_cast<uint64_t>(front->code);
+	program_lo->value     = static_cast<uint32_t>(address >> 8u);
+	(program_lo + 1)->value &= 0xffffff00u;
+	(program_lo + 1)->value |= static_cast<uint32_t>((address >> 40u) & 0xffu);
+
+	fused_result->user_data = front->user_data;
+
+	return OK;
+}
+
 static constexpr int GRAPHICS5_ERROR_INVALID_PACKET = static_cast<int>(0x8a6c000cu);
 
 enum class RegIndirectPacket : uint32_t {
@@ -1347,7 +1393,7 @@ int KYTY_SYSV_ABI GraphicsWriteDataPatchSetAddressOrOffset(uint32_t* cmd,
 		return OK;
 	}
 
-	return static_cast<int>(0x8a6c000cu);
+	return GRAPHICS5_ERROR_INVALID_PACKET;
 }
 
 int KYTY_SYSV_ABI GraphicsUnknownJumpPatchSetTarget(uint32_t* cmd, const volatile uint32_t* target,
@@ -1820,7 +1866,6 @@ uint32_t* KYTY_SYSV_ABI GraphicsCbReleaseMem(CommandBuffer* buf, uint8_t action,
 	cmd[5] = static_cast<uint32_t>(packet_data & 0xffffffffu);
 	cmd[6] = static_cast<uint32_t>((packet_data >> 32u) & 0xffffffffu);
 	cmd[7] = interrupt_ctx_id & 0x07ffffffu;
-
 	return cmd;
 }
 
@@ -2380,7 +2425,7 @@ int KYTY_SYSV_ABI GraphicsUnknownIkfdtRIqCE(uint32_t* cmd, uint64_t arg1,
 
 	auto op = (cmd[0] >> 8u) & 0xffu;
 	if (op != Pm4::IT_INDIRECT_BUFFER) {
-		return 0x8a6c000c;
+		return GRAPHICS5_ERROR_INVALID_PACKET;
 	}
 
 	auto vaddr        = reinterpret_cast<uint64_t>(target);
@@ -2800,6 +2845,10 @@ uint32_t KYTY_SYSV_ABI GraphicsAcbCondExecGetSize() {
 	return GraphicsDcbCondExecGetSize();
 }
 
+uint32_t KYTY_SYSV_ABI GraphicsAcbJumpGetSize() {
+	return 0x10u;
+}
+
 uint32_t* KYTY_SYSV_ABI GraphicsAcbWaitRegMem(CommandBuffer* buf, uint8_t size,
                                               uint8_t compare_function, uint8_t cache_policy,
                                               const volatile void* address, uint64_t reference,
@@ -3119,7 +3168,7 @@ int KYTY_SYSV_ABI GraphicsDmaDataPatchSetDstAddressOrOffset(uint32_t* cmd,
 		return OK;
 	}
 
-	return static_cast<int>(0x8a6c000cu);
+	return GRAPHICS5_ERROR_INVALID_PACKET;
 }
 
 int KYTY_SYSV_ABI GraphicsDmaDataPatchSetSrcAddressOrOffsetOrImmediate(
@@ -3133,7 +3182,7 @@ int KYTY_SYSV_ABI GraphicsDmaDataPatchSetSrcAddressOrOffsetOrImmediate(
 		return OK;
 	}
 
-	return static_cast<int>(0x8a6c000cu);
+	return GRAPHICS5_ERROR_INVALID_PACKET;
 }
 
 uint32_t KYTY_SYSV_ABI GraphicsGetPacketSize(uint32_t* packet) {
@@ -3197,6 +3246,15 @@ int KYTY_SYSV_ABI GraphicsSetRangePredication(uint32_t* start, const volatile ui
 	return OK;
 }
 
+int KYTY_SYSV_ABI GraphicsRewindPatchSetRewindState(uint32_t* cmd, uint8_t state) {
+	if (((cmd[0] >> 8u) & 0xffu) != Pm4::IT_REWIND) {
+		return GRAPHICS5_ERROR_INVALID_PACKET;
+	}
+
+	cmd[1] = (cmd[1] & 0x7fffffffu) | (static_cast<uint32_t>(state) << 31u);
+	return OK;
+}
+
 int KYTY_SYSV_ABI GraphicsCondExecPatchSetEnd(uint32_t* cmd, const volatile uint32_t* buffer) {
 	PRINT_NAME();
 
@@ -3205,23 +3263,23 @@ int KYTY_SYSV_ABI GraphicsCondExecPatchSetEnd(uint32_t* cmd, const volatile uint
 	     reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(buffer));
 
 	if (cmd == nullptr || buffer == nullptr) {
-		return static_cast<int>(0x8a6c000cu);
+		return GRAPHICS5_ERROR_INVALID_PACKET;
 	}
 
 	auto op = (cmd[0] >> 8u) & 0xffu;
 	if (op != Pm4::IT_COND_EXEC) {
-		return static_cast<int>(0x8a6c000cu);
+		return GRAPHICS5_ERROR_INVALID_PACKET;
 	}
 
 	auto* packet_end = cmd + 5;
 	auto* range_end  = const_cast<uint32_t*>(reinterpret_cast<const volatile uint32_t*>(buffer));
 	if (range_end < packet_end) {
-		return static_cast<int>(0x8a6c000cu);
+		return GRAPHICS5_ERROR_INVALID_PACKET;
 	}
 
 	auto num_dwords = static_cast<uint64_t>(range_end - packet_end);
 	if (num_dwords > 0x3fffu) {
-		return static_cast<int>(0x8a6c000cu);
+		return GRAPHICS5_ERROR_INVALID_PACKET;
 	}
 
 	cmd[4] = (cmd[4] & ~0x3fffu) | static_cast<uint32_t>(num_dwords);
@@ -3237,12 +3295,12 @@ int KYTY_SYSV_ABI GraphicsCondExecPatchSetCommandAddress(uint32_t*
 	     reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(command));
 
 	if (cmd == nullptr || command == nullptr) {
-		return static_cast<int>(0x8a6c000cu);
+		return GRAPHICS5_ERROR_INVALID_PACKET;
 	}
 
 	auto op = (cmd[0] >> 8u) & 0xffu;
 	if (op != Pm4::IT_COND_EXEC || (reinterpret_cast<uintptr_t>(command) & 0x3u) != 0) {
-		return static_cast<int>(0x8a6c000cu);
+		return GRAPHICS5_ERROR_INVALID_PACKET;
 	}
 
 	auto addr = reinterpret_cast<uint64_t>(command);
@@ -3704,150 +3762,6 @@ static void submit_dcb(uint32_t* dcb, uint32_t size_in_dwords) {
 	EXIT_IF(g_renderer == nullptr);
 	g_renderer->GetGpu().Submit(dcb, size_in_dwords, nullptr, 0,
 	                            !dcb_has_queued_interrupt(dcb, size_in_dwords));
-	Gen5::track_pending_graphics_segment_after_submit(dcb, size_in_dwords);
-}
-
-static std::vector<uint64_t> collect_acb_wait_addresses(const uint32_t* acb,
-                                                        uint32_t        size_in_dwords) {
-	std::vector<uint64_t> addresses;
-
-	for (uint32_t offset = 0; offset < size_in_dwords;) {
-		auto cmd_id = acb[offset];
-		auto len    = KYTY_PM4_LEN(cmd_id);
-		if (len == 0 || len > size_in_dwords - offset) {
-			return addresses;
-		}
-
-		auto op = (cmd_id >> 8u) & 0xffu;
-		if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd_id) == Pm4::R_WAIT_MEM_32 && len >= 7) {
-			auto address = static_cast<uint64_t>(acb[offset + 1]) |
-			               (static_cast<uint64_t>(acb[offset + 2]) << 32u);
-			if (address != 0) {
-				addresses.push_back(address);
-			}
-		} else if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd_id) == Pm4::R_WAIT_MEM_64 && len >= 9) {
-			auto address = static_cast<uint64_t>(acb[offset + 1]) |
-			               (static_cast<uint64_t>(acb[offset + 2]) << 32u);
-			if (address != 0) {
-				addresses.push_back(address);
-			}
-		}
-
-		offset += len;
-	}
-
-	return addresses;
-}
-
-static bool acb_waits_for_address(const std::vector<uint64_t>& wait_addresses,
-                                  uint64_t                     release_address) {
-	for (auto address: wait_addresses) {
-		if (address == release_address) {
-			return true;
-		}
-	}
-
-	return false;
-}
-
-static void flush_pending_graphics_segment_before_acb(const uint32_t* acb,
-                                                      uint32_t        acb_size_in_dwords) {
-	uint32_t* dcb            = nullptr;
-	uint32_t  size_in_dwords = 0;
-	auto      wait_addresses = collect_acb_wait_addresses(acb, acb_size_in_dwords);
-
-	{
-		std::lock_guard lock(Gen5::g_pending_graphics_segment_mutex);
-		if (!wait_addresses.empty() && Gen5::g_pending_graphics_segment.start != nullptr) {
-			auto* scan        = Gen5::g_pending_graphics_segment.start;
-			auto* matched_end = Gen5::g_pending_graphics_segment.start;
-			while (scan < Gen5::g_pending_graphics_segment.end) {
-				auto cmd_id = *scan;
-				if (cmd_id == 0x80000000u) {
-					scan++;
-					continue;
-				}
-				if ((cmd_id & 0xC0000000u) != 0xC0000000u) {
-					break;
-				}
-
-				auto len = KYTY_PM4_LEN(cmd_id);
-				if (len == 0 ||
-				    len > static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end - scan)) {
-					break;
-				}
-
-				if (((cmd_id >> 8u) & 0xffu) == Pm4::IT_NOP &&
-				    KYTY_PM4_R(cmd_id) == Pm4::R_RELEASE_MEM && len >= 7) {
-					auto release_addr =
-					    static_cast<uint64_t>(scan[3]) | (static_cast<uint64_t>(scan[4]) << 32u);
-					if (acb_waits_for_address(wait_addresses, release_addr)) {
-						matched_end = scan + len;
-					}
-				}
-
-				scan += len;
-			}
-
-			if (matched_end > Gen5::g_pending_graphics_segment.start) {
-				Gen5::g_pending_graphics_segment.end = matched_end;
-			}
-		}
-
-		if (Gen5::g_pending_graphics_segment.start != nullptr &&
-		    Gen5::g_pending_graphics_segment.end > Gen5::g_pending_graphics_segment.start) {
-			auto* scan      = Gen5::g_pending_graphics_segment.start;
-			auto* valid_end = Gen5::g_pending_graphics_segment.start;
-			while (scan < Gen5::g_pending_graphics_segment.end) {
-				auto cmd_id = *scan;
-				if (cmd_id == 0x80000000u) {
-					scan++;
-					valid_end = scan;
-					continue;
-				}
-				if ((cmd_id & 0xC0000000u) != 0xC0000000u) {
-					break;
-				}
-
-				auto len = KYTY_PM4_LEN(cmd_id);
-				if (len == 0 ||
-				    len > static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end - scan)) {
-					break;
-				}
-
-				scan += len;
-				valid_end = scan;
-			}
-
-			if (valid_end < Gen5::g_pending_graphics_segment.end) {
-				static std::atomic<uint32_t> log_count {0};
-				if (log_count.fetch_add(1) < 64) {
-					LOGF("\t trimming pending graphics segment: addr = 0x%016" PRIx64
-					     ", old_dw = 0x%08" PRIx32 ", new_dw = 0x%08" PRIx32 "\n",
-					     reinterpret_cast<uint64_t>(Gen5::g_pending_graphics_segment.start),
-					     static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end -
-					                           Gen5::g_pending_graphics_segment.start),
-					     static_cast<uint32_t>(valid_end - Gen5::g_pending_graphics_segment.start));
-				}
-				Gen5::g_pending_graphics_segment.end = valid_end;
-			}
-		}
-
-		if (Gen5::g_pending_graphics_segment.start == nullptr ||
-		    Gen5::g_pending_graphics_segment.end <= Gen5::g_pending_graphics_segment.start) {
-			return;
-		}
-
-		dcb            = Gen5::g_pending_graphics_segment.start;
-		size_in_dwords = static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end -
-		                                       Gen5::g_pending_graphics_segment.start);
-	}
-
-	LOGF("\t flushing pending graphics segment before ACB: addr = 0x%016" PRIx64
-	     ", dw_num = 0x%08" PRIx32 "\n",
-	     reinterpret_cast<uint64_t>(dcb), size_in_dwords);
-
-	submit_dcb(dcb, size_in_dwords);
 }
 
 int KYTY_SYSV_ABI GraphicsDriverSubmitDcb(const Packet* packet) {
@@ -3963,8 +3877,6 @@ static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) {
 		LOGF("\t acb[%u] = 0x%08" PRIx32 "\n", i, acb[i]);
 	}
 
-	flush_pending_graphics_segment_before_acb(acb, size_in_dwords);
-
 	GraphicsDbgDumpDcb("a", size_in_dwords, acb);
 
 	const bool trigger_interrupt_on_done = !dcb_has_queued_interrupt(acb, size_in_dwords);

src/libs/agc.h

@@ -38,6 +38,8 @@ int KYTY_SYSV_ABI   GraphicsUnknownGetFusedShaderSize(SizeAlign* dst, const Shad
                                                       const Shader* back);
 int KYTY_SYSV_ABI   GraphicsUnknownFuseShaderHalves(Shader* fused_result, const Shader* front,
                                                     const Shader* back, void* scratch_mem);
+int KYTY_SYSV_ABI   GraphicsUnknownNApJjpKNBl4(Shader* fused_result, const Shader* front,
+                                               const Shader* back, void* scratch_mem);
 int KYTY_SYSV_ABI   GraphicsSetCxRegIndirectPatchSetAddress(uint32_t*                      cmd,
                                                             const volatile ShaderRegister* regs);
 int KYTY_SYSV_ABI   GraphicsSetShRegIndirectPatchSetAddress(uint32_t*                      cmd,
@@ -183,6 +185,7 @@ uint32_t KYTY_SYSV_ABI  GraphicsAcbAcquireMemGetSize();
 uint32_t* KYTY_SYSV_ABI GraphicsAcbCondExec(CommandBuffer* buf, const volatile uint32_t* address,
                                             uint32_t num_dwords);
 uint32_t KYTY_SYSV_ABI  GraphicsAcbCondExecGetSize();
+uint32_t KYTY_SYSV_ABI  GraphicsAcbJumpGetSize();
 uint32_t* KYTY_SYSV_ABI GraphicsAcbWaitRegMem(CommandBuffer* buf, uint8_t size,
                                               uint8_t compare_function, uint8_t cache_policy,
                                               const volatile void* address, uint64_t reference,
@@ -241,6 +244,7 @@ uint32_t KYTY_SYSV_ABI  GraphicsGetPacketSize(uint32_t* packet);
 int KYTY_SYSV_ABI       GraphicsSetPacketPredication(uint32_t* packet, uint32_t predication);
 int KYTY_SYSV_ABI       GraphicsSetRangePredication(uint32_t* start, const volatile uint32_t* end,
                                                     uint32_t predication);
+int KYTY_SYSV_ABI       GraphicsRewindPatchSetRewindState(uint32_t* cmd, uint8_t state);
 int KYTY_SYSV_ABI       GraphicsCondExecPatchSetEnd(uint32_t* cmd, const volatile uint32_t* buffer);
 int KYTY_SYSV_ABI       GraphicsCondExecPatchSetCommandAddress(uint32_t*                cmd,
                                                                const volatile uint32_t* command);

src/libs/audio.cpp

@@ -364,7 +364,12 @@ bool Audio::QueueSdlAudio(PortOut* port, const void* data, bool blocking) {
 	}
 
 	if (blocking) {
-		const auto min_queued_size = queue_size * 2u;
+		constexpr uint64_t target_latency_us = 40000;
+		const auto buffer_us = port->freq != 0 ? (1000000ULL * port->samples_num) / port->freq : 0;
+		const auto buffers =
+		    buffer_us != 0 ? static_cast<uint32_t>((target_latency_us + buffer_us - 1) / buffer_us)
+		                   : 2u;
+		const auto min_queued_size = queue_size * std::clamp(buffers, 2u, 16u);
 		const auto wait_start      = LibKernel::KernelGetProcessTime();
 		while (SDL_GetQueuedAudioSize(port->audio_device) > min_queued_size) {
 			if (LibKernel::KernelGetProcessTime() - wait_start > 200000) {
@@ -511,7 +516,16 @@ uint32_t Audio::AudioOutOutputs(OutputParam* params, uint32_t num, bool blocking
 		max_wait_time      = (wait_time > max_wait_time ? wait_time : max_wait_time);
 	}
 
-	if (blocking && max_wait_time != 0) {
+	bool all_ports_have_device = true;
+	for (uint32_t i = 0; i < num; i++) {
+		if (m_out_ports[params[i].handle.GetId()].audio_device == 0) {
+			all_ports_have_device = false;
+			break;
+		}
+	}
+
+	// Device-backed ports are paced by the SDL queue above.
+	if (blocking && max_wait_time != 0 && !all_ports_have_device) {
 		Common::Thread::SleepMicro(max_wait_time);
 	}
 

src/libs/libAudio2.cpp

@@ -336,18 +336,6 @@ static AudioOut2PortStateEntry* audioout2_find_port_locked(AudioOut2PortHandle p
 	return nullptr;
 }
 
-static uint32_t audioout2_context_grains(AudioOut2ContextHandle ctx) {
-	uint32_t samples_num = 512;
-
-	g_audioout2_context_mutex.Lock();
-	if (auto* state = audioout2_find_context_locked(ctx); state != nullptr) {
-		samples_num = (state->num_grains == 0 ? 512u : state->num_grains);
-	}
-	g_audioout2_context_mutex.Unlock();
-
-	return samples_num;
-}
-
 static void audioout2_queue_context_audio(AudioOut2ContextHandle ctx, bool blocking) {
 	std::vector<AudioInternal::OutputParam> params;
 	params.reserve(AudioInternal::OUT_PORTS_MAX);
@@ -455,6 +443,12 @@ int KYTY_SYSV_ABI AudioOut2ContextDestroy(AudioOut2ContextHandle ctx) {
 	PRINT_NAME();
 	LOGF("\t ctx = 0x%016" PRIx64 "\n", ctx);
 
+	g_audioout2_context_mutex.Lock();
+	if (auto* state = audioout2_find_context_locked(ctx); state != nullptr) {
+		*state = AudioOut2ContextState {};
+	}
+	g_audioout2_context_mutex.Unlock();
+
 	std::array<int, 256> audio_handles {};
 	size_t               audio_handles_num = 0;
 
@@ -473,12 +467,6 @@ int KYTY_SYSV_ABI AudioOut2ContextDestroy(AudioOut2ContextHandle ctx) {
 		audioout2_close_audio_handle(audio_handles[i]);
 	}
 
-	g_audioout2_context_mutex.Lock();
-	if (auto* state = audioout2_find_context_locked(ctx); state != nullptr) {
-		*state = AudioOut2ContextState {};
-	}
-	g_audioout2_context_mutex.Unlock();
-
 	return OK;
 }
 
@@ -559,30 +547,45 @@ int KYTY_SYSV_ABI AudioOut2PortCreate(AudioOut2ContextHandle ctx, const AudioOut
 	EXIT_NOT_IMPLEMENTED(params == nullptr);
 	EXIT_NOT_IMPLEMENTED(port == nullptr);
 
-	const auto next_port = g_audioout2_next_port.fetch_add(1, std::memory_order_relaxed);
+	const auto next_port    = g_audioout2_next_port.fetch_add(1, std::memory_order_relaxed);
+	const auto audio_format = audioout2_data_format_to_audio_format(params->data_format);
+	const auto audio_type   = audioout2_port_type_to_audio_out_type(params->port_type);
+
+	g_audioout2_context_mutex.Lock();
+	const auto* context_state = audioout2_find_context_locked(ctx);
+	if (context_state == nullptr) {
+		g_audioout2_context_mutex.Unlock();
+		return AUDIO_OUT2_ERROR_INVALID_PARAM;
+	}
+	const auto samples_num = context_state->num_grains == 0 ? 512u : context_state->num_grains;
 
 	g_audioout2_port_mutex.Lock();
-	auto* port_state = audioout2_find_port_locked(0);
-	if (port_state == nullptr) {
-		for (auto& candidate: g_audioout2_ports) {
-			if (!candidate.used) {
-				port_state = &candidate;
-				break;
-			}
+	AudioOut2PortStateEntry* port_state = nullptr;
+	for (auto& candidate: g_audioout2_ports) {
+		if (!candidate.used) {
+			port_state = &candidate;
+			break;
 		}
 	}
+	if (port_state != nullptr) {
+		*port_state               = AudioOut2PortStateEntry {};
+		port_state->used          = true;
+		port_state->handle        = next_port;
+		port_state->context       = ctx;
+		port_state->port_type     = params->port_type;
+		port_state->data_format   = params->data_format;
+		port_state->sampling_freq = params->sampling_freq;
+		port_state->samples_num   = samples_num;
+		port_state->audio_format  = audio_format;
+	}
 	g_audioout2_port_mutex.Unlock();
+	g_audioout2_context_mutex.Unlock();
 
-	if (next_port > g_audioout2_ports.size() || port_state == nullptr) {
+	if (port_state == nullptr) {
 		return AUDIO_OUT2_ERROR_PORT_FULL;
 	}
 
-	*port = next_port;
-
-	const auto samples_num  = audioout2_context_grains(ctx);
-	const auto audio_format = audioout2_data_format_to_audio_format(params->data_format);
-	const auto audio_type   = audioout2_port_type_to_audio_out_type(params->port_type);
-	int        audio_handle = 0;
+	int audio_handle = 0;
 
 	if (audio_format != AudioInternal::Format::Unknown &&
 	    !audioout2_port_type_is_object(params->port_type)) {
@@ -591,17 +594,17 @@ int KYTY_SYSV_ABI AudioOut2PortCreate(AudioOut2ContextHandle ctx, const AudioOut
 	}
 
 	g_audioout2_port_mutex.Lock();
-	*port_state               = AudioOut2PortStateEntry {};
-	port_state->used          = true;
-	port_state->handle        = *port;
-	port_state->context       = ctx;
-	port_state->port_type     = params->port_type;
-	port_state->data_format   = params->data_format;
-	port_state->sampling_freq = params->sampling_freq;
-	port_state->samples_num   = samples_num;
-	port_state->audio_format  = audio_format;
-	port_state->audio_handle  = audio_handle;
+	const bool reserved = port_state->used && port_state->handle == next_port;
+	if (reserved) {
+		port_state->audio_handle = audio_handle;
+	}
 	g_audioout2_port_mutex.Unlock();
+	if (!reserved) {
+		audioout2_close_audio_handle(audio_handle);
+		return AUDIO_OUT2_ERROR_INVALID_PARAM;
+	}
+
+	*port = next_port;
 
 	if (next_port <= 16 || (next_port % 600) == 0) {
 		PRINT_NAME();

src/libs/libC.cpp

@@ -633,6 +633,15 @@ LIB_VERSION("LibcInternalExt", 1, "LibcInternal", 1, 1);
 static uint64_t g_mspace_atomic_id_mask = 0;
 static uint64_t g_mstate_table[64]      = {0};
 
+using thread_atexit_destructor_t = KYTY_SYSV_ABI void (*)(void*);
+
+struct ThreadAtexitDestructor {
+	thread_atexit_destructor_t destructor;
+	void*                      object;
+};
+
+static thread_local std::vector<ThreadAtexitDestructor> g_thread_atexit_destructors;
+
 struct Info {
 	uint64_t  size;
 	uint32_t  unknown1;
@@ -650,25 +659,29 @@ void KYTY_SYSV_ABI LibcHeapGetTraceInfo(Info* info) {
 	info->mstate_table          = g_mstate_table;
 }
 
-uint64_t KYTY_SYSV_ABI LibcInternalExtUnknownQBS714Jr3g(uint64_t arg0, uint64_t arg1, uint64_t arg2,
-                                                        uint64_t arg3, uint64_t arg4,
-                                                        uint64_t arg5) {
+int KYTY_SYSV_ABI LibcInternalExtCxaThreadAtexit(thread_atexit_destructor_t destructor, void* object,
+                                                 void* /*module_id*/) {
 	PRINT_NAME();
 
-	LOGF("\t arg0 = 0x%016" PRIx64 "\n"
-	     "\t arg1 = 0x%016" PRIx64 "\n"
-	     "\t arg2 = 0x%016" PRIx64 "\n"
-	     "\t arg3 = 0x%016" PRIx64 "\n"
-	     "\t arg4 = 0x%016" PRIx64 "\n"
-	     "\t arg5 = 0x%016" PRIx64 "\n",
-	     arg0, arg1, arg2, arg3, arg4, arg5);
+	g_thread_atexit_destructors.push_back({destructor, object});
 
 	return 0;
 }
 
+void RunThreadAtexitDestructors() {
+	while (!g_thread_atexit_destructors.empty()) {
+		auto destructor = g_thread_atexit_destructors.back();
+		g_thread_atexit_destructors.pop_back();
+
+		if (destructor.destructor != nullptr) {
+			destructor.destructor(destructor.object);
+		}
+	}
+}
+
 LIB_DEFINE(InitLibcInternalExt_1) {
 	LIB_FUNC("NWtTN10cJzE", LibcInternalExt::LibcHeapGetTraceInfo);
-	LIB_FUNC("qBS714-Jr3g", LibcInternalExt::LibcInternalExtUnknownQBS714Jr3g);
+	LIB_FUNC("qBS714-Jr3g", LibcInternalExt::LibcInternalExtCxaThreadAtexit);
 }
 
 } // namespace LibcInternalExt

src/libs/libGraphicsDriver.cpp

@@ -159,6 +159,7 @@ LIB_DEFINE(InitGraphicsDriver_1) {
 	LIB_FUNC("f3dg2CSgRKY", Gen5::GraphicsCreateShader);
 	LIB_FUNC("dolOmWH+huQ", Gen5::GraphicsUnknownGetFusedShaderSize);
 	LIB_FUNC("fd5Bp5tGTgo", Gen5::GraphicsUnknownFuseShaderHalves);
+	LIB_FUNC("nApJjpKNBl4", Gen5::GraphicsUnknownNApJjpKNBl4);
 	LIB_FUNC("vcmNN+AAXnY", Gen5::GraphicsSetCxRegIndirectPatchSetAddress);
 	LIB_FUNC("Qrj4c+61z4A", Gen5::GraphicsSetShRegIndirectPatchSetAddress);
 	LIB_FUNC("6lNcCp+fxi4", Gen5::GraphicsSetUcRegIndirectPatchSetAddress);
@@ -207,6 +208,7 @@ LIB_DEFINE(InitGraphicsDriver_1) {
 	LIB_FUNC("ewobAQeMo5k", Gen5::GraphicsAcbAcquireMemGetSize);
 	LIB_FUNC("qyM2bxYFPAk", Gen5::GraphicsAcbCondExec);
 	LIB_FUNC("ozKzBP4aki4", Gen5::GraphicsAcbCondExecGetSize);
+	LIB_FUNC("b-oySn+G2tE", Gen5::GraphicsAcbJumpGetSize);
 	LIB_FUNC("htn36gPnBk4", Gen5::GraphicsAcbWaitRegMem);
 	LIB_FUNC("-RnpfpxIhec", Gen5::GraphicsAcbDmaData);
 	LIB_FUNC("qzMN2XKGA4k", Gen5::GraphicsAcbCopyData);
@@ -282,6 +284,7 @@ LIB_DEFINE(InitGraphicsDriver_1) {
 	LIB_FUNC("YWTKOju587o", Gen5::GraphicsCondExecPatchSetCommandAddress);
 	LIB_FUNC("k-JpyR2dYAM", Gen5::GraphicsCondExecPatchSetEnd);
 	LIB_FUNC("3ZWa3AoyWZQ", Gen5::GraphicsCondExecPatchSetCommandAddress);
+	LIB_FUNC("ziVA3whp3p4", Gen5::GraphicsRewindPatchSetRewindState);
 	LIB_FUNC("YUeqkyT7mEQ", Gen5::GraphicsDcbSetFlip);
 }
 

src/libs/libKernel.cpp

@@ -2021,6 +2021,12 @@ int64_t KYTY_SYSV_ABI fstat(int d, LibKernel::FileSystem::FileStat* sb) {
 	return POSIX_N_CALL(LibKernel::FileSystem::KernelFstat(d, sb));
 }
 
+int KYTY_SYSV_ABI ftruncate(int d, int64_t length) {
+	PRINT_NAME();
+
+	return POSIX_CALL(LibKernel::FileSystem::KernelFtruncate(d, length));
+}
+
 int KYTY_SYSV_ABI socket(int family, int type, int protocol) {
 	PRINT_NAME();
 	return Network::Net::Socket(family, type, protocol);
@@ -2159,6 +2165,7 @@ LIB_DEFINE(InitLibKernel_1_Posix) {
 	LIB_FUNC("yS8U2TGCe1A", nanosleep);
 	LIB_FUNC("E6ao34wPw+U", stat);
 	LIB_FUNC("JGMio+21L4c", mkdir);
+	LIB_FUNC("ih4CD9-gghM", Posix::ftruncate);
 	LIB_FUNC("pDuPEf3m4fI", Posix::sem_init);
 	LIB_FUNC("cDW233RAwWo", Posix::sem_destroy);
 	LIB_FUNC("YCV5dGGBcCo", Posix::sem_wait);

src/libs/libNet.cpp

@@ -97,6 +97,10 @@ int KYTY_SYSV_ABI NetShutdown(int s, int how) {
 	return FinishSocketCall(Net::Shutdown(s, how));
 }
 
+int KYTY_SYSV_ABI NetGetsockname(int s, void* addr, uint32_t* addrlen) {
+	return FinishSocketCall(Net::Getsockname(s, addr, addrlen));
+}
+
 int KYTY_SYSV_ABI NetPoolCreate(const char* name, int size, int flags) {
 	return NET_CALL(Net::NetPoolCreate(name, size, flags));
 }
@@ -196,6 +200,7 @@ LIB_DEFINE(InitNet_1_Net) {
 	LIB_FUNC("v6M4txecCuo", LibNet::NetEtherNtostr);
 	LIB_FUNC("6Oc0bLsIYe0", LibNet::NetGetMacAddress);
 	LIB_FUNC("hLuXdjHnhiI", LibNet::NetGetSockInfo);
+	LIB_FUNC("hoOAofhhRvE", LibNet::NetGetsockname);
 	LIB_FUNC("SF47kB2MNTo", LibNet::NetEpollCreate);
 	LIB_FUNC("ZVw46bsasAk", LibNet::NetEpollControl);
 	LIB_FUNC("drjIbDbA7UQ", LibNet::NetEpollWait);

src/libs/libRtc.cpp

@@ -4,6 +4,7 @@
 #include "libs/libs.h"
 #include "loader/symbolDatabase.h"
 
+#include <chrono>
 #include <cstdint>
 #include <cstdio>
 #include <limits>
@@ -247,20 +248,12 @@ static int KYTY_SYSV_ABI RtcGetCurrentTick(RtcTick* tick) {
 		return RTC_ERROR_DATETIME_UNINITIALIZED;
 	}
 
-	const auto now  = Common::DateTime::FromSystemUTC();
-	const auto date = now.GetDate();
-	const auto tod  = now.GetTime();
-
-	RtcDateTime time {};
-	time.year        = static_cast<uint16_t>(date.Year());
-	time.month       = static_cast<uint16_t>(date.Month());
-	time.day         = static_cast<uint16_t>(date.Day());
-	time.hour        = static_cast<uint16_t>(tod.Hour24());
-	time.minute      = static_cast<uint16_t>(tod.Minute());
-	time.second      = static_cast<uint16_t>(tod.Second());
-	time.microsecond = static_cast<uint32_t>(tod.Msec() * 1000);
-
-	return RtcGetTick(&time, tick);
+	const auto now_us =
+	    static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::microseconds>(
+	                              std::chrono::system_clock::now().time_since_epoch())
+	                              .count());
+	tick->tick = RTC_UNIX_EPOCH_TICKS + now_us;
+	return OK;
 }
 
 static int KYTY_SYSV_ABI RtcGetCurrentNetworkTick(RtcTick* tick) {

src/libs/libSystemService.cpp

@@ -1,6 +1,7 @@
 #include "common/abi.h"
 #include "common/assert.h"
 #include "common/common.h"
+#include "common/emulatorConfig.h"
 #include "common/logging/log.h"
 #include "common/stringUtils.h"
 #include "libs/errno.h"
@@ -25,37 +26,6 @@ namespace SystemService {
 [[maybe_unused]] constexpr int PARAM_ID_SCREEN_READER       = 208;
 [[maybe_unused]] constexpr int PARAM_ID_ENTER_BUTTON_ASSIGN = 1000;
 
-[[maybe_unused]] constexpr int PARAM_LANG_JAPANESE      = 0;
-[[maybe_unused]] constexpr int PARAM_LANG_ENGLISH_US    = 1;
-[[maybe_unused]] constexpr int PARAM_LANG_FRENCH        = 2;
-[[maybe_unused]] constexpr int PARAM_LANG_SPANISH       = 3;
-[[maybe_unused]] constexpr int PARAM_LANG_GERMAN        = 4;
-[[maybe_unused]] constexpr int PARAM_LANG_ITALIAN       = 5;
-[[maybe_unused]] constexpr int PARAM_LANG_DUTCH         = 6;
-[[maybe_unused]] constexpr int PARAM_LANG_PORTUGUESE_PT = 7;
-[[maybe_unused]] constexpr int PARAM_LANG_RUSSIAN       = 8;
-[[maybe_unused]] constexpr int PARAM_LANG_KOREAN        = 9;
-[[maybe_unused]] constexpr int PARAM_LANG_CHINESE_T     = 10;
-[[maybe_unused]] constexpr int PARAM_LANG_CHINESE_S     = 11;
-[[maybe_unused]] constexpr int PARAM_LANG_FINNISH       = 12;
-[[maybe_unused]] constexpr int PARAM_LANG_SWEDISH       = 13;
-[[maybe_unused]] constexpr int PARAM_LANG_DANISH        = 14;
-[[maybe_unused]] constexpr int PARAM_LANG_NORWEGIAN     = 15;
-[[maybe_unused]] constexpr int PARAM_LANG_POLISH        = 16;
-[[maybe_unused]] constexpr int PARAM_LANG_PORTUGUESE_BR = 17;
-[[maybe_unused]] constexpr int PARAM_LANG_ENGLISH_GB    = 18;
-[[maybe_unused]] constexpr int PARAM_LANG_TURKISH       = 19;
-[[maybe_unused]] constexpr int PARAM_LANG_SPANISH_LA    = 20;
-[[maybe_unused]] constexpr int PARAM_LANG_ARABIC        = 21;
-[[maybe_unused]] constexpr int PARAM_LANG_FRENCH_CA     = 22;
-[[maybe_unused]] constexpr int PARAM_LANG_CZECH         = 23;
-[[maybe_unused]] constexpr int PARAM_LANG_HUNGARIAN     = 24;
-[[maybe_unused]] constexpr int PARAM_LANG_GREEK         = 25;
-[[maybe_unused]] constexpr int PARAM_LANG_ROMANIAN      = 26;
-[[maybe_unused]] constexpr int PARAM_LANG_THAI          = 27;
-[[maybe_unused]] constexpr int PARAM_LANG_VIETNAMESE    = 28;
-[[maybe_unused]] constexpr int PARAM_LANG_INDONESIAN    = 29;
-
 [[maybe_unused]] constexpr int PARAM_DATE_FORMAT_YYYYMMDD = 0;
 [[maybe_unused]] constexpr int PARAM_DATE_FORMAT_DDMMYYYY = 1;
 [[maybe_unused]] constexpr int PARAM_DATE_FORMAT_MMDDYYYY = 2;
@@ -121,7 +91,7 @@ static int KYTY_SYSV_ABI SystemServiceParamGetInt(int param_id, int* value) {
 	int v = 0;
 
 	switch (param_id) {
-		case PARAM_ID_LANG: v = PARAM_LANG_ENGLISH_US; break;
+		case PARAM_ID_LANG: v = static_cast<int>(Config::GetConsoleLanguage()); break;
 		case PARAM_ID_DATE_FORMAT: v = PARAM_DATE_FORMAT_DDMMYYYY; break;
 		case PARAM_ID_TIME_FORMAT: v = PARAM_TIME_FORMAT_24HOUR; break;
 		case PARAM_ID_TIME_ZONE: v = +180; break;

src/loader/runtimeLinker.cpp

@@ -148,7 +148,7 @@ static bool PatchGuestMemory64(uint64_t vaddr, uint64_t value) {
 }
 
 static uint64_t AllocateUnresolvedImportThunk(uint64_t record_id) {
-	constexpr uint64_t thunk_size = 162;
+	constexpr uint64_t thunk_size = 165;
 
 	if (g_unresolved_stub_thunk_pages.empty() ||
 	    g_unresolved_stub_thunk_offset + thunk_size > UNRESOLVED_STUB_PAGE_SIZE) {
@@ -254,6 +254,10 @@ static uint64_t AllocateUnresolvedImportThunk(uint64_t record_id) {
 	emit(0x41);
 	emit(0xff);
 	emit(0xe3); // jmp r11
+	// Match the integer fallback for floating-point return values.
+	emit(0x0f);
+	emit(0x57);
+	emit(0xc0); // xorps xmm0, xmm0
 	emit(0x31);
 	emit(0xc0); // xor eax, eax
 	emit(0xc3); // ret
@@ -356,9 +360,10 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
 	auto* func = reinterpret_cast<entry_func_t>(addr);
 
 	if (stack_top != nullptr) {
-		const auto guest_rsp =
+		const auto aligned_stack_top =
 		    reinterpret_cast<uintptr_t>(stack_top) & ~static_cast<uintptr_t>(0x0f);
-		const auto guest_rbp = guest_rsp - 4u * sizeof(uint64_t);
+		const auto guest_rsp = aligned_stack_top - 2u * sizeof(uintptr_t);
+		const auto guest_rbp = guest_rsp;
 
 		auto* guest_root_frame = reinterpret_cast<uintptr_t*>(guest_rbp);
 		guest_root_frame[0]    = 0;
@@ -503,6 +508,13 @@ struct MainEntryStackTestState {
 static KYTY_SYSV_ABI void TestMainEntryStackCallback(EntryParams* params,
                                                      atexit_func_t /*atexit_func*/) {
 	auto* state = reinterpret_cast<MainEntryStackTestState*>(const_cast<char*>(params->argv[0]));
+	asm volatile("pushq %%r15\n\t"
+	             "pushq %%r14\n\t"
+	             "popq %%r14\n\t"
+	             "popq %%r15\n\t"
+	             :
+	             :
+	             : "memory");
 	asm volatile("movq %%rsp, %0" : "=r"(state->rsp) : : "memory");
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	asm volatile("movq %%gs:0x08, %0\n\t"
@@ -525,6 +537,8 @@ bool TestMainEntryUsesGuestStack() {
 	MainEntryStackTestState state {};
 	EntryParams             params {};
 	params.argv[0] = reinterpret_cast<const char*>(&state);
+	std::memset(reinterpret_cast<void*>(stack_base), 0xcd, stack_size);
+	auto* root_frame = reinterpret_cast<const uintptr_t*>(stack_base + stack_size) - 2;
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	uintptr_t original_teb_stack_base  = 0;
@@ -554,9 +568,10 @@ bool TestMainEntryUsesGuestStack() {
 	constexpr bool teb_ok = true;
 #endif
 
-	const bool rsp_ok = state.rsp >= stack_base && state.rsp < stack_base + stack_size;
-	const bool freed  = Libs::LibKernel::Memory::FreeGuestMemory(stack_base, stack_size);
-	return state.called && rsp_ok && teb_ok && freed;
+	const bool rsp_ok  = state.rsp >= stack_base && state.rsp < stack_base + stack_size;
+	const bool root_ok = root_frame[0] == 0 && root_frame[1] == 0;
+	const bool freed   = Libs::LibKernel::Memory::FreeGuestMemory(stack_base, stack_size);
+	return state.called && rsp_ok && root_ok && teb_ok && freed;
 }
 
 bool TestModuleRelocationUsesWritableHostMapping() {

src/main.cpp

@@ -10,6 +10,7 @@
 #include "emulator.h"
 #include "kytyGitVersion.h"
 
+#include <charconv>
 #include <cstdio>
 #include <fmt/format.h>
 
@@ -47,6 +48,7 @@ static void PrintUsage() {
 	::printf("  --screen-width <num>                 Window width. Default: 1280.\n");
 	::printf("  --screen-height <num>                Window height. Default: 720.\n");
 	::printf("  --vblank-frequency <num>             Virtual vblank frequency. Default: 60.\n");
+	::printf("  --console-language <0-29>            Console language. Default: 1 (English US).\n");
 	::printf("  --vulkan-validation <true|false>     Enable Vulkan validation.\n");
 	::printf("  --shader-validation <true|false>     Enable shader validation.\n");
 	::printf("  --shader-optimization-type <value>   None, Size, or Performance.\n");
@@ -59,8 +61,6 @@ static void PrintUsage() {
 	::printf("  --printf-output-file <path>          Guest printf output file.\n");
 	::printf("  --profiler-direction <value>         None or Network.\n");
 	::printf("  --spirv-debug-printf <true|false>    Enable SPIR-V debug printf.\n");
-	::printf("  --ngg-rectlist-draw <true|false>     Draw rect-list auto draws using the NGG "
-	         "4-vertex path.\n");
 	::printf(
 	    "  --readback-linear-images <true|false> Read back writable linear images on submit.\n");
 	::printf("  --rd                                 Enable RenderDoc capture.\n");
@@ -103,6 +103,17 @@ static bool ParseEnum(const std::string& value, E& out) {
 	return true;
 }
 
+static bool ParseConsoleLanguage(const std::string& value, uint32_t& out) {
+	uint32_t language = 0;
+	auto [end, error] = std::from_chars(value.data(), value.data() + value.size(), language);
+	if (error != std::errc {} || end != value.data() + value.size() ||
+	    language > Config::MAX_CONSOLE_LANGUAGE) {
+		return false;
+	}
+	out = language;
+	return true;
+}
+
 static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_help) {
 	show_help = false;
 
@@ -169,6 +180,11 @@ static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_he
 			const int32_t vblank_frequency = Common::ToInt32(value);
 			options.config.vblank_frequency =
 			    static_cast<uint32_t>(vblank_frequency < 0 ? 0 : vblank_frequency);
+		} else if (arg == "--console-language") {
+			if (!ParseConsoleLanguage(value, options.config.console_language)) {
+				::printf("invalid console language: %s\n", value.c_str());
+				return false;
+			}
 		} else if (arg == "--vulkan-validation") {
 			if (!ParseBool(value, options.config.vulkan_validation_enabled)) {
 				::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str());
@@ -220,11 +236,6 @@ static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_he
 				::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str());
 				return false;
 			}
-		} else if (arg == "--ngg-rectlist-draw") {
-			if (!ParseBool(value, options.config.ngg_rectlist_draw_enabled)) {
-				::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str());
-				return false;
-			}
 		} else if (arg == "--readback-linear-images") {
 			if (!ParseBool(value, options.config.readback_linear_images)) {
 				::printf("invalid boolean for %s: %s\n", arg.c_str(), value.c_str());

tests/AudioOut2PortTests.cpp

@@ -0,0 +1,249 @@
+#include "libs/audio.h"
+#include "libs/audio_internal.h"
+#include "libs/errno.h"
+
+#include <algorithm>
+#include <atomic>
+#include <condition_variable>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <mutex>
+#include <thread>
+#include <vector>
+
+namespace {
+
+namespace AudioOut2 = Libs::Audio::AudioOut2;
+
+std::mutex              g_device_mutex;
+std::condition_variable g_device_cv;
+std::vector<int>        g_live_devices;
+int                     g_next_device  = 1;
+int                     g_open_waiters = 0;
+bool                    g_block_opens  = false;
+
+void Check(bool value, const char* text) {
+	if (!value) {
+		std::fprintf(stderr, "AudioOut2PortTests: failed: %s\n", text);
+		std::abort();
+	}
+}
+
+struct PortParam {
+	uint16_t port_type;
+	uint16_t pad;
+	uint32_t data_format;
+	uint32_t sampling_freq;
+	uint32_t flags;
+	uint64_t user_handle;
+	uint32_t reserved[10];
+};
+
+struct ContextParam {
+	uint32_t max_ports;
+	uint32_t max_object_ports;
+	uint32_t guarantee_object_ports;
+	uint32_t queue_depth;
+	uint32_t num_grains;
+	uint32_t flags;
+	uint32_t reserved[10];
+};
+
+struct PortState {
+	uint16_t output;
+	uint8_t  num_channels;
+	uint8_t  pad1;
+	int16_t  volume;
+	uint16_t reroute_counter;
+	uint32_t flags;
+	uint32_t pad2;
+	uint64_t reserved[6];
+};
+
+const auto* AsParam(const PortParam* param) {
+	return reinterpret_cast<const AudioOut2::AudioOut2PortParam*>(param);
+}
+
+const auto* AsParam(const ContextParam* param) {
+	return reinterpret_cast<const AudioOut2::AudioOut2ContextParam*>(param);
+}
+
+auto* AsState(PortState* state) {
+	return reinterpret_cast<AudioOut2::AudioOut2PortState*>(state);
+}
+
+PortParam MakeParam(uint32_t data_format = 0x200) {
+	PortParam param {};
+	param.data_format   = data_format;
+	param.sampling_freq = 48000;
+	return param;
+}
+
+AudioOut2::AudioOut2ContextHandle CreateContext() {
+	ContextParam param {};
+	param.queue_depth                         = 4;
+	param.num_grains                          = 512;
+	AudioOut2::AudioOut2ContextHandle context = 0;
+	Check(AudioOut2::AudioOut2ContextCreate(AsParam(&param), nullptr, 0, &context) == OK,
+	      "context create failed");
+	return context;
+}
+
+void BlockDeviceOpens() {
+	std::lock_guard lock(g_device_mutex);
+	g_open_waiters = 0;
+	g_block_opens  = true;
+}
+
+void WaitForDeviceOpens(int count) {
+	std::unique_lock lock(g_device_mutex);
+	g_device_cv.wait(lock, [count]() { return g_open_waiters >= count; });
+}
+
+void ReleaseDeviceOpens() {
+	std::lock_guard lock(g_device_mutex);
+	g_block_opens = false;
+	g_device_cv.notify_all();
+}
+
+int LiveDeviceCount() {
+	std::lock_guard lock(g_device_mutex);
+	return static_cast<int>(g_live_devices.size());
+}
+
+void TestSlotReuse() {
+	const auto context = CreateContext();
+	const auto param   = MakeParam();
+	for (int i = 0; i < 300; i++) {
+		AudioOut2::AudioOut2PortHandle port = 0;
+		Check(AudioOut2::AudioOut2PortCreate(context, AsParam(&param), &port) == OK,
+		      "port slot was not reusable");
+		Check(port != 0, "port handle is zero");
+		AudioOut2::AudioOut2PortDestroy(port);
+	}
+	AudioOut2::AudioOut2ContextDestroy(context);
+}
+
+void TestFullTableRecovers() {
+	const auto                                  context = CreateContext();
+	const auto                                  param   = MakeParam();
+	std::vector<AudioOut2::AudioOut2PortHandle> ports;
+	ports.reserve(256);
+
+	for (int i = 0; i < 256; i++) {
+		AudioOut2::AudioOut2PortHandle port = 0;
+		Check(AudioOut2::AudioOut2PortCreate(context, AsParam(&param), &port) == OK,
+		      "port table filled early");
+		ports.push_back(port);
+	}
+
+	AudioOut2::AudioOut2PortHandle overflow = 0;
+	Check(AudioOut2::AudioOut2PortCreate(context, AsParam(&param), &overflow) != OK,
+	      "full port table accepted another port");
+
+	for (auto port: ports) {
+		AudioOut2::AudioOut2PortDestroy(port);
+	}
+
+	AudioOut2::AudioOut2PortHandle port = 0;
+	Check(AudioOut2::AudioOut2PortCreate(context, AsParam(&param), &port) == OK,
+	      "port table did not recover");
+	AudioOut2::AudioOut2PortDestroy(port);
+	AudioOut2::AudioOut2ContextDestroy(context);
+}
+
+void TestConcurrentCreates() {
+	constexpr int                               thread_count = 8;
+	const auto                                  context      = CreateContext();
+	const auto                                  param        = MakeParam(0x800);
+	std::vector<AudioOut2::AudioOut2PortHandle> ports(thread_count);
+	std::vector<int>                            results(thread_count);
+	std::vector<std::thread>                    threads;
+
+	BlockDeviceOpens();
+	for (int i = 0; i < thread_count; i++) {
+		threads.emplace_back([&, i]() {
+			results[i] = AudioOut2::AudioOut2PortCreate(context, AsParam(&param), &ports[i]);
+		});
+	}
+	WaitForDeviceOpens(thread_count);
+	ReleaseDeviceOpens();
+	for (auto& thread: threads) {
+		thread.join();
+	}
+
+	for (int i = 0; i < thread_count; i++) {
+		Check(results[i] == OK, "concurrent port create failed");
+		PortState state {};
+		AudioOut2::AudioOut2PortGetState(ports[i], AsState(&state));
+		Check(state.num_channels == 8, "concurrent create lost its reserved slot");
+		AudioOut2::AudioOut2PortDestroy(ports[i]);
+	}
+	Check(LiveDeviceCount() == 0, "concurrent create leaked a device");
+	AudioOut2::AudioOut2ContextDestroy(context);
+}
+
+void TestContextDestroyCancelsPendingCreate() {
+	const auto                     context = CreateContext();
+	const auto                     param   = MakeParam();
+	AudioOut2::AudioOut2PortHandle port    = 0;
+	int                            result  = OK;
+
+	BlockDeviceOpens();
+	std::thread creator(
+	    [&]() { result = AudioOut2::AudioOut2PortCreate(context, AsParam(&param), &port); });
+	WaitForDeviceOpens(1);
+	AudioOut2::AudioOut2ContextDestroy(context);
+	ReleaseDeviceOpens();
+	creator.join();
+
+	Check(result != OK, "destroyed context retained a pending port create");
+	Check(LiveDeviceCount() == 0, "cancelled port create leaked a device");
+}
+
+} // namespace
+
+namespace Libs::Audio::AudioInternal {
+
+int AudioOutOpen(int /*type*/, uint32_t /*samples_num*/, uint32_t /*freq*/, Format /*format*/) {
+	std::unique_lock lock(g_device_mutex);
+	const int        handle = g_next_device++;
+	g_live_devices.push_back(handle);
+	g_open_waiters++;
+	g_device_cv.notify_all();
+	g_device_cv.wait(lock, []() { return !g_block_opens; });
+	return handle;
+}
+
+void AudioOutClose(int handle) {
+	std::lock_guard lock(g_device_mutex);
+	const auto      it = std::find(g_live_devices.begin(), g_live_devices.end(), handle);
+	if (it != g_live_devices.end()) {
+		g_live_devices.erase(it);
+	}
+}
+
+uint32_t AudioOutOutputs(const OutputParam* /*params*/, uint32_t /*num*/, bool /*blocking*/) {
+	return 0;
+}
+
+} // namespace Libs::Audio::AudioInternal
+
+namespace Libs::LibKernel {
+
+uint64_t KYTY_SYSV_ABI KernelGetProcessTime() {
+	static std::atomic_uint64_t now {0};
+	return now.fetch_add(1000);
+}
+
+} // namespace Libs::LibKernel
+
+int main() {
+	TestSlotReuse();
+	TestFullTableRecovers();
+	TestConcurrentCreates();
+	TestContextDestroyCancelsPendingCreate();
+	std::printf("AudioOut2PortTests: all cases passed\n");
+	return 0;
+}

tests/BitArrayTests.cpp

@@ -0,0 +1,310 @@
+#include "common/bitArray.h"
+
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <utility>
+
+namespace {
+
+using Bits = Common::BitArray<128>;
+
+static_assert(sizeof(Common::BitArray<1024>) == 128);
+
+void Check(bool value, const char *message) {
+  if (!value) {
+    std::fprintf(stderr, "BitArrayTests: failed: %s\n", message);
+    std::abort();
+  }
+}
+
+void TestPointAndRangeOperations() {
+  Bits bits;
+  Check(bits.None() && !bits.Any(), "default state is not empty");
+
+  for (const auto index : {size_t{0}, size_t{63}, size_t{64}, size_t{127}}) {
+    bits.Set(index);
+    Check(bits.Get(index), "Set did not set a boundary bit");
+    bits.Unset(index);
+    Check(!bits.Get(index), "Unset did not clear a boundary bit");
+  }
+
+  bits.SetRange(60, 68);
+  for (size_t index = 0; index < 128; index++) {
+    Check(bits.Get(index) == (index >= 60 && index < 68),
+          "cross-word SetRange changed the wrong bits");
+  }
+  bits.Fill();
+  bits.UnsetRange(60, 68);
+  for (size_t index = 0; index < 128; index++) {
+    Check(bits.Get(index) == !(index >= 60 && index < 68),
+          "cross-word UnsetRange changed the wrong bits");
+  }
+  bits.Clear();
+  bits.SetRange(0, 128);
+  Check(!bits.None(), "full SetRange left the array empty");
+  bits.UnsetRange(0, 128);
+  Check(bits.None(), "full UnsetRange left set bits");
+
+  bits.Set(7);
+  bits.SetRange(9, 9);
+  bits.SetRange(0, 129);
+  bits.UnsetRange(9, 9);
+  bits.UnsetRange(0, 129);
+  Check(bits.Get(7), "invalid or empty range modified the array");
+}
+
+void TestMaskedConstructionAndBitwiseOperations() {
+  Bits source;
+  source.Fill();
+  const Bits masked(source, 31, 97);
+  for (size_t index = 0; index < 128; index++) {
+    Check(masked.Get(index) == (index >= 31 && index < 97),
+          "masked constructor retained a bit outside its range");
+  }
+  Check(Bits(source, 12, 12).None(), "empty masked constructor produced bits");
+  Check(Bits(source, 0, 129).None(),
+        "invalid masked constructor produced bits");
+
+  Bits left;
+  left.SetRange(0, 80);
+  Bits right;
+  right.SetRange(40, 120);
+  const auto exclusive = left ^ right;
+  for (size_t index = 0; index < 128; index++) {
+    const bool expected = (index < 80) != (index >= 40 && index < 120);
+    Check(exclusive.Get(index) == expected, "XOR produced the wrong bit");
+  }
+  const auto inverted = ~left;
+  for (size_t index = 0; index < 128; index++) {
+    Check(inverted.Get(index) == (index >= 80), "NOT produced the wrong bit");
+  }
+}
+
+void TestRangeDiscoveryAndIteration() {
+  Bits bits;
+  Check(bits.FirstRange() == Bits::Range{128, 128},
+        "empty FirstRange is wrong");
+  Check(bits.LastRange() == Bits::Range{0, 0}, "empty LastRange is wrong");
+
+  bits.SetRange(3, 8);
+  bits.SetRange(63, 70);
+  bits.Set(127);
+  Check(bits.FirstRange() == Bits::Range{3, 8}, "FirstRange is wrong");
+  Check(bits.FirstRangeFrom(5) == Bits::Range{5, 8},
+        "FirstRangeFrom inside a run is wrong");
+  Check(bits.FirstRangeFrom(8) == Bits::Range{63, 70},
+        "FirstRangeFrom gap is wrong");
+  Check(bits.LastRange() == Bits::Range{127, 128}, "LastRange is wrong");
+  Check(bits.LastRangeFrom(69) == Bits::Range{63, 69},
+        "LastRangeFrom inside a run is wrong");
+  Check(bits.LastRangeFrom(63) == Bits::Range{3, 8},
+        "LastRangeFrom gap is wrong");
+
+  constexpr std::array expected{Bits::Range{3, 8}, Bits::Range{63, 70},
+                                Bits::Range{127, 128}};
+  size_t range_index = 0;
+  for (const auto range : bits) {
+    Check(range_index < expected.size() && range == expected[range_index],
+          "range iterator produced the wrong run");
+    range_index++;
+  }
+  Check(range_index == expected.size(), "range iterator omitted a run");
+}
+
+void TestRandomizedDifferential() {
+  Bits bits;
+  std::array<bool, 128> reference{};
+  uint64_t random = 0x53a9'7f11'ced4'29b5ull;
+  const auto next_random = [&random] {
+    random ^= random << 13;
+    random ^= random >> 7;
+    random ^= random << 17;
+    return random;
+  };
+
+  for (size_t operation = 0; operation < 10000; operation++) {
+    const auto first = static_cast<size_t>(next_random() % 128);
+    const auto last =
+        first + 1 + static_cast<size_t>(next_random() % (128 - first));
+    if ((next_random() & 1) != 0) {
+      bits.SetRange(first, last);
+      for (auto index = first; index < last; index++) {
+        reference[index] = true;
+      }
+    } else {
+      bits.UnsetRange(first, last);
+      for (auto index = first; index < last; index++) {
+        reference[index] = false;
+      }
+    }
+
+    bool any = false;
+    for (size_t index = 0; index < reference.size(); index++) {
+      Check(bits.Get(index) == reference[index],
+            "randomized bit state diverged");
+      any |= reference[index];
+    }
+    Check(bits.Any() == any && bits.None() == !any,
+          "randomized Any/None diverged");
+
+    const auto range_start = static_cast<size_t>(next_random() % 129);
+    auto expected_first_begin = range_start;
+    while (expected_first_begin < reference.size() &&
+           !reference[expected_first_begin]) {
+      expected_first_begin++;
+    }
+    if (expected_first_begin == reference.size()) {
+      Check(bits.FirstRangeFrom(range_start) == Bits::Range{128, 128},
+            "randomized FirstRangeFrom empty suffix diverged");
+    } else {
+      auto expected_first_end = expected_first_begin;
+      while (expected_first_end < reference.size() &&
+             reference[expected_first_end]) {
+        expected_first_end++;
+      }
+      Check(bits.FirstRangeFrom(range_start) ==
+                Bits::Range{expected_first_begin, expected_first_end},
+            "randomized FirstRangeFrom diverged");
+    }
+
+    const auto range_end = static_cast<size_t>(next_random() % 129);
+    auto expected_last_end = range_end;
+    while (expected_last_end != 0 && !reference[expected_last_end - 1]) {
+      expected_last_end--;
+    }
+    if (expected_last_end == 0) {
+      Check(bits.LastRangeFrom(range_end) == Bits::Range{0, 0},
+            "randomized LastRangeFrom empty prefix diverged");
+    } else {
+      auto expected_last_begin = expected_last_end;
+      while (expected_last_begin != 0 && reference[expected_last_begin - 1]) {
+        expected_last_begin--;
+      }
+      Check(bits.LastRangeFrom(range_end) ==
+                Bits::Range{expected_last_begin, expected_last_end},
+            "randomized LastRangeFrom diverged");
+    }
+
+    const auto masked_start = static_cast<size_t>(next_random() % 129);
+    const auto masked_end =
+        masked_start +
+        static_cast<size_t>(next_random() % (129 - masked_start));
+    const Bits masked(bits, masked_start, masked_end);
+    for (size_t index = 0; index < reference.size(); index++) {
+      Check(masked.Get(index) == (index >= masked_start && index < masked_end &&
+                                  reference[index]),
+            "randomized masked constructor diverged");
+    }
+
+    size_t first_begin = 0;
+    while (first_begin < reference.size() && !reference[first_begin]) {
+      first_begin++;
+    }
+    if (first_begin == reference.size()) {
+      Check(bits.FirstRange() == Bits::Range{128, 128},
+            "randomized empty FirstRange diverged");
+      Check(bits.LastRange() == Bits::Range{0, 0},
+            "randomized empty LastRange diverged");
+    } else {
+      auto first_end = first_begin;
+      while (first_end < reference.size() && reference[first_end]) {
+        first_end++;
+      }
+      Check(bits.FirstRange() == Bits::Range{first_begin, first_end},
+            "randomized FirstRange diverged");
+
+      auto last_end = reference.size();
+      while (!reference[last_end - 1]) {
+        last_end--;
+      }
+      auto last_begin = last_end;
+      while (last_begin != 0 && reference[last_begin - 1]) {
+        last_begin--;
+      }
+      Check(bits.LastRange() == Bits::Range{last_begin, last_end},
+            "randomized LastRange diverged");
+    }
+
+    size_t expected_begin = 0;
+    for (const auto [begin, end] : bits) {
+      while (expected_begin < reference.size() && !reference[expected_begin]) {
+        expected_begin++;
+      }
+      Check(begin == expected_begin, "randomized iterator run start diverged");
+      while (expected_begin < reference.size() && reference[expected_begin]) {
+        expected_begin++;
+      }
+      Check(end == expected_begin, "randomized iterator run end diverged");
+    }
+    while (expected_begin < reference.size() && !reference[expected_begin]) {
+      expected_begin++;
+    }
+    Check(expected_begin == reference.size(),
+          "randomized iterator omitted a run");
+  }
+}
+
+void TestTrackerSizedRandomizedDifferential() {
+  using TrackerBits = Common::BitArray<1024>;
+  TrackerBits bits;
+  std::array<bool, 1024> reference{};
+  uint64_t random = 0x9e37'79b9'7f4a'7c15ull;
+  const auto next_random = [&random] {
+    random ^= random << 13;
+    random ^= random >> 7;
+    random ^= random << 17;
+    return random;
+  };
+
+  for (size_t operation = 0; operation < 4096; operation++) {
+    const auto first = static_cast<size_t>(next_random() % reference.size());
+    const auto last =
+        first + 1 +
+        static_cast<size_t>(next_random() % (reference.size() - first));
+    const bool set = (next_random() & 1) != 0;
+    if (set) {
+      bits.SetRange(first, last);
+    } else {
+      bits.UnsetRange(first, last);
+    }
+    for (auto index = first; index < last; index++) {
+      reference[index] = set;
+    }
+
+    for (size_t index = 0; index < reference.size(); index++) {
+      Check(bits.Get(index) == reference[index],
+            "tracker-sized randomized bit state diverged");
+    }
+
+    size_t expected = 0;
+    for (const auto [begin, end] : bits) {
+      while (expected < reference.size() && !reference[expected]) {
+        expected++;
+      }
+      Check(begin == expected, "tracker-sized randomized range start diverged");
+      while (expected < reference.size() && reference[expected]) {
+        expected++;
+      }
+      Check(end == expected, "tracker-sized randomized range end diverged");
+    }
+    while (expected < reference.size() && !reference[expected]) {
+      expected++;
+    }
+    Check(expected == reference.size(),
+          "tracker-sized randomized iterator omitted a run");
+  }
+}
+
+} // namespace
+
+int main() {
+  TestPointAndRangeOperations();
+  TestMaskedConstructionAndBitwiseOperations();
+  TestRangeDiscoveryAndIteration();
+  TestRandomizedDifferential();
+  TestTrackerSizedRandomizedDifferential();
+  std::puts("BitArrayTests: all cases passed");
+  return 0;
+}

tests/ImagePageTableTests.cpp

@@ -3,13 +3,15 @@
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
+#include <utility>
 #include <vector>
 
 namespace {
 
 using Owners     = std::vector<uint32_t>;
 using Table      = Libs::Graphics::MultiLevelPageTable<Owners>;
-using OwnerIndex = Libs::Graphics::MultiRangePageOwnerIndex<uint32_t>;
+using PageOwners = Libs::Graphics::InlinePageOwnerList<uint32_t, 16>;
+using OwnerTable = Libs::Graphics::MultiLevelPageTable<PageOwners, 20, 40, 10>;
 
 void Check(bool value, const char* text) {
 	if (!value) {
@@ -77,59 +79,93 @@ void TestAddressSpaceBoundaries() {
 	      "final page supports allocating and nonallocating access");
 }
 
-void TestMultiRangeRegistrationDeduplicatesPages() {
-	OwnerIndex index;
-	// Depth and stencil-like planes overlap tracking pages and share one 1 MiB bucket.
-	Check(index.Register(7, {{0x101000, 0x2800}, {0x102000, 0x3000}}),
-	      "multi-range owner registers");
-	Check(index.CoarseMembershipCount(1) == 1,
-	      "one owner is inserted once in a shared 1 MiB bucket");
-	Check(index.TrackingMembershipCount(0x102) == 1,
-	      "overlapping planes insert one 4 KiB membership");
-	Check(!index.Register(7, {{0x101000, 0x1000}}), "duplicate owner registration hard-fails");
+void TestInlineOwnerStorageAndOverflow() {
+	PageOwners owners;
+	for (uint32_t owner = 1; owner <= 18; ++owner) {
+		owners.push_back(owner);
+	}
+	Check(owners.size() == 18 && owners.front() == 1,
+	      "inline owner storage grows past its 16-owner capacity");
+	uint32_t expected = 1;
+	for (const uint32_t owner: owners) {
+		Check(owner == expected++, "overflow preserves registration order");
+	}
+	Check(owners.Erase(5) && owners.size() == 17 && !owners.Contains(5),
+	      "overflow erase removes only the requested owner");
+	Check(owners.Erase(18) && owners.size() == 16,
+	      "overflow storage shrinks back to inline capacity");
+	const std::vector<uint32_t> remaining {1, 2, 3, 4, 6, 7, 8, 9,
+	                                       10, 11, 12, 13, 14, 15, 16, 17};
+	size_t                      remaining_index = 0;
+	for (const uint32_t owner: owners) {
+		Check(owner == remaining[remaining_index++],
+		      "overflow-to-inline shrink preserves every owner");
+	}
+	Check(!owners.Erase(99), "missing inline owner is reported without mutation");
 
-	const auto owners = index.Query(0x100000, 0x10000);
-	Check(owners.size() == 1 && owners.front() == 7, "multi-page query returns an owner once");
+	PageOwners moved = std::move(owners);
+	Check(owners.empty() && moved.size() == remaining.size() && moved.front() == 1,
+	      "moving a full inline owner list leaves the source empty");
+	PageOwners partial;
+	partial.push_back(41);
+	partial.push_back(42);
+	PageOwners partial_moved = std::move(partial);
+	Check(partial.empty() && partial_moved.size() == 2 && partial_moved[1] == 42,
+	      "moving a partially populated list copies only live owners");
+	PageOwners assigned;
+	assigned.push_back(99);
+	assigned = std::move(partial_moved);
+	Check(partial_moved.empty() && assigned.size() == 2 && assigned.front() == 41,
+	      "move assignment replaces an inline list without reading inactive slots");
+	PageOwners empty;
+	PageOwners empty_moved = std::move(empty);
+	Check(empty.empty() && empty_moved.empty(), "moving an empty owner list is safe");
 }
 
-void TestSharedPageUnregisterLifecycle() {
-	OwnerIndex                               index;
-	const std::vector<OwnerIndex::ByteRange> ranges {{0x202000, 0x2000}};
-	Check(index.Register(11, ranges) && index.Register(22, ranges),
-	      "two owners register on identical pages");
-	Check(index.CoarseMembershipCount(2) == 2 && index.TrackingMembershipCount(0x202) == 2,
-	      "coarse and tracking pages retain both owners");
-
-	std::vector<OwnerIndex::ByteRange> releases;
-	Check(index.Unregister(11, releases), "first owner unregisters");
-	Check(releases.empty(), "shared tracking pages are not released with one owner remaining");
-	Check(index.Query(0x202000, 1).size() == 1 && index.Query(0x202000, 1).front() == 22,
-	      "unregistering one owner preserves the other");
-	Check(!index.Unregister(11, releases), "missing membership hard-fails without mutation");
-
-	Check(index.Unregister(22, releases), "final owner unregisters");
-	Check(releases.size() == 1 && releases.front().address == 0x202000 &&
-	          releases.front().size == 0x2000,
-	      "adjacent final-owner tracking pages return one contiguous release");
+void TestOneMiBRegistrationGranularity() {
+	static_assert(OwnerTable::kPageBits == 20);
+	OwnerTable            table;
+	OwnerTable::PageRange pages {};
+	constexpr uint64_t    range_size = 64ull * 1024 * 1024;
+	Check(OwnerTable::TryGetPageRange(0, range_size, pages), "large owner range is valid");
+	Check(pages.first == 0 && pages.last_exclusive == 64,
+	      "64 MiB registration touches exactly 64 one-MiB entries");
+	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+		table[page].push_back(7);
+	}
+	Check(table.AllocatedBucketCount() == 1,
+	      "large registration uses one sparse second-level bucket");
+	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+		auto* owners = table.Find(page);
+		Check(owners != nullptr && owners->size() == 1 && owners->front() == 7,
+		      "each touched one-MiB entry retains the owner once");
+		Check(owners->Erase(7), "large owner unregisters by coarse page");
+	}
 }
 
-void TestStrictByteFilteringAndPredicate() {
-	OwnerIndex index;
-	Check(index.Register(31, {{0x300100, 0x100}}), "first byte-disjoint owner registers");
-	Check(index.Register(32, {{0x300800, 0x100}}), "second byte-disjoint owner registers");
-	Check(index.TrackingMembershipCount(0x300) == 2,
-	      "byte-disjoint owners share one tracking page");
-	Check(index.Query(0x300400, 0x40).empty(), "page hit without byte overlap is filtered out");
-	const auto page_candidates = index.QueryCandidates(0x300400, 0x40);
-	Check(page_candidates.size() == 2,
-	      "fault candidate query retains byte-disjoint owners on the touched page");
-	const auto first = index.Query(0x300180, 0x10);
-	Check(first.size() == 1 && first.front() == 31,
-	      "strict byte overlap selects only the matching owner");
-	const auto predicate_filtered =
-	    index.Query(0x300000, 0x1000, [](uint32_t owner) { return owner == 32; });
-	Check(predicate_filtered.size() == 1 && predicate_filtered.front() == 32,
-	      "supplied predicate filters query owners");
+void TestSharedCoarsePageLifecycle() {
+	OwnerTable table;
+	auto&      owners = table[2];
+	owners.push_back(11);
+	owners.push_back(22);
+	Check(owners.size() == 2, "two images share one coarse page");
+	Check(owners.Erase(11) && owners.size() == 1 && owners.front() == 22,
+	      "unregistering one image preserves its coarse-page neighbor");
+	Check(!owners.Erase(11), "double unregister is rejected without mutation");
+	Check(owners.Erase(22) && owners.empty(), "final coarse-page owner unregisters");
+}
+
+void TestOneMiBBoundaries() {
+	OwnerTable::PageRange range {};
+	Check(OwnerTable::TryGetPageRange(0x0fffff, 2, range),
+	      "range crossing a one-MiB boundary is valid");
+	Check(range.first == 0 && range.last_exclusive == 2,
+	      "cross-boundary registration touches both coarse pages");
+	Check(OwnerTable::TryGetPageRange(OwnerTable::kAddressSpaceSize - 1, 1, range),
+	      "final guest byte maps to a coarse owner page");
+	Check(range.first == OwnerTable::kPageCount - 1 &&
+	          range.last_exclusive == OwnerTable::kPageCount,
+	      "final guest byte uses the final one-MiB page");
 }
 
 } // namespace
@@ -139,9 +175,10 @@ int main() {
 	TestCrossBucketRange();
 	TestQueriesDoNotAllocate();
 	TestAddressSpaceBoundaries();
-	TestMultiRangeRegistrationDeduplicatesPages();
-	TestSharedPageUnregisterLifecycle();
-	TestStrictByteFilteringAndPredicate();
+	TestInlineOwnerStorageAndOverflow();
+	TestOneMiBRegistrationGranularity();
+	TestSharedCoarsePageLifecycle();
+	TestOneMiBBoundaries();
 	std::printf("ImagePageTableTests: all cases passed\n");
 	return 0;
 }

tests/MemoryTrackerTests.cpp

@@ -27,7 +27,6 @@ namespace {
 
 using Libs::Graphics::MemoryTracker;
 using Libs::Graphics::PageManager;
-using Libs::Graphics::PageWatchMode;
 using Libs::Graphics::RangeSet;
 
 void Check(bool value, const char *text) {
@@ -127,6 +126,21 @@ bool IsWritable(const void *address) {
   return Protection(address) == PAGE_READWRITE;
 }
 
+uint64_t g_protection_calls = 0;
+
+struct ProtectionCall {
+  uint64_t address;
+  uint64_t size;
+  Common::VirtualMemory::Mode mode;
+};
+
+std::vector<ProtectionCall> g_protection_log;
+
+void ResetProtectionLog() {
+  g_protection_calls = 0;
+  g_protection_log.clear();
+}
+
 bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
                          Common::VirtualMemory::Mode mode) {
   uint32_t protection = PAGE_NOACCESS;
@@ -136,14 +150,14 @@ bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
     protection = PAGE_READWRITE;
   }
   DWORD old_protection = 0;
+  g_protection_calls++;
+  g_protection_log.push_back({vaddr, size, mode});
   return VirtualProtect(reinterpret_cast<void *>(vaddr), size, protection,
                         &old_protection) != 0;
 }
 
 struct TrackerHarness {
-  explicit TrackerHarness(
-      PageWatchMode gpu_watch_mode = PageWatchMode::ReadWrite)
-      : tracker(page_manager, gpu_watch_mode) {}
+  TrackerHarness() : tracker(page_manager) {}
 
   PageManager page_manager;
   MemoryTracker tracker;
@@ -287,6 +301,124 @@ void TestGpuDirtyBits() {
   Release(page_manager, memory, page_size * 2);
 }
 
+void TestExactDirtyIntervalsSharingTrackerPage() {
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto &page_manager = harness.page_manager;
+  const auto page_size = page_manager.GetPageSize();
+  auto *memory = Allocate(page_manager, 1);
+  const auto address = reinterpret_cast<uint64_t>(memory);
+
+  tracker.ForEachUploadRange(
+      address, page_size, false, [](uint64_t, uint64_t) noexcept {},
+      []() noexcept {});
+  RangeSet exact_dirty;
+  exact_dirty.Add(address + 64, 16);
+  exact_dirty.Add(address + 192, 32);
+
+  ResetProtectionLog();
+  tracker.MarkRegionAsGpuModified(address + 64, 16);
+  tracker.MarkRegionAsGpuModified(address + 192, 32);
+  Check(g_protection_calls == 1 &&
+            tracker.IsRegionGpuModified(address, page_size) &&
+            Protection(memory) == PAGE_NOACCESS,
+        "disjoint byte dirtiness duplicated the page watcher");
+
+  exact_dirty.Subtract(address + 64, 16);
+  if (exact_dirty.Intersections(address, page_size).empty()) {
+    tracker.UnmarkRegionAsGpuModified(address, page_size);
+  }
+  Check(g_protection_calls == 1 &&
+            tracker.IsRegionGpuModified(address, page_size) &&
+            Protection(memory) == PAGE_NOACCESS,
+        "draining one exact interval prematurely released its shared page");
+
+  exact_dirty.Subtract(address + 192, 32);
+  if (exact_dirty.Intersections(address, page_size).empty()) {
+    tracker.UnmarkRegionAsGpuModified(address, page_size);
+  }
+  Check(g_protection_calls == 2 &&
+            !tracker.IsRegionGpuModified(address, page_size) &&
+            Protection(memory) == PAGE_READONLY,
+        "draining the final exact interval did not release its tracker page");
+
+  tracker.UntrackMemory(address, page_size);
+  Release(page_manager, memory, page_size);
+}
+
+void TestGpuDownloadProtectionMirrors() {
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto &page_manager = harness.page_manager;
+  const auto page_size = page_manager.GetPageSize();
+  auto *memory = Allocate(page_manager, 4);
+  const auto address = reinterpret_cast<uint64_t>(memory);
+
+  tracker.ForEachUploadRange(
+      address, page_size * 4, false, [](uint64_t, uint64_t) noexcept {},
+      []() noexcept {});
+  tracker.MarkRegionAsGpuModified(address + 16, 32);
+  tracker.MarkRegionAsGpuModified(address + page_size * 2 + 16, 32);
+
+  std::vector<RangeSet::Range> visited;
+  ResetProtectionLog();
+  tracker.ForEachDownloadRange<false>(
+      address, page_size * 3,
+      [&](uint64_t range_address, uint64_t range_size) noexcept {
+        visited.push_back({range_address, range_size});
+      });
+  Check(visited.size() == 2 && visited[0].address == address &&
+            visited[0].size == page_size &&
+            visited[1].address == address + page_size * 2 &&
+            visited[1].size == page_size && g_protection_calls == 0 &&
+            tracker.IsRegionGpuModified(address, page_size * 3),
+        "non-clearing download changed protection or lost sparse ranges");
+
+  visited.clear();
+  tracker.ForEachDownloadRange<true>(
+      address + 16, 32,
+      [&](uint64_t range_address, uint64_t range_size) noexcept {
+        visited.push_back({range_address, range_size});
+      });
+  Check(visited.size() == 1 && visited[0].address == address &&
+            visited[0].size == page_size && g_protection_log.size() == 1 &&
+            g_protection_log[0].address == address &&
+            g_protection_log[0].size == page_size &&
+            g_protection_log[0].mode == Common::VirtualMemory::Mode::Read &&
+            !tracker.IsRegionGpuModified(address, page_size) &&
+            tracker.IsRegionGpuModified(address + page_size * 2, page_size) &&
+            Protection(memory) == PAGE_READONLY &&
+            Protection(memory + page_size * 2) == PAGE_NOACCESS,
+        "partial download did not preserve the CPU/GPU protection mirrors");
+
+  visited.clear();
+  ResetProtectionLog();
+  tracker.ForEachDownloadRange<true>(
+      address + 16, 32,
+      [&](uint64_t range_address, uint64_t range_size) noexcept {
+        visited.push_back({range_address, range_size});
+      });
+  Check(visited.empty() && g_protection_calls == 0 &&
+            tracker.IsRegionGpuModified(address + page_size * 2, page_size),
+        "idempotent partial download disturbed another GPU-owned page");
+
+  tracker.UnmarkRegionAsGpuModified(address, page_size * 3);
+  Check(!tracker.IsRegionGpuModified(address, page_size * 3) &&
+            Protection(memory + page_size * 2) == PAGE_READONLY,
+        "broad final unmark did not restore write-only tracking");
+  ResetProtectionLog();
+  tracker.MarkRegionAsCpuModified(address + 16, 32);
+  Check(
+      g_protection_log.size() == 1 && g_protection_log[0].address == address &&
+          g_protection_log[0].size == page_size &&
+          g_protection_log[0].mode == Common::VirtualMemory::Mode::ReadWrite &&
+          IsWritable(memory) && !IsWritable(memory + page_size),
+      "CPU-dirty transition did not release only its write watcher");
+
+  tracker.UntrackMemory(address, page_size * 4);
+  Release(page_manager, memory, page_size * 4);
+}
+
 void TestCrossRegionUpload() {
   constexpr uintptr_t base = 0x0000000200010000ull;
   constexpr uint64_t region_size = 4ull * 1024ull * 1024ull;
@@ -315,35 +447,106 @@ void TestCrossRegionUpload() {
   Release(page_manager, memory, region_size * 2);
 }
 
-void TestBackingWritePublication() {
+void TestGpuUnmarkUsesRegionMask() {
+  constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
+  constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
   TrackerHarness harness;
   auto &tracker = harness.tracker;
   auto &page_manager = harness.page_manager;
-  const auto page_size = page_manager.GetPageSize();
-  auto *memory = Allocate(page_manager, 1);
-  const auto address = reinterpret_cast<uint64_t>(memory);
+  auto *memory = Allocate(page_manager, region_size * 2 / page_size);
+  const auto allocation_base = reinterpret_cast<uint64_t>(memory);
+  const auto region_base =
+      (allocation_base + region_size - 1) & ~(region_size - 1);
+  Check(region_base + region_size + page_size <=
+            allocation_base + region_size * 2,
+        "test allocation does not span two complete tracker regions");
+
+  const auto sparse_begin = region_base + page_size;
+  tracker.ForEachUploadRange(
+      sparse_begin, page_size * 3, false, [](uint64_t, uint64_t) noexcept {},
+      []() noexcept {});
+  tracker.MarkRegionAsGpuModified(sparse_begin, page_size);
+  tracker.MarkRegionAsGpuModified(sparse_begin + page_size * 2, page_size);
+  ResetProtectionLog();
+  tracker.UnmarkRegionAsGpuModified(sparse_begin, page_size * 3);
+  Check(
+      g_protection_calls == 1 && g_protection_log.size() == 1 &&
+          g_protection_log[0].address == sparse_begin &&
+          g_protection_log[0].size == page_size * 3 &&
+          g_protection_log[0].mode == Common::VirtualMemory::Mode::Read &&
+          !tracker.IsRegionGpuModified(sparse_begin, page_size * 3) &&
+          Protection(reinterpret_cast<void *>(sparse_begin)) == PAGE_READONLY &&
+          Protection(reinterpret_cast<void *>(sparse_begin + page_size)) ==
+              PAGE_READONLY &&
+          Protection(reinterpret_cast<void *>(sparse_begin + page_size * 2)) ==
+              PAGE_READONLY,
+      "GPU unmark did not coalesce a sparse 4 MiB region mask");
+  ResetProtectionLog();
+  tracker.UnmarkRegionAsGpuModified(sparse_begin, page_size * 3);
+  Check(g_protection_calls == 0,
+        "idempotent GPU unmark performed a protection call");
+
+  const auto boundary = region_base + region_size;
+  const auto cross_begin = boundary - page_size;
+  tracker.ForEachUploadRange(
+      cross_begin, page_size * 2, false, [](uint64_t, uint64_t) noexcept {},
+      []() noexcept {});
+  tracker.MarkRegionAsGpuModified(cross_begin, page_size * 2);
+  ResetProtectionLog();
+  tracker.UnmarkRegionAsGpuModified(cross_begin, page_size * 2);
+  Check(g_protection_calls == 2 && g_protection_log.size() == 2 &&
+            g_protection_log[0].address == cross_begin &&
+            g_protection_log[0].size == page_size &&
+            g_protection_log[0].mode == Common::VirtualMemory::Mode::Read &&
+            g_protection_log[1].address == boundary &&
+            g_protection_log[1].size == page_size &&
+            g_protection_log[1].mode == Common::VirtualMemory::Mode::Read &&
+            !tracker.IsRegionGpuModified(cross_begin, page_size * 2),
+        "cross-region GPU unmark did not use one update per 4 MiB region");
+
+  tracker.UntrackMemory(allocation_base, region_size * 2);
+  Release(page_manager, memory, region_size * 2);
+}
+
+void TestFullRegionGpuUnmarkBatching() {
+  constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
+  constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto &page_manager = harness.page_manager;
+  auto *memory = Allocate(page_manager, region_size * 2 / page_size);
+  const auto allocation_base = reinterpret_cast<uint64_t>(memory);
+  const auto region_base =
+      (allocation_base + region_size - 1) & ~(region_size - 1);
+  Check(region_base + region_size <= allocation_base + region_size * 2,
+        "test allocation does not contain a complete tracker region");
 
   tracker.ForEachUploadRange(
-      address, page_size, true, [](uint64_t, uint64_t) noexcept {},
+      region_base, region_size, false, [](uint64_t, uint64_t) noexcept {},
       []() noexcept {});
-  Check(tracker.IsRegionGpuModified(address, page_size) &&
-            Protection(memory) == PAGE_NOACCESS,
-        "backing publication setup did not establish GPU ownership");
-  std::vector<RangeSet::Range> dirty{{address, page_size}};
-  auto writes = page_manager.ReserveBackingWrites(dirty);
-  Check(writes.size() == 1 && Protection(memory) == PAGE_NOACCESS,
-        "backing reservation exposed protected guest memory");
-  uint32_t downloads = 0;
-  tracker.ForEachDownloadRange<true>(
-      address, page_size, [&](uint64_t, uint64_t) noexcept { downloads++; });
-  tracker.MarkRegionAsCpuModified(address, page_size);
-  writes.clear();
-  Check(downloads == 1 && !tracker.IsRegionGpuModified(address, page_size) &&
-            tracker.IsRegionCpuModified(address, page_size) &&
-            IsWritable(memory),
-        "backing publication did not restore CPU ownership");
-  tracker.UntrackMemory(address, page_size);
-  Release(page_manager, memory, page_size);
+  tracker.MarkRegionAsGpuModified(region_base, region_size);
+  Check(tracker.IsRegionGpuModified(region_base, region_size) &&
+            Protection(reinterpret_cast<void *>(region_base)) ==
+                PAGE_NOACCESS &&
+            Protection(reinterpret_cast<void *>(region_base + region_size -
+                                                page_size)) == PAGE_NOACCESS,
+        "full-region setup did not establish GPU read protection");
+
+  ResetProtectionLog();
+  tracker.UnmarkRegionAsGpuModified(region_base, region_size);
+  Check(
+      g_protection_log.size() == 1 &&
+          g_protection_log[0].address == region_base &&
+          g_protection_log[0].size == region_size &&
+          g_protection_log[0].mode == Common::VirtualMemory::Mode::Read &&
+          !tracker.IsRegionGpuModified(region_base, region_size) &&
+          Protection(reinterpret_cast<void *>(region_base)) == PAGE_READONLY &&
+          Protection(reinterpret_cast<void *>(region_base + region_size -
+                                              page_size)) == PAGE_READONLY,
+      "full-region GPU unmark did not use one exact 4 MiB protection request");
+
+  tracker.UntrackMemory(allocation_base, region_size * 2);
+  Release(page_manager, memory, region_size * 2);
 }
 
 [[noreturn]] void RunDeathCase(const char *name) {
@@ -433,8 +636,11 @@ int main(int argc, char **argv) {
   TestCpuDirtyUpload();
   TestRangeInvalidation();
   TestGpuDirtyBits();
+  TestExactDirtyIntervalsSharingTrackerPage();
+  TestGpuDownloadProtectionMirrors();
   TestCrossRegionUpload();
-  TestBackingWritePublication();
+  TestGpuUnmarkUsesRegionMask();
+  TestFullRegionGpuUnmarkBatching();
   TestFatalPaths();
   std::puts("MemoryTrackerTests: all cases passed");
   return 0;

tests/PageManagerTests.cpp

@@ -26,7 +26,9 @@
 namespace {
 
 using Libs::Graphics::PageManager;
-using Libs::Graphics::PageWatchMode;
+using Libs::Graphics::RegionBits;
+using Libs::Graphics::TRACKER_PAGE_SIZE;
+using Libs::Graphics::TRACKER_REGION_SIZE;
 
 void Check(bool value, const char *text) {
   if (!value) {
@@ -113,6 +115,11 @@ bool IsWritable(const void *address) {
 }
 
 uint64_t g_protection_calls = 0;
+struct ProtectionCall {
+  uint64_t address;
+  uint64_t size;
+};
+std::vector<ProtectionCall> g_protection_ranges;
 
 bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
                          Common::VirtualMemory::Mode mode) {
@@ -124,6 +131,7 @@ bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
   }
   DWORD old_protection = 0;
   g_protection_calls++;
+  g_protection_ranges.push_back({vaddr, size});
   return VirtualProtect(reinterpret_cast<void *>(vaddr), size, protection,
                         &old_protection) != 0;
 }
@@ -155,12 +163,12 @@ void TestWatchAndUnwatch() {
   const auto address = reinterpret_cast<uint64_t>(memory);
 
   manager.OnGpuMap(address, page_size * 2);
-  manager.UpdatePageWatchers(true, address, page_size);
+  manager.UpdatePageWatchers<true>(address, page_size);
   Check(Protection(memory) == PAGE_READONLY && IsWritable(memory + page_size),
         "write watch installed incorrect protections");
   Check(g_protection_calls != 0,
         "watch protection bypassed the address-space owner callback");
-  manager.UpdatePageWatchers(false, address, page_size);
+  manager.UpdatePageWatchers<false>(address, page_size);
   Check(IsWritable(memory), "write unwatch did not restore access");
   manager.OnGpuUnmap(address, page_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
@@ -173,39 +181,17 @@ void TestSharedWatcherCounts() {
   const auto address = reinterpret_cast<uint64_t>(memory);
 
   manager.OnGpuMap(address, page_size);
-  manager.UpdatePageWatchers(true, address + 8, 32);
-  manager.UpdatePageWatchers(true, address + 128, 64);
-  manager.UpdatePageWatchers(false, address + 8, 32);
+  manager.UpdatePageWatchers<true>(address + 8, 32);
+  manager.UpdatePageWatchers<true>(address + 128, 64);
+  manager.UpdatePageWatchers<false>(address + 8, 32);
   Check(Protection(memory) == PAGE_READONLY,
         "first unwatch released a shared watcher");
-  manager.UpdatePageWatchers(false, address + 128, 64);
+  manager.UpdatePageWatchers<false>(address + 128, 64);
   Check(IsWritable(memory), "last unwatch did not restore access");
   manager.OnGpuUnmap(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
-void TestMixedWatcherModes() {
-  PageManager manager;
-  const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size);
-  const auto address = reinterpret_cast<uint64_t>(memory);
-
-  manager.OnGpuMap(address, page_size);
-  manager.UpdatePageWatchers(true, address, page_size, PageWatchMode::Write);
-  manager.UpdatePageWatchers(true, address, page_size,
-                             PageWatchMode::ReadWrite);
-  Check(Protection(memory) == PAGE_NOACCESS,
-        "read/write watcher did not deny access");
-  manager.UpdatePageWatchers(false, address, page_size, PageWatchMode::Write);
-  Check(Protection(memory) == PAGE_NOACCESS,
-        "write unwatch released a read/write watcher");
-  manager.UpdatePageWatchers(false, address, page_size,
-                             PageWatchMode::ReadWrite);
-  Check(IsWritable(memory), "read/write unwatch did not restore access");
-  manager.OnGpuUnmap(address, page_size);
-  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
-}
-
 void TestCrossRegionRange() {
   PageManager manager;
   const auto page_size = manager.GetPageSize();
@@ -218,11 +204,27 @@ void TestCrossRegionRange() {
         "test allocation does not contain a region boundary");
 
   manager.OnGpuMap(base, region_size * 2);
-  manager.UpdatePageWatchers(true, boundary - page_size, page_size * 2);
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchers<true>(boundary - page_size, page_size * 2);
+  Check(g_protection_calls == 2 && g_protection_ranges.size() == 2 &&
+            g_protection_ranges[0].address == boundary - page_size &&
+            g_protection_ranges[0].size == page_size &&
+            g_protection_ranges[1].address == boundary &&
+            g_protection_ranges[1].size == page_size,
+        "cross-region watch was not split only at the region boundary");
   Check(!IsWritable(reinterpret_cast<void *>(boundary - page_size)) &&
             !IsWritable(reinterpret_cast<void *>(boundary)),
         "cross-region watch did not protect both pages");
-  manager.UpdatePageWatchers(false, boundary - page_size, page_size * 2);
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchers<false>(boundary - page_size, page_size * 2);
+  Check(g_protection_calls == 2 && g_protection_ranges.size() == 2 &&
+            g_protection_ranges[0].address == boundary - page_size &&
+            g_protection_ranges[0].size == page_size &&
+            g_protection_ranges[1].address == boundary &&
+            g_protection_ranges[1].size == page_size,
+        "cross-region unwatch was not split only at the region boundary");
   Check(IsWritable(reinterpret_cast<void *>(boundary - page_size)) &&
             IsWritable(reinterpret_cast<void *>(boundary)),
         "cross-region unwatch did not restore both pages");
@@ -239,26 +241,26 @@ void TestBatchedWatcherRanges() {
   const auto address = reinterpret_cast<uint64_t>(memory);
 
   manager.OnGpuMap(address, allocation_size);
-  manager.UpdatePageWatchers(true, address + page_size, page_size);
-  manager.UpdatePageWatchers(true, address + page_size * 3, page_size);
-  manager.UpdatePageWatchers(true, address, page_size * 5);
-  manager.UpdatePageWatchers(false, address, page_size * 5);
+  manager.UpdatePageWatchers<true>(address + page_size, page_size);
+  manager.UpdatePageWatchers<true>(address + page_size * 3, page_size);
+  manager.UpdatePageWatchers<true>(address, page_size * 5);
+  manager.UpdatePageWatchers<false>(address, page_size * 5);
   Check(IsWritable(memory) && Protection(memory + page_size) == PAGE_READONLY &&
             IsWritable(memory + page_size * 2) &&
             Protection(memory + page_size * 3) == PAGE_READONLY &&
             IsWritable(memory + page_size * 4),
         "fragmented unwatch lost overlapping watcher counts");
-  manager.UpdatePageWatchers(false, address + page_size, page_size);
-  manager.UpdatePageWatchers(false, address + page_size * 3, page_size);
+  manager.UpdatePageWatchers<false>(address + page_size, page_size);
+  manager.UpdatePageWatchers<false>(address + page_size * 3, page_size);
 
   g_protection_calls = 0;
-  manager.UpdatePageWatchers(true, address, allocation_size);
+  manager.UpdatePageWatchers<true>(address, allocation_size);
   Check(g_protection_calls == 4 && Protection(memory) == PAGE_READONLY &&
             Protection(memory + region_size) == PAGE_READONLY &&
             Protection(memory + region_size * 2) == PAGE_READONLY &&
             Protection(memory + allocation_size - page_size) == PAGE_READONLY,
         "large watch was not batched and protected by tracking region");
-  manager.UpdatePageWatchers(false, address, allocation_size);
+  manager.UpdatePageWatchers<false>(address, allocation_size);
   Check(IsWritable(memory) && IsWritable(memory + region_size) &&
             IsWritable(memory + region_size * 2) &&
             IsWritable(memory + allocation_size - page_size),
@@ -268,20 +270,266 @@ void TestBatchedWatcherRanges() {
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
+void TestRegionMaskWatcherRanges() {
+  PageManager manager;
+  constexpr auto page_size = TRACKER_PAGE_SIZE;
+  constexpr auto region_size = TRACKER_REGION_SIZE;
+  auto *memory = Allocate(region_size * 2);
+  const auto allocation_base = reinterpret_cast<uint64_t>(memory);
+  const auto region_base =
+      (allocation_base + region_size - 1) & ~(region_size - 1);
+  Check(region_base + region_size <= allocation_base + region_size * 2,
+        "test allocation does not contain a complete tracking region");
+  manager.OnGpuMap(allocation_base, region_size * 2);
+
+  RegionBits full_mask;
+  full_mask.Fill();
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<true>(region_base, full_mask);
+  Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 &&
+            g_protection_ranges[0].address == region_base &&
+            g_protection_ranges[0].size == region_size,
+        "full region mask did not use one protection span");
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<false>(region_base, full_mask);
+  Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 &&
+            g_protection_ranges[0].address == region_base &&
+            g_protection_ranges[0].size == region_size,
+        "full region unmask did not use one protection span");
+
+  RegionBits sparse_mask;
+  sparse_mask.Set(1);
+  sparse_mask.Set(3);
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<true>(region_base, sparse_mask);
+  Check(g_protection_calls == 2 &&
+            Protection(reinterpret_cast<void *>(region_base + page_size)) ==
+                PAGE_READONLY &&
+            IsWritable(reinterpret_cast<void *>(region_base + page_size * 2)) &&
+            Protection(reinterpret_cast<void *>(region_base + page_size * 3)) ==
+                PAGE_READONLY,
+        "sparse region mask installed incorrect write watchers");
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<false>(region_base, sparse_mask);
+  Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 &&
+            g_protection_ranges[0].address == region_base + page_size &&
+            g_protection_ranges[0].size == page_size * 3,
+        "sparse unmask did not bridge a compatible gap");
+
+  manager.UpdatePageWatchersForRegion<true>(region_base, sparse_mask);
+  g_protection_calls = 0;
+  manager.UpdatePageWatchersForRegion<true>(region_base, sparse_mask);
+  Check(g_protection_calls == 0,
+        "duplicate sparse watch changed an already protected range");
+  manager.UpdatePageWatchersForRegion<false>(region_base, sparse_mask);
+  Check(g_protection_calls == 0,
+        "first sparse unwatch released a duplicate watcher");
+  manager.UpdatePageWatchersForRegion<false>(region_base, sparse_mask);
+
+  manager.UpdatePageWatchers<true>(region_base + page_size * 2, page_size);
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<true>(region_base, sparse_mask);
+  Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 &&
+            g_protection_ranges[0].address == region_base + page_size &&
+            g_protection_ranges[0].size == page_size * 3,
+        "sparse mask did not bridge a compatible protected gap");
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<false>(region_base, sparse_mask);
+  Check(g_protection_calls == 2 && g_protection_ranges.size() == 2 &&
+            g_protection_ranges[0].address == region_base + page_size &&
+            g_protection_ranges[0].size == page_size &&
+            g_protection_ranges[1].address == region_base + page_size * 3 &&
+            g_protection_ranges[1].size == page_size,
+        "sparse unmask crossed an incompatible protected gap");
+  manager.UpdatePageWatchers<false>(region_base + page_size * 2, page_size);
+
+  g_protection_calls = 0;
+  manager.UpdatePageWatchersForRegion<true, true>(region_base, sparse_mask);
+  Check(g_protection_calls == 2 &&
+            Protection(reinterpret_cast<void *>(region_base + page_size)) ==
+                PAGE_NOACCESS &&
+            Protection(reinterpret_cast<void *>(region_base + page_size * 3)) ==
+                PAGE_NOACCESS,
+        "sparse read mask did not deny access");
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<false, true>(region_base, sparse_mask);
+  Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 &&
+            g_protection_ranges[0].size == page_size * 3,
+        "sparse read unmask did not bridge a compatible gap");
+
+  manager.OnGpuUnmap(allocation_base, region_size * 2);
+  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
+}
+
+void TestRegionEndpointBatching() {
+  PageManager manager;
+  constexpr auto page_size = TRACKER_PAGE_SIZE;
+  constexpr auto region_size = TRACKER_REGION_SIZE;
+  constexpr auto last_page = region_size / page_size - 1;
+  auto *memory = Allocate(region_size * 2);
+  const auto allocation_base = reinterpret_cast<uint64_t>(memory);
+  const auto region_base =
+      (allocation_base + region_size - 1) & ~(region_size - 1);
+  Check(region_base + region_size <= allocation_base + region_size * 2,
+        "test allocation does not contain a complete tracking region");
+  manager.OnGpuMap(allocation_base, region_size * 2);
+
+  RegionBits endpoints;
+  endpoints.Set(0);
+  endpoints.Set(last_page);
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<true>(region_base, endpoints);
+  Check(g_protection_calls == 2 && g_protection_ranges.size() == 2 &&
+            g_protection_ranges[0].address == region_base &&
+            g_protection_ranges[0].size == page_size &&
+            g_protection_ranges[1].address ==
+                region_base + region_size - page_size &&
+            g_protection_ranges[1].size == page_size,
+        "endpoint watch did not protect only the selected pages");
+
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<false>(region_base, endpoints);
+  Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 &&
+            g_protection_ranges[0].address == region_base &&
+            g_protection_ranges[0].size == region_size,
+        "endpoint unwatch did not coalesce the compatible 4 MiB span");
+
+  RegionBits full_mask;
+  full_mask.Fill();
+  manager.UpdatePageWatchersForRegion<true>(region_base, full_mask);
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<true>(region_base, full_mask);
+  Check(g_protection_calls == 0,
+        "duplicate full-region watch issued a redundant protection call");
+  manager.UpdatePageWatchersForRegion<false>(region_base, full_mask);
+  Check(g_protection_calls == 0,
+        "first full-region unwatch released overlapping watcher counts");
+  manager.UpdatePageWatchersForRegion<false>(region_base, full_mask);
+  Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 &&
+            g_protection_ranges[0].address == region_base &&
+            g_protection_ranges[0].size == region_size,
+        "last full-region unwatch did not use one 4 MiB protection call");
+
+  manager.OnGpuUnmap(allocation_base, region_size * 2);
+  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
+}
+
+void TestReadWriteWatcherInteractions() {
+  PageManager manager;
+  constexpr auto page_size = TRACKER_PAGE_SIZE;
+  constexpr auto region_size = TRACKER_REGION_SIZE;
+  auto *memory = Allocate(region_size * 2);
+  const auto allocation_base = reinterpret_cast<uint64_t>(memory);
+  const auto region_base =
+      (allocation_base + region_size - 1) & ~(region_size - 1);
+  Check(region_base + region_size <= allocation_base + region_size * 2,
+        "test allocation does not contain a complete tracking region");
+  manager.OnGpuMap(allocation_base, region_size * 2);
+
+  RegionBits write_mask;
+  write_mask.SetRange(10, 15);
+  RegionBits read_mask;
+  read_mask.Set(11);
+  read_mask.Set(13);
+
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<true>(region_base, write_mask);
+  Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 &&
+            g_protection_ranges[0].address == region_base + page_size * 10 &&
+            g_protection_ranges[0].size == page_size * 5,
+        "contiguous write watch was not batched");
+
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<true, true>(region_base, read_mask);
+  Check(
+      g_protection_calls == 2 &&
+          Protection(reinterpret_cast<void *>(region_base + page_size * 11)) ==
+              PAGE_NOACCESS &&
+          Protection(reinterpret_cast<void *>(region_base + page_size * 12)) ==
+              PAGE_READONLY &&
+          Protection(reinterpret_cast<void *>(region_base + page_size * 13)) ==
+              PAGE_NOACCESS,
+      "read watchers did not compose with write-only watchers");
+
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<false>(region_base, write_mask);
+  Check(
+      g_protection_calls == 3 &&
+          IsWritable(reinterpret_cast<void *>(region_base + page_size * 10)) &&
+          Protection(reinterpret_cast<void *>(region_base + page_size * 11)) ==
+              PAGE_NOACCESS &&
+          IsWritable(reinterpret_cast<void *>(region_base + page_size * 12)) &&
+          Protection(reinterpret_cast<void *>(region_base + page_size * 13)) ==
+              PAGE_NOACCESS &&
+          IsWritable(reinterpret_cast<void *>(region_base + page_size * 14)),
+      "write unwatch changed pages still owned by read watchers");
+
+  g_protection_calls = 0;
+  g_protection_ranges.clear();
+  manager.UpdatePageWatchersForRegion<false, true>(region_base, read_mask);
+  Check(
+      g_protection_calls == 1 && g_protection_ranges.size() == 1 &&
+          g_protection_ranges[0].address == region_base + page_size * 11 &&
+          g_protection_ranges[0].size == page_size * 3 &&
+          IsWritable(reinterpret_cast<void *>(region_base + page_size * 11)) &&
+          IsWritable(reinterpret_cast<void *>(region_base + page_size * 13)),
+      "read unwatch did not coalesce through compatible writable state");
+
+  manager.OnGpuUnmap(allocation_base, region_size * 2);
+  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
+}
+
 [[noreturn]] void RunDeathCase(const char *name) {
   PageManager manager;
   const auto page_size = manager.GetPageSize();
   if (std::strcmp(name, "invalid-range") == 0) {
-    manager.UpdatePageWatchers(true, (1ull << 40u) - 1, 2);
+    manager.UpdatePageWatchers<true>((1ull << 40u) - 1, 2);
   } else if (std::strcmp(name, "unknown-untrack") == 0) {
-    manager.UpdatePageWatchers(false, 0x1000, page_size);
+    manager.UpdatePageWatchers<false>(0x1000, page_size);
   } else if (std::strcmp(name, "destructor-watch") == 0) {
     auto doomed = std::make_unique<PageManager>();
     auto *memory = Allocate(page_size);
     const auto address = reinterpret_cast<uint64_t>(memory);
     doomed->OnGpuMap(address, page_size);
-    doomed->UpdatePageWatchers(true, address, page_size);
+    doomed->UpdatePageWatchers<true>(address, page_size);
     doomed.reset();
+  } else if (std::strcmp(name, "known-write-underflow") == 0) {
+    auto *memory = Allocate(page_size);
+    const auto address = reinterpret_cast<uint64_t>(memory);
+    manager.OnGpuMap(address, page_size);
+    manager.UpdatePageWatchers<true>(address, page_size);
+    manager.UpdatePageWatchers<false>(address, page_size);
+    manager.UpdatePageWatchers<false>(address, page_size);
+  } else if (std::strcmp(name, "read-overflow") == 0) {
+    auto *memory = Allocate(page_size);
+    const auto address = reinterpret_cast<uint64_t>(memory);
+    manager.OnGpuMap(address, page_size);
+    RegionBits mask;
+    const auto region_base = address & ~(TRACKER_REGION_SIZE - 1);
+    const auto page = static_cast<size_t>((address - region_base) / page_size);
+    mask.Set(page);
+    manager.UpdatePageWatchersForRegion<true, true>(region_base, mask);
+    manager.UpdatePageWatchersForRegion<true, true>(region_base, mask);
+  } else if (std::strcmp(name, "write-overflow") == 0) {
+    auto *memory = Allocate(page_size);
+    const auto address = reinterpret_cast<uint64_t>(memory);
+    manager.OnGpuMap(address, page_size);
+    for (uint32_t count = 0; count < 128; count++) {
+      manager.UpdatePageWatchers<true>(address, page_size);
+    }
   }
   std::_Exit(0x7f);
 }
@@ -328,7 +576,8 @@ void CheckDeathCase(const char *name) {
 
 void TestFatalPaths() {
   for (const char *name :
-       {"invalid-range", "unknown-untrack", "destructor-watch"}) {
+       {"invalid-range", "unknown-untrack", "destructor-watch",
+        "known-write-underflow", "read-overflow", "write-overflow"}) {
     CheckDeathCase(name);
   }
 }
@@ -350,9 +599,11 @@ int main(int argc, char **argv) {
   }
   TestWatchAndUnwatch();
   TestSharedWatcherCounts();
-  TestMixedWatcherModes();
   TestCrossRegionRange();
   TestBatchedWatcherRanges();
+  TestRegionMaskWatcherRanges();
+  TestRegionEndpointBatching();
+  TestReadWriteWatcherInteractions();
   TestFatalPaths();
   std::puts("PageManagerTests: all cases passed");
   return 0;

tests/shaderCfgTests.cpp

@@ -5394,7 +5394,224 @@ void TestNewShaderRecompilerCfgSharedOuterAndLoopMerge() {
 	CheckSpirvBinaryValidates(result.spirv);
 }
 
-void TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges() {
+void TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),    // loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 4),         // loop exit -> end
+	    EncodeSopc(0x06, 1, 1),      // s_cmp_eq_u32 s1, s1
+	    EncodeSopp(0x04, 2),         // early break -> same loop end
+	    EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1
+	    EncodeSopp(0x02, 0xfffau),   // backedge -> loop header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "loop early-break CFG did not stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0,
+	      "loop early-break SPIR-V lacks OpLoopMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "loop early-break SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "loop early-break CFG unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),    // outer loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 9),         // outer exit -> end
+	    EncodeSopc(0x0a, 1, 129),    // inner loop: s_cmp_lt_u32 s1, 1
+	    EncodeSopp(0x04, 5),         // inner exit -> outer continue
+	    EncodeSopc(0x06, 2, 2),      // s_cmp_eq_u32 s2, s2
+	    EncodeSopp(0x05, 5),         // nonlocal exit -> outer end
+	    EncodeSMovB32(3, 129),       // inner work
+	    EncodeSop2(0x00, 1, 1, 129), // s_add_u32 s1, s1, 1
+	    EncodeSopp(0x02, 0xfff9u),   // inner backedge
+	    EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1
+	    EncodeSopp(0x02, 0xfff5u),   // outer backedge
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"),
+	      "nested-loop nonlocal exit did not select dispatcher fallback");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0,
+	      "nested-loop nonlocal exit dispatcher SPIR-V lacks OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),    // outer loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 6),         // outer exit -> end
+	    EncodeSopc(0x0a, 1, 129),    // inner loop: s_cmp_lt_u32 s1, 1
+	    EncodeSopp(0x04, 2),         // inner exit -> outer continue
+	    EncodeSMovB32(2, 129),       // inner work
+	    EncodeSopp(0x02, 0xfffcu),   // inner backedge
+	    EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1
+	    EncodeSopp(0x02, 0xfff8u),   // outer backedge
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "nested local loop exit did not stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) >= 2,
+	      "nested local loop exit SPIR-V lacks both OpLoopMerge instructions");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "nested local loop exit SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "nested local loop exit unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),    // outer loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 11),        // outer exit -> end
+	    EncodeSopc(0x06, 1, 1),      // inner loop first exit condition
+	    EncodeSopp(0x05, 3),         // first inner exit -> tail A
+	    EncodeSopc(0x06, 2, 2),      // inner loop second exit condition
+	    EncodeSopp(0x05, 3),         // second inner exit -> tail B
+	    EncodeSopp(0x02, 0xfffbu),   // inner backedge
+	    EncodeSMovB32(3, 129),       // tail A
+	    EncodeSopp(0x02, 2),         // tail A -> outer continue
+	    EncodeSMovB32(4, 129),       // tail B
+	    EncodeSopp(0x02, 0),         // tail B -> outer continue
+	    EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1
+	    EncodeSopp(0x02, 0xfff3u),   // outer backedge
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::Decoder::Program program;
+	std::string                        error;
+	Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, program, &error),
+	      error.c_str());
+
+	ShaderRecompiler::CFG::Graph graph;
+	Check(ShaderRecompiler::CFG::BuildGraph(program, graph, &error), error.c_str());
+	const auto original_block_count = graph.blocks.size();
+	Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
+	Check(graph.blocks.size() > original_block_count,
+	      "nested loop exit tails did not create a private inner merge");
+
+	const auto* outer_header = graph.FindBlockByPc(0);
+	const auto* inner_header = graph.FindBlockByPc(8);
+	Check(outer_header != nullptr && inner_header != nullptr &&
+	          outer_header->terminator.loop_header && inner_header->terminator.loop_header,
+	      "nested loop exit-tail fixture did not retain both loop headers");
+	Check(inner_header->terminator.merge_block != outer_header->terminator.continue_block,
+	      "inner loop merge still aliases the outer continue target");
+	const auto* inner_merge = graph.FindBlock(inner_header->terminator.merge_block);
+	Check(inner_merge != nullptr && inner_merge->inst_begin == inner_merge->inst_end &&
+	          inner_merge->terminator.kind == ShaderRecompiler::CFG::TerminatorKind::Branch &&
+	          inner_merge->terminator.true_block == outer_header->terminator.continue_block,
+	      "private inner merge does not forward to the outer continue target");
+}
+
+void TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x06, 7, 7),      // entry branch bypasses loop -> exit X
+	    EncodeSopp(0x05, 5),         // entry -> X
+	    EncodeSopc(0x0a, 0, 129),    // loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 5),         // loop exit -> Y
+	    EncodeSopc(0x06, 1, 1),      // inner condition
+	    EncodeSopp(0x05, 1),         // nonmerge exit -> X, else continue
+	    EncodeSopp(0x02, 0xfffbu),   // loop backedge
+	    EncodeSMovB32(2, 129),       // X
+	    EncodeSopp(0x02, 2),         // X -> end
+	    EncodeSMovB32(3, 129),       // Y
+	    EncodeSopp(0x02, 0),         // Y -> end
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"),
+	      "mixed continue/nonmerge exit did not select dispatcher fallback");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0,
+	      "mixed continue/nonmerge exit dispatcher SPIR-V lacks OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgConditionalLatchNoSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopp(0x02, 0),       // loop header -> conditional block
+	    EncodeSopc(0x06, 0, 0),    // s_cmp_eq_u32 s0, s0
+	    EncodeSopp(0x05, 1),       // loop exit -> end
+	    EncodeSopp(0x02, 0xfffcu), // separate latch -> loop header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "conditional latch did not stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0,
+	      "conditional latch SPIR-V lacks OpLoopMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "conditional latch SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "conditional latch unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopp(0x02, 0),       // loop header -> conditional latch
+	    EncodeSopc(0x06, 0, 0),    // s_cmp_eq_u32 s0, s0
+	    EncodeSopp(0x05, 0xfffdu), // direct latch backedge -> loop header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "direct conditional latch did not stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0,
+	      "direct conditional latch SPIR-V lacks OpLoopMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "direct conditional latch SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "direct conditional latch unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection() {
 	const uint32_t shader[] = {
 	    EncodeSMovB32(0, 128),       // s0 = 0
 	    EncodeSopc(0x0a, 0, 130),    // loop: s_cmp_lt_u32 s0, 2
@@ -5419,15 +5636,111 @@ void TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges() {
 	std::string                     error;
 	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
 	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
-	      "shared loop continue selections should stay on structured path");
-	Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"),
-	      "shared loop continue selections were not split before structurization");
-	Check(SpirvContainsOpcode(result.spirv, 246),
-	      "shared loop continue selections SPIR-V lacks OpLoopMerge");
-	Check(SpirvContainsOpcode(result.spirv, 247),
-	      "shared loop continue selections SPIR-V lacks OpSelectionMerge");
-	Check(!SpirvContainsOpcode(result.spirv, 251),
-	      "shared loop continue selections unexpectedly used dispatcher OpSwitch");
+	      "loop early continues should stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0,
+	      "loop early continues SPIR-V lacks OpLoopMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "loop early continues SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "loop early continues unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgConditionalLoopHeaderSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x06, 0, 0),    // loop body selection condition
+	    EncodeSopp(0x05, 2),       // select path B
+	    EncodeSMovB32(1, 129),     // path A
+	    EncodeSopp(0x02, 1),       // path A -> join
+	    EncodeSMovB32(2, 129),     // path B
+	    EncodeSMovB32(3, 129),     // join
+	    EncodeSopc(0x06, 4, 4),    // repeat condition
+	    EncodeSopp(0x05, 0xfff8u), // repeat -> guest header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::Decoder::Program decoded;
+	std::string                        error;
+	Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error),
+	      error.c_str());
+	ShaderRecompiler::CFG::Graph graph;
+	Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str());
+	const auto original_block_count = graph.blocks.size();
+	Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
+	Check(graph.blocks.size() > original_block_count,
+	      "conditional guest loop header did not create a synthetic header");
+
+	uint32_t loop_headers      = 0;
+	uint32_t selection_headers = 0;
+	for (const auto& block: graph.blocks) {
+		if (block.terminator.loop_header) {
+			loop_headers++;
+			Check(block.inst_begin == block.inst_end &&
+			          block.terminator.kind == ShaderRecompiler::CFG::TerminatorKind::Branch,
+			      "canonical loop header is not an empty unconditional block");
+		} else if (block.terminator.kind ==
+		               ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch &&
+		           block.terminator.merge_block != UINT32_MAX) {
+			selection_headers++;
+		}
+	}
+	Check(loop_headers == 1u && selection_headers == 1u,
+	      "guest conditional was not separated from the loop header");
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage = ShaderType::Compute;
+	ShaderRecompiler::CompileResult result;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u,
+	      "conditional loop-header SPIR-V has the wrong loop-merge count");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 1u,
+	      "conditional loop-header SPIR-V has the wrong selection-merge count");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u,
+	      "conditional loop-header unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgMultipleLoopLatches() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),  // loop condition
+	    EncodeSopp(0x04, 5),       // loop exit -> end
+	    EncodeSopc(0x06, 1, 1),    // early repeat condition
+	    EncodeSopp(0x05, 0xfffcu), // early repeat -> header
+	    EncodeSMovB32(2, 129),     // body
+	    EncodeSMovB32(3, 129),     // body tail
+	    EncodeSopp(0x02, 0xfff9u), // ordinary latch -> header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::Decoder::Program decoded;
+	std::string                        error;
+	Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error),
+	      error.c_str());
+	ShaderRecompiler::CFG::Graph graph;
+	Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str());
+	const auto original_block_count = graph.blocks.size();
+	Check(graph.back_edges.size() == 2u, "multiple-latch fixture lacks two native backedges");
+	Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
+	Check(graph.blocks.size() == original_block_count + 1u,
+	      "multiple native latches did not create one synthetic continue");
+	Check(graph.back_edges.size() == 1u && graph.natural_loops.size() == 1u,
+	      "multiple native latches were not coalesced to one SPIR-V backedge");
+	const auto& loop           = graph.natural_loops.front();
+	const auto* continue_block = graph.FindBlock(loop.continue_block);
+	Check(continue_block != nullptr && continue_block->inst_begin == continue_block->inst_end &&
+	          continue_block->predecessors.size() == 2u,
+	      "canonical continue does not join both native latches");
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage = ShaderType::Compute;
+	ShaderRecompiler::CompileResult result;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u,
+	      "multiple-latch SPIR-V has the wrong loop-merge count");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0u,
+	      "multiple-latch SPIR-V unexpectedly used a selection merge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u,
+	      "multiple-latch SPIR-V unexpectedly used dispatcher OpSwitch");
 	CheckSpirvBinaryValidates(result.spirv);
 }
 
@@ -5457,6 +5770,85 @@ void TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit() {
 	CheckSpirvBinaryValidates(result.spirv);
 }
 
+void TestNewShaderRecompilerCfgOverlappingEarlyExitLadder() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x06, 0, 0), // block 0
+	    EncodeSopp(0x04, 2),    // block 0 -> 2 or 1
+	    EncodeSopc(0x06, 1, 1), // block 1
+	    EncodeSopp(0x04, 6),    // block 1 -> 5 or 2
+	    EncodeSopc(0x06, 2, 2), // block 2
+	    EncodeSopp(0x04, 4),    // block 2 -> 5 or 3
+	    EncodeSopc(0x06, 3, 3), // block 3
+	    EncodeSopp(0x04, 2),    // block 3 -> 5 or 4
+	    EncodeSMovB32(4, 129),  // block 4
+	    0xbf810000u,             // block 4 -> 6
+	    EncodeSMovB32(5, 129),  // block 5
+	    0xbf810000u,             // block 5 -> 6
+	};
+
+	ShaderRecompiler::Decoder::Program decoded;
+	std::string                        error;
+	Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error),
+	      error.c_str());
+	ShaderRecompiler::CFG::Graph graph;
+	Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str());
+	Check(graph.blocks.size() == 7u && graph.blocks[0].successors == std::vector<uint32_t>({1, 2}) &&
+	          graph.blocks[0].terminator.true_block == 2u &&
+	          graph.blocks[0].terminator.false_block == 1u &&
+	          graph.blocks[1].successors == std::vector<uint32_t>({2, 5}) &&
+	          graph.blocks[1].terminator.true_block == 5u &&
+	          graph.blocks[1].terminator.false_block == 2u &&
+	          graph.blocks[2].successors == std::vector<uint32_t>({3, 5}) &&
+	          graph.blocks[2].terminator.true_block == 5u &&
+	          graph.blocks[2].terminator.false_block == 3u &&
+	          graph.blocks[3].successors == std::vector<uint32_t>({4, 5}) &&
+	          graph.blocks[3].terminator.true_block == 5u &&
+	          graph.blocks[3].terminator.false_block == 4u &&
+	          graph.blocks[4].successors == std::vector<uint32_t>({6}) &&
+	          graph.blocks[5].successors == std::vector<uint32_t>({6}),
+	      "overlapping early-exit fixture does not match the observed shader CFG");
+	Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
+	std::vector<bool>     reachable(graph.blocks.size());
+	std::vector<uint32_t> pending = {graph.entry_block};
+	while (!pending.empty()) {
+		const auto block_id = pending.back();
+		pending.pop_back();
+		if (reachable[block_id]) {
+			continue;
+		}
+		reachable[block_id] = true;
+		pending.insert(pending.end(), graph.blocks[block_id].successors.begin(),
+		               graph.blocks[block_id].successors.end());
+	}
+	Check(std::all_of(reachable.begin(), reachable.end(), [](bool value) { return value; }),
+	      "overlapping early-exit structurization left unreachable blocks");
+	std::vector<uint32_t> merges;
+	for (const auto& block: graph.blocks) {
+		if (block.terminator.kind == ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch) {
+			Check(block.terminator.merge_block != UINT32_MAX &&
+			          std::find(merges.begin(), merges.end(), block.terminator.merge_block) ==
+			              merges.end(),
+			      "overlapping early-exit structurization retained a shared merge");
+			merges.push_back(block.terminator.merge_block);
+		}
+	}
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Pixel;
+	options.dump_ir = true;
+	ShaderRecompiler::CompileResult result;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "overlapping early-exit ladder did not stay on the structured path");
+	Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"),
+	      "overlapping early-exit ladder retained a shared merge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) >= 4u,
+	      "overlapping early-exit ladder lost its selections");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u,
+	      "overlapping early-exit ladder used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
 void TestNewShaderRecompilerCfgIrreducibleDispatcher() {
 	const uint32_t shader[] = {
 	    EncodeSopp(0x05, 2),       // entry -> B, fallthrough A
@@ -7014,7 +7406,6 @@ int main() {
 	using namespace Libs::Graphics;
 
 	EnsureConfigInitialized();
-
 	TestResourceDescriptorClassification();
 	TestNativeShaderResourceDependencies();
 	TestNormalizedImageContracts();
@@ -7086,8 +7477,18 @@ int main() {
 	TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher();
 	TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeDispatcher();
 	TestNewShaderRecompilerCfgSharedOuterAndLoopMerge();
-	TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges();
+	TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection();
+	TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher();
+	TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection();
+	TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit();
+	TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher();
+	TestNewShaderRecompilerCfgConditionalLatchNoSelection();
+	TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection();
+	TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection();
+	TestNewShaderRecompilerCfgConditionalLoopHeaderSelection();
+	TestNewShaderRecompilerCfgMultipleLoopLatches();
 	TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit();
+	TestNewShaderRecompilerCfgOverlappingEarlyExitLadder();
 	TestNewShaderRecompilerCfgIrreducibleDispatcher();
 	TestNewShaderRecompilerExecMaskHelpers();
 	TestComputeShaderInputWaveSize();