Implement ftruncate abi

Extracted from Stefanos Costa's pthread fix in #147.

.github/workflows/build-linux.yml

@@ -1,82 +0,0 @@
-name: Build KytyPS5 (Linux)
-
-on:
-  workflow_dispatch:
-  push:
-    branches: [ main, master ]
-  pull_request:
-
-jobs:
-  build:
-    runs-on: ubuntu-24.04
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        shell: bash
-        run: |
-          sudo apt-get update
-          sudo apt-get install --no-install-recommends --yes \
-            clang \
-            glslang-tools \
-            libgl1-mesa-dev \
-            libx11-dev \
-            libxcursor-dev \
-            libxext-dev \
-            libxfixes-dev \
-            libxi-dev \
-            libxrandr-dev \
-            libxss-dev \
-            lld \
-            ninja-build
-
-      - name: Install Qt 6.10.3
-        uses: jurplel/install-qt-action@v4
-        with:
-          version: "6.10.3"
-          host: linux
-          target: desktop
-          arch: linux_gcc_64
-          cache: true
-
-      - name: Verify toolchain
-        shell: bash
-        run: |
-          git --version
-          cmake --version
-          ninja --version
-          clang++ --version
-          ld.lld --version
-          glslangValidator --version
-          echo "Qt6_DIR=$Qt6_DIR"
-
-      - name: Configure
-        shell: bash
-        run: |
-          cmake -S src -B _Build/linux \
-            -G Ninja \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_C_COMPILER=clang \
-            -DCMAKE_CXX_COMPILER=clang++ \
-            -DCMAKE_PREFIX_PATH="$Qt6_DIR"
-
-      - name: Build
-        shell: bash
-        run: |
-          cmake --build _Build/linux --target launcher --parallel
-
-      - name: Install
-        shell: bash
-        run: |
-          cmake --install _Build/linux --prefix _Build/linux/install
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: KytyPS5-Linux
-          path: _Build/linux/install/**
-          if-no-files-found: error

.github/workflows/build.yml

@@ -73,7 +73,7 @@ jobs:
       - name: Configure
         shell: cmd
         run: |
-          cmake -S src -B _Build/windows ^
+          cmake -S . -B _Build/windows ^
             -G Ninja ^
             -DCMAKE_BUILD_TYPE=Release ^
             -DCMAKE_C_COMPILER=clang-cl ^
@@ -83,7 +83,12 @@ jobs:
       - name: Build
         shell: cmd
         run: |
-          cmake --build _Build/windows --target launcher --parallel
+          cmake --build _Build/windows --target launcher audio_out2_port_tests virtual_memory_allocation_tests --parallel
+
+      - name: Test
+        shell: cmd
+        run: |
+          ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|virtual_memory_allocation)$"
 
       - name: Install
         shell: cmd
@@ -142,7 +147,7 @@ jobs:
       - name: Configure
         shell: bash
         run: |
-          cmake -S src -B _Build/macos \
+          cmake -S . -B _Build/macos \
             -G Ninja \
             -DCMAKE_BUILD_TYPE=Release \
             -DCMAKE_OSX_ARCHITECTURES=x86_64 \
@@ -153,7 +158,15 @@ jobs:
       - name: Build
         shell: bash
         run: |
-          cmake --build _Build/macos --target launcher --parallel
+          cmake --build _Build/macos \
+            --target launcher audio_out2_port_tests virtual_memory_allocation_tests \
+            --parallel
+
+      - name: Test
+        shell: bash
+        run: |
+          ctest --test-dir _Build/macos --output-on-failure \
+            -R '^(audio_out2_port|virtual_memory_allocation)$'
 
       - name: Install
         shell: bash
@@ -198,10 +211,138 @@ jobs:
           path: _Build/macos/install/**
           if-no-files-found: error
 
+  linux:
+    name: Build KytyPS5 (Linux)
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        shell: bash
+        run: |
+          sudo apt-get update
+          sudo apt-get install --no-install-recommends --yes \
+            clang \
+            glslang-tools \
+            libasound2-dev \
+            libdbus-1-dev \
+            libgl1-mesa-dev \
+            libpulse-dev \
+            libudev-dev \
+            libwayland-dev \
+            libx11-dev \
+            libxcursor-dev \
+            libxext-dev \
+            libxfixes-dev \
+            libxi-dev \
+            libxkbcommon-dev \
+            libxrandr-dev \
+            libxss-dev \
+            lld \
+            ninja-build \
+            wayland-protocols
+
+      - name: Install Qt 6.10.3
+        uses: jurplel/install-qt-action@v4
+        with:
+          version: "6.10.3"
+          host: linux
+          target: desktop
+          arch: linux_gcc_64
+          cache: true
+
+      - name: Verify toolchain
+        shell: bash
+        run: |
+          git --version
+          cmake --version
+          ninja --version
+          clang++ --version
+          ld.lld --version
+          glslangValidator --version
+          echo "Qt6_DIR=$Qt6_DIR"
+
+      - name: Configure
+        shell: bash
+        run: |
+          mkdir -p _Build
+          cmake -S . -B _Build/linux \
+            -G Ninja \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_C_COMPILER=clang \
+            -DCMAKE_CXX_COMPILER=clang++ \
+            -DCMAKE_PREFIX_PATH="$Qt6_DIR" 2>&1 | tee _Build/configure.log
+          exit "${PIPESTATUS[0]}"
+
+      - name: Verify SDL2 backends
+        shell: bash
+        run: |
+          status=0
+          for feature in SDL_ALSA SDL_PULSEAUDIO SDL_WAYLAND SDL_X11 SDL_LIBUDEV SDL_DBUS; do
+            if grep -qE "^--   ${feature} +\\(Wanted: ON\\): ON" _Build/configure.log; then
+              echo "ok   ${feature}"
+            else
+              echo "FAIL ${feature} is not enabled"
+              status=1
+            fi
+          done
+          exit "$status"
+
+      - name: Build
+        shell: bash
+        run: |
+          cmake --build _Build/linux \
+            --target launcher page_manager_tests memory_tracker_tests \
+              audio_out2_port_tests virtual_memory_allocation_tests \
+            --parallel
+
+      - name: Test
+        shell: bash
+        run: |
+          ctest --test-dir _Build/linux --output-on-failure \
+            -R '^(audio_out2_port|page_manager|memory_tracker|virtual_memory_allocation)$'
+
+      - name: Install
+        shell: bash
+        run: |
+          cmake --install _Build/linux --prefix _Build/linux/install
+
+      - name: Verify artifacts
+        shell: bash
+        run: |
+          file _Build/linux/install/launcher
+          file _Build/linux/install/kyty_emulator
+          file _Build/linux/install/kyty_emulator | grep -q "ELF 64-bit LSB .*x86-64"
+          ldd _Build/linux/install/kyty_emulator > /dev/null
+          readelf -d _Build/linux/install/launcher | grep -q 'RPATH.*\$ORIGIN/lib'
+          while IFS= read -r -d '' binary; do
+            while read -r dependency; do
+              test -e "_Build/linux/install/lib/$dependency"
+            done < <(
+              readelf -d "$binary" |
+                sed -n 's/.*Shared library: \[\(libQt6[^]]*\|libicu[^]]*\)\].*/\1/p'
+            )
+          done < <(
+            find _Build/linux/install/launcher _Build/linux/install/plugins \
+              -type f \( -name launcher -o -name '*.so' \) -print0
+          )
+          _Build/linux/install/kyty_emulator --help > /dev/null
+
+      - name: Upload Linux artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: KytyPS5-Linux-x86_64
+          path: _Build/linux/install/**
+          if-no-files-found: error
+
   release:
     name: Release KytyPS5
-    if: github.event_name == 'push'
-    needs: [windows, macos]
+    if: github.event_name == 'push' && github.repository == 'KytyPS5/KytyPS5'
+    needs: [windows, macos, linux]
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -222,12 +363,16 @@ jobs:
         run: |
           windows_dir="artifacts/KytyPS5-Windows-x64"
           macos_dir="artifacts/KytyPS5-macOS-x86_64"
+          linux_dir="artifacts/KytyPS5-Linux-x86_64"
           test -d "$windows_dir"
           test -d "$macos_dir"
+          test -d "$linux_dir"
           chmod a+x \
             "$macos_dir/launcher" \
             "$macos_dir/kyty_emulator" \
-            "$macos_dir/libMoltenVK.dylib"
+            "$macos_dir/libMoltenVK.dylib" \
+            "$linux_dir/launcher" \
+            "$linux_dir/kyty_emulator"
           (
             cd "$windows_dir"
             zip -r "$GITHUB_WORKSPACE/$RELEASE_NAME-Windows-x64.zip" .
@@ -236,6 +381,8 @@ jobs:
             cd "$macos_dir"
             zip -r "$GITHUB_WORKSPACE/$RELEASE_NAME-macOS-x86_64.zip" .
           )
+          tar -C "$linux_dir" -czf \
+            "$GITHUB_WORKSPACE/$RELEASE_NAME-Linux-x86_64.tar.gz" .
 
       - name: Create release
         shell: bash
@@ -245,6 +392,7 @@ jobs:
           assets=(
             "$RELEASE_NAME-Windows-x64.zip"
             "$RELEASE_NAME-macOS-x86_64.zip"
+            "$RELEASE_NAME-Linux-x86_64.tar.gz"
           )
           if gh release view "$RELEASE_NAME" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then
             gh release upload "$RELEASE_NAME" "${assets[@]}" \

.gitignore

@@ -4,3 +4,4 @@
 build/
 _Build/vscode-clang/
 _Build/
+cmake-build-debug/

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-    "cmake.sourceDirectory": "${workspaceFolder}/src",
+    "cmake.sourceDirectory": "${workspaceFolder}",
     "cmake.buildDirectory": "${workspaceFolder}/_Build/vscode-clang",
     "cmake.generator": "Ninja",
     "cmake.environment": {

CMakeLists.txt

@@ -6,6 +6,10 @@ endif()
 
 project(Kyty)
 
+set(KYTY_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src")
+set(KYTY_TESTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests")
+set(KYTY_THIRD_PARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/3rdparty")
+
 if(CMAKE_SYSTEM_NAME MATCHES ".*Linux")
 	set(LINUX TRUE)
 endif()
@@ -19,13 +23,11 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_CXX_SCAN_FOR_MODULES OFF)
 
-include(utils.cmake)
+include("${KYTY_SOURCE_DIR}/utils.cmake")
 include(CTest)
 
 option(KYTY_ENABLE_CLANG_TIDY "Run clang-tidy checks during builds" OFF)
 
-set(KYTY_THIRD_PARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../3rdparty")
-
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 
 include(TestBigEndian)
@@ -103,13 +105,13 @@ include_directories(
 	${KYTY_THIRD_PARTY_DIR}/fmt/include
 	${KYTY_THIRD_PARTY_DIR}/magic_enum/include/magic_enum
 	${PROJECT_BINARY_DIR}
-	${CMAKE_CURRENT_SOURCE_DIR}
+	${KYTY_SOURCE_DIR}
 )
 
 set(KYTY_VERSION "${PROJECT_VERSION}")
 
 configure_file(
-  ${PROJECT_SOURCE_DIR}/cmake_config.h.in
+  ${KYTY_SOURCE_DIR}/cmake_config.h.in
   ${PROJECT_BINARY_DIR}/cmake_config.h
   )
 
@@ -117,11 +119,11 @@ find_package(Git)
 
 add_custom_target( KytyGitVersion
     COMMAND ${CMAKE_COMMAND}
-	-D INPUT_FILE=${CMAKE_CURRENT_SOURCE_DIR}/kytyGitVersion.h.in
-    -D OUTPUT_FILE=${CMAKE_CURRENT_BINARY_DIR}/kytyGitVersion.h
+	-D INPUT_FILE=${KYTY_SOURCE_DIR}/kytyGitVersion.h.in
+	-D OUTPUT_FILE=${PROJECT_BINARY_DIR}/kytyGitVersion.h
 	-D GIT_EXECUTABLE=${GIT_EXECUTABLE}
 	-D GIT_WORKING_DIRECTORY=${CMAKE_CURRENT_SOURCE_DIR}
-	-P ${CMAKE_CURRENT_SOURCE_DIR}/generate_version.cmake
+	-P ${KYTY_SOURCE_DIR}/generate_version.cmake
 	COMMENT "Generate kytyGitVersion.h"
 )
 
@@ -143,43 +145,51 @@ option(KYTY_BUILD_LAUNCHER "Build Qt launcher" ON)
 config_compiler_and_linker()
 
 add_subdirectory("${KYTY_THIRD_PARTY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/3rdparty")
-add_subdirectory(common)
+add_subdirectory("${KYTY_SOURCE_DIR}/common" "${CMAKE_CURRENT_BINARY_DIR}/common")
 
 file(GLOB kyty_emulator_src CONFIGURE_DEPENDS
-	libs/*.cpp
-	libs/*.h
-	graphics/*.cpp
-	graphics/*.h
-	graphics/guest_gpu/*.cpp
-	graphics/guest_gpu/*.h
-	graphics/guest_gpu/command_processor/*.cpp
-	graphics/guest_gpu/command_processor/*.h
-	graphics/host_gpu/*.cpp
-	graphics/host_gpu/*.h
-	graphics/host_gpu/renderer/*.cpp
-	graphics/host_gpu/renderer/*.h
-	graphics/shader/*.cpp
-	graphics/shader/*.h
-	graphics/shader/recompiler/*.cpp
-	graphics/shader/recompiler/*.h
-	graphics/shader/recompiler/shaderIR/*.cpp
-	graphics/shader/recompiler/shaderIR/*.h
-	graphics/shader/recompiler/spirvEmitter/*.cpp
-	graphics/shader/recompiler/spirvEmitter/*.h
-	graphics/host_gpu/objects/*.cpp
-	graphics/host_gpu/objects/*.h
-	graphics/presentation/*.cpp
-	graphics/presentation/*.h
-	graphics/presentation/window/*.cpp
-	graphics/presentation/window/*.h
-	kernel/*.cpp
-	kernel/*.h
-	loader/*.cpp
-	loader/*.h
+	"${KYTY_SOURCE_DIR}/libs/*.cpp"
+	"${KYTY_SOURCE_DIR}/libs/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/command_processor/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/command_processor/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/cache/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/cache/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/image/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/image/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/pipeline/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/pipeline/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/cfg/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/cfg/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/decompiler/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/decompiler/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/emitter/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/emitter/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/presentation/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/presentation/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/presentation/window/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/presentation/window/*.h"
+	"${KYTY_SOURCE_DIR}/kernel/*.cpp"
+	"${KYTY_SOURCE_DIR}/kernel/*.h"
+	"${KYTY_SOURCE_DIR}/loader/*.cpp"
+	"${KYTY_SOURCE_DIR}/loader/*.h"
 )
 
 find_program(KYTY_GLSLANG_VALIDATOR glslangValidator REQUIRED)
-set(gpu_tiler_shader_dir "${CMAKE_CURRENT_SOURCE_DIR}/graphics/host_gpu/shaders")
+set(gpu_tiler_shader_dir "${KYTY_SOURCE_DIR}/graphics/host_gpu/shaders")
 set(gpu_tiler_generated_dir "${PROJECT_BINARY_DIR}/gpu_tiler_shaders")
 set(gpu_tiler_shader_names
 	standard256
@@ -207,7 +217,7 @@ foreach(shader_name IN LISTS gpu_tiler_shader_names)
 		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
 		        "-I${gpu_tiler_shader_dir}" -o "${shader_spv}" "${shader_source}"
 		COMMAND ${CMAKE_COMMAND} -DINPUT=${shader_spv} -DOUTPUT=${shader_header}
-		        -DSYMBOL=${shader_symbol} -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+		        -DSYMBOL=${shader_symbol} -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
 		DEPENDS "${shader_source}" ${gpu_tiler_shader_includes}
 		VERBATIM
 	)
@@ -231,7 +241,7 @@ foreach(shader_source IN LISTS gpu_blit_shader_sources)
 		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
 		        "-I${gpu_tiler_shader_dir}" -o "${shader_spv}" "${shader_source}"
 		COMMAND ${CMAKE_COMMAND} -DINPUT=${shader_spv} -DOUTPUT=${shader_header}
-		        -DSYMBOL=${shader_symbol} -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+		        -DSYMBOL=${shader_symbol} -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
 		DEPENDS "${shader_source}"
 		VERBATIM
 	)
@@ -240,8 +250,8 @@ endforeach()
 list(APPEND kyty_emulator_src ${gpu_blit_shader_headers})
 
 list(APPEND kyty_emulator_src
-	emulator.h
-	emulator.cpp
+	"${KYTY_SOURCE_DIR}/emulator.h"
+	"${KYTY_SOURCE_DIR}/emulator.cpp"
 )
 list(REMOVE_DUPLICATES kyty_emulator_src)
 
@@ -251,8 +261,28 @@ endif()
 
 set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL2-static xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9)
 
+# Linux system libraries required by the static FFmpeg archive.
+if(LINUX)
+	find_package(Threads REQUIRED)
+	list(APPEND kyty_emulator_link_libraries m ${CMAKE_DL_LIBS} Threads::Threads)
+
+	# Optional FFmpeg dependencies.
+	find_package(ZLIB)
+	if(ZLIB_FOUND)
+		list(APPEND kyty_emulator_link_libraries ZLIB::ZLIB)
+	endif()
+	find_library(KYTY_BZ2_LIBRARY bz2)
+	if(KYTY_BZ2_LIBRARY)
+		list(APPEND kyty_emulator_link_libraries ${KYTY_BZ2_LIBRARY})
+	endif()
+	find_library(KYTY_LZMA_LIBRARY lzma)
+	if(KYTY_LZMA_LIBRARY)
+		list(APPEND kyty_emulator_link_libraries ${KYTY_LZMA_LIBRARY})
+	endif()
+endif()
+
 set(inc_headers
-	${CMAKE_CURRENT_SOURCE_DIR}
+	${KYTY_SOURCE_DIR}
 	${KYTY_THIRD_PARTY_DIR}/SDL2/include
 	${KYTY_THIRD_PARTY_DIR}/VulkanMemoryAllocator/include
 	${KYTY_THIRD_PARTY_DIR}/SPIRV-Tools/include
@@ -270,7 +300,7 @@ if (KYTY_CLANG_CL)
 endif()
 
 list(APPEND check_headers
-	${CMAKE_CURRENT_SOURCE_DIR}
+	${KYTY_SOURCE_DIR}
 )
 
 function(add_kyty_full_emulator_test target source)
@@ -284,90 +314,116 @@ function(add_kyty_full_emulator_test target source)
 		target_link_libraries(${target} onecore)
 		add_custom_command(TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $<TARGET_FILE_DIR:${target}>/libwinpthread-1.dll)
 	endif()
+	# The macOS x86_64 guest address space needs its .zerofill segments anchored
+	# by linker flags, or the kernel kills the binary on load (posix_spawn EIO).
+	configure_macos_guest_address_space(${target})
 endfunction()
 
-add_kyty_full_emulator_test(shader_cfg_tests ../tests/shaderCfgTests.cpp)
+function(configure_macos_guest_address_space target)
+	if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
+			(NOT CMAKE_OSX_ARCHITECTURES AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$")))
+		target_sources(${target} PRIVATE "${KYTY_SOURCE_DIR}/kernel/macosGuestAddressSpace.cpp")
+		target_compile_definitions(${target} PRIVATE KYTY_LINKED_GUEST_ADDRESS_SPACE=1)
+		target_link_options(${target} PRIVATE
+			-Wl,-ld_classic,-no_pie,-no_fixup_chains,-no_huge,-pagezero_size,0x40000,-segaddr,SYSTEM_MANAGED,0x40000,-segaddr,SYSTEM_RESERVED,0x7ffffc000,-segaddr,USER_AREA,0x7000000000,-image_base,0x700000000000)
+	endif()
+endfunction()
+
+add_kyty_full_emulator_test(shader_cfg_tests "${KYTY_TESTS_DIR}/shaderCfgTests.cpp")
 
 add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL
-	../tests/ScalarProvenanceTests.cpp
-	graphics/host_gpu/hostMemory.cpp
-	graphics/shader/recompiler/ScalarProvenance.cpp
-	graphics/shader/recompiler/SrtWalker.cpp
+	"${KYTY_TESTS_DIR}/ScalarProvenanceTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ReadLaneElimination.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp"
 )
 target_link_libraries(scalar_provenance_tests fmt::fmt)
 target_include_directories(scalar_provenance_tests PRIVATE ${inc_headers})
 
 add_executable(page_manager_tests EXCLUDE_FROM_ALL
-	../tests/PageManagerTests.cpp
-	graphics/host_gpu/pageManager.cpp
+	"${KYTY_TESTS_DIR}/PageManagerTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/pageManager.cpp"
 )
 target_include_directories(page_manager_tests PRIVATE ${inc_headers})
 
+add_executable(bit_array_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/BitArrayTests.cpp"
+)
+target_include_directories(bit_array_tests PRIVATE ${inc_headers})
+
 add_executable(memory_tracker_tests EXCLUDE_FROM_ALL
-	../tests/MemoryTrackerTests.cpp
-	graphics/host_gpu/pageManager.cpp
-	graphics/host_gpu/memoryTracker.cpp
+	"${KYTY_TESTS_DIR}/MemoryTrackerTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/pageManager.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/memoryTracker.cpp"
 )
 target_link_libraries(memory_tracker_tests fmt::fmt common)
 target_include_directories(memory_tracker_tests PRIVATE ${inc_headers})
-target_compile_definitions(memory_tracker_tests PRIVATE KYTY_MEMORY_TRACKER_TESTS=1)
 
 add_executable(shader_vertex_metadata_tests EXCLUDE_FROM_ALL
-	../tests/ShaderVertexMetadataTests.cpp
-	graphics/host_gpu/hostMemory.cpp
-	graphics/shader/shaderVertexMetadata.cpp
+	"${KYTY_TESTS_DIR}/ShaderVertexMetadataTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/shaderVertexMetadata.cpp"
 )
 target_include_directories(shader_vertex_metadata_tests PRIVATE ${inc_headers})
 
 add_executable(shader_stage_runtime_tests EXCLUDE_FROM_ALL
-	../tests/ShaderStageRuntimeTests.cpp
-	graphics/guest_gpu/gpu_format.cpp
-	graphics/host_gpu/hostMemory.cpp
-	graphics/shader/shaderStageRuntime.cpp
-	graphics/shader/recompiler/ResourceMaterialization.cpp
-	graphics/shader/recompiler/ScalarProvenance.cpp
-	graphics/shader/recompiler/SrtWalker.cpp
+	"${KYTY_TESTS_DIR}/ShaderStageRuntimeTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/shaderStageRuntime.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceMaterialization.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp"
 )
 target_link_libraries(shader_stage_runtime_tests fmt::fmt)
 target_include_directories(shader_stage_runtime_tests PRIVATE ${inc_headers})
 
 add_executable(resource_tracking_tests EXCLUDE_FROM_ALL
-	../tests/ResourceTrackingTests.cpp
-	graphics/guest_gpu/gpu_format.cpp
-	graphics/host_gpu/hostMemory.cpp
-	graphics/shader/recompiler/ScalarProvenance.cpp
-	graphics/shader/recompiler/SrtWalker.cpp
-	graphics/shader/recompiler/SrtPatcher.cpp
-	graphics/shader/recompiler/ResourceTracking.cpp
-	graphics/shader/recompiler/ResourceMaterialization.cpp
-	graphics/shader/recompiler/ShaderInfoCollection.cpp
-	graphics/shader/recompiler/BindingLayout.cpp
+	"${KYTY_TESTS_DIR}/ResourceTrackingTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtPatcher.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceTracking.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceMaterialization.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ShaderInfoCollection.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/BindingLayout.cpp"
 )
 target_link_libraries(resource_tracking_tests fmt::fmt)
 target_include_directories(resource_tracking_tests PRIVATE ${inc_headers})
 
 add_executable(resource_mutex_tests EXCLUDE_FROM_ALL
-	../tests/ResourceMutexTests.cpp
-	graphics/host_gpu/renderer/resourceMutex.cpp
+	"${KYTY_TESTS_DIR}/ResourceMutexTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/host_gpu/renderer/cache/resourceMutex.cpp"
 )
 target_link_libraries(resource_mutex_tests common)
 target_include_directories(resource_mutex_tests PRIVATE ${inc_headers})
 
+add_executable(audio_out2_port_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/AudioOut2PortTests.cpp"
+	"${KYTY_SOURCE_DIR}/libs/libAudio2.cpp"
+	"${KYTY_SOURCE_DIR}/loader/timer.cpp"
+)
+target_link_libraries(audio_out2_port_tests common fmt::fmt)
+target_include_directories(audio_out2_port_tests PRIVATE ${inc_headers})
+
 add_executable(event_queue_lifetime_tests EXCLUDE_FROM_ALL
-	../tests/EventQueueLifetimeTests.cpp
-	kernel/eventQueue.cpp
-	loader/timer.cpp
+	"${KYTY_TESTS_DIR}/EventQueueLifetimeTests.cpp"
+	"${KYTY_SOURCE_DIR}/kernel/eventQueue.cpp"
+	"${KYTY_SOURCE_DIR}/loader/timer.cpp"
 )
 target_link_libraries(event_queue_lifetime_tests common fmt::fmt)
 target_include_directories(event_queue_lifetime_tests PRIVATE ${inc_headers})
 
 add_executable(image_page_table_tests EXCLUDE_FROM_ALL
-	../tests/ImagePageTableTests.cpp
+	"${KYTY_TESTS_DIR}/ImagePageTableTests.cpp"
 )
 target_link_libraries(image_page_table_tests common fmt::fmt)
 target_include_directories(image_page_table_tests PRIVATE ${inc_headers})
 
-add_kyty_full_emulator_test(shader_recompiler_compute_tests ../tests/ShaderRecompilerComputeTests.cpp)
+add_kyty_full_emulator_test(shader_recompiler_compute_tests "${KYTY_TESTS_DIR}/ShaderRecompilerComputeTests.cpp")
 
 set(gpu_test_generated_dir "${PROJECT_BINARY_DIR}/gpu_test_shaders")
 set(gpu_test_ms_depth_source
@@ -383,24 +439,41 @@ add_custom_command(
 	        -o "${gpu_test_ms_depth_spv}" "${gpu_test_ms_depth_source}"
 	COMMAND ${CMAKE_COMMAND} -DINPUT=${gpu_test_ms_depth_spv}
 	        -DOUTPUT=${gpu_test_ms_depth_header} -DSYMBOL=GPU_TEST_MS_DEPTH_SPV
-	        -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+	        -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
 	DEPENDS "${gpu_test_ms_depth_source}"
 	VERBATIM
 )
 target_sources(shader_recompiler_compute_tests PRIVATE
 	"${gpu_test_ms_depth_header}")
 
-add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp)
+add_kyty_full_emulator_test(virtual_memory_allocation_tests "${KYTY_TESTS_DIR}/VirtualMemoryAllocationTests.cpp")
 target_compile_definitions(virtual_memory_allocation_tests PRIVATE
 	KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1)
 
+# These tests use exceptions.
+if(NOT KYTY_CLANG_CL)
+	foreach(kyty_exception_test scalar_provenance_tests resource_tracking_tests
+			virtual_memory_allocation_tests)
+		target_compile_options(${kyty_exception_test} PRIVATE -fexceptions)
+	endforeach()
+endif()
+
 if(BUILD_TESTING)
+	add_test(NAME shader_cfg COMMAND $<TARGET_FILE:shader_cfg_tests>)
+	add_test(NAME scalar_provenance COMMAND $<TARGET_FILE:scalar_provenance_tests>)
 	add_test(NAME image_page_table COMMAND $<TARGET_FILE:image_page_table_tests>)
 	add_test(NAME memory_tracker COMMAND $<TARGET_FILE:memory_tracker_tests>)
 	add_test(NAME page_manager COMMAND $<TARGET_FILE:page_manager_tests>)
+	add_test(NAME bit_array COMMAND $<TARGET_FILE:bit_array_tests>)
+	add_test(NAME shader_vertex_metadata COMMAND $<TARGET_FILE:shader_vertex_metadata_tests>)
+	add_test(NAME shader_stage_runtime COMMAND $<TARGET_FILE:shader_stage_runtime_tests>)
+	add_test(NAME resource_tracking COMMAND $<TARGET_FILE:resource_tracking_tests>)
 	add_test(NAME resource_mutex COMMAND $<TARGET_FILE:resource_mutex_tests>)
 	add_test(NAME event_queue_lifetime COMMAND $<TARGET_FILE:event_queue_lifetime_tests>)
+	add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
+	add_test(NAME virtual_memory_allocation
+		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
 	add_test(NAME command_scheduler_timeline
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --scheduler-only)
 	add_test(NAME stream_buffer_ring
@@ -410,11 +483,6 @@ if(BUILD_TESTING)
 	add_test(NAME gpu_tiler
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --gpu-tiler-only)
 
-	if(WIN32)
-		add_test(NAME texture_cache_image_overlap
-			COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --image-overlap-only)
-		add_test(NAME texture_cache_htile_clear
-			COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --htile-clear-only)
 	add_test(NAME texture_cache_layered_image
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --layered-image-only)
 	add_test(NAME texture_cache_image_views
@@ -423,15 +491,39 @@ if(BUILD_TESTING)
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --storage-sampled-only)
 	add_test(NAME texture_cache_depth_readback
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --depth-readback-only)
-		add_test(NAME buffer_cache_ranges
-			COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-range-only)
 	add_test(NAME buffer_cache_dirty_gc
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-gc-only)
+
+	if(WIN32)
+		# These tests still depend on the Windows multisample-depth path.
+		add_test(NAME texture_cache_image_overlap
+			COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --image-overlap-only)
+		add_test(NAME texture_cache_htile_clear
+			COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --htile-clear-only)
+		add_test(NAME buffer_cache_ranges
+			COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-range-only)
 	endif()
+
+	add_custom_target(kyty_tests DEPENDS
+		shader_cfg_tests
+		scalar_provenance_tests
+		image_page_table_tests
+		memory_tracker_tests
+		page_manager_tests
+		bit_array_tests
+		shader_vertex_metadata_tests
+		shader_stage_runtime_tests
+		resource_tracking_tests
+		resource_mutex_tests
+		event_queue_lifetime_tests
+		shader_recompiler_compute_tests
+		virtual_memory_allocation_tests
+	)
 endif()
 
 
-add_executable(kyty_emulator main.cpp ${kyty_emulator_src})
+add_executable(kyty_emulator "${KYTY_SOURCE_DIR}/main.cpp" ${kyty_emulator_src})
+configure_macos_guest_address_space(kyty_emulator)
 
 target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries})
 if (WIN32)
@@ -446,6 +538,8 @@ endif()
 if (CLANG AND NOT KYTY_CLANG_CL)
 	target_link_libraries(kyty_emulator pthread)
 endif()
+# dlopen/dlsym/dladdr for RenderDoc.
+target_link_libraries(kyty_emulator ${CMAKE_DL_LIBS})
 target_include_directories(kyty_emulator PRIVATE ${inc_headers})
 
 clang_tidy_check(kyty_emulator "" "${check_headers}" "${inc_headers}")
@@ -458,7 +552,12 @@ set(KYTY_EMULATOR_MAP_LINK_PATH "${CMAKE_CURRENT_BINARY_DIR}/${KYTY_EMULATOR_MAP
 set(KYTY_EMULATOR_PDB_LINK_PATH "${CMAKE_CURRENT_BINARY_DIR}/kyty_emulator.pdb")
 
 if(KYTY_CLANG_CL)
-	set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "/DYNAMICBASE:NO /DEBUG:FULL /PDB:${KYTY_EMULATOR_PDB_LINK_PATH} /lldmap:${KYTY_EMULATOR_MAP_LINK_PATH}")
+	target_link_options(kyty_emulator PRIVATE
+		"/DYNAMICBASE:NO"
+		"/DEBUG:FULL"
+		"/PDB:${KYTY_EMULATOR_PDB_LINK_PATH}"
+		"/lldmap:${KYTY_EMULATOR_MAP_LINK_PATH}"
+	)
 	add_custom_command(TARGET kyty_emulator POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $<TARGET_FILE_DIR:kyty_emulator>/libwinpthread-1.dll)
 elseif(WIN32 OR LINUX)
 	set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "${KYTY_LD_OPTIONS} -Wl,-Map=${KYTY_EMULATOR_MAP_LINK_PATH}")
@@ -473,14 +572,14 @@ if(APPLE)
 	# reverts to the hardened defaults and aborts when it first executes written code).
 	add_custom_command(TARGET kyty_emulator POST_BUILD
 		COMMAND codesign -s - --force --options runtime
-			--entitlements "${CMAKE_CURRENT_SOURCE_DIR}/macos_jit.entitlements"
+			--entitlements "${KYTY_SOURCE_DIR}/macos_jit.entitlements"
 			$<TARGET_FILE:kyty_emulator>
 		COMMENT "Codesign kyty_emulator with JIT entitlements (macOS)")
 endif()
 
 install(TARGETS kyty_emulator DESTINATION .)
 if(KYTY_BUILD_LAUNCHER)
-	add_subdirectory(launcher)
+	add_subdirectory("${KYTY_SOURCE_DIR}/launcher" "${CMAKE_CURRENT_BINARY_DIR}/launcher")
 endif()
 if(KYTY_CLANG_CL)
 	install(FILES "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" DESTINATION .)

README.md

@@ -1,14 +1,14 @@
 # KytyPS5
 
 [![Build KytyPS5 (Windows)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28Windows%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
-[![Build KytyPS5 (Linux)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build-linux.yml/badge.svg)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build-linux.yml)
+[![Build KytyPS5 (Linux)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28Linux%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
 [![Build KytyPS5 (macOS)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28macOS%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
-[![Platform](https://img.shields.io/badge/platform-Windows%20x64%20%7C%20macOS%20x86__64-0078D4.svg)](#system-requirements)
+[![Platform](https://img.shields.io/badge/platform-Windows%20x64%20%7C%20Linux%20x64%20%7C%20macOS%20x86__64-0078D4.svg)](#system-requirements)
 [![Status](https://img.shields.io/badge/status-early%20development-orange.svg)](#current-status)
 [![License](https://img.shields.io/badge/license-GPL--2.0-blue.svg)](LICENSE)
 
-KytyPS5 is a free and open-source PlayStation 5 emulator written in C++ for Windows, with
-experimental macOS support. It is based on a heavily modified version of
+KytyPS5 is a free and open-source PlayStation 5 emulator written in C++ for Windows and Linux,
+with experimental macOS support. It is based on a heavily modified version of
 [Kyty](https://github.com/InoriRus/Kyty). The project is in an early stage of development, so
 compatibility is limited and behavior may change significantly between builds.
 
@@ -24,10 +24,12 @@ KytyPS5 can boot 2D games and a selection of 3D games, including titles built wi
 
 Development is focused on compatibility and boot reliability.
 
-Linux support is planned. Windows remains the primary supported platform.
+Windows is the primary platform and receives the most testing. Linux builds and runs; see
+[Building on Linux](#building-on-linux).
 
-macOS support is experimental. Compatibility with the same games on Windows and macOS has not yet
-been tested.
+macOS support is experimental. The emulator is built for x86-64 and runs on Apple Silicon under
+Rosetta 2, with Vulkan provided by MoltenVK. A small number of titles have been verified in-game
+on Apple Silicon hardware; see [Building on macOS](#building-on-macos).
 
 ## Bugs and Issues
 
@@ -49,7 +51,7 @@ graphical glitches, low compatibility, and poor performance.
   </tr>
   <tr>
     <td align="center">
-      <strong>Minecraft Legends</strong><br>
+      <strong>Neptunia ReVerse</strong><br>
       <img src="docs/screenshots/ps5-04.png" width="300" alt="Minecraft Legends running in KytyPS5">
     </td>
     <td align="center">
@@ -57,15 +59,28 @@ graphical glitches, low compatibility, and poor performance.
       <img src="docs/screenshots/ps5-05.png" width="300" alt="SILENT HILL: The Short Message running in KytyPS5">
     </td>
   </tr>
+  <tr>
+    <td align="center">
+      <strong>Hellboy</strong><br>
+      <img src="docs/screenshots/ps5-02.png" width="300" alt="Disgaea 6 running in KytyPS5">
+    </td>
+    <td align="center">
+      <strong>Paleo Pines</strong><br>
+      <img src="docs/screenshots/ps5-06.png" width="300" alt="Dreaming Sarah running in KytyPS5">
+    </td>
+  </tr>
 </table>
 
+<p align="center"><em>And many more...</em></p>
+
 ## Contributing
 
 Testing games and submitting detailed bug reports are useful ways to contribute. Search existing
 issues first, then use the **Game Emulation Bug Report** template and attach the complete log file.
 
-Code contributions should be focused, build successfully on Windows, and include relevant tests
-where practical. Because KytyPS5 is still evolving quickly, consider opening an issue before
+Code contributions should be focused, build successfully on the platforms they touch, and include
+relevant tests where practical. Windows is the primary target, so a change that alters shared code
+should not regress it; changes confined to a platform's own code paths only need to build there. Because KytyPS5 is still evolving quickly, consider opening an issue before
 starting a large change.
 
 ### Formatting
@@ -100,11 +115,12 @@ the Vulkan/SPIR-V validation rules.
 
 ### System requirements
 
-- Windows 10 version 1803
-- A 64-bit x86 processor
-- A Vulkan 1.3-capable GPU with current drivers
+- Windows 10 version 1803, a current Linux distribution, or macOS on Apple Silicon
+- A 64-bit x86 processor (on macOS, an Apple Silicon processor with Rosetta 2)
+- A Vulkan 1.3-capable GPU with current drivers (on macOS, Vulkan is provided by the bundled
+  MoltenVK)
 
-### Build requirements
+### Build requirements (Windows)
 
 - Git
 - CMake 3.12 or newer
@@ -125,7 +141,7 @@ git submodule update --init --recursive
 Configure the project. Replace the Qt path with the version installed on your system:
 
 ```powershell
-cmake -S src -B _Build/windows -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_PREFIX_PATH="C:/Qt/6.x.x/msvc2022_64"
+cmake -S . -B _Build/windows -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_PREFIX_PATH="C:/Qt/6.x.x/msvc2022_64"
 ```
 
 Build the launcher and stage a runnable installation:
@@ -138,11 +154,98 @@ cmake --install _Build/windows --prefix _Build/windows/install
 The finished application and its runtime dependencies will be placed in
 `_Build/windows/install`.
 
+### Building on Linux
+
+Install the toolchain and the libraries the bundled SDL2 needs. Without the audio, Wayland and
+udev development packages SDL2 quietly configures itself without those backends, and the resulting
+build has no working sound and no gamepad hotplug:
+
+```bash
+sudo apt-get install --no-install-recommends \
+  clang lld ninja-build cmake git glslang-tools \
+  libgl1-mesa-dev libx11-dev libxcursor-dev libxext-dev libxfixes-dev \
+  libxi-dev libxrandr-dev libxss-dev libxkbcommon-dev \
+  libasound2-dev libpulse-dev libudev-dev libdbus-1-dev libwayland-dev wayland-protocols
+```
+
+Qt 6 (Concurrent, Network, Widgets) is also required — either the distribution packages
+(`qt6-base-dev`) or an official Qt installation.
+
+```bash
+git submodule update --init --recursive
+
+cmake -S . -B _Build/linux -G Ninja -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
+  -DCMAKE_PREFIX_PATH="$Qt6_DIR"
+
+cmake --build _Build/linux --target launcher --parallel
+cmake --install _Build/linux --prefix _Build/linux/install
+```
+
+The install step copies the Qt libraries and plugins next to the binaries, so
+`_Build/linux/install` runs without a matching system Qt.
+
+As on Windows, the MSVC compiler is not used; Clang is required. `cl.exe` is rejected at configure
+time.
+
+The CMake source root is the repository root.
+
+### Building on macOS
+
+macOS builds target x86-64 and run under Rosetta 2 on Apple Silicon, so the PS5's x86-64 game
+code executes through the same translation layer as the emulator itself. Prebuilt archives are
+attached to releases; the steps below are for building from source.
+
+Requirements:
+
+- An Apple Silicon Mac with Rosetta 2 installed (`softwareupdate --install-rosetta`)
+- Xcode (or the Command Line Tools)
+- Homebrew packages: `brew install cmake ninja glslang`
+- Qt 6 (Concurrent, Network, Widgets) with x86-64 support. The official Qt installation is
+  universal and works; Homebrew's Qt is arm64-only and will not link
+
+```bash
+git submodule update --init --recursive
+
+cmake -S . -B _Build/macos -G Ninja -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_OSX_ARCHITECTURES=x86_64 \
+  -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
+  -DCMAKE_PREFIX_PATH="$Qt6_DIR"
+
+cmake --build _Build/macos --target launcher --parallel
+cmake --install _Build/macos --prefix _Build/macos/install
+```
+
+The build re-signs `kyty_emulator` with the JIT entitlements it needs to execute translated
+guest code; no manual signing step is required.
+
+Vulkan comes from MoltenVK. Download `MoltenVK-macos.tar` from the
+[MoltenVK releases](https://github.com/KhronosGroup/MoltenVK/releases), then copy
+`MoltenVK/dynamic/dylib/macOS/libMoltenVK.dylib` next to `kyty_emulator` and ad-hoc sign it:
+
+```bash
+codesign --force --sign - _Build/macos/install/libMoltenVK.dylib
+```
+
+Release archives already include a signed `libMoltenVK.dylib`.
+
+### Regression tests
+
+Build every regression executable and run the registered tests with:
+
+```powershell
+cmake --build _Build/windows --target kyty_tests
+ctest --test-dir _Build/windows --output-on-failure
+```
+
+Use `_Build/linux` instead of `_Build/windows` for a Linux build.
+
 ### Visual Studio Code
 
 A ready-made Visual Studio Code setup is included in [`.vscode`](.vscode). It configures CMake
 Tools to build the project with Ninja and `clang-cl` and provides launch profiles for both
-`launcher.exe` and `kyty_emulator.exe`.
+`launcher.exe` and `kyty_emulator.exe`. It is Windows-only: VS Code settings cannot select a
+compiler per platform, so on Linux configure from the command line as shown above.
 
 Before using it:
 
@@ -164,6 +267,10 @@ To use the graphical launcher:
 .\_Build\windows\install\launcher.exe
 ```
 
+```bash
+./_Build/linux/install/launcher
+```
+
 On first launch, add one or more game folders in the global settings. The launcher searches those
 folders recursively for game directories containing `eboot.bin`. Select a detected game and run it
 from the game list.
@@ -174,7 +281,19 @@ The emulator can also be started directly with a legally obtained game directory
 .\_Build\windows\install\kyty_emulator.exe --game "D:\Games\ExampleGame"
 ```
 
-Run `kyty_emulator.exe --help` to see the available graphics, logging, validation, profiling, and
+```bash
+./_Build/linux/install/kyty_emulator --game "/games/ExampleGame"
+```
+
+On macOS, point SDL at the MoltenVK library explicitly; the hardened runtime prevents it from
+being picked up from the executable's directory:
+
+```bash
+cd _Build/macos/install
+SDL_VULKAN_LIBRARY="$PWD/libMoltenVK.dylib" ./kyty_emulator --game "/games/ExampleGame"
+```
+
+Run `kyty_emulator --help` to see the available graphics, logging, validation, profiling, and
 debugging options.
 
 ### AI Use

src/common/CMakeLists.txt

@@ -13,7 +13,7 @@ endif()
 add_library(common STATIC ${common_src} ${common_headers})
 
 target_include_directories(common
-	PUBLIC ${PROJECT_SOURCE_DIR}
+	PUBLIC ${KYTY_SOURCE_DIR}
 	PRIVATE
 		${KYTY_THIRD_PARTY_DIR}/cpuinfo/include
 )

src/common/bitArray.h

@@ -0,0 +1,260 @@
+#ifndef EMULATOR_SRC_COMMON_BITARRAY_H_
+#define EMULATOR_SRC_COMMON_BITARRAY_H_
+
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+#include <utility>
+
+namespace Common {
+
+template <size_t N>
+class BitArray final {
+	static_assert(N != 0, "BitArray size must be nonzero");
+	static_assert(N % 64 == 0, "BitArray size must be a multiple of 64 bits");
+
+	static constexpr size_t BITS_PER_WORD = 64;
+	static constexpr size_t WORD_COUNT    = N / BITS_PER_WORD;
+
+public:
+	using Range = std::pair<size_t, size_t>;
+
+	class Iterator final {
+	public:
+		using iterator_category = std::forward_iterator_tag;
+		using value_type        = Range;
+		using difference_type   = std::ptrdiff_t;
+		using pointer           = const Range*;
+		using reference         = const Range&;
+
+		Iterator(const BitArray& bits, size_t start)
+		    : m_bits(bits), m_range(bits.FirstRangeFrom(start)) {}
+
+		Iterator& operator++() {
+			m_range = m_bits.FirstRangeFrom(m_range.second);
+			return *this;
+		}
+
+		[[nodiscard]] bool operator==(const Iterator& other) const {
+			return &m_bits == &other.m_bits && m_range == other.m_range;
+		}
+
+		[[nodiscard]] bool operator!=(const Iterator& other) const { return !(*this == other); }
+
+		[[nodiscard]] reference operator*() const { return m_range; }
+		[[nodiscard]] pointer   operator->() const { return &m_range; }
+
+	private:
+		const BitArray& m_bits;
+		Range           m_range;
+	};
+
+	using const_iterator = Iterator;
+
+	constexpr BitArray() = default;
+
+	constexpr BitArray(const BitArray& other, size_t start, size_t end) {
+		if (start >= end || end > N) {
+			return;
+		}
+
+		const auto first_word = start / BITS_PER_WORD;
+		const auto last_word  = (end - 1) / BITS_PER_WORD;
+		const auto start_bit  = start % BITS_PER_WORD;
+		const auto end_bit    = (end - 1) % BITS_PER_WORD;
+		const auto start_mask = ~uint64_t {0} << start_bit;
+		const auto end_mask =
+		    end_bit == BITS_PER_WORD - 1 ? ~uint64_t {0} : (uint64_t {1} << (end_bit + 1)) - 1;
+
+		if (first_word == last_word) {
+			m_data[first_word] = other.m_data[first_word] & start_mask & end_mask;
+			return;
+		}
+
+		m_data[first_word] = other.m_data[first_word] & start_mask;
+		for (auto word = first_word + 1; word < last_word; word++) {
+			m_data[word] = other.m_data[word];
+		}
+		m_data[last_word] = other.m_data[last_word] & end_mask;
+	}
+
+	[[nodiscard]] constexpr bool Get(size_t index) const {
+		return (m_data[index / BITS_PER_WORD] & (uint64_t {1} << (index % BITS_PER_WORD))) != 0;
+	}
+
+	constexpr void Set(size_t index) {
+		m_data[index / BITS_PER_WORD] |= uint64_t {1} << (index % BITS_PER_WORD);
+	}
+
+	constexpr void Unset(size_t index) {
+		m_data[index / BITS_PER_WORD] &= ~(uint64_t {1} << (index % BITS_PER_WORD));
+	}
+
+	constexpr void SetRange(size_t start, size_t end) {
+		if (start >= end || end > N) {
+			return;
+		}
+
+		const auto first_word = start / BITS_PER_WORD;
+		const auto last_word  = (end - 1) / BITS_PER_WORD;
+		const auto start_bit  = start % BITS_PER_WORD;
+		const auto end_bit    = (end - 1) % BITS_PER_WORD;
+		const auto start_mask = ~uint64_t {0} << start_bit;
+		const auto end_mask =
+		    end_bit == BITS_PER_WORD - 1 ? ~uint64_t {0} : (uint64_t {1} << (end_bit + 1)) - 1;
+
+		if (first_word == last_word) {
+			m_data[first_word] |= start_mask & end_mask;
+			return;
+		}
+
+		m_data[first_word] |= start_mask;
+		for (auto word = first_word + 1; word < last_word; word++) {
+			m_data[word] = ~uint64_t {0};
+		}
+		m_data[last_word] |= end_mask;
+	}
+
+	constexpr void UnsetRange(size_t start, size_t end) {
+		if (start >= end || end > N) {
+			return;
+		}
+
+		const auto first_word = start / BITS_PER_WORD;
+		const auto last_word  = (end - 1) / BITS_PER_WORD;
+		const auto start_bit  = start % BITS_PER_WORD;
+		const auto end_bit    = (end - 1) % BITS_PER_WORD;
+		const auto start_mask = (uint64_t {1} << start_bit) - 1;
+		const auto end_mask =
+		    end_bit == BITS_PER_WORD - 1 ? uint64_t {0} : ~((uint64_t {1} << (end_bit + 1)) - 1);
+
+		if (first_word == last_word) {
+			m_data[first_word] &= start_mask | end_mask;
+			return;
+		}
+
+		m_data[first_word] &= start_mask;
+		for (auto word = first_word + 1; word < last_word; word++) {
+			m_data[word] = 0;
+		}
+		m_data[last_word] &= end_mask;
+	}
+
+	constexpr void Clear() { m_data.fill(0); }
+	constexpr void Fill() { m_data.fill(~uint64_t {0}); }
+
+	[[nodiscard]] constexpr bool None() const {
+		uint64_t combined = 0;
+		for (const auto word: m_data) {
+			combined |= word;
+		}
+		return combined == 0;
+	}
+
+	[[nodiscard]] constexpr bool Any() const { return !None(); }
+
+	[[nodiscard]] constexpr Range FirstRangeFrom(size_t start) const {
+		if (start >= N) {
+			return {N, N};
+		}
+
+		auto word_index = start / BITS_PER_WORD;
+		auto word       = m_data[word_index] & (~uint64_t {0} << (start % BITS_PER_WORD));
+		while (word == 0) {
+			word_index++;
+			if (word_index == WORD_COUNT) {
+				return {N, N};
+			}
+			word = m_data[word_index];
+		}
+
+		const auto first     = word_index * BITS_PER_WORD + std::countr_zero(word);
+		const auto first_bit = first % BITS_PER_WORD;
+		const auto first_ones =
+		    static_cast<size_t>(std::countr_one(m_data[word_index] >> first_bit));
+		if (first_bit + first_ones < BITS_PER_WORD) {
+			return {first, first + first_ones};
+		}
+
+		for (word_index++; word_index < WORD_COUNT; word_index++) {
+			word = m_data[word_index];
+			if (word != ~uint64_t {0}) {
+				return {first, word_index * BITS_PER_WORD + std::countr_one(word)};
+			}
+		}
+		return {first, N};
+	}
+
+	[[nodiscard]] constexpr Range FirstRange() const { return FirstRangeFrom(0); }
+
+	[[nodiscard]] constexpr Range LastRangeFrom(size_t end) const {
+		if (end == 0) {
+			return {0, 0};
+		}
+		if (end > N) {
+			end = N;
+		}
+
+		auto       word_index = (end - 1) / BITS_PER_WORD;
+		const auto end_bit    = (end - 1) % BITS_PER_WORD;
+		const auto end_mask =
+		    end_bit == BITS_PER_WORD - 1 ? ~uint64_t {0} : (uint64_t {1} << (end_bit + 1)) - 1;
+		auto word = m_data[word_index] & end_mask;
+		while (word == 0) {
+			if (word_index == 0) {
+				return {0, 0};
+			}
+			word = m_data[--word_index];
+		}
+
+		const auto empty_bits = static_cast<size_t>(std::countl_zero(word));
+		const auto ones       = static_cast<size_t>(std::countl_one(word << empty_bits));
+		const auto last       = (word_index + 1) * BITS_PER_WORD - empty_bits;
+		if (empty_bits + ones < BITS_PER_WORD) {
+			return {last - ones, last};
+		}
+
+		while (word_index != 0) {
+			word = m_data[--word_index];
+			if (word != ~uint64_t {0}) {
+				return {(word_index + 1) * BITS_PER_WORD - std::countl_one(word), last};
+			}
+		}
+		return {0, last};
+	}
+
+	[[nodiscard]] constexpr Range LastRange() const { return LastRangeFrom(N); }
+
+	[[nodiscard]] const_iterator begin() const { return Iterator(*this, 0); }
+	[[nodiscard]] const_iterator end() const { return Iterator(*this, N); }
+
+	constexpr BitArray& operator^=(const BitArray& other) {
+		for (size_t word = 0; word < WORD_COUNT; word++) {
+			m_data[word] ^= other.m_data[word];
+		}
+		return *this;
+	}
+
+	[[nodiscard]] constexpr BitArray operator^(const BitArray& other) const {
+		auto result = *this;
+		result ^= other;
+		return result;
+	}
+
+	[[nodiscard]] constexpr BitArray operator~() const {
+		auto result = *this;
+		for (auto& word: result.m_data) {
+			word = ~word;
+		}
+		return result;
+	}
+
+private:
+	std::array<uint64_t, WORD_COUNT> m_data {};
+};
+
+} // namespace Common
+
+#endif // EMULATOR_SRC_COMMON_BITARRAY_H_

src/common/emulatorConfig.cpp

@@ -37,6 +37,10 @@ uint32_t GetVblankFrequency() {
 	return std::clamp(g_config->vblank_frequency, 30u, 360u);
 }
 
+uint32_t GetConsoleLanguage() {
+	return g_config->console_language;
+}
+
 bool VulkanValidationEnabled() {
 	return g_config->vulkan_validation_enabled;
 }
@@ -89,10 +93,6 @@ bool RenderDocEnabled() {
 	return g_config->renderdoc_enabled;
 }
 
-bool NggRectlistDrawEnabled() {
-	return g_config->ngg_rectlist_draw_enabled;
-}
-
 bool ReadbackLinearImagesEnabled() {
 	return g_config->readback_linear_images;
 }

src/common/emulatorConfig.h

@@ -18,10 +18,14 @@ enum class ProfilerDirection { None, Network };
 
 enum class OutputDirection { Silent, Console, File };
 
+constexpr uint32_t DEFAULT_CONSOLE_LANGUAGE = 1;
+constexpr uint32_t MAX_CONSOLE_LANGUAGE     = 29;
+
 struct ConfigOptions {
 	uint32_t               screen_width                = 1280;
 	uint32_t               screen_height               = 720;
 	uint32_t               vblank_frequency            = 60;
+	uint32_t               console_language            = DEFAULT_CONSOLE_LANGUAGE;
 	bool                   vulkan_validation_enabled   = false;
 	bool                   shader_validation_enabled   = false;
 	ShaderOptimizationType shader_optimization_type    = ShaderOptimizationType::None;
@@ -35,7 +39,6 @@ struct ConfigOptions {
 	ProfilerDirection      profiler_direction          = ProfilerDirection::None;
 	bool                   spirv_debug_printf_enabled  = false;
 	bool                   renderdoc_enabled           = false;
-	bool                   ngg_rectlist_draw_enabled   = true;
 	bool                   readback_linear_images      = false;
 };
 
@@ -44,6 +47,7 @@ void Load(const ConfigOptions& cfg);
 uint32_t GetScreenWidth();
 uint32_t GetScreenHeight();
 uint32_t GetVblankFrequency();
+uint32_t GetConsoleLanguage();
 bool     VulkanValidationEnabled();
 
 bool                   ShaderValidationEnabled();
@@ -64,7 +68,6 @@ ProfilerDirection GetProfilerDirection();
 bool SpirvDebugPrintfEnabled();
 
 bool RenderDocEnabled();
-bool NggRectlistDrawEnabled();
 bool ReadbackLinearImagesEnabled();
 
 } // namespace Config

src/common/hostException.cpp

@@ -9,6 +9,11 @@
 #elif defined(__APPLE__)
 #include <csignal>
 #include <sys/ucontext.h>
+#else
+#include <csignal>
+#include <initializer_list>
+#include <ucontext.h> // IWYU pragma: keep
+#include <unistd.h>
 #endif
 
 // IWYU pragma: no_include <errhandlingapi.h>
@@ -19,7 +24,7 @@
 
 namespace Common::HostException {
 
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+#if !defined(__APPLE__)
 
 static std::atomic<Handler> g_handler {nullptr};
 static std::atomic_uint32_t g_install_state {0};
@@ -33,7 +38,9 @@ static_assert(decltype(g_install_state)::is_always_lock_free);
 	std::fputs(reason != nullptr ? reason : "unspecified", stderr);
 	std::fputc('\n', stderr);
 	std::fflush(stderr);
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	TerminateProcess(GetCurrentProcess(), static_cast<UINT>(EXCEPTION_NONCONTINUABLE_EXCEPTION));
+#endif
 	std::_Exit(321);
 }
 
@@ -51,6 +58,21 @@ public:
 	KYTY_CLASS_NO_COPY(FilterScope);
 };
 
+static Handler LoadInstalledHandler() noexcept {
+	if (g_install_state.load(std::memory_order_acquire) == 0) {
+		FailFast("host exception handler is not installed");
+	}
+
+	const auto handler = g_handler.load(std::memory_order_acquire);
+	if (handler == nullptr) {
+		FailFast("host exception callback is null");
+	}
+	return handler;
+}
+#endif
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+
 static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
 	FilterScope filter_scope;
 
@@ -109,14 +131,7 @@ static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
 	info.r14 = exception->ContextRecord->R14;
 	info.r15 = exception->ContextRecord->R15;
 
-	if (g_install_state.load(std::memory_order_acquire) == 0) {
-		FailFast("host exception handler is not installed");
-	}
-
-	const auto handler = g_handler.load(std::memory_order_acquire);
-	if (handler == nullptr) {
-		FailFast("host exception callback is null");
-	}
+	const auto handler = LoadInstalledHandler();
 
 	return handler(info) ? EXCEPTION_CONTINUE_EXECUTION : EXCEPTION_CONTINUE_SEARCH;
 }
@@ -213,10 +228,79 @@ static void SignalHandler(int sig, siginfo_t* si, void* uctx) {
 	sigaction(sig, &dfl, nullptr);
 }
 
+#else
+
+// x86-64 page-fault error bits.
+constexpr uint64_t PAGE_FAULT_ERROR_WRITE       = 0x02;
+constexpr uint64_t PAGE_FAULT_ERROR_INSTRUCTION = 0x10;
+
+// Let the kernel handle an unresolved fault on retry.
+static void ChainToDefault(int signal_number) noexcept {
+	struct sigaction restore {};
+	restore.sa_handler = SIG_DFL;
+	sigemptyset(&restore.sa_mask);
+	restore.sa_flags = 0;
+	::sigaction(signal_number, &restore, nullptr);
+}
+
+static void SignalHandler(int signal_number, siginfo_t* signal_info, void* native_context) {
+	FilterScope filter_scope;
+
+	auto* context = static_cast<ucontext_t*>(native_context);
+	auto* gregs   = context->uc_mcontext.gregs;
+
+	ExceptionInfo info {};
+	info.exception_address = static_cast<uint64_t>(gregs[REG_RIP]);
+	info.native_code       = static_cast<uint32_t>(signal_number);
+	info.native_context    = context;
+
+	if (signal_number == SIGSEGV || signal_number == SIGBUS) {
+		info.type             = ExceptionType::AccessViolation;
+		const auto error_code = static_cast<uint64_t>(gregs[REG_ERR]);
+		if ((error_code & PAGE_FAULT_ERROR_INSTRUCTION) != 0) {
+			info.access_violation_type = AccessViolationType::Execute;
+		} else if ((error_code & PAGE_FAULT_ERROR_WRITE) != 0) {
+			info.access_violation_type = AccessViolationType::Write;
+		} else {
+			info.access_violation_type = AccessViolationType::Read;
+		}
+		info.access_violation_vaddr = reinterpret_cast<uint64_t>(signal_info->si_addr);
+	} else if (signal_number == SIGILL) {
+		info.type = ExceptionType::IllegalInstruction;
+	} else {
+		ChainToDefault(signal_number);
+		return;
+	}
+
+	info.rax = static_cast<uint64_t>(gregs[REG_RAX]);
+	info.rbx = static_cast<uint64_t>(gregs[REG_RBX]);
+	info.rcx = static_cast<uint64_t>(gregs[REG_RCX]);
+	info.rdx = static_cast<uint64_t>(gregs[REG_RDX]);
+	info.rsi = static_cast<uint64_t>(gregs[REG_RSI]);
+	info.rdi = static_cast<uint64_t>(gregs[REG_RDI]);
+	info.rbp = static_cast<uint64_t>(gregs[REG_RBP]);
+	info.rsp = static_cast<uint64_t>(gregs[REG_RSP]);
+	info.r8  = static_cast<uint64_t>(gregs[REG_R8]);
+	info.r9  = static_cast<uint64_t>(gregs[REG_R9]);
+	info.r10 = static_cast<uint64_t>(gregs[REG_R10]);
+	info.r11 = static_cast<uint64_t>(gregs[REG_R11]);
+	info.r12 = static_cast<uint64_t>(gregs[REG_R12]);
+	info.r13 = static_cast<uint64_t>(gregs[REG_R13]);
+	info.r14 = static_cast<uint64_t>(gregs[REG_R14]);
+	info.r15 = static_cast<uint64_t>(gregs[REG_R15]);
+
+	const auto handler = LoadInstalledHandler();
+
+	if (handler(info)) {
+		return;
+	}
+
+	ChainToDefault(signal_number);
+}
+
 #endif
 
 bool InstallHandler(Handler handler) {
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	if (handler == nullptr) {
 		return false;
 	}
@@ -228,31 +312,22 @@ bool InstallHandler(Handler handler) {
 
 	g_handler.store(handler, std::memory_order_release);
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	if (AddVectoredExceptionHandler(1, ExceptionFilter) == nullptr) {
 		g_handler.store(nullptr, std::memory_order_release);
 		g_install_state.store(0, std::memory_order_release);
 		printf("AddVectoredExceptionHandler() failed\n");
 		return false;
 	}
-
-	g_install_state.store(2, std::memory_order_release);
-	return true;
 #elif defined(__APPLE__)
-	if (handler == nullptr) {
-		return false;
-	}
-
-	uint32_t expected_state = 0;
-	if (!g_install_state.compare_exchange_strong(expected_state, 1, std::memory_order_acq_rel)) {
-		return expected_state == 2 && g_handler.load(std::memory_order_acquire) == handler;
-	}
-
-	g_handler.store(handler, std::memory_order_release);
-
 	struct sigaction sa {};
 	sa.sa_sigaction = SignalHandler;
 	sa.sa_flags     = SA_SIGINFO;
 	sigemptyset(&sa.sa_mask);
+	// The guest signal-dispatch path (KernelRaiseException) interrupts threads with
+	// SIGUSR1; block it while a fault is being resolved so a stop-the-world request
+	// cannot preempt the handler between the protection fix and the retry.
+	sigaddset(&sa.sa_mask, SIGUSR1);
 
 	// macOS raises SIGBUS for protection faults on some paths and SIGSEGV on others;
 	// SIGILL covers instructions the host cannot execute (routed to the x64 emulator).
@@ -264,13 +339,25 @@ bool InstallHandler(Handler handler) {
 		printf("sigaction() failed to install the host fault handler\n");
 		return false;
 	}
+#else
+	struct sigaction action {};
+	action.sa_sigaction = SignalHandler;
+	sigemptyset(&action.sa_mask);
+	// Fault resolution needs the normal thread stack.
+	action.sa_flags = SA_SIGINFO | SA_RESTART;
+
+	for (const int signal_number: {SIGSEGV, SIGBUS, SIGILL}) {
+		if (::sigaction(signal_number, &action, nullptr) != 0) {
+			g_handler.store(nullptr, std::memory_order_release);
+			g_install_state.store(0, std::memory_order_release);
+			printf("sigaction(%d) failed\n", signal_number);
+			return false;
+		}
+	}
+#endif
 
 	g_install_state.store(2, std::memory_order_release);
 	return true;
-#else
-	(void)handler;
-	return false;
-#endif
 }
 
 } // namespace Common::HostException

src/common/lruCache.h

@@ -49,8 +49,7 @@ public:
 
 	template <typename Function>
 	void ForEachItemBelow(Tick tick, Function&& function) {
-		constexpr bool ReturnsBool =
-		    std::is_same_v<std::invoke_result_t<Function, Object>, bool>;
+		constexpr bool ReturnsBool = std::is_same_v<std::invoke_result_t<Function, Object>, bool>;
 		for (auto* item = m_first; item != nullptr;) {
 			if (item->tick > tick) {
 				return;

src/common/platform/sysDbg.h

@@ -23,6 +23,10 @@ struct sys_dbg_stack_info_t {
 	size_t    commited_size;
 	size_t    total_size;
 	size_t    code_size;
+
+	// Full stack reservation reported by pthread.
+	uintptr_t reserved_addr;
+	size_t    reserved_size;
 #endif
 };
 

src/common/platform/sysLinuxDbg.cpp

@@ -8,6 +8,8 @@
 
 #include <cstdlib>
 #include <cstring>
+#include <execinfo.h>
+#include <pthread.h>
 #include <sys/param.h>
 #include <sys/types.h>
 #include <unistd.h>
@@ -15,8 +17,47 @@
 #include <libgen.h> // POSIX basename() lives here on macOS, not in <cstring>
 #endif
 
-void SysStackWalk(void** /*stack*/, int* depth) {
+// Avoid unwinding a guest-owned stack.
+static bool OnOwnStack() {
+	const char* probe = reinterpret_cast<const char*>(&probe);
+
+	pthread_attr_t attr {};
+#if defined(__APPLE__)
+	const auto* top  = static_cast<const char*>(pthread_get_stackaddr_np(pthread_self()));
+	const auto  size = pthread_get_stacksize_np(pthread_self());
+	(void)attr;
+	return top != nullptr && size != 0 && probe < top && probe >= top - size;
+#else
+	if (pthread_getattr_np(pthread_self(), &attr) != 0) {
+		return false;
+	}
+	void*      base = nullptr;
+	size_t     size = 0;
+	const bool ok = pthread_attr_getstack(&attr, &base, &size) == 0 && base != nullptr && size != 0;
+	pthread_attr_destroy(&attr);
+	if (!ok) {
+		return false;
+	}
+	const auto* low = static_cast<const char*>(base);
+	return probe >= low && probe < low + size;
+#endif
+}
+
+void SysStackWalk(void** stack, int* depth) {
+	if (stack == nullptr || depth == nullptr || *depth <= 0) {
+		if (depth != nullptr) {
 			*depth = 0;
+		}
+		return;
+	}
+
+	if (!OnOwnStack()) {
+		*depth = 0;
+		return;
+	}
+
+	const int n = ::backtrace(stack, *depth);
+	*depth      = (n < 0 ? 0 : n);
 }
 
 void SysStackUsagePrint(sys_dbg_stack_info_t& stack) {
@@ -33,6 +74,33 @@ void SysStackUsage(sys_dbg_stack_info_t& s) {
 
 	[[maybe_unused]] int result = 0;
 
+	memset(&s, 0, sizeof(sys_dbg_stack_info_t));
+
+	// Record the reservation before the Linux /proc walk.
+	{
+		pthread_attr_t self_attr {};
+#if defined(__APPLE__)
+		void*        stack_top  = pthread_get_stackaddr_np(pthread_self());
+		const size_t stack_size = pthread_get_stacksize_np(pthread_self());
+		if (stack_top != nullptr && stack_size != 0) {
+			s.reserved_addr = reinterpret_cast<uintptr_t>(stack_top) - stack_size;
+			s.reserved_size = stack_size;
+		}
+		(void)self_attr;
+#else
+		if (pthread_getattr_np(pthread_self(), &self_attr) == 0) {
+			void*  stack_base = nullptr;
+			size_t stack_size = 0;
+			if (pthread_attr_getstack(&self_attr, &stack_base, &stack_size) == 0 &&
+			    stack_base != nullptr && stack_size != 0) {
+				s.reserved_addr = reinterpret_cast<uintptr_t>(stack_base);
+				s.reserved_size = stack_size;
+			}
+			pthread_attr_destroy(&self_attr);
+		}
+#endif
+	}
+
 	char str[1024];
 	char str2[1024];
 	result = sprintf(str, "/proc/%d/exe", static_cast<int>(pid));
@@ -46,8 +114,6 @@ void SysStackUsage(sys_dbg_stack_info_t& s) {
 
 	result = sprintf(str, "/proc/%d/maps", static_cast<int>(pid));
 
-	memset(&s, 0, sizeof(sys_dbg_stack_info_t));
-
 	FILE* f = fopen(str, "r");
 
 	if (f == nullptr) {
@@ -121,6 +187,11 @@ void SysStackUsage(sys_dbg_stack_info_t& s) {
 	}
 
 	result = fclose(f);
+
+	if (s.reserved_addr == 0) {
+		s.reserved_addr = s.addr;
+		s.reserved_size = s.total_size;
+	}
 }
 
 #endif

src/common/platform/sysLinuxFileIO.cpp

@@ -12,7 +12,10 @@
 #include <cerrno>
 #include <cstdlib>
 #include <dirent.h>
+#include <fcntl.h>
+#include <filesystem>
 #include <sys/stat.h>
+#include <system_error>
 #include <unistd.h>
 #include <utime.h>
 
@@ -40,10 +43,43 @@ struct sys_file_t {
 	};
 };
 
+// Darwin uses BSD timestamp member names.
+#if defined(__APPLE__)
+#define KYTY_STAT_ATIME_NS(st) ((st).st_atimespec.tv_nsec)
+#define KYTY_STAT_MTIME_NS(st) ((st).st_mtimespec.tv_nsec)
+#else
+#define KYTY_STAT_ATIME_NS(st) ((st).st_atim.tv_nsec)
+#define KYTY_STAT_MTIME_NS(st) ((st).st_mtim.tv_nsec)
+#endif
+
 static std::filesystem::path get_internal_name(const std::filesystem::path& name) {
 	return name.is_absolute() ? name : (std::filesystem::path(".") / name);
 }
 
+// Pass access-pattern hints to the host.
+static void apply_cache_hint(FILE* f, sys_file_cache_type_t cache_type) {
+	if (f == nullptr) {
+		return;
+	}
+
+#if !defined(__APPLE__)
+	int advice = POSIX_FADV_NORMAL;
+	switch (cache_type) {
+		case SYS_FILE_CACHE_RANDOM_ACCESS: advice = POSIX_FADV_RANDOM; break;
+		case SYS_FILE_CACHE_SEQUENTIAL_SCAN: advice = POSIX_FADV_SEQUENTIAL; break;
+		case SYS_FILE_CACHE_AUTO:
+		default: return;
+	}
+	::posix_fadvise(fileno(f), 0, 0, advice);
+#else
+	if (cache_type == SYS_FILE_CACHE_SEQUENTIAL_SCAN) {
+		::fcntl(fileno(f), F_RDAHEAD, 1);
+	} else if (cache_type == SYS_FILE_CACHE_RANDOM_ACCESS) {
+		::fcntl(fileno(f), F_RDAHEAD, 0);
+	}
+#endif
+}
+
 void SysFileRead(void* data, uint32_t size, sys_file_t& f, uint32_t* bytes_read) {
 	if (f.type == SYS_FILE_FILE) {
 		size_t w = fread(data, 1, size, f.f);
@@ -136,8 +172,7 @@ sys_file_t* SysFileCreate(const std::filesystem::path& file_name) {
 	return ret;
 }
 
-sys_file_t* SysFileOpenR(const std::filesystem::path& file_name,
-                         sys_file_cache_type_t /*cache_type*/) {
+sys_file_t* SysFileOpenR(const std::filesystem::path& file_name, sys_file_cache_type_t cache_type) {
 	auto* ret = new sys_file_t;
 
 	ret->type = SYS_FILE_FILE;
@@ -151,6 +186,8 @@ sys_file_t* SysFileOpenR(const std::filesystem::path& file_name,
 		ret->type = SYS_FILE_ERROR;
 	}
 
+	apply_cache_hint(f, cache_type);
+
 	ret->f = f;
 
 	return ret;
@@ -180,8 +217,7 @@ sys_file_t* SysFileCreate() {
 	return ret;
 }
 
-sys_file_t* SysFileOpenW(const std::filesystem::path& file_name,
-                         sys_file_cache_type_t /*cache_type*/) {
+sys_file_t* SysFileOpenW(const std::filesystem::path& file_name, sys_file_cache_type_t cache_type) {
 	auto* ret = new sys_file_t;
 
 	auto real_name     = get_internal_name(file_name);
@@ -195,13 +231,15 @@ sys_file_t* SysFileOpenW(const std::filesystem::path& file_name,
 		ret->type = SYS_FILE_FILE;
 	}
 
+	apply_cache_hint(f, cache_type);
+
 	ret->f = f;
 
 	return ret;
 }
 
 sys_file_t* SysFileOpenRw(const std::filesystem::path& file_name,
-                          sys_file_cache_type_t /*cache_type*/) {
+                          sys_file_cache_type_t        cache_type) {
 	auto* ret = new sys_file_t;
 
 	auto real_name     = get_internal_name(file_name);
@@ -215,6 +253,8 @@ sys_file_t* SysFileOpenRw(const std::filesystem::path& file_name,
 		ret->type = SYS_FILE_FILE;
 	}
 
+	apply_cache_hint(f, cache_type);
+
 	ret->f = f;
 
 	return ret;
@@ -240,11 +280,17 @@ uint64_t SysFileSize(sys_file_t& f) {
 	[[maybe_unused]] int result = 0;
 
 	if (f.type == SYS_FILE_FILE) {
-		uint32_t pos  = ftell(f.f);
-		result        = fseek(f.f, 0, SEEK_END);
-		uint32_t size = ftell(f.f);
-		result        = fseek(f.f, pos, SEEK_SET);
-		return size;
+		// Preserve sizes above 4 GiB.
+		const off_t pos = ftello(f.f);
+		if (pos < 0) {
+			return 0;
+		}
+		if (fseeko(f.f, 0, SEEK_END) != 0) {
+			return 0;
+		}
+		const off_t size = ftello(f.f);
+		result           = fseeko(f.f, pos, SEEK_SET);
+		return (size < 0 ? 0 : static_cast<uint64_t>(size));
 	}
 
 	if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) {
@@ -261,8 +307,18 @@ uint64_t SysFileSize(const std::filesystem::path& file_name) {
 	return size;
 }
 
-bool SysFileTruncate(sys_file_t& /*f*/, uint64_t /*size*/) {
-	return false;
+bool SysFileTruncate(sys_file_t& f, uint64_t size) {
+	bool ok = false;
+	if (f.type == SYS_FILE_FILE) {
+		// Flush before resizing and restore the caller's position.
+		const auto position = ftell(f.f);
+		fflush(f.f);
+		ok = (ftruncate(fileno(f.f), static_cast<off_t>(size)) == 0);
+		if (position >= 0) {
+			fseek(f.f, position, SEEK_SET);
+		}
+	}
+	return ok;
 }
 
 bool SysFileUnlink(sys_file_t& /*f*/, const std::filesystem::path& name) {
@@ -383,6 +439,7 @@ SysFileTimeStruct SysFileGetLastAccessTimeUtc(const std::filesystem::path& name)
 	} else {
 		r.is_invalid = false;
 		r.time       = s.st_atime;
+		r.nanos      = KYTY_STAT_ATIME_NS(s);
 	}
 
 	return r;
@@ -401,6 +458,7 @@ SysFileTimeStruct SysFileGetLastWriteTimeUtc(const std::filesystem::path& name)
 	} else {
 		r.is_invalid = false;
 		r.time       = s.st_mtime;
+		r.nanos      = KYTY_STAT_MTIME_NS(s);
 	}
 
 	return r;
@@ -420,13 +478,36 @@ void SysFileGetLastAccessAndWriteTimeUtc(const std::filesystem::path& name, SysF
 		a.is_invalid = false;
 		w.is_invalid = false;
 		a.time       = s.st_atime;
+		a.nanos      = KYTY_STAT_ATIME_NS(s);
 		w.time       = s.st_mtime;
+		w.nanos      = KYTY_STAT_MTIME_NS(s);
 	}
 }
 
-void SysFileGetLastAccessAndWriteTimeUtc(sys_file_t& /*f*/, SysFileTimeStruct& /*a*/,
-                                         SysFileTimeStruct& /*w*/) {
-	EXIT("not implemented\n");
+void SysFileGetLastAccessAndWriteTimeUtc(sys_file_t& f, SysFileTimeStruct& a,
+                                         SysFileTimeStruct& w) {
+	if (f.type == SYS_FILE_FILE) {
+		struct stat s {};
+
+		const bool ok = (0 == fstat(fileno(f.f), &s));
+
+		a.is_invalid = w.is_invalid = !ok;
+
+		if (ok) {
+			a.time  = s.st_atime;
+			a.nanos = KYTY_STAT_ATIME_NS(s);
+			w.time  = s.st_mtime;
+			w.nanos = KYTY_STAT_MTIME_NS(s);
+		}
+	} else if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) {
+		// Memory-backed files use the current time.
+		SysTimeStruct t {};
+		SysGetSystemTimeUtc(t);
+		SysSystemToFileTimeUtc(t, a);
+		SysSystemToFileTimeUtc(t, w);
+	} else {
+		a.is_invalid = w.is_invalid = true;
+	}
 }
 
 bool SysFileSetLastAccessTimeUtc(const std::filesystem::path& name, SysFileTimeStruct& access) {
@@ -532,45 +613,110 @@ bool SysFileSetLastAccessAndWriteTimeUtc(const std::filesystem::path& name,
 	//	}
 }
 
-void SysFileFindFiles(const std::filesystem::path& /*path*/,
-                      std::vector<sys_file_find_t>& /*out*/) {
-	EXIT("not implemented\n");
-}
+// Recursively collect regular files.
+void SysFileFindFiles(const std::filesystem::path& path, std::vector<sys_file_find_t>& out) {
+	auto real_path = get_internal_name(path);
 
-void SysFileGetDents(const std::filesystem::path& path, std::vector<sys_dir_entry_t>& out) {
-	DIR* dir = opendir(path.c_str());
+	DIR* dir = opendir(real_path.string().c_str());
 	if (dir == nullptr) {
 		return;
 	}
-	for (dirent* entry = readdir(dir); entry != nullptr; entry = readdir(dir)) {
-		sys_dir_entry_t r {};
-		r.name = entry->d_name;
-		if (entry->d_type == DT_REG) {
-			r.is_file = true;
-		} else if (entry->d_type == DT_DIR) {
-			r.is_file = false;
-		} else {
-			// DT_UNKNOWN / symlink: resolve with stat.
-			struct stat st {};
-			r.is_file = (stat((path / entry->d_name).c_str(), &st) == 0) ? S_ISREG(st.st_mode) : true;
+
+	for (const dirent* entry = readdir(dir); entry != nullptr; entry = readdir(dir)) {
+		const std::string file_name(entry->d_name);
+
+		if (file_name == "." || file_name == "..") {
+			continue;
 		}
-		out.push_back(std::move(r));
+
+		auto        child = real_path / file_name;
+		struct stat s {};
+
+		// lstat, so a symlink is never followed into a cycle during the recursive walk.
+		if (0 != lstat(child.string().c_str(), &s)) {
+			continue;
 		}
+
+		if (S_ISDIR(s.st_mode)) {
+			SysFileFindFiles(child, out);
+		} else if (S_ISREG(s.st_mode)) {
+			sys_file_find_t r {};
+
+			r.path_with_name              = child;
+			r.size                        = static_cast<uint64_t>(s.st_size);
+			r.last_access_time.is_invalid = false;
+			r.last_access_time.time       = s.st_atime;
+			r.last_access_time.nanos      = KYTY_STAT_ATIME_NS(s);
+			r.last_write_time.is_invalid  = false;
+			r.last_write_time.time        = s.st_mtime;
+			r.last_write_time.nanos       = KYTY_STAT_MTIME_NS(s);
+
+			out.push_back(r);
+		}
+	}
+
 	closedir(dir);
 }
 
-bool SysFileCopyFile(const std::filesystem::path& /*src*/, const std::filesystem::path& /*dst*/) {
-	EXIT("not implemented\n");
-	return false;
+// Keep "." and ".." to match FindFirstFileW.
+void SysFileGetDents(const std::filesystem::path& path, std::vector<sys_dir_entry_t>& out) {
+	auto real_path = get_internal_name(path);
+
+	DIR* dir = opendir(real_path.string().c_str());
+	if (dir == nullptr) {
+		return;
+	}
+
+	for (const dirent* entry = readdir(dir); entry != nullptr; entry = readdir(dir)) {
+		sys_dir_entry_t r {};
+
+		r.name = entry->d_name;
+
+		if (entry->d_type == DT_UNKNOWN) {
+			// Some filesystems do not populate d_type.
+			struct stat s {};
+			r.is_file = 0 == lstat((real_path / r.name).string().c_str(), &s) && S_ISREG(s.st_mode);
+		} else {
+			r.is_file = entry->d_type != DT_DIR;
+		}
+
+		out.push_back(r);
+	}
+
+	closedir(dir);
 }
 
-bool SysFileMoveFile(const std::filesystem::path& /*src*/, const std::filesystem::path& /*dst*/) {
-	EXIT("not implemented\n");
-	return false;
+bool SysFileCopyFile(const std::filesystem::path& src, const std::filesystem::path& dst) {
+	std::error_code error;
+	return std::filesystem::copy_file(get_internal_name(src), get_internal_name(dst),
+	                                  std::filesystem::copy_options::overwrite_existing, error) &&
+	       !error;
 }
 
-void SysFileRemoveReadonly(const std::filesystem::path& /*name*/) {
-	EXIT("not implemented\n");
+bool SysFileMoveFile(const std::filesystem::path& src, const std::filesystem::path& dst) {
+	auto real_src = get_internal_name(src);
+	auto real_dst = get_internal_name(dst);
+
+	// Match MoveFileW: fail when the destination exists.
+	std::error_code error;
+	if (std::filesystem::exists(real_dst, error)) {
+		return false;
+	}
+
+	return 0 == rename(real_src.string().c_str(), real_dst.string().c_str());
+}
+
+void SysFileRemoveReadonly(const std::filesystem::path& name) {
+	auto real_name     = get_internal_name(name);
+	auto real_name_str = real_name.string();
+
+	struct stat s {};
+
+	if (0 != stat(real_name_str.c_str(), &s)) {
+		return;
+	}
+
+	chmod(real_name_str.c_str(), s.st_mode | S_IWUSR);
 }
 
 #endif

src/common/platform/sysLinuxVirtual.cpp

@@ -8,9 +8,11 @@
 #include "common/platform/sysVirtual.h"
 #include "common/virtualMemory.h"
 
+#include <atomic>
 #include <map>
 #include <pthread.h>
 #include <sys/mman.h>
+#include <unistd.h>
 
 #if defined(__APPLE__)
 #include <mach/mach.h>
@@ -81,6 +83,70 @@ static VirtualMemory::Mode get_protection_flag(int mode) {
 	}
 }
 
+// Keep automatic mappings inside the guest and GPU-addressable low window.
+#ifdef KYTY_FIXED_NOREPLACE
+static constexpr uintptr_t LOW_ARENA_LIMIT = 0x000000FC00000000ULL; // libc mspace window ceiling
+static constexpr uintptr_t LOW_ARENA_FLOOR = 0x000000A000000000ULL; // 640 GiB
+static constexpr uintptr_t LOW_ARENA_GRAIN = 0x0000000000010000ULL; // 64 KiB
+
+static_assert(LOW_ARENA_LIMIT <= 0x0000010000000000ULL,
+              "arena must stay inside the GPU page tracker's 1<<40 window");
+static_assert(LOW_ARENA_FLOOR < LOW_ARENA_LIMIT, "arena floor must sit below its ceiling");
+
+static std::atomic<uintptr_t> g_low_arena_next {LOW_ARENA_LIMIT};
+#endif
+
+// Caller holds g_virtual_mutex.
+static void record_alloc(uintptr_t addr, size_t size) {
+	auto next = g_allocs->upper_bound(addr);
+	if (next != g_allocs->begin()) {
+		auto       it         = std::prev(next);
+		const auto alloc_addr = it->first;
+		const auto alloc_end  = alloc_addr + it->second;
+		if (alloc_addr <= addr && addr + size <= alloc_end) {
+			g_allocs->erase(it);
+			if (alloc_addr < addr) {
+				(*g_allocs)[alloc_addr] = addr - alloc_addr;
+			}
+			if (addr + size < alloc_end) {
+				(*g_allocs)[addr + size] = alloc_end - (addr + size);
+			}
+		}
+	}
+	(*g_allocs)[addr] = size;
+}
+
+#ifdef KYTY_FIXED_NOREPLACE
+static uintptr_t align_up_to(uintptr_t addr, uint64_t alignment) {
+	return (addr + alignment - 1) & ~(alignment - 1);
+}
+#endif
+
+// Freed arena addresses are not reused while GPU caches remain keyed by address.
+static void* map_anonymous(uintptr_t addr, size_t size, int protect, int flags) {
+	if (addr != 0) {
+		return mmap(reinterpret_cast<void*>(addr), size, protect, flags, -1, 0); // NOLINT
+	}
+
+#ifdef KYTY_FIXED_NOREPLACE
+	const auto step = align_up_to(size, LOW_ARENA_GRAIN);
+	for (int attempt = 0; attempt < 256; attempt++) {
+		const auto top = g_low_arena_next.fetch_sub(step, std::memory_order_relaxed);
+		if (top < step || top - step < LOW_ARENA_FLOOR) {
+			break;
+		}
+		const auto hint = (top - step) & ~(LOW_ARENA_GRAIN - 1);
+		void* ptr = mmap(reinterpret_cast<void*>(hint), size, protect, flags | MAP_FIXED_NOREPLACE,
+		                 -1, 0); // NOLINT
+		if (ptr != MAP_FAILED) {
+			return ptr;
+		}
+	}
+#endif
+
+	return mmap(nullptr, size, protect, flags, -1, 0); // NOLINT
+}
+
 uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
 	EXIT_IF(g_allocs == nullptr);
 
@@ -88,14 +154,13 @@ uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mo
 
 	int protect = get_protection_flag(mode);
 
-	void* ptr =
-	    mmap(reinterpret_cast<void*>(addr), size, protect, MAP_PRIVATE | MAP_ANON, -1, 0); // NOLINT
+	void* ptr = map_anonymous(addr, size, protect, MAP_PRIVATE | MAP_ANON);
 
 	auto ret_addr = reinterpret_cast<uintptr_t>(ptr);
 
 	if (ptr != MAP_FAILED) {
 		pthread_mutex_lock(&g_virtual_mutex);
-		(*g_allocs)[ret_addr] = size;
+		record_alloc(ret_addr, size);
 		uintptr_t page_start = ret_addr >> 12u;
 		uintptr_t page_end   = (ret_addr + size - 1) >> 12u;
 		for (uintptr_t page = page_start; page <= page_end; page++) {
@@ -122,16 +187,15 @@ uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::
 	auto addr    = static_cast<uintptr_t>(address);
 	int  protect = get_protection_flag(mode);
 
-	void* ptr =
-	    mmap(reinterpret_cast<void*>(addr), size, protect, MAP_PRIVATE | MAP_ANON, -1, 0); // NOLINT
+	void* ptr = map_anonymous(addr, size, protect, MAP_PRIVATE | MAP_ANON);
 
 	auto ret_addr = reinterpret_cast<uintptr_t>(ptr);
 
 	if (ptr != MAP_FAILED && ((ret_addr & (alignment - 1)) != 0)) {
 		munmap(ptr, size);
 
-		ptr      = mmap(reinterpret_cast<void*>(addr), size + alignment, protect,
-		                MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); // NOLINT
+		ptr =
+		    map_anonymous(addr, size + alignment, protect, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE);
 		ret_addr = reinterpret_cast<uintptr_t>(ptr);
 		if (ptr != MAP_FAILED) {
 #if defined(__APPLE__)
@@ -186,7 +250,7 @@ uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::
 	}
 
 	pthread_mutex_lock(&g_virtual_mutex);
-	(*g_allocs)[ret_addr] = size;
+	record_alloc(ret_addr, size);
 	uintptr_t page_start = ret_addr >> 12u;
 	uintptr_t page_end   = (ret_addr + size - 1) >> 12u;
 	for (uintptr_t page = page_start; page <= page_end; page++) {
@@ -272,7 +336,7 @@ bool SysVirtualAllocFixed(uint64_t address, uint64_t size, VirtualMemory::Mode m
 
 	if (ptr != MAP_FAILED) {
 		pthread_mutex_lock(&g_virtual_mutex);
-		(*g_allocs)[ret_addr] = size;
+		record_alloc(ret_addr, size);
 		uintptr_t page_start = ret_addr >> 12u;
 		uintptr_t page_end   = (ret_addr + size - 1) >> 12u;
 		for (uintptr_t page = page_start; page <= page_end; page++) {
@@ -303,16 +367,15 @@ uint64_t SysVirtualReserveAligned(uint64_t address, uint64_t size, uint64_t alig
 
 	auto addr = static_cast<uintptr_t>(address);
 
-	void* ptr = mmap(reinterpret_cast<void*>(addr), size, PROT_NONE,
-	                 MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); // NOLINT
+	void* ptr = map_anonymous(addr, size, PROT_NONE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE);
 
 	auto ret_addr = reinterpret_cast<uintptr_t>(ptr);
 
 	if (ptr != MAP_FAILED && ((ret_addr & (alignment - 1)) != 0)) {
 		munmap(ptr, size);
 
-		ptr      = mmap(reinterpret_cast<void*>(addr), size + alignment, PROT_NONE,
-		                MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); // NOLINT
+		ptr      = map_anonymous(addr, size + alignment, PROT_NONE,
+		                         MAP_PRIVATE | MAP_ANON | MAP_NORESERVE);
 		ret_addr = reinterpret_cast<uintptr_t>(ptr);
 		if (ptr != MAP_FAILED) {
 #if defined(__APPLE__)
@@ -368,12 +431,7 @@ uint64_t SysVirtualReserveAligned(uint64_t address, uint64_t size, uint64_t alig
 	}
 
 	pthread_mutex_lock(&g_virtual_mutex);
-	(*g_allocs)[ret_addr] = size;
-	uintptr_t page_start  = ret_addr >> 12u;
-	uintptr_t page_end    = (ret_addr + size - 1) >> 12u;
-	for (uintptr_t page = page_start; page <= page_end; page++) {
-		(*g_protects)[page] = PROT_NONE;
-	}
+	record_alloc(ret_addr, size);
 	pthread_mutex_unlock(&g_virtual_mutex);
 
 	return ret_addr;
@@ -406,12 +464,7 @@ bool SysVirtualReserveFixed(uint64_t address, uint64_t size) {
 
 	if (ptr != MAP_FAILED) {
 		pthread_mutex_lock(&g_virtual_mutex);
-		(*g_allocs)[ret_addr] = size;
-		uintptr_t page_start  = ret_addr >> 12u;
-		uintptr_t page_end    = (ret_addr + size - 1) >> 12u;
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			(*g_protects)[page] = PROT_NONE;
-		}
+		record_alloc(ret_addr, size);
 		pthread_mutex_unlock(&g_virtual_mutex);
 
 		return true;
@@ -421,7 +474,29 @@ bool SysVirtualReserveFixed(uint64_t address, uint64_t size) {
 }
 
 bool SysVirtualDecommit(uint64_t address, uint64_t size) {
-	return SysVirtualProtect(address, size, VirtualMemory::Mode::NoAccess);
+	// Drop physical pages while preserving the reservation.
+	if (!SysVirtualProtect(address, size, VirtualMemory::Mode::NoAccess)) {
+		return false;
+	}
+
+	if (size != 0) {
+#if defined(__APPLE__)
+		constexpr int RECLAIM_ADVICE = MADV_FREE;
+#else
+		constexpr int RECLAIM_ADVICE = MADV_DONTNEED;
+#endif
+		const auto page_size = static_cast<uintptr_t>(sysconf(_SC_PAGESIZE));
+		if (page_size != 0) {
+			// Do not discard pages outside the requested range.
+			const auto begin = (static_cast<uintptr_t>(address) + page_size - 1) & ~(page_size - 1);
+			const auto end   = (static_cast<uintptr_t>(address) + size) & ~(page_size - 1);
+			if (end > begin) {
+				::madvise(reinterpret_cast<void*>(begin), end - begin, RECLAIM_ADVICE);
+			}
+		}
+	}
+
+	return true;
 }
 
 bool SysVirtualFree(uint64_t address) {
@@ -473,15 +548,34 @@ bool SysVirtualFreeRange(uint64_t address, uint64_t size) {
 		pthread_mutex_unlock(&g_virtual_mutex);
 		return false;
 	}
-	auto       it         = std::prev(next);
-	const auto alloc_addr = it->first;
-	const auto alloc_end  = alloc_addr + it->second;
-	if (addr < alloc_addr || end > alloc_end || munmap(reinterpret_cast<void*>(addr), size) != 0) {
+
+	// A reservation may have been split into several adjacent records.
+	auto       first      = std::prev(next);
+	const auto alloc_addr = first->first;
+	if (addr < alloc_addr || alloc_addr + first->second <= addr) {
 		pthread_mutex_unlock(&g_virtual_mutex);
 		return false;
 	}
 
-	g_allocs->erase(it);
+	auto      last   = first;
+	uintptr_t cursor = alloc_addr + first->second;
+	while (cursor < end) {
+		auto following = std::next(last);
+		if (following == g_allocs->end() || following->first != cursor) {
+			pthread_mutex_unlock(&g_virtual_mutex);
+			return false;
+		}
+		last   = following;
+		cursor = following->first + following->second;
+	}
+	const auto alloc_end = cursor;
+
+	if (munmap(reinterpret_cast<void*>(addr), size) != 0) {
+		pthread_mutex_unlock(&g_virtual_mutex);
+		return false;
+	}
+
+	g_allocs->erase(first, std::next(last));
 	if (alloc_addr < addr) {
 		(*g_allocs)[alloc_addr] = addr - alloc_addr;
 	}

src/common/platform/sysTimer.h

@@ -27,7 +27,9 @@ struct SysFileTimeStruct {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	FILETIME time;
 #elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
+	// Nanoseconds preserve sub-second file timestamps.
 	time_t time;
+	long   nanos;
 #endif
 	bool is_invalid;
 };
@@ -139,7 +141,7 @@ inline void SysFileToSystemTimeUtc(const SysFileTimeStruct& f, SysTimeStruct& t)
 	t.Hour         = i.tm_hour;
 	t.Minute       = i.tm_min;
 	t.Second       = (i.tm_sec == 60 ? 59 : i.tm_sec);
-	t.Milliseconds = 0;
+	t.Milliseconds = static_cast<uint16_t>((f.nanos / 1000000) % 1000);
 }
 
 inline void SysTimeTToSystem(time_t t, SysTimeStruct& s) {
@@ -168,10 +170,11 @@ inline void SysSystemToFileTimeUtc(const SysTimeStruct& f, SysFileTimeStruct& t)
 
 // Retrieves the current local date and time.
 inline void SysGetSystemTime(SysTimeStruct& t) {
-	time_t    st {};
+	// Preserve millisecond precision.
+	timespec  now {};
 	struct tm i {};
 
-	if (time(&st) == static_cast<time_t>(-1) || localtime_r(&st, &i) == nullptr) {
+	if (clock_gettime(CLOCK_REALTIME, &now) != 0 || localtime_r(&now.tv_sec, &i) == nullptr) {
 		t.is_invalid = true;
 		return;
 	}
@@ -183,15 +186,16 @@ inline void SysGetSystemTime(SysTimeStruct& t) {
 	t.Hour         = i.tm_hour;
 	t.Minute       = i.tm_min;
 	t.Second       = (i.tm_sec == 60 ? 59 : i.tm_sec);
-	t.Milliseconds = 0;
+	t.Milliseconds = static_cast<uint16_t>((now.tv_nsec / 1000000) % 1000);
 }
 
 // Retrieves the current system date and time in Coordinated Universal Time (UTC).
 inline void SysGetSystemTimeUtc(SysTimeStruct& t) {
-	time_t    st {};
+	// Preserve millisecond precision.
+	timespec  now {};
 	struct tm i {};
 
-	if (time(&st) == static_cast<time_t>(-1) || gmtime_r(&st, &i) == nullptr) {
+	if (clock_gettime(CLOCK_REALTIME, &now) != 0 || gmtime_r(&now.tv_sec, &i) == nullptr) {
 		t.is_invalid = true;
 		return;
 	}
@@ -203,7 +207,7 @@ inline void SysGetSystemTimeUtc(SysTimeStruct& t) {
 	t.Hour         = i.tm_hour;
 	t.Minute       = i.tm_min;
 	t.Second       = (i.tm_sec == 60 ? 59 : i.tm_sec);
-	t.Milliseconds = 0;
+	t.Milliseconds = static_cast<uint16_t>((now.tv_nsec / 1000000) % 1000);
 }
 
 inline void SysQueryPerformanceFrequency(uint64_t* freq) {

src/common/threads.cpp

@@ -5,6 +5,7 @@
 
 #include <algorithm>
 #include <atomic>
+#include <cerrno>
 #include <chrono>             // IWYU pragma: keep
 #include <condition_variable> // IWYU pragma: keep
 #include <mutex>
@@ -14,6 +15,12 @@
 #define KYTY_WIN_CS
 #endif
 
+// macOS has no clock_nanosleep.
+#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS && !defined(__APPLE__)
+#define KYTY_POSIX_HIGH_RES_SLEEP
+#include <ctime>
+#endif
+
 #include <sstream>
 #include <string>
 #include <thread>
@@ -121,6 +128,42 @@ static SleepConditionVariableCS_func_t ResolveSleepConditionVariableCS() {
 
 #endif
 
+#ifdef KYTY_POSIX_HIGH_RES_SLEEP
+// Spin for very short waits; use an absolute deadline for longer waits.
+static void SleepHighResolutionNanos(uint64_t nanos) {
+	if (nanos == 0) {
+		return;
+	}
+
+	constexpr uint64_t NANOS_PER_SEC = 1000000000;
+	constexpr uint64_t SPIN_LIMIT_NS = 50000; // below this a context switch dominates
+
+	timespec deadline {};
+	if (clock_gettime(CLOCK_MONOTONIC, &deadline) != 0) {
+		std::this_thread::sleep_for(std::chrono::nanoseconds(nanos));
+		return;
+	}
+
+	auto target_nsec = static_cast<uint64_t>(deadline.tv_nsec) + nanos;
+	deadline.tv_sec += static_cast<time_t>(target_nsec / NANOS_PER_SEC);
+	deadline.tv_nsec = static_cast<long>(target_nsec % NANOS_PER_SEC);
+
+	if (nanos <= SPIN_LIMIT_NS) {
+		timespec now {};
+		do {
+			if (clock_gettime(CLOCK_MONOTONIC, &now) != 0) {
+				return;
+			}
+		} while (now.tv_sec < deadline.tv_sec ||
+		         (now.tv_sec == deadline.tv_sec && now.tv_nsec < deadline.tv_nsec));
+		return;
+	}
+
+	while (clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &deadline, nullptr) == EINTR) {
+	}
+}
+#endif
+
 namespace Common {
 
 using thread_id_t = std::thread::id;
@@ -249,6 +292,8 @@ void Thread::Sleep(uint32_t millis) {
 void Thread::SleepMicro(uint32_t micros) {
 #ifdef KYTY_WIN_CS
 	SleepHighResolution100ns(static_cast<uint64_t>(micros) * 10);
+#elif defined(KYTY_POSIX_HIGH_RES_SLEEP)
+	SleepHighResolutionNanos(static_cast<uint64_t>(micros) * 1000);
 #else
 	std::this_thread::sleep_for(std::chrono::microseconds(micros));
 #endif
@@ -257,6 +302,8 @@ void Thread::SleepMicro(uint32_t micros) {
 void Thread::SleepNano(uint64_t nanos) {
 #ifdef KYTY_WIN_CS
 	SleepHighResolution100ns((nanos + 99) / 100);
+#elif defined(KYTY_POSIX_HIGH_RES_SLEEP)
+	SleepHighResolutionNanos(nanos);
 #else
 	std::this_thread::sleep_for(std::chrono::nanoseconds(nanos));
 #endif

src/common/uniqueFunction.h

@@ -20,9 +20,7 @@ class UniqueFunction {
 	public:
 		explicit Callable(Function function): m_function(std::move(function)) {}
 
-		Result Invoke(Args&&... args) override {
-			return m_function(std::forward<Args>(args)...);
-		}
+		Result Invoke(Args&&... args) override { return m_function(std::forward<Args>(args)...); }
 
 	private:
 		Function m_function;

src/common/virtualMemory.cpp

@@ -58,25 +58,6 @@ bool FlushInstructionCache(uint64_t address, uint64_t size) {
 	return SysVirtualFlushInstructionCache(address, size);
 }
 
-bool PatchReplace(uint64_t vaddr, uint64_t value) {
-	Mode old_mode {};
-	Protect(vaddr, 8, Mode::ReadWrite, &old_mode);
-
-	auto* ptr = reinterpret_cast<uint64_t*>(vaddr);
-
-	bool ret = (*ptr != value);
-
-	*ptr = value;
-
-	Protect(vaddr, 8, old_mode);
-
-	if (IsExecute(old_mode)) {
-		FlushInstructionCache(vaddr, 8);
-	}
-
-	return ret;
-}
-
 } // namespace VirtualMemory
 
 } // namespace Common

src/common/virtualMemory.h

@@ -37,7 +37,6 @@ bool     Free(uint64_t address);
 bool     FreeRange(uint64_t address, uint64_t size);
 bool     Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode = nullptr);
 bool     FlushInstructionCache(uint64_t address, uint64_t size);
-bool     PatchReplace(uint64_t vaddr, uint64_t value);
 
 } // namespace VirtualMemory
 

src/emulator.cpp

@@ -105,7 +105,7 @@ static void ClearDebugTextureFolder() {
 	}
 }
 
-static void Init(const Config::ConfigOptions& cfg) {
+static void Init(const Config::ConfigOptions& cfg, const std::filesystem::path& param_json) {
 	EXIT_IF(!Common::Thread::IsMainThread());
 
 	auto* slist = Common::SubsystemsList::Instance();
@@ -127,12 +127,21 @@ static void Init(const Config::ConfigOptions& cfg) {
 	slist->InitAll(true);
 
 	Config::Load(cfg);
+	slist->Add(log, {core, config});
+	slist->InitAll(true);
+
+	if (Common::File::IsFileExisting(param_json)) {
+		Loader::SystemContentLoadParamSfo(param_json);
+		if (const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
+		    flexible_memory_size != 0) {
+			Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
+		}
+	}
 
 	slist->Add(audio, {core, log, pthread, memory});
 	slist->Add(controller, {core, log, config});
 	slist->Add(file_system, {core, log, pthread});
 	slist->Add(graphics, {core, log, pthread, memory, config, profiler, controller});
-	slist->Add(log, {core, config});
 	slist->Add(memory, {core, log});
 	slist->Add(network, {core, log, pthread});
 	slist->Add(profiler, {core, config});
@@ -180,7 +189,8 @@ void Run(const RunOptions& options) {
 		EXIT("ELF is required\n");
 	}
 
-	Init(options.config);
+	const auto param_json = options.app0_dir / "sce_sys" / "param.json";
+	Init(options.config, param_json);
 
 	ClearDebugTextureFolder();
 
@@ -192,15 +202,6 @@ void Run(const RunOptions& options) {
 	Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/app0");
 	Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/hostapp");
 
-	auto param_json = options.app0_dir / "sce_sys" / "param.json";
-	if (Common::File::IsFileExisting(param_json)) {
-		Loader::SystemContentLoadParamSfo(param_json);
-		if (auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
-		    flexible_memory_size != 0) {
-			Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
-		}
-	}
-
 	MountSandboxDirs();
 
 	auto* rt = Common::Singleton<Loader::RuntimeLinker>::Instance();

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -83,8 +83,7 @@ public:
 	               uint32_t first_instance = 0);
 	void DrawIndexOffset(uint32_t index_offset, uint32_t index_count, uint32_t flags);
 	void DrawIndexAuto(uint32_t index_count, uint32_t flags,
-	                   uint32_t render_target_slice_offset = 0, uint32_t instance_count = 1,
-	                   uint32_t first_vertex = 0, uint32_t first_instance = 0);
+	                   uint32_t render_target_slice_offset = 0);
 	void DrawIndirect(uint32_t data_offset, uint32_t draw_initiator, bool indexed);
 	void DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count,
 	                       const volatile uint32_t* count_addr, uint32_t stride_in_bytes,
@@ -152,6 +151,9 @@ private:
 	                      uint32_t interrupt_context_id);
 	void ProcessPm4(Pm4Execution& execution, size_t stop_depth);
 	void SuspendPm4();
+	void SubmitNonIndexedDraw(uint32_t vertex_count, uint32_t flags,
+	                          uint32_t render_target_slice_offset, uint32_t first_vertex,
+	                          uint32_t first_instance);
 
 	CommandScheduler&    GetScheduler() const { return m_renderer.GetCommandScheduler(); }
 	RenderCommandBuffer& CurrentBuffer() { return GetScheduler().Current(); }
@@ -168,6 +170,7 @@ private:
 	uint64_t         m_index_base_addr                  = 0;
 	uint64_t         m_draw_indirect_args_base_addr     = 0;
 	uint64_t         m_dispatch_indirect_args_base_addr = 0;
+	// Persistent draw state: indirect draws update it for subsequent draws.
 	uint32_t m_num_instances = 1;
 
 	uint32_t m_de_count        = 0;

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -65,10 +65,23 @@ constexpr uint32_t GcrKnownMask             = GcrGl2MetadataInvalidate | GcrGl0V
                                               GcrGl2Writeback | GcrOrder012 | GcrOrder210;
 constexpr uint32_t RegisterSelectorMask     = 0x70000000u;
 
-uint32_t NormalizeRegisterOffset(uint32_t raw_offset) {
-	return (raw_offset & ~RegisterSelectorMask);
+constexpr uint32_t NormalizeRegisterOffset(uint32_t raw_offset) {
+	return raw_offset & ~RegisterSelectorMask;
 }
 
+// Indirect Cx descriptors retain their selector. Selector 1 offsets 0..31 address the
+// SPI_PS_INPUT_CNTL register bank; ordinary context-register offsets remain unchanged.
+constexpr uint32_t DecodeIndirectCxRegisterOffset(uint32_t raw_offset) {
+	const auto offset = NormalizeRegisterOffset(raw_offset);
+	return (raw_offset & RegisterSelectorMask) == Pm4::CX_PS_SHADER_USAGE_BASE && offset < 32u
+	           ? Pm4::SPI_PS_INPUT_CNTL_0 + offset
+	           : offset;
+}
+
+static_assert(DecodeIndirectCxRegisterOffset(Pm4::CX_PS_SHADER_USAGE_BASE + 2u) ==
+              Pm4::SPI_PS_INPUT_CNTL_0 + 2u);
+static_assert(DecodeIndirectCxRegisterOffset(Pm4::DB_Z_INFO) == Pm4::DB_Z_INFO);
+
 bool ReleaseMemGcrNeedsBarrier(uint32_t eop_event_type, uint32_t gcr_cntl) {
 	return eop_event_type != 0x28u ||
 	       (gcr_cntl & (GcrGl2MetadataInvalidate | GcrGl0VectorInvalidate | GcrGl1Invalidate |
@@ -1913,8 +1926,14 @@ KYTY_CP_OP_PARSER(CpOpCopyData) {
 	const uint32_t num_bytes           = ((control >> 16u) & 0x1u) != 0 ? 8u : 4u;
 	const uint64_t src                 = buffer[1] | (static_cast<uint64_t>(buffer[2]) << 32u);
 	const uint64_t dst                 = buffer[3] | (static_cast<uint64_t>(buffer[4]) << 32u);
-	if (src_sel == (9u << 1u)) {
-		if (dst_sel != (2u << 1u) || dst == 0 || (dst & (num_bytes - 1u)) != 0) {
+	uint32_t       reference_clock_dst = 0;
+	switch (src_sel) {
+		case 9u: reference_clock_dst = 2u; break;
+		case 18u: reference_clock_dst = 4u; break;
+		default: break;
+	}
+	if (reference_clock_dst != 0) {
+		if (dst_sel != reference_clock_dst || dst == 0 || (dst & (num_bytes - 1u)) != 0) {
 			EXIT("unsupported reference-clock copyData, src_sel=0x%02" PRIx32
 			     " dst_sel=0x%02" PRIx32 " dst=0x%016" PRIx64 " size=%u\n",
 			     src_sel, dst_sel, dst, num_bytes);
@@ -2321,14 +2340,18 @@ KYTY_CP_OP_PARSER(CpOpIndirectCxRegs) {
 		EXIT("indirect CX registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw);
 	}
 	for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) {
-		auto cmd_offset = indirect_buffer[0];
+		// Keep the encoded offset for packet control values, and use the decoded offset only
+		// for register dispatch.
+		auto raw_cmd_offset = indirect_buffer[0];
+		auto cmd_offset     = DecodeIndirectCxRegisterOffset(raw_cmd_offset);
 		auto value          = indirect_buffer[1];
 
 		if (HwCtxTrySetFakeRegister(cmd_offset, value)) {
 			continue;
 		}
 
-		if (cmd_offset == 0xffffffffu) {
+		// The sentinel is an encoded descriptor value and must be checked before normalization.
+		if (raw_cmd_offset == 0xffffffffu) {
 			static bool logged = false;
 			if (!logged) {
 				LOGF("\t temporary: skipping indirect CX sentinel pair offset = 0xffffffff, value "
@@ -3373,6 +3396,12 @@ void GraphicsInitJmpTablesCxIndirect() {
 	g_hw_ctx_indirect_func[Pm4::DB_COUNT_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
 		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
 	};
+	for (auto cmd_offset = Pm4::DB_SRESULTS_COMPARE_STATE0;
+	     cmd_offset <= Pm4::DB_SRESULTS_COMPARE_STATE1; cmd_offset++) {
+		g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
+			HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		};
+	}
 	g_hw_ctx_indirect_func[Pm4::DB_RENDER_OVERRIDE] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
 		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
 	};

src/graphics/guest_gpu/gpu_defs.h

@@ -72,6 +72,7 @@ enum class ChannelLayout : uint32_t {
 	k32_32       = 11,
 	k16_16_16_16 = 12,
 	k32_32_32_32 = 14,
+	k5_6_5       = 16,
 	k5_5_5_1     = 17,
 	k4_4_4_4     = 19,
 	kBc1         = 35,
@@ -374,6 +375,8 @@ enum class BufferFormat : uint32_t {
 	k32_32_32_32UInt    = 75,
 	k32_32_32_32SInt    = 76,
 	k32_32_32_32Float   = 77,
+	k8Srgb              = 128,
+	k8_8Srgb            = 129,
 	k8_8_8_8Srgb        = 130,
 	k9_9_9_5Float       = 132,
 	k5_6_5UNorm         = 133,

src/graphics/guest_gpu/gpu_format.cpp

@@ -39,6 +39,7 @@ constexpr FormatInfo kFormatInfo[] = {
     {GpuEnumValue(BufferFormat::k16_16Float), 4, 0, 4, true, false},
     {GpuEnumValue(BufferFormat::k11_11_10Float), 4, 0, 4, true, false},
     {GpuEnumValue(BufferFormat::k10_10_10_2UNorm), 4, 0, 4, true, false},
+    {GpuEnumValue(BufferFormat::k10_10_10_2UInt), 4, 0, 4, true, true},
     {GpuEnumValue(BufferFormat::k8_8_8_8UNorm), 4, 0, 4, true, false},
     {GpuEnumValue(BufferFormat::k8_8_8_8SNorm), 4, 0, 4, true, false},
     {GpuEnumValue(BufferFormat::k8_8_8_8UInt), 4, 0, 4, true, true},
@@ -57,6 +58,8 @@ constexpr FormatInfo kFormatInfo[] = {
     {GpuEnumValue(BufferFormat::k32_32_32_32UInt), 16, 0, 16, true, true},
     {GpuEnumValue(BufferFormat::k32_32_32_32SInt), 16, 0, 16, false, false},
     {GpuEnumValue(BufferFormat::k32_32_32_32Float), 16, 0, 16, true, false},
+    {GpuEnumValue(BufferFormat::k8Srgb), 1, 0, 0, true, false},
+    {GpuEnumValue(BufferFormat::k8_8Srgb), 2, 0, 0, true, false},
     {GpuEnumValue(BufferFormat::k8_8_8_8Srgb), 4, 0, 4, true, false},
     {GpuEnumValue(BufferFormat::k9_9_9_5Float), 4, 0, 0, true, false},
     {GpuEnumValue(BufferFormat::k5_6_5UNorm), 2, 0, 2, true, false},

src/graphics/guest_gpu/graphicsRun.cpp

@@ -34,7 +34,6 @@ namespace Libs::Graphics {
 
 static thread_local CommandProcessor* g_current_processor = nullptr;
 static thread_local Pm4Execution*     g_current_execution = nullptr;
-static thread_local uint32_t          g_submission_pause_depth = 0;
 static thread_local bool              g_gpu_mutex_owned   = false;
 static thread_local bool              g_gpu_thread        = false;
 
@@ -98,8 +97,6 @@ public:
 	                   bool trigger_agc_interrupt_on_done);
 	void SubmitFlipPreparation(uint64_t request_id);
 	void Done();
-	void PauseSubmissions();
-	void ResumeSubmissions();
 	void Shutdown();
 	[[nodiscard]] bool IsStopping();
 	void               SendCommand(Common::UniqueFunction<void>&& command);
@@ -410,8 +407,13 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw
 	const uint32_t increment    = (write_control >> 16u) & 0x1u;
 	const uint32_t write_confirm = (write_control >> 20u) & 0x1u;
 
-	if (dst_sel != 0 && dst_sel != 2 && dst_sel != 4 && dst_sel != 5) {
-		EXIT("unsupported writeData destination selector 0x%02" PRIx32 "\n", dst_sel);
+	switch (dst_sel) {
+		case 0:
+		case 2:
+		case 4:
+		case 5:
+		case 6: break;
+		default: EXIT("unsupported writeData destination selector 0x%02" PRIx32 "\n", dst_sel);
 	}
 	EXIT_NOT_IMPLEMENTED(increment != 0);
 
@@ -691,26 +693,6 @@ bool GpuState::Process(Submission& submission) {
 	return complete;
 }
 
-void GpuState::PauseSubmissions() {
-	if (g_gpu_mutex_owned) {
-		EXIT("GPU submissions are already paused by this thread\n");
-	}
-	g_gpu_mutex_owned = true;
-	m_submission_mutex.Lock();
-	if (!IsGpuThread()) {
-		WaitLocked();
-	}
-	m_renderer.GetCommandScheduler().DrainPriorityOperations();
-}
-
-void GpuState::ResumeSubmissions() {
-	if (!g_gpu_mutex_owned) {
-		EXIT("GPU submissions resumed without an active pause\n");
-	}
-	m_submission_mutex.Unlock();
-	g_gpu_mutex_owned = false;
-}
-
 Pm4ProcessResult CommandProcessor::Process(Pm4Execution& execution, uint32_t* buffer,
                                            uint32_t size_dw) {
 	KYTY_PROFILER_BLOCK("CommandProcessor::Process");
@@ -962,9 +944,8 @@ void CommandProcessor::DrawIndexOffset(uint32_t index_offset, uint32_t index_cou
 	auto* index_addr = reinterpret_cast<const void*>(
 	    m_index_base_addr + static_cast<uint64_t>(index_offset) * index_size);
 
-	m_renderer.GetRenderExecutor().DrawIndex(m_submit_id, CurrentBuffer(),
-	                                             m_index_type_and_size, index_count, index_addr,
-	                                             flags, 1, m_num_instances);
+	m_renderer.GetRenderExecutor().DrawIndex(m_submit_id, CurrentBuffer(), m_index_type_and_size,
+	                                         index_count, index_addr, flags, 1, m_num_instances);
 }
 
 void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiator, bool indexed) {
@@ -1002,8 +983,9 @@ void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiato
 				     args.start_vertex_location, args.start_instance_location);
 			}
 		}
-		DrawIndexAuto(args.vertex_count_per_instance, 0, 0, args.instance_count,
-		              args.start_vertex_location, args.start_instance_location);
+		m_num_instances = args.instance_count;
+		SubmitNonIndexedDraw(args.vertex_count_per_instance, 0, 0, args.start_vertex_location,
+		                     args.start_instance_location);
 		return;
 	}
 
@@ -1043,6 +1025,7 @@ void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiato
 		}
 	}
 
+	m_num_instances = args.instance_count;
 	DrawIndex(index_count, index_addr, 0, 1, args.instance_count, nullptr, 0,
 	          static_cast<int32_t>(args.base_vertex_location), args.start_instance_location);
 }
@@ -1100,8 +1083,9 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 					     args->start_vertex_location, args->start_instance_location);
 				}
 			}
-			DrawIndexAuto(args->vertex_count_per_instance, 0, 0, args->instance_count,
-			              args->start_vertex_location, args->start_instance_location);
+			m_num_instances = args->instance_count;
+			SubmitNonIndexedDraw(args->vertex_count_per_instance, 0, 0, args->start_vertex_location,
+			                     args->start_instance_location);
 			continue;
 		}
 
@@ -1142,6 +1126,7 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 			}
 		}
 
+		m_num_instances = args->instance_count;
 		DrawIndex(index_count, index_addr, 0, 1, args->instance_count, nullptr, 0,
 		          static_cast<int32_t>(args->base_vertex_location), args->start_instance_location);
 	}
@@ -1190,8 +1175,8 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g
 			}
 		}
 
-		m_renderer.GetRenderExecutor().DispatchDirect(
-		    m_submit_id, CurrentBuffer(), thread_group_x, thread_group_y, thread_group_z, mode);
+		m_renderer.GetRenderExecutor().DispatchDirect(m_submit_id, CurrentBuffer(), thread_group_x,
+		                                              thread_group_y, thread_group_z, mode);
 	}
 
 	constexpr uint32_t DispatchInitiatorUseThreadDimensions = 1u << 5u;
@@ -1233,13 +1218,18 @@ void CommandProcessor::DispatchIndirect(uint32_t data_offset, uint32_t mode) {
 }
 
 void CommandProcessor::DrawIndexAuto(uint32_t index_count, uint32_t flags,
-                                     uint32_t render_target_slice_offset, uint32_t instance_count,
+                                     uint32_t render_target_slice_offset) {
+	SubmitNonIndexedDraw(index_count, flags, render_target_slice_offset, 0, 0);
+}
+
+void CommandProcessor::SubmitNonIndexedDraw(uint32_t vertex_count, uint32_t flags,
+                                            uint32_t render_target_slice_offset,
                                             uint32_t first_vertex, uint32_t first_instance) {
 	CheckBuffer();
 
-	m_renderer.GetRenderExecutor().DrawAuto(
-	    m_submit_id, CurrentBuffer(), index_count, flags, render_target_slice_offset,
-	    instance_count, first_vertex, first_instance);
+	m_renderer.GetRenderExecutor().DrawAuto(m_submit_id, CurrentBuffer(), vertex_count, flags,
+	                                        render_target_slice_offset, m_num_instances,
+	                                        first_vertex, first_instance);
 }
 
 void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) {
@@ -1317,8 +1307,8 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 				if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) {
 					auto* dst = static_cast<uint32_t*>(dst_gpu_addr);
 					SynchronizeGpu();
-					Sync::ReadGds(m_renderer.GetBufferCache().GetGdsBuffer(), dst,
-					              value & 0xffffu, value >> 16u);
+					Sync::ReadGds(m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu,
+					              value >> 16u);
 					Sync::WriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst, value & 0xffffu,
 					                            value >> 16u);
 					return;
@@ -1486,8 +1476,7 @@ void CommandProcessor::EmitGlobalBarrier() {
 	barrier.srcStageMask  = vk::PipelineStageFlagBits2::eAllCommands;
 	barrier.srcAccessMask = vk::AccessFlagBits2::eMemoryWrite;
 	barrier.dstStageMask  = vk::PipelineStageFlagBits2::eAllCommands;
-	barrier.dstAccessMask =
-	    vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite;
+	barrier.dstAccessMask = vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite;
 
 	vk::DependencyInfo dependency {};
 	dependency.memoryBarrierCount = 1;
@@ -1690,32 +1679,6 @@ int Gpu::GetFrameNum() const {
 	return m_state->GetFrameNum();
 }
 
-void Gpu::PauseSubmissions() {
-	m_state->PauseSubmissions();
-}
-
-void Gpu::ResumeSubmissions() {
-	m_state->ResumeSubmissions();
-}
-
-Gpu::SubmissionLock::SubmissionLock(Gpu& gpu): m_gpu(gpu) {
-	if (g_current_processor != nullptr || g_submission_pause_depth == UINT32_MAX) {
-		EXIT("cannot acquire GPU submission lock in the current state\n");
-	}
-	if (g_submission_pause_depth++ == 0) {
-		m_gpu.PauseSubmissions();
-	}
-}
-
-Gpu::SubmissionLock::~SubmissionLock() {
-	if (g_submission_pause_depth == 0) {
-		EXIT("GPU submission lock released without ownership\n");
-	}
-	if (--g_submission_pause_depth == 0) {
-		m_gpu.ResumeSubmissions();
-	}
-}
-
 bool Gpu::IsCommandProcessorThread() noexcept {
 	return g_current_processor != nullptr;
 }
@@ -1724,12 +1687,4 @@ CommandProcessor* Gpu::CurrentCommandProcessor() noexcept {
 	return g_current_processor;
 }
 
-bool Gpu::SubmissionLockHeld() noexcept {
-	return g_submission_pause_depth != 0;
-}
-
-bool Gpu::MutexHeld() noexcept {
-	return g_gpu_mutex_owned;
-}
-
 } // namespace Libs::Graphics

src/graphics/guest_gpu/graphicsRun.h

@@ -35,25 +35,8 @@ public:
 
 	[[nodiscard]] static bool              IsCommandProcessorThread() noexcept;
 	[[nodiscard]] static CommandProcessor* CurrentCommandProcessor() noexcept;
-	[[nodiscard]] static bool              SubmissionLockHeld() noexcept;
-	[[nodiscard]] static bool              MutexHeld() noexcept;
-
-	class SubmissionLock final {
-	public:
-		explicit SubmissionLock(Gpu& gpu);
-		~SubmissionLock();
-		KYTY_CLASS_NO_COPY(SubmissionLock);
-
-	private:
-		Gpu& m_gpu;
-	};
 
 private:
-	friend class SubmissionLock;
-
-	void PauseSubmissions();
-	void ResumeSubmissions();
-
 	std::unique_ptr<GpuState> m_state;
 };
 } // namespace Libs::Graphics

src/graphics/guest_gpu/pm4.cpp

@@ -110,7 +110,6 @@ void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t star
 	auto* cmd = cmd_buffer + start_dw;
 	auto  dw  = num_dw;
 	while (dw != 0) {
-		EXIT_NOT_IMPLEMENTED(dw < 2);
 		EXIT_NOT_IMPLEMENTED(dw > num_dw);
 
 		auto cmd_id = *cmd++;
@@ -120,6 +119,9 @@ void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t star
 		uint32_t len = 0;
 
 		const auto packet_type = static_cast<PacketType>(cmd_id >> 30u);
+		// Type-2 packets are header-only padding; every other packet type requires a body.
+		EXIT_NOT_IMPLEMENTED(dw < 2 && packet_type != PacketType::Type2);
+
 		switch (packet_type) {
 			case PacketType::Type3: {
 				const bool    sh_gx = (cmd_id & 0x2u) == 0;

src/graphics/guest_gpu/pm4.h

@@ -385,6 +385,9 @@ constexpr uint32_t SPI_SHADER_POS_FORMAT  = 0x1C3;
 constexpr uint32_t SPI_SHADER_Z_FORMAT    = 0x1C4;
 constexpr uint32_t SPI_SHADER_COL_FORMAT  = 0x1C5;
 
+// Indirect Cx descriptor selector for the 32-entry PS input-control register bank.
+constexpr uint32_t CX_PS_SHADER_USAGE_BASE = 0x10000000u;
+
 constexpr uint32_t CB_BLEND0_CONTROL                            = 0x1E0;
 constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_SHIFT       = 0;
 constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_MASK        = 0x1F;

src/graphics/host_gpu/memoryTracker.cpp

@@ -4,25 +4,9 @@
 
 namespace Libs::Graphics {
 
-#if defined(KYTY_MEMORY_TRACKER_TESTS)
-namespace {
-std::atomic<MemoryTracker::UnmapContentionHook> g_unmap_contention_hook {nullptr};
-}
-
-void MemoryTracker::SetUnmapContentionHook(UnmapContentionHook hook) noexcept {
-	g_unmap_contention_hook.store(hook, std::memory_order_release);
-}
-#endif
-
 static_assert(std::atomic<void*>::is_always_lock_free);
 
-MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode)
-    : m_page_manager(page_manager), m_gpu_watch_mode(gpu_watch_mode) {
-	switch (m_gpu_watch_mode) {
-		case PageWatchMode::Write:
-		case PageWatchMode::ReadWrite: break;
-		default: EXIT("unsupported memory tracker GPU page-watch mode\n");
-	}
+MemoryTracker::MemoryTracker(PageManager& page_manager): m_page_manager(page_manager) {
 	m_regions = std::make_unique<std::atomic<RegionManager*>[]>(REGION_COUNT);
 	for (size_t i = 0; i < REGION_COUNT; i++) {
 		m_regions[i].store(nullptr, std::memory_order_relaxed);
@@ -31,6 +15,7 @@ MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_
 
 MemoryTracker::~MemoryTracker() = default;
 
+#if KYTY_BUILD == KYTY_BUILD_DEBUG
 void MemoryTracker::ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
                                           const char* operation) const noexcept {
 	if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr ||
@@ -68,6 +53,7 @@ void MemoryTracker::ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t va
 		}
 	}
 }
+#endif
 
 void MemoryTracker::ValidateRange(uint64_t vaddr, uint64_t size) {
 	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
@@ -94,7 +80,6 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
 bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		return manager->IsModified<DirtySource::Cpu>(offset, bytes);
@@ -104,7 +89,6 @@ bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	return Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		return manager->IsModified<DirtySource::Gpu>(offset, bytes);
@@ -114,45 +98,31 @@ bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
 void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
-		const auto       changed =
 		manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
-		manager->ApplyProtection(changed, false);
 	});
 }
 
 void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
-	Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+	Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
-		const auto       changed =
 		manager->ChangeState<DirtySource::Gpu, true>(manager->GetCpuAddr() + offset, bytes);
-		manager->ApplyGpuProtection(changed, true, m_gpu_watch_mode);
 	});
 }
 
 void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
-	Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+	Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
-		if (!manager->IsFullyModified<DirtySource::Gpu>(offset, bytes)) {
-			EXIT("cannot clear partially GPU-dirty tracking range\n");
-		}
-		const auto changed =
 		manager->ChangeState<DirtySource::Gpu, false>(manager->GetCpuAddr() + offset, bytes);
-		manager->ApplyGpuProtection(changed, false, m_gpu_watch_mode);
 	});
 }
 
 void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
-	RequireMapped(vaddr, size);
-
 	std::vector<RegionManager*> managers;
 	managers.reserve((vaddr % TRACKER_REGION_SIZE + size + TRACKER_REGION_SIZE - 1) /
 	                 TRACKER_REGION_SIZE);
@@ -171,10 +141,7 @@ void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
 		EXIT("cannot untrack GPU-dirty memory\n");
 	}
 	Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-		const auto changed =
 		manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
-		manager->ApplyProtection(changed, false);
-		manager->Untrack(manager->GetCpuAddr() + offset, bytes);
 	});
 	locks.clear();
 }
@@ -185,109 +152,4 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
 	UntrackMemoryLocked(vaddr, size);
 }
 
-void MemoryTracker::UnmapMemory(uint64_t vaddr, uint64_t size) {
-	CheckNotInUploadCallback();
-	std::unique_lock access(m_access_mutex, std::try_to_lock);
-	if (!access.owns_lock()) {
-#if defined(KYTY_MEMORY_TRACKER_TESTS)
-		if (const auto hook = g_unmap_contention_hook.load(std::memory_order_acquire);
-		    hook != nullptr) {
-			hook();
-		}
-#endif
-		access.lock();
-	}
-	UntrackMemoryLocked(vaddr, size);
-	m_page_manager.OnGpuUnmap(vaddr, size);
-}
-
-bool MemoryTracker::InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept {
-	switch (phase) {
-		case PageFaultPhase::Release: return true;
-		case PageFaultPhase::Invalidate: {
-			const auto action = BeginCpuFault(vaddr, size);
-			switch (action) {
-				case CpuFaultAction::Untracked: return false;
-				case CpuFaultAction::Continue: return true;
-				case CpuFaultAction::Download:
-					EXIT("generic region invalidation cannot download GPU-dirty memory\n");
-			}
-		}
-		case PageFaultPhase::Complete:
-			return CompleteCpuFault(vaddr, size, PageFaultAccess::Write, false);
-	}
-	EXIT("unsupported region invalidation phase\n");
-}
-
-bool MemoryTracker::InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr, uint64_t size,
-                                              PageFaultPhase phase) noexcept {
-	switch (phase) {
-		case PageFaultPhase::Release: return true;
-		case PageFaultPhase::Invalidate: {
-			const bool gpu_modified = Iterate<false>(
-			    vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-				    std::scoped_lock lock(manager->lock);
-				    return manager->IsModified<DirtySource::Gpu>(offset, bytes);
-			    });
-			if (!gpu_modified) {
-				return false;
-			}
-			const auto action = BeginCpuFault(vaddr, size);
-			if (access != PageFaultAccess::Write || action != CpuFaultAction::Download) {
-				EXIT("virtual GPU write fault requires write access to GPU-dirty memory\n");
-			}
-			return true;
-		}
-		case PageFaultPhase::Complete: {
-			if (access != PageFaultAccess::Write) {
-				EXIT("virtual GPU write completion requires write access\n");
-			}
-			bool completed = false;
-			Iterate<false>(
-			    vaddr, size, [&completed](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-				    std::scoped_lock lock(manager->lock);
-				    if (completed) {
-					    EXIT("virtual GPU write fault spans multiple tracked regions\n");
-				    }
-				    completed =
-				        manager->CompleteVirtualGpuWrite(manager->GetCpuAddr() + offset, bytes);
-			    });
-			return completed;
-		}
-	}
-	EXIT("unsupported virtual GPU write invalidation phase\n");
-}
-
-CpuFaultAction MemoryTracker::BeginCpuFault(uint64_t vaddr, uint64_t size,
-                                            PageFaultAccess access) noexcept {
-	CheckNotInUploadCallback();
-	CpuFaultAction action = CpuFaultAction::Untracked;
-	Iterate<false>(
-	    vaddr, size, [&action, access](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-		    std::scoped_lock lock(manager->lock);
-		    if (action != CpuFaultAction::Untracked) {
-			    EXIT("CPU fault spans multiple tracked regions\n");
-		    }
-		    action = manager->BeginCpuFault(manager->GetCpuAddr() + offset, bytes, access);
-	    });
-	return action;
-}
-
-bool MemoryTracker::CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access,
-                                     bool downloaded) noexcept {
-	CheckNotInUploadCallback();
-	bool found = false;
-	Iterate<false>(
-	    vaddr, size,
-	    [&found, access, downloaded](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-		    std::scoped_lock lock(manager->lock);
-		    if (found) {
-			    EXIT("CPU fault completion spans multiple tracked regions\n");
-		    }
-		    found = manager->CompleteCpuFault(manager->GetCpuAddr() + offset, bytes, access,
-		                                      downloaded);
-	    });
-	return found;
-}
-
 } // namespace Libs::Graphics

src/graphics/host_gpu/memoryTracker.h

@@ -18,8 +18,7 @@ namespace Libs::Graphics {
 
 class MemoryTracker final {
 public:
-	explicit MemoryTracker(PageManager&  page_manager,
-	                       PageWatchMode gpu_watch_mode = PageWatchMode::ReadWrite);
+	explicit MemoryTracker(PageManager& page_manager);
 	~MemoryTracker();
 
 	KYTY_CLASS_NO_COPY(MemoryTracker);
@@ -30,20 +29,55 @@ public:
 	void               MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UntrackMemory(uint64_t vaddr, uint64_t size);
-	void               UnmapMemory(uint64_t vaddr, uint64_t size);
-	[[nodiscard]] CpuFaultAction
-	                   BeginCpuFault(uint64_t vaddr, uint64_t size,
-	                                 PageFaultAccess access = PageFaultAccess::Write) noexcept;
-	[[nodiscard]] bool CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access,
-	                                    bool downloaded) noexcept;
-	[[nodiscard]] bool InvalidateRegion(uint64_t vaddr, uint64_t size,
-	                                    PageFaultPhase phase) noexcept;
-	[[nodiscard]] bool InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr,
-	                                             uint64_t size, PageFaultPhase phase) noexcept;
+	template <typename Flush>
+	void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) {
+		static_assert(std::is_invocable_v<Flush&>);
+		CheckNotInUploadCallback();
+		ValidateRange(vaddr, size);
+
+		const auto update_cpu_state = [this, vaddr, size] {
+			std::lock_guard             access(m_access_mutex);
+			std::vector<RegionManager*> managers;
+			Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
+				managers.push_back(manager);
+			});
+			std::vector<std::unique_lock<TrackingSpinLock>> locks;
+			locks.reserve(managers.size());
+			for (auto* manager: managers) {
+				locks.emplace_back(manager->lock);
+			}
+			const bool gpu_modified = Iterate<false>(
+			    vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+				    return manager->IsModified<DirtySource::Gpu>(offset, bytes);
+			    });
+			if (gpu_modified) {
+				return true;
+			}
+			Iterate<false>(vaddr, size,
+			               [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+				               manager->ChangeState<DirtySource::Cpu, true>(
+				                   manager->GetCpuAddr() + offset, bytes);
+			               });
+			return false;
+		};
+
+		if (!update_cpu_state()) {
+			return;
+		}
+		std::forward<Flush>(on_flush)();
+		if (update_cpu_state()) {
+			EXIT("memory invalidation retained GPU-owned pages\n");
+		}
+	}
+#if KYTY_BUILD == KYTY_BUILD_DEBUG
 	void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
 	                           const char* operation) const noexcept;
 	void ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
 	                               const char* operation);
+#else
+	void ValidateGpuDirtyPages(const RangeSet&, uint64_t, uint64_t, const char*) const noexcept {}
+	void ValidateGpuDirtyOwnership(const RangeSet&, uint64_t, uint64_t, const char*) {}
+#endif
 
 	template <bool clear, typename Preflight, typename Func>
 	void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Preflight&& preflight, Func&& func) {
@@ -51,7 +85,6 @@ public:
 		static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
 		CheckNotInUploadCallback();
 		std::lock_guard             access(m_access_mutex);
-		RequireMapped(vaddr, size);
 		std::vector<RegionManager*> managers;
 		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
 			managers.push_back(manager);
@@ -63,9 +96,6 @@ public:
 		}
 		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 			const auto address = manager->GetCpuAddr() + offset;
-			if (manager->HasPendingFault(address, bytes)) {
-				EXIT("GPU download synchronization raced a pending CPU fault\n");
-			}
 			manager->template ForEachModifiedRange<DirtySource::Gpu, false>(address, bytes,
 			                                                                preflight);
 		});
@@ -77,10 +107,8 @@ public:
 			Iterate<false>(vaddr, size,
 			               [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 				               const auto address = manager->GetCpuAddr() + offset;
-				               const auto changed =
 				               manager->template ForEachModifiedRange<DirtySource::Gpu, true>(
 				                   address, bytes, [](uint64_t, uint64_t) noexcept {});
-				               manager->ApplyGpuProtection(changed, false, m_gpu_watch_mode);
 			               });
 		}
 	}
@@ -91,11 +119,6 @@ public:
 		    vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward<Func>(func));
 	}
 
-#if defined(KYTY_MEMORY_TRACKER_TESTS)
-	using UnmapContentionHook = void (*)() noexcept;
-	static void SetUnmapContentionHook(UnmapContentionHook hook) noexcept;
-#endif
-
 	template <typename RangeFunc, typename UploadFunc>
 	void ForEachUploadRange(uint64_t vaddr, uint64_t size, bool is_written, RangeFunc&& range_func,
 	                        UploadFunc&& upload_func) {
@@ -103,12 +126,10 @@ public:
 		static_assert(std::is_nothrow_invocable_v<UploadFunc&>);
 		CheckNotInUploadCallback();
 		std::unique_lock access(m_access_mutex);
-		RequireMapped(vaddr, size);
 		Iterate<true>(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {});
 		const auto* previous_upload_owner = std::exchange(s_upload_owner, this);
 		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 			manager->lock.lock();
-			manager->Track(manager->GetCpuAddr() + offset, bytes);
 			manager->ForEachModifiedRange<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset,
 			                                                      bytes, range_func);
 			if (!is_written) {
@@ -117,11 +138,10 @@ public:
 		});
 		upload_func();
 		if (is_written) {
-			Iterate<false>(
-			    vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-				    const auto changed = manager->template ChangeState<DirtySource::Gpu, true>(
+			Iterate<false>(vaddr, size,
+			               [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+				               manager->template ChangeState<DirtySource::Gpu, true>(
 				                   manager->GetCpuAddr() + offset, bytes);
-				    manager->ApplyGpuProtection(changed, true, m_gpu_watch_mode);
 				               manager->lock.unlock();
 			               });
 		}
@@ -170,14 +190,6 @@ private:
 
 	static void    ValidateRange(uint64_t vaddr, uint64_t size);
 	void           UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
-	void        RequireMapped(uint64_t vaddr, uint64_t size) const {
-		ValidateRange(vaddr, size);
-		if (!m_page_manager.IsMapped(vaddr, size)) {
-			EXIT("memory tracker range [0x%llx, 0x%llx) is not mapped\n",
-			     static_cast<unsigned long long>(vaddr),
-			     static_cast<unsigned long long>(vaddr + size));
-		}
-	}
 	RegionManager* GetOrCreateRegion(uint64_t index);
 
 	std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;
@@ -185,7 +197,6 @@ private:
 	std::mutex                                     m_region_mutex;
 	std::mutex                                     m_access_mutex;
 	PageManager&                                   m_page_manager;
-	PageWatchMode                                  m_gpu_watch_mode = PageWatchMode::ReadWrite;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/pageManager.cpp

@@ -1,13 +1,14 @@
 #include "graphics/host_gpu/pageManager.h"
 
 #include "graphics/host_gpu/regionDefinitions.h"
+#include "kernel/memory.h"
 
+#include <algorithm>
 #include <array>
 #include <atomic>
 #include <cstdarg>
 #include <cstdio>
 #include <cstdlib>
-#include <limits>
 #include <memory>
 #include <mutex>
 #include <vector>
@@ -20,10 +21,9 @@
 #undef min
 #undef max
 #elif defined(__APPLE__)
-#include <mach/mach.h>
-#include <mach/mach_vm.h>
-#include <pthread.h>
-#include <sys/mman.h>
+#include <unistd.h>
+#else
+#include <execinfo.h>
 #include <unistd.h>
 #endif
 
@@ -36,9 +36,8 @@ constexpr uint64_t ADDRESS_SIZE = TRACKER_ADDRESS_SIZE;
 constexpr uint64_t REGION_COUNT = ADDRESS_SIZE / REGION_SIZE;
 
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
-// The tracker reuses Win32 memory-protection tags as internal page-state values (on
-// Windows they come from <windows.h> and are what VirtualQuery returns). Mirror the
-// canonical Win32 numeric values so the shared state-machine logic is identical.
+// The tracker reuses Win32 memory-protection tags as internal page-state values.
+// Mirror their canonical numeric values so the shared state-machine logic is identical.
 constexpr uint32_t PAGE_NOACCESS  = 0x01;
 constexpr uint32_t PAGE_READONLY  = 0x02;
 constexpr uint32_t PAGE_READWRITE = 0x04;
@@ -49,44 +48,6 @@ constexpr uint32_t NO_ACCESS_PROTECTION  = PAGE_NOACCESS;
 constexpr uint32_t READ_ONLY_PROTECTION  = PAGE_READONLY;
 constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
 
-#if defined(__APPLE__)
-// Map the tracker's Win32-style protection tags to POSIX mprotect flags.
-static int PageProtToPosix(uint32_t protection) {
-	switch (protection) {
-		case PAGE_NOACCESS: return PROT_NONE;
-		case PAGE_READONLY: return PROT_READ;
-		case PAGE_READWRITE: return PROT_READ | PROT_WRITE;
-		default: return PROT_NONE;
-	}
-}
-
-// Query the current protection of the page containing vaddr via the Mach VM map and
-// collapse it to the tracker's read/write tags (execute is irrelevant to write tracking).
-static uint32_t MachQueryPageProt(uint64_t vaddr) {
-	auto              region_addr = static_cast<mach_vm_address_t>(vaddr);
-	mach_vm_size_t    region_size = 0;
-	vm_region_basic_info_data_64_t info {};
-	mach_msg_type_number_t         count       = VM_REGION_BASIC_INFO_COUNT_64;
-	mach_port_t                    object_name = MACH_PORT_NULL;
-
-	kern_return_t kr =
-	    mach_vm_region(mach_task_self(), &region_addr, &region_size, VM_REGION_BASIC_INFO_64,
-	                   reinterpret_cast<vm_region_info_t>(&info), &count, &object_name);
-	if (kr != KERN_SUCCESS || region_addr > vaddr) {
-		return PAGE_NOACCESS; // no region covering vaddr
-	}
-	if ((info.protection & VM_PROT_WRITE) != 0) {
-		return PAGE_READWRITE;
-	}
-	if ((info.protection & VM_PROT_READ) != 0) {
-		return PAGE_READONLY;
-	}
-	return PAGE_NOACCESS;
-}
-#endif
-
-thread_local bool g_in_fault_resolution = false;
-
 [[noreturn]] void FailFast(const char* reason = nullptr) noexcept {
 	std::fputs("PageManager fail-fast: ", stderr);
 	std::fputs(reason != nullptr ? reason : "invalid page state", stderr);
@@ -101,6 +62,10 @@ thread_local bool g_in_fault_resolution = false;
 		std::fprintf(stderr, "  frame[%u]=0x%016" PRIxPTR " image_rva=0x%016" PRIxPTR "\n", i,
 		             address, address >= image_base ? address - image_base : 0);
 	}
+#elif !defined(__APPLE__)
+	void*     frames[16] {};
+	const int frame_count = ::backtrace(frames, static_cast<int>(std::size(frames)));
+	::backtrace_symbols_fd(frames, frame_count, STDERR_FILENO);
 #endif
 	std::fflush(stderr);
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
@@ -120,14 +85,13 @@ thread_local bool g_in_fault_resolution = false;
 	std::_Exit(322);
 }
 
-uint32_t CurrentThread() noexcept {
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	return GetCurrentThreadId();
-#elif defined(__APPLE__)
-	return static_cast<uint32_t>(pthread_mach_thread_np(pthread_self()));
-#else
-	FailFast();
-#endif
+Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
+	switch (protection) {
+		case NO_ACCESS_PROTECTION: return Common::VirtualMemory::Mode::NoAccess;
+		case READ_ONLY_PROTECTION: return Common::VirtualMemory::Mode::Read;
+		case READ_WRITE_PROTECTION: return Common::VirtualMemory::Mode::ReadWrite;
+		default: Fatal("unmappable protection 0x%08" PRIx32, protection);
+	}
 }
 
 class SpinGuard final {
@@ -163,28 +127,61 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
 
 struct PageManager::Impl {
 	struct PageState {
-		std::atomic_flag lock                 = ATOMIC_FLAG_INIT;
-		uint32_t         mappings             = 0;
-		uint32_t         gpu_read_mappings    = 0;
-		uint32_t         gpu_write_mappings   = 0;
-		uint32_t         write_watchers       = 0;
-		uint32_t         access_watchers      = 0;
-		uint32_t         original_protection  = 0;
-		uint32_t         backing_writer       = 0;
-		bool             resolving            = false;
-		bool             resolving_read_write = false;
-		bool             late_read_pending    = false;
-		bool             late_write_pending   = false;
+		uint8_t write_watchers  : 7 = 0;
+		uint8_t access_watchers : 1 = 0;
+
+		[[nodiscard]] uint32_t Perms() const noexcept {
+			if (access_watchers != 0) {
+				return NO_ACCESS_PROTECTION;
+			}
+			if (write_watchers != 0) {
+				return READ_ONLY_PROTECTION;
+			}
+			return READ_WRITE_PROTECTION;
+		}
+
+		template <int delta, bool is_read>
+		uint32_t AddDelta(uint64_t address) {
+			static_assert(delta >= -1 && delta <= 1);
+			if constexpr (is_read) {
+				if constexpr (delta == 1) {
+					if (access_watchers != 0) {
+						Fatal("read-watcher overflow at 0x%016" PRIx64, address);
+					}
+					return ++access_watchers;
+				} else if constexpr (delta == -1) {
+					if (access_watchers == 0) {
+						Fatal("read-watcher underflow at 0x%016" PRIx64, address);
+					}
+					return --access_watchers;
+				} else {
+					return access_watchers;
+				}
+			} else {
+				if constexpr (delta == 1) {
+					if (write_watchers == 0x7f) {
+						Fatal("write-watcher overflow at 0x%016" PRIx64, address);
+					}
+					return ++write_watchers;
+				} else if constexpr (delta == -1) {
+					if (write_watchers == 0) {
+						Fatal("write-watcher underflow at 0x%016" PRIx64, address);
+					}
+					return --write_watchers;
+				} else {
+					return write_watchers;
+				}
+			}
+		}
 	};
+	static_assert(sizeof(PageState) == 1);
 
 	struct Region {
+		std::atomic_flag                    lock = ATOMIC_FLAG_INIT;
 		std::array<PageState, REGION_PAGES> pages;
 	};
 
-	Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) {
-		if (fault_handler == nullptr) {
-			Fatal("null fault handler");
-		}
+	Impl() {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 		SYSTEM_INFO info {};
 		GetSystemInfo(&info);
@@ -195,11 +192,13 @@ struct PageManager::Impl {
 #elif defined(__APPLE__)
 		// Under Rosetta the host page size is 4 KB, matching TRACKER_PAGE_SIZE.
 		if (static_cast<uint64_t>(getpagesize()) != PAGE_SIZE) {
-			Fatal("unsupported host page size 0x%08" PRIx32,
-			      static_cast<uint32_t>(getpagesize()));
+			Fatal("unsupported host page size 0x%08" PRIx32, static_cast<uint32_t>(getpagesize()));
 		}
 #else
-		Fatal("page-fault invalidation is not implemented on this platform");
+		const auto host_page_size = ::sysconf(_SC_PAGESIZE);
+		if (host_page_size < 0 || static_cast<uint64_t>(host_page_size) != PAGE_SIZE) {
+			Fatal("unsupported host page size %ld", static_cast<long>(host_page_size));
+		}
 #endif
 		regions = std::make_unique<std::atomic<Region*>[]>(REGION_COUNT);
 		for (uint64_t i = 0; i < REGION_COUNT; i++) {
@@ -209,11 +208,9 @@ struct PageManager::Impl {
 
 	~Impl() {
 		for (const auto& region: region_storage) {
+			SpinGuard lock(region->lock);
 			for (auto& page: region->pages) {
-				SpinGuard lock(page.lock);
-				if (page.mappings != 0 || page.gpu_read_mappings != 0 ||
-				    page.gpu_write_mappings != 0 || page.write_watchers != 0 ||
-				    page.access_watchers != 0 || page.backing_writer != 0 || page.resolving) {
+				if (page.write_watchers != 0 || page.access_watchers != 0) {
 					FailFast("PageManager destroyed with live page state");
 				}
 			}
@@ -241,564 +238,140 @@ struct PageManager::Impl {
 		return ptr;
 	}
 
-	PageState& GetPage(Region& region, uint64_t vaddr) const {
-		return region.pages[(vaddr % REGION_SIZE) / PAGE_SIZE];
-	}
-
-	static uint32_t WatcherProtection(const PageState& page) {
-		if (page.access_watchers != 0) {
-			return NO_ACCESS_PROTECTION;
-		}
-		if (page.write_watchers != 0) {
-			return READ_ONLY_PROTECTION;
-		}
-		return page.original_protection;
-	}
-
-	static void PublishDelayedFaults(PageState& page, uint32_t old_protection,
-	                                 uint32_t new_protection) {
-		if (old_protection == NO_ACCESS_PROTECTION && new_protection != NO_ACCESS_PROTECTION) {
-			page.late_read_pending = true;
-		}
-		if ((old_protection == NO_ACCESS_PROTECTION ||
-		     old_protection == READ_ONLY_PROTECTION) &&
-		    new_protection == READ_WRITE_PROTECTION) {
-			page.late_write_pending = true;
+	void Protect(uint64_t vaddr, uint64_t size, uint32_t protection) noexcept {
+		if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
+		                                                     ToMemoryMode(protection))) {
+			Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
+			      protection);
 		}
 	}
 
-	static uint32_t QueryProtection(uint64_t vaddr) {
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		MEMORY_BASIC_INFORMATION info {};
-		if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
-		                 sizeof(info)) == 0 ||
-		    info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) {
-			Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32
-			      ", protection=0x%08" PRIx32 ")",
-			      vaddr, static_cast<uint32_t>(info.State), static_cast<uint32_t>(info.Protect));
+	template <bool track, bool is_read, bool masked>
+	void UpdateRegionWatchers(Region& region, uint64_t base_addr, size_t first, size_t last,
+	                          const RegionBits* mask = nullptr) {
+		SpinGuard lock(region.lock);
+		auto      perms                 = region.pages[first].Perms();
+		uint64_t  range_begin           = 0;
+		uint64_t  range_bytes           = 0;
+		uint64_t  potential_range_bytes = 0;
+
+		const auto release_pending = [&] {
+			if (range_bytes != 0) {
+				Protect(base_addr + range_begin * PAGE_SIZE, range_bytes, perms);
+				range_bytes           = 0;
+				potential_range_bytes = 0;
 			}
-		return info.Protect;
-#elif defined(__APPLE__)
-		const uint32_t protection = MachQueryPageProt(vaddr);
-		if (protection != PAGE_READWRITE) {
-			Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (protection=0x%08" PRIx32
-			      ")",
-			      vaddr, protection);
-		}
-		return protection;
-#else
-		(void)vaddr;
-		Fatal("page query is unsupported on this platform");
-#endif
+		};
+
+		for (size_t page_index = first; page_index < last; page_index++) {
+			auto&      page    = region.pages[page_index];
+			const auto address = base_addr + page_index * PAGE_SIZE;
+			const bool update  = !masked || mask->Get(page_index);
+
+			const auto old_perms = page.Perms();
+			const auto new_count = update ? page.AddDelta<track ? 1 : -1, is_read>(address)
+			                              : page.AddDelta<0, is_read>(address);
+			const auto new_perms = page.Perms();
+
+			if (new_perms != perms) [[unlikely]] {
+				release_pending();
+				perms = new_perms;
+			} else if (range_bytes != 0) {
+				potential_range_bytes += PAGE_SIZE;
 			}
 
-	static bool AllowsAccess(uint64_t vaddr, PageFaultAccess access) noexcept {
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		MEMORY_BASIC_INFORMATION info {};
-		if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
-		                 sizeof(info)) == 0 ||
-		    info.State != MEM_COMMIT) {
-			return false;
-		}
-		switch (access) {
-			case PageFaultAccess::Read:
-				return info.Protect == PAGE_READONLY || info.Protect == PAGE_READWRITE;
-			case PageFaultAccess::Write: return info.Protect == PAGE_READWRITE;
-			default: return false;
-		}
-#elif defined(__APPLE__)
-		const uint32_t protection = MachQueryPageProt(vaddr);
-		switch (access) {
-			case PageFaultAccess::Read:
-				return protection == PAGE_READONLY || protection == PAGE_READWRITE;
-			case PageFaultAccess::Write: return protection == PAGE_READWRITE;
-			default: return false;
-		}
-#else
-		(void)vaddr;
-		return false;
-#endif
+			if (!update) {
+				continue;
 			}
 
-	static void Protect(uint64_t vaddr, uint32_t protection, uint32_t expected_old,
-	                    bool fault_path) noexcept {
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		DWORD old_protection = 0;
-		if (VirtualProtect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), PAGE_SIZE,
-		                   protection, &old_protection) == 0 ||
-		    old_protection != expected_old) {
-			if (fault_path) {
-				FailFast("VirtualProtect fault transition did not match expected protection");
+			const bool watcher_edge = (track && new_count == 1) || (!track && new_count == 0);
+			if (watcher_edge && old_perms != new_perms) {
+				if (range_bytes == 0) {
+					range_begin           = page_index;
+					potential_range_bytes = PAGE_SIZE;
 				}
-			Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
-			      ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
-			      vaddr, static_cast<uint32_t>(old_protection), expected_old, protection);
+				range_bytes = potential_range_bytes;
 			}
-#elif defined(__APPLE__)
-		// mprotect cannot report the previous protection, so the expected_old comparison
-		// is dropped; the tracker is the sole mutator of these pages and drives the
-		// transition from its own shadow state.
-		(void)expected_old;
-		if (mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), PAGE_SIZE,
-		             PageProtToPosix(protection)) != 0) {
-			if (fault_path) {
-				FailFast("mprotect fault transition failed");
 		}
-			Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, protection);
+
+		release_pending();
+	}
+
+	template <bool track, bool is_read>
+	void UpdatePageWatchers(uint64_t vaddr, uint64_t size) {
+		const auto begin = PageStart(vaddr);
+		const auto end   = PageEnd(vaddr, size);
+		for (auto chunk_begin = begin; chunk_begin < end;) {
+			const auto chunk_end   = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE);
+			const auto region_base = chunk_begin / REGION_SIZE * REGION_SIZE;
+			auto*      region = track ? GetOrCreateRegion(chunk_begin) : FindRegion(chunk_begin);
+			if (region == nullptr) {
+				Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin);
+			}
+			const auto first = static_cast<size_t>((chunk_begin - region_base) / PAGE_SIZE);
+			const auto last  = static_cast<size_t>((chunk_end - region_base) / PAGE_SIZE);
+			UpdateRegionWatchers<track, is_read, false>(*region, region_base, first, last);
+			chunk_begin = chunk_end;
 		}
-#else
-		(void)vaddr;
-		(void)protection;
-		(void)fault_path;
-		FailFast("page protection is unsupported on this platform");
-#endif
 	}
 
 	std::unique_ptr<std::atomic<Region*>[]> regions;
 	std::vector<std::unique_ptr<Region>>    region_storage;
 	std::mutex                              region_mutex;
-	PageFaultHandler                        fault_handler = nullptr;
-	void*                                   fault_context = nullptr;
 };
 
 static_assert(std::atomic<void*>::is_always_lock_free);
 
-PageManager::PageManager(PageFaultHandler fault_handler, void* fault_context)
-    : m_impl(std::make_unique<Impl>(fault_handler, fault_context)) {}
+PageManager::PageManager(): m_impl(std::make_unique<Impl>()) {}
 
 PageManager::~PageManager() = default;
 
 uint64_t PageManager::GetPageSize() const {
-	if (g_in_fault_resolution) {
-		FailFast("nested page fault while resolving a watched page");
-	}
 	return PAGE_SIZE;
 }
 
-bool PageManager::IsTracked(uint64_t vaddr) const noexcept {
-	if (g_in_fault_resolution) {
-		FailFast("IsTracked called during fault resolution");
+template <bool track>
+void PageManager::UpdatePageWatchers(uint64_t vaddr, uint64_t size) {
+	m_impl->UpdatePageWatchers<track, false>(vaddr, size);
+}
+
+template void PageManager::UpdatePageWatchers<true>(uint64_t, uint64_t);
+template void PageManager::UpdatePageWatchers<false>(uint64_t, uint64_t);
+
+template <bool track, bool is_read>
+void PageManager::UpdatePageWatchersForRegion(uint64_t base_addr, RegionBits& mask) {
+	if (base_addr % REGION_SIZE != 0 || base_addr >= ADDRESS_SIZE ||
+	    REGION_SIZE > ADDRESS_SIZE - base_addr) {
+		Fatal("invalid tracking region base 0x%016" PRIx64, base_addr);
 	}
-	auto* region = m_impl->FindRegion(vaddr);
+
+	const auto start_range = mask.FirstRange();
+	const auto end_range   = mask.LastRange();
+	if (start_range.first == REGION_PAGES) {
+		FailFast("empty region watcher mask");
+	}
+	const auto first = start_range.first;
+	const auto last  = end_range.second;
+	if (start_range.second == end_range.second) {
+		m_impl->UpdatePageWatchers<track, is_read>(base_addr + first * PAGE_SIZE,
+		                                           (last - first) * PAGE_SIZE);
+		return;
+	}
+
+	auto* region = track ? m_impl->GetOrCreateRegion(base_addr) : m_impl->FindRegion(base_addr);
 	if (region == nullptr) {
-		return false;
+		Fatal("untracking unknown region 0x%016" PRIx64, base_addr);
 	}
-	auto&     page = m_impl->GetPage(*region, vaddr);
-	SpinGuard lock(page.lock);
-	return page.write_watchers != 0 || page.access_watchers != 0;
+	m_impl->UpdateRegionWatchers<track, is_read, true>(*region, base_addr, first, last, &mask);
 }
 
-bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
-	if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
-		return false;
-	}
-	const auto end = PageStart(vaddr + size - 1) + PAGE_SIZE;
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(page_vaddr);
-		if (region == nullptr) {
-			return false;
-		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
-		SpinGuard lock(page.lock);
-		if (page.mappings == 0) {
-			return false;
-		}
-	}
-	return true;
-}
+template void PageManager::UpdatePageWatchersForRegion<true, true>(uint64_t, RegionBits&);
+template void PageManager::UpdatePageWatchersForRegion<true, false>(uint64_t, RegionBits&);
+template void PageManager::UpdatePageWatchersForRegion<false, true>(uint64_t, RegionBits&);
+template void PageManager::UpdatePageWatchersForRegion<false, false>(uint64_t, RegionBits&);
 
-bool PageManager::HasAnyMapping(uint64_t vaddr, uint64_t size) const noexcept {
-	if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE ||
-	    size > ADDRESS_SIZE - vaddr) {
-		return false;
-	}
-	const auto end = PageEnd(vaddr, size);
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(page_vaddr);
-		if (region == nullptr) {
-			continue;
-		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
-		SpinGuard lock(page.lock);
-		if (page.mappings != 0) {
-			return true;
-		}
-	}
-	return false;
-}
+void PageManager::OnGpuMap(uint64_t, uint64_t) {}
 
-bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept {
-	if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
-		FailFast("HasGpuAccess received an invalid GPU access mode");
-	}
-	const bool need_read  = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
-	const bool need_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
-	if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
-		return false;
-	}
-	const auto end = PageEnd(vaddr, size);
-	for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(addr);
-		if (region == nullptr) {
-			return false;
-		}
-		auto&     page = m_impl->GetPage(*region, addr);
-		SpinGuard lock(page.lock);
-		if ((need_read && page.gpu_read_mappings == 0) ||
-		    (need_write && page.gpu_write_mappings == 0)) {
-			return false;
-		}
-	}
-	return true;
-}
-
-void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
-                                     PageWatchMode mode) {
-	if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
-		Fatal("invalid watcher mode");
-	}
-	const auto end = PageEnd(vaddr, size);
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		auto* region =
-		    track ? m_impl->GetOrCreateRegion(page_vaddr) : m_impl->FindRegion(page_vaddr);
-		if (region == nullptr) {
-			Fatal("untracking unknown page 0x%016" PRIx64, page_vaddr);
-		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
-		SpinGuard lock(page.lock);
-		if (page.resolving && track) {
-			FailFast("new page watcher raced active fault resolution");
-		}
-		if (page.mappings == 0) {
-			Fatal("watching unmapped page 0x%016" PRIx64, page_vaddr);
-		}
-		auto& watchers =
-		    (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
-		if (track) {
-			if (watchers == std::numeric_limits<uint32_t>::max()) {
-				Fatal("watcher overflow at 0x%016" PRIx64, page_vaddr);
-			}
-			const bool first_watcher = page.write_watchers == 0 && page.access_watchers == 0;
-			if (first_watcher) {
-				page.original_protection = Impl::QueryProtection(page_vaddr);
-			}
-			const auto old_protection = Impl::WatcherProtection(page);
-			watchers++;
-			const auto new_protection = Impl::WatcherProtection(page);
-			if (new_protection != old_protection) {
-				Impl::Protect(page_vaddr, new_protection, old_protection, false);
-			}
-			switch (new_protection) {
-				case NO_ACCESS_PROTECTION:
-					page.late_read_pending  = false;
-					page.late_write_pending = false;
-					break;
-				case READ_ONLY_PROTECTION: page.late_write_pending = false; break;
-				default: break;
-			}
-		} else {
-			if (watchers == 0) {
-				Fatal("watcher underflow at 0x%016" PRIx64, page_vaddr);
-			}
-			if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) {
-				Fatal("backing write ownership changed at 0x%016" PRIx64, page_vaddr);
-			}
-			const auto old_protection = Impl::WatcherProtection(page);
-			watchers--;
-			const auto new_protection = Impl::WatcherProtection(page);
-			if (page.backing_writer == 0 && new_protection != old_protection) {
-				Impl::Protect(page_vaddr, new_protection, old_protection, false);
-			}
-			if (page.backing_writer == 0) {
-				Impl::PublishDelayedFaults(page, old_protection, new_protection);
-			}
-			if (page.backing_writer == 0 && page.write_watchers == 0 && page.access_watchers == 0) {
-				page.original_protection = 0;
-			}
-		}
-	}
-}
-
-void PageManager::OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access) {
-	if (g_in_fault_resolution) {
-		FailFast("GPU mapping changed during fault resolution");
-	}
-	if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
-		FailFast("GPU map received an invalid access mode");
-	}
-	const bool gpu_read  = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
-	const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
-	const auto end       = PageEnd(vaddr, size);
-	for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
-		auto&     page = m_impl->GetPage(*m_impl->GetOrCreateRegion(addr), addr);
-		SpinGuard lock(page.lock);
-		if (page.resolving || page.mappings == std::numeric_limits<uint32_t>::max() ||
-		    (gpu_read && page.gpu_read_mappings == std::numeric_limits<uint32_t>::max()) ||
-		    (gpu_write && page.gpu_write_mappings == std::numeric_limits<uint32_t>::max())) {
-			Fatal("invalid map state at 0x%016" PRIx64, addr);
-		}
-		page.mappings++;
-		page.gpu_read_mappings += gpu_read ? 1u : 0u;
-		page.gpu_write_mappings += gpu_write ? 1u : 0u;
-	}
-}
-
-void PageManager::OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access) {
-	if (g_in_fault_resolution) {
-		FailFast("GPU unmapping changed during fault resolution");
-	}
-	if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
-		FailFast("GPU unmap received an invalid access mode");
-	}
-	const bool gpu_read  = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
-	const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
-	const auto end       = PageEnd(vaddr, size);
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(page_vaddr);
-		if (region == nullptr) {
-			Fatal("unmapping unknown page 0x%016" PRIx64, page_vaddr);
-		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
-		SpinGuard lock(page.lock);
-		if (page.resolving || page.mappings == 0 || (gpu_read && page.gpu_read_mappings == 0) ||
-		    (gpu_write && page.gpu_write_mappings == 0) ||
-		    (page.mappings == 1 && (page.write_watchers != 0 || page.access_watchers != 0))) {
-			Fatal("invalid unmap state at 0x%016" PRIx64, page_vaddr);
-		}
-		page.mappings--;
-		page.gpu_read_mappings -= gpu_read ? 1u : 0u;
-		page.gpu_write_mappings -= gpu_write ? 1u : 0u;
-		if (page.mappings == 0) {
-			if (page.gpu_read_mappings != 0 || page.gpu_write_mappings != 0) {
-				FailFast("GPU unmap left nonzero GPU mapping counts");
-			}
-			page.late_read_pending  = false;
-			page.late_write_pending = false;
-		}
-	}
-}
-
-PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr,
-                                        uint64_t size) noexcept
-    : m_manager(manager), m_vaddr(vaddr), m_size(size) {
-	m_manager.BeginBackingWrite(vaddr, size);
-}
-
-PageManager::BackingWrite::~BackingWrite() {
-	m_manager.EndBackingWrite(m_vaddr, m_size);
-}
-
-std::vector<std::unique_ptr<PageManager::BackingWrite>>
-PageManager::ReserveBackingWrites(std::span<const RangeSet::Range> ranges) {
-	if (ranges.empty()) {
-		Fatal("cannot reserve empty backing-write ranges");
-	}
-	std::vector<std::unique_ptr<BackingWrite>> writes;
-	writes.reserve(ranges.size());
-	uint64_t begin = 0;
-	uint64_t end   = 0;
-	for (const auto& range: ranges) {
-		if (range.address == 0 || range.size == 0 || range.size > UINT64_MAX - range.address ||
-		    range.address + range.size > UINT64_MAX - (PAGE_SIZE - 1)) {
-			Fatal("invalid backing-write range");
-		}
-		const auto page_begin = PageStart(range.address);
-		const auto page_end   = PageStart(range.address + range.size + PAGE_SIZE - 1);
-		if (begin != 0 && page_begin > end) {
-			writes.push_back(std::make_unique<BackingWrite>(*this, begin, end - begin));
-			begin = 0;
-		}
-		if (begin == 0) {
-			begin = page_begin;
-			end   = page_end;
-		} else {
-			end = std::max(end, page_end);
-		}
-	}
-	writes.push_back(std::make_unique<BackingWrite>(*this, begin, end - begin));
-	return writes;
-}
-
-void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
-	if (g_in_fault_resolution) {
-		FailFast("backing write began during fault resolution");
-	}
-	const auto end    = PageEnd(vaddr, size);
-	const auto writer = CurrentThread();
-	for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(address);
-		if (region == nullptr) {
-			Fatal("backing write reserves an unknown page at 0x%016" PRIx64, address);
-		}
-		auto&     page = m_impl->GetPage(*region, address);
-		SpinGuard lock(page.lock);
-		if (page.mappings == 0 || page.resolving || page.backing_writer != 0 ||
-		    page.access_watchers == 0) {
-			Fatal("backing write races page resolution at 0x%016" PRIx64, address);
-		}
-		page.resolving            = true;
-		page.resolving_read_write = true;
-		page.backing_writer       = writer;
-	}
-}
-
-void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
-	if (g_in_fault_resolution) {
-		FailFast("backing write ended during fault resolution");
-	}
-	const auto end    = PageEnd(vaddr, size);
-	const auto writer = CurrentThread();
-	for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(address);
-		if (region == nullptr) {
-			FailFast("backing write ended for an unknown page");
-		}
-		auto&     page = m_impl->GetPage(*region, address);
-		SpinGuard lock(page.lock);
-		if (!page.resolving || page.backing_writer != writer) {
-			FailFast("backing write ended without matching owner and resolving state");
-		}
-		const auto old_protection = NO_ACCESS_PROTECTION;
-		const auto new_protection = Impl::WatcherProtection(page);
-		if (new_protection != old_protection) {
-			Impl::Protect(address, new_protection, old_protection, false);
-		}
-		Impl::PublishDelayedFaults(page, old_protection, new_protection);
-		if (page.write_watchers == 0 && page.access_watchers == 0) {
-			page.original_protection = 0;
-		}
-		page.backing_writer       = 0;
-		page.resolving            = false;
-		page.resolving_read_write = false;
-	}
-}
-
-bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
-	if (g_in_fault_resolution) {
-		FailFast("nested HandleFault call");
-	}
-	auto* region = m_impl->FindRegion(fault_vaddr);
-	if (region == nullptr) {
-		return false;
-	}
-	auto& page   = m_impl->GetPage(*region, fault_vaddr);
-	bool  waited = false;
-	while (true) {
-		SpinGuard lock(page.lock);
-		if (access == PageFaultAccess::Read && page.late_read_pending &&
-		    Impl::AllowsAccess(fault_vaddr, access)) {
-			page.late_read_pending = false;
-			return true;
-		}
-		if (access == PageFaultAccess::Write && page.late_write_pending &&
-		    Impl::AllowsAccess(fault_vaddr, access)) {
-			page.late_write_pending = false;
-			return true;
-		}
-		if (page.resolving) {
-			if (page.backing_writer == CurrentThread()) {
-				FailFast("backing writer faulted on its own reserved page");
-			}
-			if ((!page.resolving_read_write && access != PageFaultAccess::Write) ||
-			    (page.resolving_read_write && access != PageFaultAccess::Read &&
-			     access != PageFaultAccess::Write)) {
-				FailFast("fault access is incompatible with the active resolver");
-			}
-			waited = true;
-			continue;
-		}
-		if (page.write_watchers == 0 && page.access_watchers == 0) {
-			if (access != PageFaultAccess::Read && access != PageFaultAccess::Write) {
-				return false;
-			}
-			bool&      pending = (access == PageFaultAccess::Read ? page.late_read_pending
-			                                                      : page.late_write_pending);
-			const bool allowed = Impl::AllowsAccess(fault_vaddr, access);
-			pending            = false;
-			if (waited && !allowed) {
-				FailFast("page remained inaccessible after waiting for its resolver");
-			}
-			// More than one CPU can fault before a protection transition becomes visible. The first
-			// delayed fault consumes the hint bit; later faults must also resume once the mapped
-			// page already permits the requested access. A genuinely read-only/no-access page still
-			// falls through to the guest exception path.
-			return allowed;
-		}
-		if ((access != PageFaultAccess::Read && access != PageFaultAccess::Write) ||
-		    (access == PageFaultAccess::Read && page.access_watchers == 0)) {
-			FailFast("fault access is incompatible with active page watchers");
-		}
-		page.resolving            = true;
-		page.resolving_read_write = page.access_watchers != 0;
-		break;
-	}
-	g_in_fault_resolution = true;
-	const bool handled    = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1,
-	                                              PageFaultPhase::Invalidate);
-	g_in_fault_resolution = false;
-	{
-		SpinGuard lock(page.lock);
-		if (!handled || !page.resolving) {
-			FailFast("fault invalidation did not preserve the resolving state");
-		}
-	}
-	g_in_fault_resolution = true;
-	const bool completed  = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1,
-	                                              PageFaultPhase::Complete);
-	g_in_fault_resolution = false;
-	{
-		SpinGuard lock(page.lock);
-		if (!completed || !page.resolving) {
-			FailFast("fault completion did not preserve the resolving state");
-		}
-		if (page.write_watchers != 0 || page.access_watchers != 0) {
-			const auto old_protection  = Impl::WatcherProtection(page);
-			const bool read_only_fault = access == PageFaultAccess::Read;
-			if (read_only_fault && page.access_watchers == 0) {
-				FailFast("read fault completed without a read/write watcher");
-			}
-			page.access_watchers = 0;
-			if (!read_only_fault) {
-				page.write_watchers = 0;
-			}
-			const auto restored_protection = Impl::WatcherProtection(page);
-			Impl::Protect(PageStart(fault_vaddr), restored_protection, old_protection, true);
-			if (page.write_watchers == 0) {
-				page.original_protection = 0;
-			}
-			Impl::PublishDelayedFaults(page, old_protection, restored_protection);
-		} else if (!Impl::AllowsAccess(fault_vaddr, access)) {
-			FailFast("fault completion left the page inaccessible");
-		}
-		page.resolving            = false;
-		page.resolving_read_write = false;
-	}
-	g_in_fault_resolution = true;
-	const bool released   = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1,
-	                                              PageFaultPhase::Release);
-	g_in_fault_resolution = false;
-	if (!released) {
-		FailFast("fault release callback failed");
-	}
-	return true;
-}
-
-bool PageManager::HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept {
-	if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE ||
-	    size > ADDRESS_SIZE - vaddr) {
-		return false;
-	}
-	const auto end = PageEnd(vaddr, size);
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		if (!IsMapped(page_vaddr, 1)) {
-			continue;
-		}
-		const auto fault_vaddr = std::max(page_vaddr, vaddr);
-		if (!HandleFault(PageFaultAccess::Write, fault_vaddr)) {
-			return false;
-		}
-	}
-	return true;
-}
+void PageManager::OnGpuUnmap(uint64_t, uint64_t) {}
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/pageManager.h

@@ -2,62 +2,32 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_PAGEMANAGER_H_
 
 #include "common/common.h"
-#include "graphics/host_gpu/rangeSet.h"
+#include "graphics/host_gpu/regionDefinitions.h"
 
 #include <memory>
-#include <span>
-#include <vector>
 
 namespace Libs::Graphics {
 
 enum class PageFaultAccess { Read, Write, Execute, Unknown };
-enum class PageFaultPhase { Invalidate, Complete, Release };
-enum class PageWatchMode { Write, ReadWrite };
-enum class GpuAccess { Read, Write, ReadWrite };
-
-using PageFaultHandler = bool (*)(void* context, PageFaultAccess access, uint64_t vaddr,
-                                  uint64_t size, PageFaultPhase phase) noexcept;
 
 class PageManager final {
 public:
-	class BackingWrite final {
-	public:
-		BackingWrite(PageManager& manager, uint64_t vaddr, uint64_t size) noexcept;
-		~BackingWrite();
-		KYTY_CLASS_NO_COPY(BackingWrite);
-
-	private:
-		PageManager& m_manager;
-		uint64_t     m_vaddr = 0;
-		uint64_t     m_size  = 0;
-	};
-
-	PageManager(PageFaultHandler fault_handler, void* fault_context);
+	PageManager();
 	// The owner must stop all PageManager callers before destruction.
 	~PageManager();
 
 	KYTY_CLASS_NO_COPY(PageManager);
 
 	[[nodiscard]] uint64_t GetPageSize() const;
-	[[nodiscard]] bool     IsTracked(uint64_t vaddr) const noexcept;
-	[[nodiscard]] bool     IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
-	[[nodiscard]] bool     HasAnyMapping(uint64_t vaddr, uint64_t size) const noexcept;
-	[[nodiscard]] bool HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept;
 
-	void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
-	                        PageWatchMode mode = PageWatchMode::Write);
-	void OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
-	void OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
-
-	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
-	[[nodiscard]] bool HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept;
-	[[nodiscard]] std::vector<std::unique_ptr<BackingWrite>>
-	     ReserveBackingWrites(std::span<const RangeSet::Range> ranges);
+	template <bool track>
+	void UpdatePageWatchers(uint64_t vaddr, uint64_t size);
+	template <bool track, bool is_read = false>
+	void UpdatePageWatchersForRegion(uint64_t base_addr, RegionBits& mask);
+	void OnGpuMap(uint64_t vaddr, uint64_t size);
+	void OnGpuUnmap(uint64_t vaddr, uint64_t size);
 
 private:
-	void BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept;
-	void EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept;
-
 	struct Impl;
 	std::unique_ptr<Impl> m_impl;
 };

src/graphics/host_gpu/rangeSet.h

@@ -59,6 +59,16 @@ public:
 		return result;
 	}
 
+	[[nodiscard]] bool Contains(uint64_t address, uint64_t size) const {
+		const auto end = End(address, size);
+		auto       it  = m_ranges.upper_bound(address);
+		if (it == m_ranges.begin()) {
+			return false;
+		}
+		--it;
+		return it->first <= address && it->second >= end;
+	}
+
 	template <typename Func>
 	void ForEachIntersection(uint64_t address, uint64_t size, Func&& func) const {
 		const auto end = End(address, size);

src/graphics/host_gpu/regionDefinitions.h

@@ -1,10 +1,9 @@
 #ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONDEFINITIONS_H_
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONDEFINITIONS_H_
 
+#include "common/bitArray.h"
 #include "common/common.h"
 
-#include <bitset>
-
 namespace Libs::Graphics {
 
 constexpr uint64_t TRACKER_PAGE_SIZE    = 4ull * 1024ull;
@@ -13,7 +12,8 @@ constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 40u;
 constexpr size_t   TRACKER_REGION_PAGES = TRACKER_REGION_SIZE / TRACKER_PAGE_SIZE;
 
 enum class DirtySource { Cpu, Gpu };
-using RegionBits = std::bitset<TRACKER_REGION_PAGES>;
+using RegionBits = Common::BitArray<TRACKER_REGION_PAGES>;
+static_assert(sizeof(RegionBits) == TRACKER_REGION_PAGES / 8);
 
 } // namespace Libs::Graphics
 

src/graphics/host_gpu/regionManager.h

@@ -18,12 +18,13 @@
 #undef max
 #elif defined(__APPLE__)
 #include <pthread.h>
+#elif defined(__linux__)
+#include <sys/syscall.h>
+#include <unistd.h>
 #endif
 
 namespace Libs::Graphics {
 
-enum class CpuFaultAction { Untracked, Continue, Download };
-
 class TrackingSpinLock final {
 public:
 	void lock() noexcept {
@@ -54,6 +55,9 @@ private:
 #elif defined(__APPLE__)
 		// mach thread port is a nonzero per-thread id (0 is the "no owner" sentinel).
 		return static_cast<uint32_t>(pthread_mach_thread_np(pthread_self()));
+#elif defined(__linux__)
+		static thread_local const uint32_t tid = static_cast<uint32_t>(::syscall(SYS_gettid));
+		return tid;
 #else
 		EXIT("region tracking thread identity is unsupported on this platform\n");
 #endif
@@ -72,229 +76,93 @@ public:
 		if (m_cpu_addr % TRACKER_REGION_SIZE != 0) {
 			EXIT("invalid region tracking manager construction\n");
 		}
-		m_cpu_dirty.set();
-		m_writable.set();
+		m_cpu_dirty.Fill();
+		m_writable.Fill();
+		m_readable.Fill();
 	}
 
 	KYTY_CLASS_NO_COPY(RegionManager);
 
 	[[nodiscard]] uint64_t GetCpuAddr() const { return m_cpu_addr; }
-	void                   Track(uint64_t vaddr, uint64_t size) {
-		const auto [start, end] = GetPageRange(vaddr, size);
-		for (auto page = start; page < end; page++) {
-			m_tracked.set(page);
-		}
-	}
-	void Untrack(uint64_t vaddr, uint64_t size) {
-		const auto [start, end] = GetPageRange(vaddr, size);
-		for (auto page = start; page < end; page++) {
-			m_tracked.reset(page);
-		}
-	}
 	template <DirtySource source>
 	[[nodiscard]] bool IsModified(uint64_t offset, uint64_t size) const {
 		const auto [start, end] = GetPageRange(m_cpu_addr + offset, size);
 		const auto& bits        = GetBits<source>();
-		for (auto page = start; page < end; page++) {
-			if (bits.test(page)) {
-				return true;
-			}
-		}
-		return false;
-	}
-
-	template <DirtySource source>
-	[[nodiscard]] bool IsFullyModified(uint64_t offset, uint64_t size) const {
-		const auto [start, end] = GetPageRange(m_cpu_addr + offset, size);
-		const auto& bits        = GetBits<source>();
-		for (auto page = start; page < end; page++) {
-			if (!bits.test(page)) {
-				return false;
-			}
-		}
-		return true;
+		return RegionBits(bits, start, end).Any();
 	}
 
 	template <DirtySource source, bool enable>
-	RegionBits ChangeState(uint64_t vaddr, uint64_t size) {
+	void ChangeState(uint64_t vaddr, uint64_t size) {
 		const auto [start, end] = GetPageRange(vaddr, size);
 		if constexpr (source == DirtySource::Cpu && enable) {
-			for (auto page = start; page < end; page++) {
-				if (m_gpu_dirty.test(page) || m_fault_pending.test(page)) {
-					EXIT("CPU dirty state conflicts with GPU dirty or pending fault state\n");
-				}
+			if (RegionBits(m_gpu_dirty, start, end).Any()) {
+				EXIT("CPU dirty state conflicts with GPU dirty state\n");
 			}
 		}
 		if constexpr (source == DirtySource::Gpu && enable) {
-			for (auto page = start; page < end; page++) {
-				if (m_cpu_dirty.test(page) || m_fault_pending.test(page)) {
-					EXIT("GPU dirty state conflicts with CPU dirty or pending fault state\n");
-				}
+			if (RegionBits(m_cpu_dirty, start, end).Any()) {
+				EXIT("GPU dirty state conflicts with CPU dirty state\n");
 			}
 		}
 		auto& bits = GetBits<source>();
-		auto  changed = bits;
-		for (auto page = start; page < end; page++) {
-			bits.set(page, enable);
+		if constexpr (enable) {
+			bits.SetRange(start, end);
+		} else {
+			bits.UnsetRange(start, end);
 		}
-		changed ^= bits;
 		if constexpr (source == DirtySource::Cpu) {
-			changed    = m_cpu_dirty ^ m_writable;
-			m_writable = m_cpu_dirty;
+			UpdateCpuProtection<!enable>();
+		} else {
+			UpdateGpuProtection<enable>();
 		}
-		return changed;
-	}
-
-	[[nodiscard]] CpuFaultAction BeginCpuFault(uint64_t vaddr, uint64_t size,
-	                                           PageFaultAccess access = PageFaultAccess::Write) {
-		if (access != PageFaultAccess::Read && access != PageFaultAccess::Write) {
-			EXIT("unsupported CPU fault access while beginning ownership transfer\n");
-		}
-		const auto [start, end] = GetPageRange(vaddr, size);
-		const bool tracked      = m_tracked.test(start);
-		for (auto page = start; page < end; page++) {
-			if (m_tracked.test(page) != tracked) {
-				EXIT("CPU fault spans mixed tracked and untracked pages\n");
-			}
-			if (m_fault_pending.test(page)) {
-				return CpuFaultAction::Untracked;
-			}
-			if (m_cpu_dirty.test(page) != m_writable.test(page) ||
-			    (m_gpu_dirty.test(page) && (m_cpu_dirty.test(page) || m_writable.test(page)))) {
-				EXIT("inconsistent CPU fault page state\n");
-			}
-		}
-		if (!tracked) {
-			return CpuFaultAction::Untracked;
-		}
-		bool gpu_dirty = m_gpu_dirty.test(start);
-		bool writable  = m_writable.test(start);
-		for (auto page = start + 1; page < end; page++) {
-			if (m_gpu_dirty.test(page) != gpu_dirty || m_writable.test(page) != writable) {
-				EXIT("CPU fault spans pages with incompatible dirty or writable state\n");
-			}
-		}
-		for (auto page = start; page < end; page++) {
-			if (!gpu_dirty && access == PageFaultAccess::Write) {
-				m_cpu_dirty.set(page);
-				m_writable.set(page);
-			}
-			m_fault_pending.set(page);
-		}
-		return gpu_dirty ? CpuFaultAction::Download : CpuFaultAction::Continue;
-	}
-
-	[[nodiscard]] bool CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access,
-	                                    bool downloaded) {
-		const auto [start, end] = GetPageRange(vaddr, size);
-		for (auto page = start; page < end; page++) {
-			if (!m_fault_pending.test(page)) {
-				return false;
-			}
-		}
-		for (auto page = start; page < end; page++) {
-			const bool gpu_dirty = m_gpu_dirty.test(page);
-			if (gpu_dirty != downloaded) {
-				EXIT("CPU fault download result disagrees with GPU dirty state\n");
-			}
-			if (gpu_dirty) {
-				m_gpu_dirty.reset(page);
-				switch (access) {
-					case PageFaultAccess::Read: break;
-					case PageFaultAccess::Write:
-						m_cpu_dirty.set(page);
-						m_writable.set(page);
-						break;
-					default: EXIT("unsupported CPU fault access after GPU download\n");
-				}
-			}
-			m_fault_pending.reset(page);
-		}
-		return true;
-	}
-
-	[[nodiscard]] bool HasPendingFault(uint64_t vaddr, uint64_t size) const {
-		const auto [start, end] = GetPageRange(vaddr, size);
-		for (auto page = start; page < end; page++) {
-			if (m_fault_pending.test(page)) {
-				return true;
-			}
-		}
-		return false;
-	}
-
-	[[nodiscard]] bool CompleteVirtualGpuWrite(uint64_t vaddr, uint64_t size) {
-		const auto [start, end] = GetPageRange(vaddr, size);
-		for (auto page = start; page < end; page++) {
-			if (!m_fault_pending.test(page)) {
-				return false;
-			}
-			if (!m_gpu_dirty.test(page)) {
-				EXIT("virtual GPU write completion found a non-GPU-dirty page\n");
-			}
-		}
-		for (auto page = start; page < end; page++) {
-			m_gpu_dirty.reset(page);
-			m_cpu_dirty.set(page);
-			m_writable.set(page);
-			m_fault_pending.reset(page);
-		}
-		return true;
 	}
 
 	template <DirtySource source, bool clear, typename Func>
-	RegionBits ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) {
+	void ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) {
 		const auto [start, end] = GetPageRange(vaddr, size);
-		auto mask               = GetBits<source>();
-		if constexpr (source == DirtySource::Cpu) {
-			mask &= ~m_fault_pending;
-		}
-		for (auto page = 0u; page < start; page++) {
-			mask.reset(page);
-		}
-		for (auto page = end; page < TRACKER_REGION_PAGES; page++) {
-			mask.reset(page);
-		}
+		RegionBits mask(GetBits<source>(), start, end);
 		if constexpr (clear) {
-			auto& bits = GetBits<source>();
-			for (auto page = start; page < end; page++) {
-				if (mask.test(page)) {
-					bits.reset(page);
-				}
-			}
+			GetBits<source>().UnsetRange(start, end);
 		}
 		if constexpr (source == DirtySource::Cpu && clear) {
-			auto changed = m_cpu_dirty ^ m_writable;
-			m_writable   = m_cpu_dirty;
-			ApplyProtection(changed, true);
+			UpdateCpuProtection<true>();
 			ForEachRange(mask, std::forward<Func>(func));
-			return changed;
+			return;
+		}
+		if constexpr (source == DirtySource::Gpu && clear) {
+			UpdateGpuProtection<false>();
 		}
 		ForEachRange(mask, std::forward<Func>(func));
-		if constexpr (clear) {
-			return mask;
-		}
-		return {};
-	}
-
-	void ApplyProtection(const RegionBits& changed, bool track) {
-		ForEachRange(changed, [this, track](uint64_t vaddr, uint64_t size) {
-			m_page_manager.UpdatePageWatchers(track, vaddr, size);
-		});
-	}
-
-	void ApplyGpuProtection(const RegionBits& changed, bool track, PageWatchMode mode) {
-		if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
-			EXIT("unsupported GPU page-watch mode\n");
-		}
-		ForEachRange(changed, [this, track, mode](uint64_t vaddr, uint64_t size) {
-			m_page_manager.UpdatePageWatchers(track, vaddr, size, mode);
-		});
 	}
 
 	TrackingSpinLock lock;
 
 private:
+	template <bool track>
+	void UpdateCpuProtection() {
+		auto mask  = m_cpu_dirty ^ m_writable;
+		m_writable = m_cpu_dirty;
+		if (mask.None()) {
+			return;
+		}
+		m_page_manager.UpdatePageWatchersForRegion<track>(m_cpu_addr, mask);
+	}
+
+	template <bool track>
+	void UpdateGpuProtection() {
+		auto readable = ~m_gpu_dirty;
+		auto mask     = readable ^ m_readable;
+		m_readable    = readable;
+		if (mask.None()) {
+			return;
+		}
+		if constexpr (track) {
+			m_page_manager.UpdatePageWatchersForRegion<true, true>(m_cpu_addr, mask);
+		} else {
+			m_page_manager.UpdatePageWatchersForRegion<false, true>(m_cpu_addr, mask);
+		}
+	}
+
 	template <DirtySource source>
 	RegionBits& GetBits() {
 		if constexpr (source == DirtySource::Cpu) {
@@ -325,18 +193,8 @@ private:
 
 	template <typename Func>
 	void ForEachRange(const RegionBits& bits, Func&& func) const {
-		size_t page = 0;
-		while (page < TRACKER_REGION_PAGES) {
-			while (page < TRACKER_REGION_PAGES && !bits.test(page)) {
-				page++;
-			}
-			const auto start = page;
-			while (page < TRACKER_REGION_PAGES && bits.test(page)) {
-				page++;
-			}
-			if (start != page) {
-				func(m_cpu_addr + start * TRACKER_PAGE_SIZE, (page - start) * TRACKER_PAGE_SIZE);
-			}
+		for (const auto [start, end]: bits) {
+			func(m_cpu_addr + start * TRACKER_PAGE_SIZE, (end - start) * TRACKER_PAGE_SIZE);
 		}
 	}
 
@@ -345,8 +203,7 @@ private:
 	RegionBits   m_cpu_dirty;
 	RegionBits   m_gpu_dirty;
 	RegionBits   m_writable;
-	RegionBits   m_fault_pending;
-	RegionBits   m_tracked;
+	RegionBits   m_readable;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -1,13 +1,13 @@
-#include "graphics/host_gpu/renderer/bufferCache.h"
+#include "graphics/host_gpu/renderer/cache/bufferCache.h"
 
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
 #include "graphics/host_gpu/graphicContext.h"
+#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
+#include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 #include "graphics/host_gpu/renderer/render.h"
-#include "graphics/host_gpu/renderer/resourceMutex.h"
-#include "graphics/host_gpu/renderer/textureCache.h"
 #include "kernel/memory.h"
 
 #include <algorithm>
@@ -123,30 +123,6 @@ struct BufferCache::RetiredBuffer {
 	std::shared_ptr<Buffer> owner;
 };
 
-struct BufferCache::FaultReadback {
-	PageFaultAccess            access = PageFaultAccess::Unknown;
-	uint64_t                   vaddr  = 0;
-	uint64_t                   size   = 0;
-	std::vector<DownloadRange> ranges;
-	bool                       installed = false;
-
-	[[nodiscard]] bool Active() const noexcept { return !ranges.empty(); }
-
-	void Reset() {
-		access    = PageFaultAccess::Unknown;
-		vaddr     = 0;
-		size      = 0;
-		installed = false;
-		ranges.clear();
-	}
-};
-
-struct BufferCache::PendingBackingPublication {
-	uint64_t address = 0;
-	uint64_t size    = 0;
-	uint64_t tick    = 0;
-};
-
 std::pair<uint64_t, uint64_t> BufferCache::DownloadEnvelope(const DownloadCopy& copy) {
 	if (copy.owner == nullptr || copy.size == 0 || copy.source_offset > copy.owner->Size() ||
 	    copy.size > copy.owner->Size() - copy.source_offset) {
@@ -184,8 +160,7 @@ BufferCache::RecordDownloads(std::span<const DownloadCopy> copies) {
 	}
 
 	auto& download                   = m_download_buffer;
-	const auto [mapped, base_offset] =
-	    download.Map(reservation_size, DOWNLOAD_ALIGNMENT);
+	const auto [mapped, base_offset] = download.Map(reservation_size, DOWNLOAD_ALIGNMENT);
 	if (mapped == nullptr) {
 		EXIT("BufferCache: download batch could not reserve the shared stream\n");
 	}
@@ -215,39 +190,29 @@ void BufferCache::PublishDownloads(std::span<const DownloadRange> downloads) {
 	}
 }
 
-void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies,
-                                       RetiredBuffer retire) {
+void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire) {
 	if (copies.empty()) {
 		return;
 	}
 	auto downloads = RecordDownloads(copies);
-	const auto tick = m_scheduler.CurrentTick();
-	BeginBackingPublication(retire.address, retire.size, tick);
 	m_scheduler.DeferOperation(
-	    [this, downloads = std::move(downloads), retire = std::move(retire), tick]() mutable {
+	    [this, downloads = std::move(downloads), retire = std::move(retire)]() mutable {
 		    PublishDownloads(downloads);
 		    {
 			    FaultSafeCacheLock lock(this, m_mutex);
-			    if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
-				    m_memory_tracker.ForEachDownloadRange<true>(
-				        retire.address, retire.size,
-				        [&](uint64_t address, uint64_t size) noexcept {
-					        m_memory_tracker.ValidateGpuDirtyPages(
-					            m_gpu_modified_ranges, address, size,
-					            "asynchronous garbage retirement");
-				        },
-				        [](uint64_t, uint64_t) noexcept {});
-			    }
 			    for (const auto& range: downloads) {
 				    m_gpu_modified_ranges.Subtract(range.address, range.size);
 			    }
+			    // ForEachDownloadRange reports full tracker pages, and every exact GPU-owned
+			    // interval on those pages was downloaded and removed. Clearing the original
+			    // query therefore cannot orphan a dirty sibling on an edge page.
+			    m_memory_tracker.UnmarkRegionAsGpuModified(retire.address, retire.size);
 			    if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
 			        !m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
 				    EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
 			    }
 			    m_memory_tracker.UntrackMemory(retire.address, retire.size);
 		    }
-		    CompleteBackingPublication(retire.address, retire.size, tick);
 	    });
 }
 
@@ -256,13 +221,12 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
                          ResourceMutex& resource_mutex)
     : m_graphics(graphics), m_scheduler(scheduler),
       m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize),
-      m_fault_readback(std::make_unique<FaultReadback>()), m_memory_tracker(page_manager),
+      m_memory_tracker(page_manager),
       m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB),
       m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
       m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
       m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB),
-      m_page_manager(page_manager), m_texture_cache(texture_cache),
-      m_resource_mutex(resource_mutex) {
+      m_texture_cache(texture_cache), m_resource_mutex(resource_mutex) {
 	std::memset(m_gds_buffer.Mapped().data(), 0, static_cast<size_t>(m_gds_buffer.Size()));
 	m_gds_buffer.Flush(0, m_gds_buffer.Size());
 	if (!m_graphics.CanReportMemoryUsage()) {
@@ -280,15 +244,9 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
 }
 
 BufferCache::~BufferCache() {
-	if (m_fault_readback->Active()) {
-		EXIT("BufferCache: destroyed with an active fault readback\n");
-	}
 	if (!m_gpu_modified_ranges.Empty()) {
 		EXIT("BufferCache: destroyed with pending GPU-modified ranges\n");
 	}
-	if (!m_pending_backing_publications.empty()) {
-		EXIT("BufferCache: destroyed with pending backing publications\n");
-	}
 	for (const auto& [vaddr, cached]: m_buffers) {
 		(void)vaddr;
 		if (m_memory_tracker.IsRegionGpuModified(cached->vaddr, cached->size)) {
@@ -298,69 +256,6 @@ BufferCache::~BufferCache() {
 	m_buffers.clear();
 }
 
-bool BufferCache::SynchronizeBacking(uint64_t vaddr, uint64_t size) {
-	bool waited = false;
-	for (;;) {
-		uint64_t tick = 0;
-		const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-		const auto page_end =
-		    (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
-		CacheRange affected {.address = page_begin, .size = page_end - page_begin};
-		{
-			FaultSafeCacheLock lock(this, m_mutex);
-			bool changed = true;
-			while (changed) {
-				changed = false;
-				for (const auto& [address, cached]: m_buffers) {
-					const CacheRange previous = affected;
-					if (ResolveOverlap(affected, {address, cached->size}) &&
-					    (previous.address != affected.address || previous.size != affected.size)) {
-						changed = true;
-					}
-				}
-			}
-		}
-		{
-			std::lock_guard lock(m_publication_mutex);
-			for (const auto& publication: m_pending_backing_publications) {
-				if (publication.address < affected.address + affected.size &&
-				    affected.address < publication.address + publication.size) {
-					tick = std::max(tick, publication.tick);
-				}
-			}
-		}
-		if (tick == 0) {
-			return waited;
-		}
-		waited = true;
-		m_scheduler.Wait(tick);
-		m_scheduler.WaitPriorityOperations(tick);
-	}
-}
-
-void BufferCache::RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached,
-                                           uint64_t vaddr, uint64_t size, bool upload) {
-	const auto invalidated = m_image_invalidated_ranges.Intersections(vaddr, size);
-	if (upload) {
-		std::array<uint8_t, 64 * 1024> bytes;
-		for (const auto& range: invalidated) {
-			for (uint64_t copied = 0; copied < range.size;) {
-				const auto chunk = std::min<uint64_t>(range.size - copied, bytes.size());
-				if (!Libs::LibKernel::Memory::TryReadBacking(range.address + copied, bytes.data(),
-				                                             chunk)) {
-					EXIT("BufferCache: failed to refresh an invalidated image alias\n");
-				}
-				Upload(command, *cached.buffer, cached.buffer->Offset(range.address + copied),
-				       bytes.data(), chunk);
-				copied += chunk;
-			}
-		}
-	}
-	if (!invalidated.empty()) {
-		m_image_invalidated_ranges.Subtract(vaddr, size);
-	}
-}
-
 StreamBuffer& BufferCache::GetUtilityBuffer(MemoryUsage usage) noexcept {
 	switch (usage) {
 		case MemoryUsage::Upload: return m_staging_buffer;
@@ -384,103 +279,72 @@ BufferBinding BufferCache::UploadTransient(const void* data, uint64_t size, uint
 	return {owner, owner->Handle(), 0};
 }
 
-bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
-                                   PageFaultPhase phase) noexcept {
-	const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-	if (size == 0 || size > page + TRACKER_PAGE_SIZE - vaddr) {
-		EXIT("BufferCache: invalid page-fault range\n");
+void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
+	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
+	    size > TRACKER_ADDRESS_SIZE - vaddr) {
+		EXIT("BufferCache: invalid memory-invalidation range\n");
 	}
+	if (!HasPageOverlap(vaddr, size)) {
+		return;
+	}
+	m_memory_tracker.InvalidateRegion(vaddr, size,
+	                                  [this, vaddr, size] { ReadMemory(vaddr, size); });
+}
 
-	if (phase == PageFaultPhase::Complete) {
-		FaultSafeCacheLock lock(this, m_mutex);
-		auto&              fault = *m_fault_readback;
-		if (!fault.Active()) {
-			return m_memory_tracker.CompleteCpuFault(vaddr, size, access, false);
-		}
-		if (fault.access != access || fault.vaddr != vaddr || fault.size != size ||
-		    fault.installed) {
-			EXIT("BufferCache: mismatched fault readback completion\n");
-		}
-		PublishDownloads(fault.ranges);
-		if (!m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) {
-			EXIT("BufferCache: failed to complete downloaded CPU fault\n");
-		}
-		fault.installed = true;
-		return true;
-	}
-
-	if (phase == PageFaultPhase::Release) {
-		FaultSafeCacheLock lock(this, m_mutex);
-		auto&              fault = *m_fault_readback;
-		if (fault.Active()) {
-			if (fault.access != access || fault.vaddr != vaddr || fault.size != size ||
-			    !fault.installed) {
-				EXIT("BufferCache: mismatched fault readback release\n");
-			}
-			for (const auto& range: fault.ranges) {
-				m_gpu_modified_ranges.Subtract(range.address, range.size);
-			}
-			fault.Reset();
-		}
-		return true;
-	}
-
-	if (phase != PageFaultPhase::Invalidate) {
-		EXIT("BufferCache: unsupported page-fault phase\n");
-	}
-
-	const auto action = m_memory_tracker.BeginCpuFault(vaddr, size, access);
-	if (action != CpuFaultAction::Download) {
-		return action == CpuFaultAction::Continue;
-	}
-
-	auto&                     fault = *m_fault_readback;
+void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
 	std::vector<DownloadCopy> copies;
 	{
 		FaultSafeCacheLock lock(this, m_mutex);
-		if (fault.Active()) {
-			EXIT("BufferCache: nested fault readback\n");
-		}
-		fault.access = access;
-		fault.vaddr  = vaddr;
-		fault.size   = size;
-
-		m_gpu_modified_ranges.ForEachIntersection(
-		    page, TRACKER_PAGE_SIZE, [&](RangeSet::Range range) {
-			    auto owner = m_buffers.upper_bound(range.address);
+		m_memory_tracker.ForEachDownloadRange<false>(
+		    vaddr, size,
+		    [&](uint64_t address, uint64_t bytes) noexcept {
+			    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
+			                                           "memory invalidation");
+		    },
+		    [&](uint64_t address, uint64_t bytes) noexcept {
+			    for (const auto range: m_gpu_modified_ranges.Intersections(address, bytes)) {
+				    for (uint64_t copied = 0; copied < range.size;) {
+					    const auto copy_address = range.address + copied;
+					    auto       owner        = m_buffers.upper_bound(copy_address);
 					    if (owner == m_buffers.begin()) {
-				    EXIT("BufferCache: fault readback has no buffer owner\n");
+						    EXIT("BufferCache: invalidation readback has no buffer owner\n");
+					    }
+					    auto& cached = *std::prev(owner)->second;
+					    if (!cached.buffer->IsInBounds(copy_address, 1)) {
+						    EXIT(
+						        "BufferCache: invalidation readback is outside its buffer owner\n");
+					    }
+					    const auto copy_size = std::min(range.size - copied,
+					                                    cached.vaddr + cached.size - copy_address);
+					    copies.push_back({cached.buffer, cached.buffer->Offset(copy_address),
+					                      copy_address, copy_size});
+					    copied += copy_size;
 				    }
-			    --owner;
-			    auto& cached = *owner->second;
-			    if (!cached.buffer->IsInBounds(range.address, range.size)) {
-				    EXIT("BufferCache: fault readback is outside its buffer owner\n");
 			    }
-			    copies.push_back({cached.buffer, cached.buffer->Offset(range.address),
-			                      range.address, range.size});
 		    });
+	}
 	if (copies.empty()) {
-			EXIT("BufferCache: GPU-dirty fault page has no dirty byte ranges\n");
-		}
-	}
-	fault.ranges = RecordDownloads(copies);
-	if (!fault.Active()) {
-		EXIT("BufferCache: GPU-dirty fault page has no dirty byte ranges\n");
+		return;
 	}
+	auto downloads = RecordDownloads(copies);
 	m_scheduler.FinishCurrent();
-	return true;
+	PublishDownloads(downloads);
+	{
+		FaultSafeCacheLock lock(this, m_mutex);
+		for (const auto& range: downloads) {
+			m_gpu_modified_ranges.Subtract(range.address, range.size);
+		}
+		// The enumeration above covered whole dirty pages and every exact interval on them.
+		m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size);
+	}
 }
 
 void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 	if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
 		EXIT("BufferCache: invalid unmap range\n");
 	}
-	(void)SynchronizeBacking(vaddr, size);
-
 	std::vector<DownloadCopy>                  copies;
-	std::vector<RangeSet::Range>                            dirty_ranges;
 	std::vector<std::pair<uint64_t, uint64_t>> modified_buffers;
-	std::vector<std::unique_ptr<PageManager::BackingWrite>> backing_writes;
 	std::vector<std::pair<uint64_t, uint64_t>> retired_buffers;
 	{
 		FaultSafeCacheLock lock(this, m_mutex);
@@ -498,12 +362,8 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 			if (dirty.empty()) {
 				EXIT("BufferCache: GPU-modified buffer has no dirty ranges\n");
 			}
-			dirty_ranges.insert(dirty_ranges.end(), dirty.begin(), dirty.end());
 			modified_buffers.emplace_back(begin, cached->size);
 		}
-		if (!dirty_ranges.empty()) {
-			backing_writes = m_page_manager.ReserveBackingWrites(dirty_ranges);
-		}
 		for (const auto& [begin, bytes]: modified_buffers) {
 			auto owner = m_buffers.find(begin);
 			if (owner == m_buffers.end() || owner->second->size != bytes) {
@@ -533,24 +393,11 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 		// command stream before removing such backing.
 		m_scheduler.FinishCurrent();
 	}
-	backing_writes.clear();
-
 	{
 		FaultSafeCacheLock lock(this, m_mutex);
-		for (const auto& [begin, bytes]: modified_buffers) {
-			if (!m_memory_tracker.IsRegionGpuModified(begin, bytes)) {
-				continue;
-			}
-			m_memory_tracker.ForEachDownloadRange<true>(
-			    begin, bytes,
-			    [&](uint64_t address, uint64_t download_size) noexcept {
-				    m_memory_tracker.ValidateGpuDirtyPages(
-				        m_gpu_modified_ranges, address, download_size, "unmap retirement");
-			    },
-			    [](uint64_t, uint64_t) noexcept {});
-		}
 		for (const auto& [begin, bytes]: modified_buffers) {
 			m_gpu_modified_ranges.Subtract(begin, bytes);
+			m_memory_tracker.UnmarkRegionAsGpuModified(begin, bytes);
 		}
 		for (const auto& [begin, bytes]: retired_buffers) {
 			m_memory_tracker.MarkRegionAsCpuModified(begin, bytes);
@@ -558,7 +405,6 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 		if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) {
 			EXIT("BufferCache: unmap retained dirty byte ranges\n");
 		}
-		m_image_invalidated_ranges.Subtract(vaddr, size);
 		m_memory_tracker.UntrackMemory(vaddr, size);
 		for (auto it = m_buffers.begin(); it != m_buffers.end();) {
 			if (vaddr < it->first + it->second->size && it->first < vaddr + size) {
@@ -651,22 +497,30 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
 	if (command.IsInvalid() || command.IsExecute()) {
 		EXIT("BufferCache: buffer request requires a recording command buffer\n");
 	}
-	ValidateGpuAccess(vaddr, size, is_read, is_written);
 	std::lock_guard transaction(m_resource_mutex);
-	(void)SynchronizeBacking(vaddr, size);
 
 	if (is_read && !is_written && size <= CACHING_PAGE_SIZE &&
 	    !m_memory_tracker.IsRegionGpuModified(vaddr, size) &&
 	    m_memory_tracker.IsRegionCpuModified(vaddr, size)) {
-		std::vector<uint8_t> data(size);
-		if (Libs::LibKernel::Memory::TryReadBacking(vaddr, data.data(), size)) {
-			return UploadTransient(data.data(), size, 16);
+		const auto alignment = std::max<uint64_t>(
+		    m_graphics.physical_device_properties.limits.minUniformBufferOffsetAlignment, 1);
+		if (auto [mapped, offset] = m_stream_buffer.Map(size, alignment, false);
+		    mapped != nullptr) {
+			if (Libs::LibKernel::Memory::TryReadBacking(vaddr, mapped, size)) {
+				m_stream_buffer.Commit();
+				return {{}, m_stream_buffer.Handle(), offset};
+			}
+		} else {
+			auto owner = std::make_shared<Buffer>(m_graphics, m_scheduler, MemoryUsage::Upload, 0,
+			                                      AllFlags, size);
+			if (Libs::LibKernel::Memory::TryReadBacking(vaddr, owner->Mapped().data(), size)) {
+				owner->Flush(0, size);
+				return {owner, owner->Handle(), 0};
+			}
 		}
 	}
 
-	if (is_formatted && is_read && !is_written) {
-		(void)m_texture_cache.SynchronizeImageToBuffer(vaddr, size);
-	} else if (is_formatted && is_written) {
+	if (is_formatted && is_written) {
 		(void)m_texture_cache.InvalidateMemoryFromGPU(vaddr, size, true);
 	}
 
@@ -682,10 +536,12 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
 			           reinterpret_cast<const void*>(address), bytes);
 		    }
 	    });
-	RefreshInvalidatedRanges(command, cached, vaddr, size, is_read);
 	if (is_written) {
 		m_gpu_modified_ranges.Add(vaddr, size);
 	}
+	if (is_formatted && is_read && !is_written) {
+		(void)SynchronizeBufferFromImage(*cached.buffer, vaddr, size);
+	}
 	return {cached.buffer, cached.buffer->Handle(), cached.buffer->Offset(vaddr)};
 }
 
@@ -710,7 +566,6 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 	    size > TRACKER_ADDRESS_SIZE - vaddr) {
 		EXIT("BufferCache: invalid image source\n");
 	}
-	(void)SynchronizeBacking(vaddr, size);
 	auto find_owner = [&]() {
 		auto owner = m_buffers.upper_bound(vaddr);
 		if (owner == m_buffers.begin()) {
@@ -725,14 +580,12 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 		const bool         cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size);
 		const bool         gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size);
 		const auto         dirty        = m_gpu_modified_ranges.Intersections(vaddr, size);
-		const bool invalidated =
-		    !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
-		const bool         requested_gpu_owned = !dirty.empty();
+		const bool         has_dirty_buffer_source = !dirty.empty();
 		m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
 		                                           "image source");
 
 		auto owner = find_owner();
-		if (requested_gpu_owned && owner == m_buffers.end()) {
+		if (has_dirty_buffer_source && owner == m_buffers.end()) {
 			CacheRange merged {.address = AlignDown(vaddr),
 			                   .size    = AlignUp(vaddr + size) - AlignDown(vaddr)};
 			using Iterator = decltype(m_buffers.begin());
@@ -782,42 +635,32 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 				EXIT("BufferCache: merged image source does not contain the requested range\n");
 			}
 		}
-		if (owner != m_buffers.end() && !cpu_modified && !invalidated &&
-		    (!gpu_modified || requested_gpu_owned)) {
-			DiscardGpuDirtyBytesLocked(vaddr, size, "image source transfer");
+		if (owner != m_buffers.end() && !cpu_modified &&
+		    (!gpu_modified || has_dirty_buffer_source)) {
 			owner->second->tick_accessed_last = m_gc_tick;
-			return {owner->second->buffer.get(), owner->second->buffer->Offset(vaddr),
-			        requested_gpu_owned};
+			return {owner->second->buffer.get(), owner->second->buffer->Offset(vaddr)};
 		}
-		if (requested_gpu_owned && owner == m_buffers.end()) {
+		if (has_dirty_buffer_source && owner == m_buffers.end()) {
 			EXIT("BufferCache: GPU-dirty image source could not resolve its native owner\n");
 		}
 	}
 
-	// Direct-memory backing remains readable while PageManager protects the guest mapping. The
-	// fallback exists for plain host mappings used by standalone renderer tests and is deliberately
-	// performed outside the cache lock so a page fault cannot recurse into BufferCache.
-	const auto stage_address = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-	const auto stage_end     = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
-	const auto stage_size    = stage_end - stage_address;
-	(void)SynchronizeBacking(stage_address, stage_size);
-	std::vector<uint8_t> bytes(stage_size);
-	if (!Libs::LibKernel::Memory::TryReadBacking(stage_address, bytes.data(), stage_size)) {
+	auto [staging, stage_offset] = m_staging_buffer.Map(size, 16);
+	if (staging == nullptr || !Libs::LibKernel::Memory::TryReadBacking(vaddr, staging, size)) {
 		EXIT("BufferCache: failed to read mapped guest image backing\n");
 	}
+	m_staging_buffer.Commit();
 
 	FaultSafeCacheLock lock(this, m_mutex);
 	const auto         dirty                   = m_gpu_modified_ranges.Intersections(vaddr, size);
-	const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
-	const bool         requested_gpu_owned = !dirty.empty();
+	const bool         has_dirty_buffer_source = !dirty.empty();
 	auto               owner                   = find_owner();
-	if (requested_gpu_owned && owner == m_buffers.end()) {
+	if (has_dirty_buffer_source && owner == m_buffers.end()) {
 		EXIT("BufferCache: GPU-dirty image source lost its native owner\n");
 	}
-	const auto stage_offset = m_staging_buffer.Copy(bytes.data(), stage_size, 16);
-	if (owner == m_buffers.end() || invalidated ||
-	    (m_memory_tracker.IsRegionGpuModified(vaddr, size) && !requested_gpu_owned)) {
-		return {&m_staging_buffer, stage_offset + vaddr - stage_address, false};
+	if (owner == m_buffers.end() ||
+	    (m_memory_tracker.IsRegionGpuModified(vaddr, size) && !has_dirty_buffer_source)) {
+		return {&m_staging_buffer, stage_offset};
 	}
 
 	auto& cached              = *owner->second;
@@ -831,43 +674,17 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 	    [&]() noexcept {
 		    for (const auto& [address, upload_size]: uploads) {
 			    cached.buffer->CopyFrom(
-			        m_scheduler.Current(), m_staging_buffer,
-			        stage_offset + address - stage_address, cached.buffer->Offset(address),
-			        upload_size, vk::AccessFlagBits::eHostWrite);
+			        m_scheduler.Current(), m_staging_buffer, stage_offset + address - vaddr,
+			        cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
 		    }
 	    });
-	DiscardGpuDirtyBytesLocked(vaddr, size, "staged image source transfer");
-	return {cached.buffer.get(), cached.buffer->Offset(vaddr), requested_gpu_owned};
-}
-
-void BufferCache::DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation) {
-	m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, operation);
-	m_gpu_modified_ranges.Subtract(vaddr, size);
-	const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-	const auto page_end   = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
-	for (auto page = page_begin; page < page_end; page += TRACKER_PAGE_SIZE) {
-		if (m_gpu_modified_ranges.Intersections(page, TRACKER_PAGE_SIZE).empty() &&
-		    m_memory_tracker.IsRegionGpuModified(page, TRACKER_PAGE_SIZE)) {
-			m_memory_tracker.UnmarkRegionAsGpuModified(page, TRACKER_PAGE_SIZE);
-		}
-	}
-	m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, operation);
-}
-
-void BufferCache::DiscardGpuDirtyBytes(uint64_t vaddr, uint64_t size) {
-	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
-	    size > TRACKER_ADDRESS_SIZE - vaddr) {
-		EXIT("BufferCache: invalid dirty-byte discard range\n");
-	}
-	FaultSafeCacheLock lock(this, m_mutex);
-	DiscardGpuDirtyBytesLocked(vaddr, size, "image output supersession");
+	return {cached.buffer.get(), cached.buffer->Offset(vaddr)};
 }
 
 void BufferCache::WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data) {
 	if (vaddr == 0 || data.empty() || data.size() > UINT64_MAX - vaddr) {
 		EXIT("BufferCache: invalid host DMA write\n");
 	}
-	(void)SynchronizeBacking(vaddr, data.size());
 	Libs::LibKernel::Memory::WriteBacking(vaddr, data.data(), data.size());
 
 	FaultSafeCacheLock lock(this, m_mutex);
@@ -883,46 +700,6 @@ void BufferCache::WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data)
 		       data.data() + begin - vaddr, range_end - begin);
 		cached->tick_accessed_last = m_gc_tick;
 	}
-	m_image_invalidated_ranges.Subtract(vaddr, data.size());
-}
-
-std::pair<std::shared_ptr<Buffer>, uint64_t> BufferCache::ObtainBufferForImageWrite(uint64_t vaddr,
-                                                                                    uint64_t size) {
-	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
-	    size > TRACKER_ADDRESS_SIZE - vaddr) {
-		EXIT("BufferCache: invalid image destination\n");
-	}
-	const auto stage_address = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-	const auto stage_end     = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
-	const auto stage_size    = stage_end - stage_address;
-	(void)SynchronizeBacking(stage_address, stage_size);
-	std::vector<uint8_t> bytes(stage_size);
-	if (!Libs::LibKernel::Memory::TryReadBacking(stage_address, bytes.data(), stage_size)) {
-		EXIT("BufferCache: failed to preserve guest bytes around an image mirror\n");
-	}
-	FaultSafeCacheLock lock(this, m_mutex);
-	auto&              cached = GetOrCreateBuffer(m_scheduler.Current(), vaddr, size);
-	m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
-	                                           "image destination");
-	if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) {
-		EXIT("BufferCache: image destination aliases GPU-owned buffer bytes\n");
-	}
-	const auto stage_offset = m_staging_buffer.Copy(bytes.data(), stage_size, 16);
-	std::vector<std::pair<uint64_t, uint64_t>> uploads;
-	m_memory_tracker.ForEachUploadRange(
-	    vaddr, size, false,
-	    [&](uint64_t address, uint64_t upload_size) noexcept {
-		    uploads.emplace_back(address, upload_size);
-	    },
-	    [&]() noexcept {
-		    for (const auto& [address, upload_size]: uploads) {
-			    cached.buffer->CopyFrom(
-			        m_scheduler.Current(), m_staging_buffer,
-			        stage_offset + address - stage_address, cached.buffer->Offset(address),
-			        upload_size, vk::AccessFlagBits::eHostWrite);
-		    }
-	    });
-	return {cached.buffer, cached.buffer->Offset(vaddr)};
 }
 
 void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds) {
@@ -939,19 +716,18 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
 	if (vaddr == 0) {
 		EXIT("BufferCache: invalid fill memory address\n");
 	}
-	ValidateGpuAccess(vaddr, size, false, true);
 	(void)m_texture_cache.ClearMeta(vaddr);
 	{
 		std::lock_guard transaction(m_resource_mutex);
 		const auto      region = m_texture_cache.QueryRegion(vaddr, size);
 		if (!HasGpuDirtyBytes(vaddr, size) && !region.gpu_image_bytes) {
 			if (region.image_bytes) {
-				m_texture_cache.PrepareHostWrite(vaddr, size);
+				m_texture_cache.InvalidateMemory(vaddr, size);
 			}
 			std::array<uint32_t, 4096> values;
 			values.fill(value);
-			const std::span<const uint8_t> bytes {
-			    reinterpret_cast<const uint8_t*>(values.data()), sizeof(values)};
+			const std::span<const uint8_t> bytes {reinterpret_cast<const uint8_t*>(values.data()),
+			                                      sizeof(values)};
 			for (uint64_t offset = 0; offset < size;) {
 				const auto chunk = std::min<uint64_t>(size - offset, bytes.size());
 				WriteHostMemory(vaddr + offset, bytes.first(chunk));
@@ -981,30 +757,17 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	    (src_gds && (src_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - src_vaddr))) {
 		EXIT("BufferCache: invalid or overlapping copy range\n");
 	}
-	if (src_memory) {
-		ValidateGpuAccess(src_vaddr, size, true, false);
-	}
-	if (dst_memory) {
-		ValidateGpuAccess(dst_vaddr, size, false, true);
-	}
 	if (src_memory || dst_memory) {
 		std::lock_guard transaction(m_resource_mutex);
-		if (src_memory) {
-			(void)SynchronizeBacking(src_vaddr, size);
-		}
 		const auto      src_region =
 		    src_memory ? m_texture_cache.QueryRegion(src_vaddr, size) : TextureCache::RegionInfo {};
 		const auto dst_region =
 		    dst_memory ? m_texture_cache.QueryRegion(dst_vaddr, size) : TextureCache::RegionInfo {};
-		if (src_memory && src_region.gpu_image_bytes &&
-		    !m_texture_cache.SynchronizeImageToBuffer(src_vaddr, size)) {
-			EXIT("BufferCache: GPU copy source image could not be synchronized\n");
-		}
 		if (src_memory && dst_memory && !HasGpuDirtyBytes(src_vaddr, size) &&
 		    !HasGpuDirtyBytes(dst_vaddr, size) && !src_region.gpu_image_bytes &&
 		    !dst_region.gpu_image_bytes) {
 			if (dst_region.image_bytes) {
-				m_texture_cache.PrepareHostWrite(dst_vaddr, size);
+				m_texture_cache.InvalidateMemory(dst_vaddr, size);
 			}
 			std::array<uint8_t, 64 * 1024> bytes;
 			for (uint64_t offset = 0; offset < size;) {
@@ -1021,7 +784,7 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	}
 
 	auto& command = m_scheduler.Current();
-	auto  src = src_memory ? ObtainBuffer(command, src_vaddr, size, false, true)
+	auto  src = src_memory ? ObtainBuffer(command, src_vaddr, size, false, true, true)
 	                       : BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = src_vaddr};
 	auto  dst = dst_memory ? ObtainBuffer(command, dst_vaddr, size, true, false, true)
 	                       : BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = dst_vaddr};
@@ -1039,8 +802,8 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	auto& source = src.owner != nullptr ? *std::static_pointer_cast<Buffer>(src.owner)
 	               : src_gds            ? m_gds_buffer
 	                                    : m_stream_buffer;
-	auto& destination = dst.owner != nullptr ? *std::static_pointer_cast<Buffer>(dst.owner)
-	                                         : m_gds_buffer;
+	auto& destination =
+	    dst.owner != nullptr ? *std::static_pointer_cast<Buffer>(dst.owner) : m_gds_buffer;
 	if (source.Handle() != src.buffer || destination.Handle() != dst.buffer) {
 		EXIT("BufferCache: resolved copy owner does not match its Vulkan handle\n");
 	}
@@ -1074,95 +837,13 @@ bool BufferCache::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 	return m_memory_tracker.IsRegionCpuModified(vaddr, size);
 }
 
-void BufferCache::InvalidateImageAliases(uint64_t vaddr, uint64_t size) {
-	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
-	    size > TRACKER_ADDRESS_SIZE - vaddr) {
-		EXIT("BufferCache: invalid image-alias invalidation\n");
-	}
-	FaultSafeCacheLock lock(this, m_mutex);
-	const auto         end = vaddr + size;
-	for (const auto& [address, cached]: m_buffers) {
-		const auto cached_end = address + cached->size;
-		const auto begin      = std::max(vaddr, address);
-		const auto range_end  = std::min(end, cached_end);
-		if (begin >= range_end) {
-			continue;
-		}
-		const auto bytes = range_end - begin;
-		if (!m_gpu_modified_ranges.Intersections(begin, bytes).empty()) {
-			EXIT("BufferCache: image ownership overlaps exact dirty buffer bytes\n");
-		}
-		m_image_invalidated_ranges.Add(begin, bytes);
-	}
-}
-
-void BufferCache::BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) {
-	if (vaddr == 0 || size == 0 || tick == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
-	    size > TRACKER_ADDRESS_SIZE - vaddr) {
-		EXIT("BufferCache: invalid pending backing publication\n");
-	}
-	std::lock_guard lock(m_publication_mutex);
-	m_pending_backing_publications.push_back({vaddr, size, tick});
-}
-
-void BufferCache::CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) {
-	std::lock_guard lock(m_publication_mutex);
-	const auto publication =
-	    std::ranges::find_if(m_pending_backing_publications, [&](const auto& pending) {
-		    return pending.address == vaddr && pending.size == size && pending.tick == tick;
-	    });
-	if (publication == m_pending_backing_publications.end()) {
-		EXIT("BufferCache: completed an unknown backing publication\n");
-	}
-	m_pending_backing_publications.erase(publication);
-}
-
-void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) {
-	FaultSafeCacheLock lock(this, m_mutex);
-	auto               owner = m_buffers.end();
-	for (auto it = m_buffers.begin(); it != m_buffers.end(); ++it) {
-		if (!PageOverlaps(vaddr, size, it->second->vaddr, it->second->size)) {
-			continue;
-		}
-		if (owner != m_buffers.end() || !it->second->buffer->IsInBounds(vaddr, size)) {
-			EXIT("BufferCache: image destination aliases a non-containing cached buffer\n");
-		}
-		owner = it;
-	}
-	m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
-	                                           "image destination publication");
-	if (owner == m_buffers.end() || m_memory_tracker.IsRegionCpuModified(vaddr, size) ||
-	    !m_gpu_modified_ranges.Intersections(vaddr, size).empty()) {
-		EXIT("BufferCache: image destination requires clean buffer ownership\n");
-	}
-	m_memory_tracker.MarkRegionAsGpuModified(vaddr, size);
-	m_gpu_modified_ranges.Add(vaddr, size);
-	m_image_invalidated_ranges.Subtract(vaddr, size);
-	m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
-	                                           "published image destination");
-	owner->second->tick_accessed_last = m_gc_tick;
-}
-
-void BufferCache::ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read,
-                                    bool is_written) const {
-	if ((!is_read && !is_written) || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
-		EXIT("BufferCache: invalid GPU access request\n");
-	}
-	if (is_read && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Read)) {
-		EXIT("BufferCache: GPU-read access denied\n");
-	}
-	if (is_written && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Write)) {
-		EXIT("BufferCache: GPU-write access denied\n");
-	}
-}
-
 void BufferCache::RunGarbageCollector() {
 	std::lock_guard transaction(m_resource_mutex);
 	const auto      tick = m_gc_tick++;
 	if (m_graphics.CanReportMemoryUsage()) {
 		m_total_used_memory = m_graphics.GetDeviceMemoryUsage();
 	}
-	if (m_total_used_memory < m_trigger_gc_memory || m_fault_readback->Active()) {
+	if (m_total_used_memory < m_trigger_gc_memory) {
 		return;
 	}
 
@@ -1190,8 +871,8 @@ void BufferCache::RunGarbageCollector() {
 		}
 		for (const auto address: candidates) {
 			auto& cached = *m_buffers.at(address);
-			m_memory_tracker.ValidateGpuDirtyOwnership(
-			    m_gpu_modified_ranges, cached.vaddr, cached.size, "garbage collection");
+			m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, cached.vaddr,
+			                                           cached.size, "garbage collection");
 			retires.push_back({address, cached.size, cached.buffer});
 			// GC runs immediately before submission. Preserve every source referenced by commands
 			// already recorded in the active batch.
@@ -1205,13 +886,13 @@ void BufferCache::RunGarbageCollector() {
 			m_memory_tracker.ForEachDownloadRange<false>(
 			    retire.address, retire.size,
 			    [&](uint64_t address, uint64_t size) noexcept {
-				    m_memory_tracker.ValidateGpuDirtyPages(
-				        m_gpu_modified_ranges, address, size, "garbage collection");
+				    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
+				                                           "garbage collection");
 			    },
 			    [&](uint64_t address, uint64_t size) noexcept {
 				    for (const auto range: m_gpu_modified_ranges.Intersections(address, size)) {
-					    copies.push_back({retire.owner, range.address - retire.address, range.address,
-					                      range.size});
+					    copies.push_back({retire.owner, range.address - retire.address,
+					                      range.address, range.size});
 				    }
 			    });
 		}
@@ -1231,7 +912,6 @@ void BufferCache::RunGarbageCollector() {
 		if (!m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
 			m_memory_tracker.UntrackMemory(retire.address, retire.size);
 		}
-		m_image_invalidated_ranges.Subtract(retire.address, retire.size);
 		if (retire.size > m_total_used_memory) {
 			EXIT("BufferCache: allocation accounting underflow\n");
 		}

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -6,11 +6,10 @@
 #include "common/threads.h"
 #include "graphics/host_gpu/memoryTracker.h"
 #include "graphics/host_gpu/rangeSet.h"
-#include "graphics/host_gpu/renderer/streamBuffer.h"
+#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 
 #include <map>
 #include <memory>
-#include <mutex>
 #include <span>
 #include <utility>
 #include <vector>
@@ -32,7 +31,6 @@ struct BufferBinding {
 struct ImageBufferSource {
 	Buffer*  buffer = nullptr;
 	uint64_t offset = 0;
-	bool     gpu_owned = false;
 };
 
 class BufferCache {
@@ -47,8 +45,8 @@ public:
 	~BufferCache();
 	KYTY_CLASS_NO_COPY(BufferCache);
 
-	[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
-	                                    PageFaultPhase phase) noexcept;
+	void                        InvalidateMemory(uint64_t vaddr, uint64_t size);
+	void                        ReadMemory(uint64_t vaddr, uint64_t size);
 	void                        UnmapMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
 	                                         bool is_written = false, bool is_read = true,
@@ -60,9 +58,6 @@ public:
 	                                            uint64_t alignment);
 	[[nodiscard]] std::shared_ptr<Buffer> ObtainNullBuffer();
 	[[nodiscard]] ImageBufferSource       ObtainBufferForImage(uint64_t vaddr, uint64_t size);
-	[[nodiscard]] std::pair<std::shared_ptr<Buffer>, uint64_t>
-	     ObtainBufferForImageWrite(uint64_t vaddr, uint64_t size);
-	void DiscardGpuDirtyBytes(uint64_t vaddr, uint64_t size);
 	void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds = false);
 	void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds = false,
 	                bool src_gds = false);
@@ -70,12 +65,6 @@ public:
 	[[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
-	void               InvalidateImageAliases(uint64_t vaddr, uint64_t size);
-	void BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
-	void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
-	[[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size);
-	void               PublishImageBuffer(uint64_t vaddr, uint64_t size);
-	void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const;
 	void               RunGarbageCollector();
 
 private:
@@ -89,8 +78,6 @@ private:
 	struct DownloadCopy;
 	struct DownloadRange;
 	struct RetiredBuffer;
-	struct FaultReadback;
-	struct PendingBackingPublication;
 	static constexpr uint64_t               DOWNLOAD_ALIGNMENT = 64;
 	[[nodiscard]] static uint64_t           AlignDown(uint64_t value) noexcept;
 	[[nodiscard]] static uint64_t           AlignUp(uint64_t value);
@@ -99,20 +86,16 @@ private:
 	}
 	[[nodiscard]] static bool PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right,
 	                                       uint64_t right_size) noexcept;
-	[[nodiscard]] static std::pair<uint64_t, uint64_t>
-	     DownloadEnvelope(const DownloadCopy& copy);
+	[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
 	[[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
 	void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset,
 	            const void* source, uint64_t size);
 	[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
 	                                              uint64_t size);
-	[[nodiscard]] std::vector<DownloadRange>
-	     RecordDownloads(std::span<const DownloadCopy> copies);
+	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
+	[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
 	void PublishDownloads(std::span<const DownloadRange> downloads);
 	void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
-	void RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, uint64_t vaddr,
-	                              uint64_t size, bool upload);
-	void DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation);
 	void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
 
 	GraphicContext&                                   m_graphics;
@@ -121,17 +104,12 @@ private:
 	Common::Mutex                                     m_mutex;
 	std::shared_ptr<Buffer>                           m_null_buffer;
 	std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
-	std::unique_ptr<FaultReadback>                    m_fault_readback;
 	RangeSet                                          m_gpu_modified_ranges;
-	RangeSet                                          m_image_invalidated_ranges;
-	std::mutex                                        m_publication_mutex;
-	std::vector<PendingBackingPublication>            m_pending_backing_publications;
 	MemoryTracker                                     m_memory_tracker;
 	StreamBuffer                                      m_staging_buffer;
 	StreamBuffer                                      m_stream_buffer;
 	StreamBuffer                                      m_download_buffer;
 	StreamBuffer                                      m_device_buffer;
-	PageManager&                                      m_page_manager;
 	TextureCache&                                     m_texture_cache;
 	ResourceMutex&                                    m_resource_mutex;
 	uint64_t                                          m_total_used_memory = 0;

src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp

@@ -0,0 +1,140 @@
+#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
+
+#include "common/assert.h"
+#include "graphics/guest_gpu/command_processor/commandProcessor.h"
+#include "graphics/guest_gpu/graphicsRun.h"
+#include "graphics/host_gpu/renderer/commandScheduler.h"
+namespace Libs::Graphics {
+
+GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
+    : m_scheduler(scheduler),
+      m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex),
+      m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache, m_resource_mutex) {}
+
+GpuResourceManager::~GpuResourceManager() = default;
+
+bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
+	constexpr uint64_t fault_size = 8;
+	if (!IsMapped(fault_vaddr, fault_size)) {
+		return false;
+	}
+	if (CommandScheduler::InDeferredOperation()) {
+		EXIT("unsupported guest-memory fault from an asynchronous GPU completion, "
+		     "addr=0x%016" PRIx64 " access=%u\n",
+		     fault_vaddr, static_cast<uint32_t>(access));
+	}
+	bool       handled = false;
+	const auto resolve = [this, access, fault_vaddr, &handled](CommandProcessor& cp) {
+		cp.BeginReadbackTransaction();
+		{
+			ResourceMutex::FaultScope fault(m_resource_mutex);
+			if (access == PageFaultAccess::Write) {
+				m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
+				m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
+			} else {
+				m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
+			}
+			handled = true;
+		}
+		cp.EndReadbackTransaction();
+	};
+	if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
+		resolve(*cp);
+		return handled;
+	}
+	if (m_resource_mutex.IsOwnedByCurrentThread()) {
+		EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64
+		     " access=%u\n",
+		     fault_vaddr, static_cast<uint32_t>(access));
+	}
+	EXIT_IF(m_gpu == nullptr);
+	m_gpu->SendCommandSyncWithProcessor(resolve);
+	return handled;
+}
+
+bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) {
+	if (!IsMapped(vaddr, size)) {
+		return false;
+	}
+	if (CommandScheduler::InDeferredOperation()) {
+		EXIT("unsupported memory invalidation from an asynchronous GPU completion, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+		     vaddr, size);
+	}
+	const auto resolve = [this, vaddr, size](CommandProcessor& cp) {
+		cp.BeginReadbackTransaction();
+		{
+			ResourceMutex::FaultScope fault(m_resource_mutex);
+			m_buffer_cache.InvalidateMemory(vaddr, size);
+			m_texture_cache.InvalidateMemory(vaddr, size);
+		}
+		cp.EndReadbackTransaction();
+	};
+	if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
+		resolve(*cp);
+		return true;
+	}
+	if (m_resource_mutex.IsOwnedByCurrentThread()) {
+		EXIT("unsupported memory invalidation from a pre-owned resource transaction, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+		     vaddr, size);
+	}
+	EXIT_IF(m_gpu == nullptr);
+	m_gpu->SendCommandSyncWithProcessor(resolve);
+	return true;
+}
+
+bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
+	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
+	    size > TRACKER_ADDRESS_SIZE - vaddr) {
+		return false;
+	}
+	std::shared_lock lock(m_mapped_ranges_mutex);
+	return m_mapped_ranges.Contains(vaddr, size);
+}
+
+void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
+	{
+		std::lock_guard lock(m_mapped_ranges_mutex);
+		m_mapped_ranges.Add(vaddr, size);
+	}
+	m_page_manager.OnGpuMap(vaddr, size);
+}
+
+void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
+	if (CommandScheduler::InDeferredOperation()) {
+		EXIT("unsupported memory unmap from an asynchronous GPU completion, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+		     vaddr, size);
+	}
+	if (m_resource_mutex.IsOwnedByCurrentThread()) {
+		EXIT("unsupported memory unmap from a pre-owned resource transaction, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+		     vaddr, size);
+	}
+	const auto unmap = [this, vaddr, size] {
+		if (m_scheduler.Active()) {
+			const auto tick = m_scheduler.CurrentTick();
+			m_scheduler.FinishCurrent();
+			m_scheduler.WaitPriorityOperations(tick);
+		}
+		m_buffer_cache.UnmapMemory(vaddr, size);
+		m_texture_cache.UnmapMemory(vaddr, size);
+		m_page_manager.OnGpuUnmap(vaddr, size);
+		std::lock_guard lock(m_mapped_ranges_mutex);
+		m_mapped_ranges.Subtract(vaddr, size);
+	};
+	if (m_gpu == nullptr) {
+		unmap();
+		return;
+	}
+	m_gpu->SendCommandSync(unmap);
+}
+
+void GpuResourceManager::RunGarbageCollector() {
+	m_texture_cache.ProcessDownloadImages();
+	m_texture_cache.RunGarbageCollector();
+	m_buffer_cache.RunGarbageCollector();
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/gpuResourceManager.h

@@ -4,11 +4,12 @@
 #include "common/abi.h"
 #include "common/common.h"
 #include "graphics/host_gpu/pageManager.h"
-#include "graphics/host_gpu/renderer/bufferCache.h"
-#include "graphics/host_gpu/renderer/resourceMutex.h"
-#include "graphics/host_gpu/renderer/textureCache.h"
+#include "graphics/host_gpu/renderer/cache/bufferCache.h"
+#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
+#include "graphics/host_gpu/renderer/cache/textureCache.h"
 
 #include <cstdint>
+#include <shared_mutex>
 
 namespace Libs::Graphics {
 
@@ -26,22 +27,20 @@ public:
 	void                        SetGpu(Gpu* gpu) noexcept { m_gpu = gpu; }
 
 	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
-	void               PrepareHostWrite(uint64_t vaddr, uint64_t size);
+	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
-	void               MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
-	void               UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
+	void               MapMemory(uint64_t vaddr, uint64_t size);
+	void               UnmapMemory(uint64_t vaddr, uint64_t size);
 	void               RunGarbageCollector();
 
 private:
-	static bool FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr, uint64_t size,
-	                       PageFaultPhase phase) noexcept;
-	[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
-	                                    PageFaultPhase phase) noexcept;
-
 	PageManager               m_page_manager;
 	ResourceMutex             m_resource_mutex;
+	CommandScheduler&         m_scheduler;
 	BufferCache               m_buffer_cache;
 	TextureCache              m_texture_cache;
+	mutable std::shared_mutex m_mapped_ranges_mutex;
+	RangeSet                  m_mapped_ranges;
 	Gpu*                      m_gpu = nullptr;
 };
 

src/graphics/host_gpu/renderer/cache/multiLevelPageTable.h

@@ -0,0 +1,272 @@
+#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_
+#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_
+
+#include "common/assert.h"
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+namespace Libs::Graphics {
+
+// Sparse two-level lookup for texture-cache page ownership.
+// Entries are selected by guest page number; queries never allocate L1 buckets.
+template <typename EntryT, size_t PageBits = 20, size_t AddressSpaceBits = 40,
+          size_t FirstLevelBits = 10>
+class MultiLevelPageTable final {
+public:
+	using Entry = EntryT;
+
+	static_assert(AddressSpaceBits < 64);
+	static_assert(PageBits < AddressSpaceBits);
+	static_assert(FirstLevelBits > 0 && FirstLevelBits <= AddressSpaceBits - PageBits);
+
+	static constexpr size_t   kPageBits          = PageBits;
+	static constexpr size_t   kAddressSpaceBits  = AddressSpaceBits;
+	static constexpr size_t   kFirstLevelBits    = FirstLevelBits;
+	static constexpr size_t   kSecondLevelBits   = AddressSpaceBits - FirstLevelBits - PageBits;
+	static constexpr size_t   kFirstLevelEntries = size_t {1} << FirstLevelBits;
+	static constexpr size_t   kBucketEntries     = size_t {1} << kSecondLevelBits;
+	static constexpr size_t   kPageCount         = size_t {1} << (AddressSpaceBits - PageBits);
+	static constexpr uint64_t kAddressSpaceSize  = uint64_t {1} << AddressSpaceBits;
+
+	struct PageRange final {
+		size_t first          = 0;
+		size_t last_exclusive = 0;
+	};
+
+	MultiLevelPageTable(): m_first_level(kFirstLevelEntries) {}
+
+	[[nodiscard]] Entry* Find(size_t page) {
+		if (!IsValidPage(page)) {
+			return nullptr;
+		}
+		auto& bucket = m_first_level[FirstLevelIndex(page)];
+		return bucket == nullptr ? nullptr : &(*bucket)[SecondLevelIndex(page)];
+	}
+
+	[[nodiscard]] const Entry* Find(size_t page) const {
+		if (!IsValidPage(page)) {
+			return nullptr;
+		}
+		const auto& bucket = m_first_level[FirstLevelIndex(page)];
+		return bucket == nullptr ? nullptr : &(*bucket)[SecondLevelIndex(page)];
+	}
+
+	[[nodiscard]] Entry& GetOrCreate(size_t page) {
+		if (!IsValidPage(page)) {
+			EXIT("MultiLevelPageTable page is outside the guest address space");
+		}
+		auto& bucket = m_first_level[FirstLevelIndex(page)];
+		if (bucket == nullptr) {
+			bucket = std::make_unique<Bucket>();
+			++m_allocated_buckets;
+		}
+		return (*bucket)[SecondLevelIndex(page)];
+	}
+
+	[[nodiscard]] Entry& operator[](size_t page) { return GetOrCreate(page); }
+
+	[[nodiscard]] static constexpr bool IsValidPage(size_t page) { return page < kPageCount; }
+
+	// Returns the half-open page interval touched by a non-empty byte range.
+	// An end exactly at 2^AddressSpaceBits is valid; wrapping or crossing it is not.
+	[[nodiscard]] static constexpr bool TryGetPageRange(uint64_t address, uint64_t size,
+	                                                    PageRange& range) {
+		if (size == 0 || address >= kAddressSpaceSize || size > kAddressSpaceSize - address) {
+			return false;
+		}
+		const uint64_t end   = address + size;
+		range.first          = static_cast<size_t>(address >> PageBits);
+		range.last_exclusive = static_cast<size_t>(((end - 1) >> PageBits) + 1);
+		return true;
+	}
+
+	[[nodiscard]] size_t AllocatedBucketCount() const { return m_allocated_buckets; }
+
+private:
+	using Bucket = std::array<Entry, kBucketEntries>;
+
+	[[nodiscard]] static constexpr size_t FirstLevelIndex(size_t page) {
+		return page >> kSecondLevelBits;
+	}
+	[[nodiscard]] static constexpr size_t SecondLevelIndex(size_t page) {
+		return page & (kBucketEntries - 1);
+	}
+
+	std::vector<std::unique_ptr<Bucket>> m_first_level;
+	size_t                               m_allocated_buckets = 0;
+};
+
+// Inline owner storage for page-table entries. Texture pages normally have only a
+// handful of owners, so avoid a heap allocation on the first 16 registrations.
+//
+// Owners are packed into live uint64_t objects instead of being constructed in
+// raw storage. This keeps sparse bucket construction cheap without relying on
+// subtle implicit-lifetime or pointer-provenance rules.
+template <typename OwnerT, size_t InlineCapacity = 16>
+class InlinePageOwnerList final {
+public:
+	using value_type = OwnerT;
+	using size_type  = size_t;
+
+	class const_iterator final {
+	public:
+		[[nodiscard]] OwnerT operator*() const noexcept { return m_owners->At(m_index); }
+		const_iterator&      operator++() noexcept {
+			++m_index;
+			return *this;
+		}
+		bool operator==(const const_iterator&) const = default;
+
+	private:
+		friend class InlinePageOwnerList;
+		const_iterator(const InlinePageOwnerList* owners, size_type index) noexcept
+		    : m_owners(owners), m_index(index) {}
+
+		const InlinePageOwnerList* m_owners = nullptr;
+		size_type                  m_index  = 0;
+	};
+
+	static_assert(InlineCapacity > 0);
+	static_assert(std::is_default_constructible_v<OwnerT>);
+	static_assert(std::is_trivially_copyable_v<OwnerT>);
+	static_assert(std::has_unique_object_representations_v<OwnerT>);
+	static_assert(sizeof(OwnerT) <= sizeof(uint64_t));
+
+	InlinePageOwnerList() noexcept {}
+	InlinePageOwnerList(const InlinePageOwnerList&)            = delete;
+	InlinePageOwnerList& operator=(const InlinePageOwnerList&) = delete;
+	InlinePageOwnerList(InlinePageOwnerList&& other) noexcept
+	    : m_overflow(std::move(other.m_overflow)),
+	      m_inline_size(std::exchange(other.m_inline_size, 0)) {
+		if (m_overflow == nullptr) {
+			std::copy_n(other.m_inline.begin(), m_inline_size, m_inline.begin());
+		}
+	}
+	InlinePageOwnerList& operator=(InlinePageOwnerList&& other) noexcept {
+		if (this != &other) {
+			m_overflow    = std::move(other.m_overflow);
+			m_inline_size = std::exchange(other.m_inline_size, 0);
+			if (m_overflow == nullptr) {
+				std::copy_n(other.m_inline.begin(), m_inline_size, m_inline.begin());
+			}
+		}
+		return *this;
+	}
+
+	[[nodiscard]] const_iterator begin() const noexcept { return {this, 0}; }
+	[[nodiscard]] const_iterator end() const noexcept { return {this, size()}; }
+
+	[[nodiscard]] bool      empty() const noexcept { return size() == 0; }
+	[[nodiscard]] size_type size() const noexcept {
+		return m_overflow == nullptr ? m_inline_size : m_overflow->size();
+	}
+	[[nodiscard]] OwnerT front() const noexcept { return At(0); }
+	[[nodiscard]] OwnerT operator[](size_type index) const noexcept { return At(index); }
+
+	void push_back(const OwnerT& owner) {
+		const uint64_t packed = Pack(owner);
+		if (m_overflow != nullptr) {
+			m_overflow->push_back(packed);
+			return;
+		}
+		if (m_inline_size < InlineCapacity) {
+			m_inline[m_inline_size] = packed;
+			++m_inline_size;
+			return;
+		}
+		auto overflow = std::make_unique<std::vector<uint64_t>>();
+		overflow->reserve(InlineCapacity * 2);
+		overflow->assign(m_inline.begin(), m_inline.end());
+		overflow->push_back(packed);
+		m_overflow = std::move(overflow);
+	}
+
+	[[nodiscard]] bool Contains(const OwnerT& owner) const noexcept {
+		const uint64_t packed = Pack(owner);
+		if (m_overflow != nullptr) {
+			return std::find(m_overflow->begin(), m_overflow->end(), packed) != m_overflow->end();
+		}
+		return std::find(m_inline.begin(), m_inline.begin() + static_cast<ptrdiff_t>(m_inline_size),
+		                 packed) != m_inline.begin() + static_cast<ptrdiff_t>(m_inline_size);
+	}
+
+	[[nodiscard]] bool Erase(const OwnerT& owner) noexcept {
+		const uint64_t packed = Pack(owner);
+		if (m_overflow != nullptr) {
+			const auto found = std::find(m_overflow->begin(), m_overflow->end(), packed);
+			if (found == m_overflow->end()) {
+				return false;
+			}
+			m_overflow->erase(found);
+			if (m_overflow->size() == InlineCapacity) {
+				std::copy(m_overflow->begin(), m_overflow->end(), m_inline.begin());
+				m_inline_size = InlineCapacity;
+				m_overflow.reset();
+			}
+			return true;
+		}
+		const auto end   = m_inline.begin() + static_cast<ptrdiff_t>(m_inline_size);
+		const auto found = std::find(m_inline.begin(), end, packed);
+		if (found == end) {
+			return false;
+		}
+		std::move(found + 1, end, found);
+		--m_inline_size;
+		return true;
+	}
+
+	template <typename Func>
+	void ForEach(Func&& func) const {
+		if (m_overflow != nullptr) {
+			for (const uint64_t owner: *m_overflow) {
+				func(Unpack(owner));
+			}
+			return;
+		}
+		for (size_type index = 0; index < m_inline_size; ++index) {
+			func(Unpack(m_inline[index]));
+		}
+	}
+
+private:
+	[[nodiscard]] OwnerT At(size_type index) const noexcept {
+		return Unpack(m_overflow == nullptr ? m_inline[index] : (*m_overflow)[index]);
+	}
+	[[nodiscard]] static uint64_t Pack(const OwnerT& owner) noexcept {
+		uint64_t packed = 0;
+		std::memcpy(&packed, &owner, sizeof(owner));
+		return packed;
+	}
+	[[nodiscard]] static OwnerT Unpack(uint64_t packed) noexcept {
+		OwnerT owner {};
+		std::memcpy(&owner, &packed, sizeof(owner));
+		return owner;
+	}
+
+	std::array<uint64_t, InlineCapacity>   m_inline;
+	std::unique_ptr<std::vector<uint64_t>> m_overflow;
+	size_type                              m_inline_size = 0;
+};
+
+// Removes only the requested owner. Other owners in the same page entry remain intact.
+template <typename Container, typename Value>
+[[nodiscard]] bool EraseExact(Container& owners, const Value& owner) {
+	const auto it = std::find(owners.begin(), owners.end(), owner);
+	if (it == owners.end()) {
+		return false;
+	}
+	owners.erase(it);
+	return true;
+}
+
+} // namespace Libs::Graphics
+
+#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_

src/graphics/host_gpu/renderer/cache/resourceMutex.cpp

@@ -1,4 +1,4 @@
-#include "graphics/host_gpu/renderer/resourceMutex.h"
+#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
 
 #include "common/assert.h"
 

src/graphics/host_gpu/renderer/cache/samplerCache.cpp

@@ -1,4 +1,4 @@
-#include "graphics/host_gpu/renderer/samplerCache.h"
+#include "graphics/host_gpu/renderer/cache/samplerCache.h"
 
 #include "common/assert.h"
 #include "common/logging/log.h"
@@ -56,7 +56,10 @@ vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) {
 			case Prospero::SamplerAnisoRatio::kFour: aniso_ratio = 4.0f; break;
 			case Prospero::SamplerAnisoRatio::kEight: aniso_ratio = 8.0f; break;
 			case Prospero::SamplerAnisoRatio::kSixteen: aniso_ratio = 16.0f; break;
-			default: EXIT("unknown ratio: %d\n", static_cast<int>(r.MaxAnisoRatio()));
+			default:
+				EXIT("unknown ratio: %d dwords=%08x,%08x,%08x,%08x\n",
+				     static_cast<int>(r.MaxAnisoRatio()), r.fields[0], r.fields[1], r.fields[2],
+				     r.fields[3]);
 		}
 	}
 

src/graphics/host_gpu/renderer/cache/streamBuffer.cpp

@@ -1,4 +1,4 @@
-#include "graphics/host_gpu/renderer/streamBuffer.h"
+#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 
 #include "common/assert.h"
 #include "common/profiler.h"
@@ -135,8 +135,8 @@ void Buffer::Write(uint64_t offset, const void* source, uint64_t size) {
 void Buffer::Flush(uint64_t offset, uint64_t size) {
 	EXIT_IF(m_mapped.empty() || offset > m_size || size > m_size - offset);
 	if (!m_is_coherent && size != 0) {
-		const auto result = vmaFlushAllocation(m_graphics->allocator, m_buffer->memory.allocation,
-		                                      offset, size);
+		const auto result =
+		    vmaFlushAllocation(m_graphics->allocator, m_buffer->memory.allocation, offset, size);
 		EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
 	}
 }
@@ -144,8 +144,8 @@ void Buffer::Flush(uint64_t offset, uint64_t size) {
 vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::AccessFlags source,
                                         vk::AccessFlags destination) const {
 	if (Handle() == nullptr || size == 0 || offset > m_size || size > m_size - offset) {
-		EXIT("Buffer: invalid DMA barrier, handle=%p offset=0x%016" PRIx64
-		     " size=0x%016" PRIx64 " capacity=0x%016" PRIx64 "\n",
+		EXIT("Buffer: invalid DMA barrier, handle=%p offset=0x%016" PRIx64 " size=0x%016" PRIx64
+		     " capacity=0x%016" PRIx64 "\n",
 		     static_cast<const void*>(Handle()), offset, size, m_size);
 	}
 	vk::BufferMemoryBarrier barrier {};
@@ -175,8 +175,7 @@ void Buffer::CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t sou
 	command.EndRendering();
 	const vk::BufferMemoryBarrier before[] = {
 	    source.Barrier(source_offset, size, source_before, vk::AccessFlagBits::eTransferRead),
-	    Barrier(destination_offset, size, destination_before,
-	            vk::AccessFlagBits::eTransferWrite),
+	    Barrier(destination_offset, size, destination_before, vk::AccessFlagBits::eTransferWrite),
 	};
 	const auto host_access  = vk::AccessFlagBits::eHostRead | vk::AccessFlagBits::eHostWrite;
 	auto       before_stage = vk::PipelineStageFlags {vk::PipelineStageFlagBits::eAllCommands};
@@ -214,8 +213,7 @@ void Buffer::Fill(uint64_t offset, uint64_t size, uint32_t value) {
 	                       vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion,
 	                       0, nullptr, 1, &before, 0, nullptr);
 	native.fillBuffer(Handle(), offset, size, value);
-	const auto after =
-	    Barrier(offset, size, vk::AccessFlagBits::eTransferWrite,
+	const auto after = Barrier(offset, size, vk::AccessFlagBits::eTransferWrite,
 	                           vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite);
 	native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
 	                       vk::PipelineStageFlagBits::eAllCommands,

src/graphics/host_gpu/renderer/cache/streamBuffer.h

@@ -54,16 +54,15 @@ public:
 	[[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept;
 	void               Write(uint64_t offset, const void* source, uint64_t size);
 	void               Flush(uint64_t offset, uint64_t size);
-	void               CopyFrom(
-	                  CommandBuffer& command, const Buffer& source, uint64_t source_offset,
+	void CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,
 	              uint64_t destination_offset, uint64_t size,
 	              vk::AccessFlags source_before      = vk::AccessFlagBits::eMemoryWrite,
-	                  vk::AccessFlags destination_before =
-	                      vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
-	                  vk::AccessFlags source_after =
-	                      vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
-	                  vk::AccessFlags destination_after =
-	                      vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite);
+	              vk::AccessFlags destination_before = vk::AccessFlagBits::eMemoryRead |
+	                                                   vk::AccessFlagBits::eMemoryWrite,
+	              vk::AccessFlags source_after       = vk::AccessFlagBits::eMemoryRead |
+	                                                   vk::AccessFlagBits::eMemoryWrite,
+	              vk::AccessFlags destination_after  = vk::AccessFlagBits::eMemoryRead |
+	                                                   vk::AccessFlagBits::eMemoryWrite);
 	void Fill(uint64_t offset, uint64_t size, uint32_t value);
 
 protected:

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -4,10 +4,11 @@
 #include "common/abi.h"
 #include "common/common.h"
 #include "common/lruCache.h"
-#include "graphics/host_gpu/memoryTracker.h"
-#include "graphics/host_gpu/renderer/blitHelper.h"
-#include "graphics/host_gpu/renderer/image.h"
-#include "graphics/host_gpu/renderer/multiLevelPageTable.h"
+#include "graphics/host_gpu/pageManager.h"
+#include "graphics/host_gpu/regionManager.h"
+#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
+#include "graphics/host_gpu/renderer/image/blitHelper.h"
+#include "graphics/host_gpu/renderer/image/image.h"
 
 #include <compare>
 #include <map>
@@ -64,8 +65,7 @@ public:
 
 	[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
 	                                        uint32_t packed_clear);
-	void               PrepareHostWrite(uint64_t address, uint64_t size);
-	[[nodiscard]] bool SynchronizeImageToBuffer(uint64_t address, uint64_t size);
+	void               InvalidateMemory(uint64_t address, uint64_t size);
 	[[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
 	                                           bool formatted_buffer_write = false);
 	[[nodiscard]] RegionInfo QueryRegion(uint64_t address, uint64_t size);
@@ -75,8 +75,6 @@ public:
 	[[nodiscard]] bool ClearMeta(uint64_t address);
 	[[nodiscard]] bool TouchMeta(uint64_t address, uint32_t slice, bool is_clear);
 
-	[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t address, uint64_t size,
-	                                    PageFaultPhase phase) noexcept;
 	void UnmapMemory(uint64_t address, uint64_t size);
 	void ProcessDownloadImages();
 	void RunGarbageCollector();
@@ -101,7 +99,8 @@ private:
 		int32_t layer = -1;
 	};
 
-	using ImageOwnerIndex = MultiRangePageOwnerIndex<ImageId>;
+	using ImageIds       = InlinePageOwnerList<ImageId, 16>;
+	using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
 
 	[[nodiscard]] Image&                 ResolveImage(ImageId id);
 	[[nodiscard]] const Image&           ResolveImage(ImageId id) const;
@@ -109,11 +108,17 @@ private:
 	[[nodiscard]] ImageId                InsertImage(const ImageInfo& info);
 	[[nodiscard]] ImageId                GetNullImage(const ImageDesc& desc);
 	void                                 RegisterImage(ImageId id);
-	void                                 UnregisterImage(ImageId id, bool release_tracking);
-	void                                 DeleteImage(ImageId id, bool release_tracking = true);
+	void                                 UnregisterImage(ImageId id);
+	void                                 DeleteImage(ImageId id);
 	void DeleteImages(std::span<const ImageId> ids, std::optional<ImageId> native_source = {});
 	void RetainImage(CommandBuffer& command, ImageId id);
 	void TouchImage(Image& image);
+	void TrackImage(ImageId id);
+	void TrackImageHead(ImageId id);
+	void TrackImageTail(ImageId id);
+	void UntrackImage(ImageId id);
+	void UntrackImageHead(ImageId id);
+	void UntrackImageTail(ImageId id);
 	void TrackImageDownload(ImageId id);
 	void TrackImageDownloadLocked(ImageId id, Image& image);
 	[[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested,
@@ -121,7 +126,8 @@ private:
 	[[nodiscard]] static BindingType UploadBinding(const Image& image);
 	[[nodiscard]] bool               SafeToDownload(const Image& image);
 
-	[[nodiscard]] std::vector<ImageId> FindImagesInRegion(uint64_t address, uint64_t size,
+	// Caller holds m_lock; it also serializes the per-image query epoch.
+	[[nodiscard]] ImageIds      FindImagesInRegion(uint64_t address, uint64_t size,
 	                                               bool page_overlap) const;
 	[[nodiscard]] OverlapResult ResolveOverlap(const ImageInfo& requested, BindingType binding,
 	                                           ImageId cached, ImageId merged);
@@ -135,7 +141,7 @@ private:
 	[[nodiscard]] DownloadPlan      BuildDownload(const Image& image) const;
 	void UploadImage(Image& image, const ImageDesc& desc, Buffer& source, uint64_t source_offset);
 	void DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset,
-	                       DownloadPlan plan);
+	                       uint64_t destination_size, DownloadPlan plan);
 	void DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset);
 	void CommitGpuWrite(Image& image);
 	void PrepareImageCopy(Image& image);
@@ -148,10 +154,8 @@ private:
 	void ValidateImageDesc(const ImageDesc& desc) const;
 
 	void InvalidateCpuAliases(uint64_t address, uint64_t size);
-	void RestoreGpuTracking(const Image& image);
-	void ReleaseGpuTracking(ImageId id);
+	void ClearGpuModified(ImageId id);
 
-	[[nodiscard]] bool                          SynchronizeImageToBuffer(ImageId id);
 	void                                        DownloadImage(ImageId id);
 	[[nodiscard]] bool                          TryDownloadImage(ImageId id);
 	[[nodiscard]] std::pair<uint8_t*, uint64_t> MapDownload(uint64_t size, uint64_t alignment);
@@ -160,14 +164,14 @@ private:
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;
 	TrackingSpinLock                                  m_lock;
-	MemoryTracker                                     m_memory_tracker;
+	PageManager&                                      m_page_manager;
 	BlitHelper                                        m_blit_helper;
 	std::unique_ptr<TileManager>                      m_tiler;
 	BufferCache&                                      m_buffer_cache;
 	ResourceMutex&                                    m_resource_mutex;
 	std::vector<Slot>                                 m_slots;
 	std::vector<uint32_t>                             m_free_slots;
-	ImageOwnerIndex                                   m_image_owner_index;
+	ImagePageTable                                    m_image_page_table;
 	std::map<vk::Format, ImageId>                     m_null_images;
 	Common::LeastRecentlyUsedCache<ImageId, uint64_t> m_lru_cache;
 	std::set<ImageId>                                 m_download_images;
@@ -177,9 +181,11 @@ private:
 	uint64_t                                          m_pressure_gc_memory = 1536ull * 1024 * 1024;
 	uint64_t         m_critical_gc_memory     = 3ull * 1024 * 1024 * 1024;
 	uint64_t         m_gc_tick                = 0;
+	mutable uint32_t m_image_query_epoch      = 0;
 	bool             m_readback_linear_images = false;
 
 	friend struct TextureCacheTestAccess;
+	friend class BufferCache;
 	friend class RenderExecutor;
 };
 

src/graphics/host_gpu/renderer/colorRenderTarget.cpp

@@ -7,9 +7,9 @@
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/objects/textureCommon.h"
 #include "graphics/host_gpu/renderer/debug.h"
-#include "graphics/host_gpu/renderer/descriptorCache.h"
+#include "graphics/host_gpu/renderer/image/textureCommon.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
@@ -79,10 +79,8 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	const auto view = ResolveTargetViewInfo(
 	    rt.view.base_array_slice_index, rt.view.last_array_slice_index, render_target_slice_offset);
 	switch (view.type) {
-		case TargetViewType::Image2D: break;
-		case TargetViewType::Image2DArray:
-			EXIT("layered render-target views are unsupported: base=%u count=%u\n", view.base_layer,
-			     view.layer_count);
+		case TargetViewType::Image2D:
+		case TargetViewType::Image2DArray: break;
 		case TargetViewType::Unsupported:
 			EXIT("invalid render-target view: base=%u last=%u draw_offset=%u\n",
 			     rt.view.base_array_slice_index, rt.view.last_array_slice_index,
@@ -121,6 +119,17 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	uint32_t   pitch  = 0;
 	uint64_t   size   = 0;
 	bool       tile   = false;
+	const bool volume = rt.attrib3.dimension == 2;
+	if (rt.attrib3.dimension != 1 && !volume) {
+		EXIT("unsupported render-target dimension: %u\n", rt.attrib3.dimension);
+	}
+	if (!volume && rt.attrib3.depth != 0) {
+		EXIT("2D render target has nonzero depth: %u\n", rt.attrib3.depth);
+	}
+	if (volume && samples != 1) {
+		EXIT("multisampled 3D render targets are unsupported\n");
+	}
+	const uint32_t depth = volume ? rt.attrib3.depth + 1u : 1u;
 	const bool     standard64 =
 	    rt.attrib3.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB);
 
@@ -147,6 +156,7 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	if (bytes_per_element == 0) {
 		EXIT("render-target format has no valid element size\n");
 	}
+	const auto transfer_format = ImageOps::RenderTargetTransferFormat(bytes_per_element);
 	if (standard64 &&
 	    (rt.attrib3.dimension != 1 || rt.attrib3.depth != 0 || levels != 1 ||
 	     rt.view.current_mip_level != 0 || view.base_layer != 0 || view.image_layers != 1 ||
@@ -167,10 +177,14 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	if (rt.pitch.pitch_div8_minus1 != 0) {
 		pitch = (rt.pitch.pitch_div8_minus1 + 1u) << 3u;
 	} else if (tile) {
-		pitch = standard64
-		            ? TileGetTexturePitch(Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float),
-		                                  width, levels, rt.attrib3.tile_mode)
-		            : TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments);
+		if (volume) {
+			pitch = TileGetTexturePitch(transfer_format, width, levels, rt.attrib3.tile_mode);
+		} else if (standard64) {
+			pitch = TileGetTexturePitch(Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float),
+			                            width, levels, rt.attrib3.tile_mode);
+		} else {
+			pitch = TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments);
+		}
 		if (pitch == 0) {
 			EXIT("unsupported render-target pitch: width=%u bytes=%u\n", width, bytes_per_element);
 		}
@@ -180,7 +194,17 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 
 	TileSizeOffset   mip_sizes[16] {};
 	TilePaddedSize   mip_padded[16] {};
-	if (tile) {
+	TileVolumeLayout volume_layout {};
+	uint64_t         backing_size = 0;
+	if (volume) {
+		if (!tile || !TileGetTextureVolumeLayout(transfer_format, width, height, depth, levels,
+		                                         rt.attrib3.tile_mode, volume_layout)) {
+			EXIT("unsupported 3D render-target layout: %ux%ux%u levels=%u tile=%u\n", width, height,
+			     depth, levels, rt.attrib3.tile_mode);
+		}
+		size         = volume_layout.block_slice_size;
+		backing_size = volume_layout.total_size;
+	} else if (tile) {
 		TileSizeAlign layout {};
 		bool          valid_layout = false;
 		if (standard64) {
@@ -205,12 +229,6 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 			mip_sizes[0]  = {static_cast<uint32_t>(size), 0, 0, 0, 0, 0};
 			mip_padded[0] = {pitch, height};
 		}
-		if (rt.slice.slice_div64_minus1 != 0 &&
-		    (static_cast<uint64_t>(rt.slice.slice_div64_minus1) + 1u) * 64u != size) {
-			EXIT("render-target slice span mismatch: encoded=0x%016" PRIx64 " derived=0x%016" PRIx64
-			     "\n",
-			     (static_cast<uint64_t>(rt.slice.slice_div64_minus1) + 1u) * 64u, size);
-		}
 	} else {
 		size = static_cast<uint64_t>(pitch) * height * bytes_per_element * samples;
 		if (size > UINT32_MAX) {
@@ -219,23 +237,40 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		mip_sizes[0]  = {static_cast<uint32_t>(size), 0, 0, 0, 0, 0};
 		mip_padded[0] = {pitch, height};
 	}
-	if (size == 0 || size > UINT64_MAX / view.image_layers) {
+	if (rt.slice.slice_div64_minus1 != 0 &&
+	    (static_cast<uint64_t>(rt.slice.slice_div64_minus1) + 1u) * 64u != size) {
+		EXIT("render-target slice span mismatch: encoded=0x%016" PRIx64 " derived=0x%016" PRIx64
+		     "\n",
+		     (static_cast<uint64_t>(rt.slice.slice_div64_minus1) + 1u) * 64u, size);
+	}
+	if (size == 0 || (!volume && size > UINT64_MAX / view.image_layers)) {
 		EXIT("render-target memory footprint is invalid\n");
 	}
-	const auto backing_size = size * view.image_layers;
+	if (!volume) {
+		backing_size = size * view.image_layers;
+	}
+	if (backing_size == 0) {
+		EXIT("render-target backing is empty\n");
+	}
 	if (backing_size > TRACKER_ADDRESS_SIZE - rt.base.addr) {
 		EXIT("render-target backing range is invalid\n");
 	}
 
 	const vk::Extent2D view_extent = {std::max(width >> rt.view.current_mip_level, 1u),
 	                                  std::max(height >> rt.view.current_mip_level, 1u)};
+	const uint32_t     view_depth  = std::max(depth >> rt.view.current_mip_level, 1u);
+	if (volume &&
+	    (view.base_layer >= view_depth || view.layer_count > view_depth - view.base_layer)) {
+		EXIT("3D render-target view exceeds mip depth: base=%u count=%u depth=%u mip=%u\n",
+		     view.base_layer, view.layer_count, view_depth, rt.view.current_mip_level);
+	}
 
 	auto decision_log_id = g_render_color_log_count.fetch_add(1);
 	if (decision_log_id < 128) {
 		LOGF("RenderColorTarget: slot=%" PRIu32 " addr=0x%010" PRIx64 " size=0x%016" PRIx64
-		     " extent=%ux%u view_mip=%u view_extent=%ux%u levels=%u pitch=%u"
+		     " extent=%ux%ux%u view_mip=%u view_extent=%ux%u levels=%u pitch=%u"
 		     " fmt=0x%08" PRIx32 " nfmt=0x%08" PRIx32 " order=0x%08" PRIx32 " samples=%u tile=%s\n",
-		     rt_slot, rt.base.addr, backing_size, width, height, rt.view.current_mip_level,
+		     rt_slot, rt.base.addr, backing_size, width, height, depth, rt.view.current_mip_level,
 		     view_extent.width, view_extent.height, levels, pitch, rt.info.format,
 		     rt.info.channel_type, rt.info.channel_order, samples, tile ? "tiled" : "linear");
 	}
@@ -244,15 +279,24 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	desc.type              = TextureCache::BindingType::RenderTarget;
 	desc.info.data         = {rt.base.addr, backing_size};
 	desc.info.pixel_format = target_format.format;
-	desc.info.guest_format = ImageOps::RenderTargetTransferFormat(bytes_per_element);
-	desc.info.type   = Prospero::ImageType::kColor2D;
-	desc.info.extent = {width, height, 1};
-	desc.info.resources       = {levels, view.image_layers};
+	desc.info.guest_format = transfer_format;
+	desc.info.type         = volume ? Prospero::ImageType::kColor3D : Prospero::ImageType::kColor2D;
+	desc.info.extent       = {width, height, depth};
+	desc.info.resources    = {levels, volume ? 1u : view.image_layers};
 	desc.info.pitch        = pitch;
 	desc.info.bytes_per_block = bytes_per_element;
 	desc.info.samples         = samples;
 	desc.info.tile_mode       = rt.attrib3.tile_mode;
 	for (uint32_t level = 0; level < levels; level++) {
+		if (volume) {
+			desc.info.mip_layout[level] = {
+			    volume_layout.level_offsets[level],
+			    volume_layout.level_sizes[level],
+			    volume_layout.level_widths[level],
+			    volume_layout.level_heights[level],
+			};
+			continue;
+		}
 		const auto level_offset =
 		    mip_sizes[level].src_size != 0 ? mip_sizes[level].src_offset : mip_sizes[level].offset;
 		const auto level_size =

src/graphics/host_gpu/renderer/colorRenderTarget.h

@@ -2,8 +2,8 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_COLORRENDERTARGET_H_
 
 #include "graphics/guest_gpu/gpu_defs.h"
+#include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
-#include "graphics/host_gpu/renderer/textureCache.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <cstdint>

src/graphics/host_gpu/renderer/commandScheduler.cpp

@@ -331,8 +331,7 @@ void CommandScheduler::WaitPriorityOperations(uint64_t tick) {
 	EXIT_IF(g_deferred_callback_scheduler == this);
 	std::unique_lock lock(m_operation_mutex);
 	m_operation_available.wait(lock, [this, tick] {
-		const bool active_before_or_at =
-		    m_priority_active && m_priority_active_tick <= tick;
+		const bool active_before_or_at = m_priority_active && m_priority_active_tick <= tick;
 		const bool queued_before_or_at =
 		    !m_priority_operations.empty() && m_priority_operations.front().tick <= tick;
 		return !active_before_or_at && !queued_before_or_at;

src/graphics/host_gpu/renderer/context.cpp

@@ -8,8 +8,8 @@
 #include "graphics/host_gpu/renderer/colorRenderTarget.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
-#include "graphics/host_gpu/renderer/descriptorCache.h"
-#include "graphics/host_gpu/renderer/imageView.h"
+#include "graphics/host_gpu/renderer/image/imageView.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vma.h"
@@ -270,8 +270,8 @@ void CommandBuffer::BeginRendering(const RenderState& state) const {
 		colors[i].sType        = vk::StructureType::eRenderingAttachmentInfo;
 		colors[i].imageView    = attachment.image_view;
 		colors[i].imageLayout  = attachment.image_layout;
-		colors[i].loadOp = attachment.is_clear ? vk::AttachmentLoadOp::eClear
-		                                      : vk::AttachmentLoadOp::eLoad;
+		colors[i].loadOp =
+		    attachment.is_clear ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad;
 		colors[i].storeOp                 = vk::AttachmentStoreOp::eStore;
 		colors[i].clearValue.color.uint32 = attachment.clear_value;
 	}
@@ -281,8 +281,8 @@ void CommandBuffer::BeginRendering(const RenderState& state) const {
 	depth.sType       = vk::StructureType::eRenderingAttachmentInfo;
 	depth.imageView   = depth_stencil.image_view;
 	depth.imageLayout = depth_stencil.image_layout;
-	depth.loadOp = depth_stencil.depth_clear ? vk::AttachmentLoadOp::eClear
-	                                        : vk::AttachmentLoadOp::eLoad;
+	depth.loadOp =
+	    depth_stencil.depth_clear ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad;
 	depth.storeOp                       = vk::AttachmentStoreOp::eStore;
 	depth.clearValue.depthStencil.depth = std::bit_cast<float>(depth_stencil.clear_value[0]);
 
@@ -290,8 +290,8 @@ void CommandBuffer::BeginRendering(const RenderState& state) const {
 	stencil.sType       = vk::StructureType::eRenderingAttachmentInfo;
 	stencil.imageView   = depth_stencil.image_view;
 	stencil.imageLayout = depth_stencil.image_layout;
-	stencil.loadOp = depth_stencil.stencil_clear ? vk::AttachmentLoadOp::eClear
-	                                            : vk::AttachmentLoadOp::eLoad;
+	stencil.loadOp =
+	    depth_stencil.stencil_clear ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad;
 	stencil.storeOp                         = vk::AttachmentStoreOp::eStore;
 	stencil.clearValue.depthStencil.stencil = depth_stencil.clear_value[1];
 

src/graphics/host_gpu/renderer/debug.cpp

@@ -359,30 +359,12 @@ static void RtCheck(const HW::RenderTarget& rt) {
 				logged = true;
 			}
 		}
-		if (rt.attrib3.depth != 0x00000000) {
-			static bool logged = false;
-			if (!logged) {
-				LOGF("RenderTarget: temporary: ignoring PS5 color target depth_minus1=0x%08" PRIx32
-				     "\n",
-				     rt.attrib3.depth);
-				logged = true;
-			}
-		}
 		if (!RenderIsColorTileMode(rt.attrib3.tile_mode)) {
 			EXIT("unknown PS5 render-target tile mode: 0x%08" PRIx32 "\n", rt.attrib3.tile_mode);
 		}
 		if (!RenderIsColorDimension(rt.attrib3.dimension)) {
 			EXIT("unknown PS5 render-target dimension: 0x%08" PRIx32 "\n", rt.attrib3.dimension);
 		}
-		if (rt.attrib3.dimension != 0x00000001) {
-			static bool logged = false;
-			if (!logged) {
-				LOGF("RenderTarget: temporary: using 2D fallback for PS5 color "
-				     "dimension=0x%08" PRIx32 "\n",
-				     rt.attrib3.dimension);
-				logged = true;
-			}
-		}
 		if (!rt.attrib3.cmask_pipe_aligned) {
 			static bool logged = false;
 			if (!logged) {
@@ -497,7 +479,15 @@ static void ZPrint(const char* func, const HW::DepthRenderTarget& z) {
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-static void ZCheck(const HW::DepthRenderTarget& z) {
+static void ZCheck(const HW::DepthRenderTarget& z, const HW::DepthControl& dc,
+                   const HW::RenderControl& rc) {
+	const bool depth_active =
+	    dc.z_enable || dc.z_write_enable || dc.depth_bounds_enable || rc.depth_clear_enable;
+	const bool stencil_active = dc.stencil_enable || rc.stencil_clear_enable;
+	if (!depth_active && !stencil_active) {
+		return;
+	}
+
 	EXIT_NOT_IMPLEMENTED(!z.z_info.HasValidTextureCompatibility());
 	EXIT_NOT_IMPLEMENTED(!z.stencil_info.HasValidTextureCompatibility());
 	if (z.z_info.format == 0) {
@@ -548,14 +538,6 @@ static void ZCheck(const HW::DepthRenderTarget& z) {
 		EXIT_NOT_IMPLEMENTED(z.htile_surface.prefetch_height != 0x00000000);
 		EXIT_NOT_IMPLEMENTED(z.htile_surface.dst_outside_zero_to_one != 0x00000000);
 
-		if (z.depth_view.slice_start != 0x00000000 || z.depth_view.slice_max != 0x00000000) {
-			static std::atomic<uint32_t> log_count {0};
-			if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
-				LOGF("DepthTarget: temporary: ignoring PS5 array slice view start=0x%08" PRIx32
-				     ", max=0x%08" PRIx32 "\n",
-				     z.depth_view.slice_start, z.depth_view.slice_max);
-			}
-		}
 		if (z.depth_view.current_mip_level != 0x00000000) {
 			static std::atomic<uint32_t> log_count {0};
 			if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
@@ -1214,7 +1196,7 @@ void hw_check(const RenderCommandBuffer& buffer) {
 	log_phase("vp");
 	VpCheck(vp, smc);
 	log_phase("z");
-	ZCheck(z);
+	ZCheck(z, d, rc);
 	log_phase("clip");
 	ClipCheck(c);
 	log_phase("rc");

src/graphics/host_gpu/renderer/depthRenderTarget.cpp

@@ -10,10 +10,10 @@
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/objects/textureCommon.h"
 #include "graphics/host_gpu/renderer/debug.h"
-#include "graphics/host_gpu/renderer/descriptorCache.h"
-#include "graphics/host_gpu/renderer/imageView.h"
+#include "graphics/host_gpu/renderer/image/imageView.h"
+#include "graphics/host_gpu/renderer/image/textureCommon.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
@@ -96,9 +96,11 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 	const auto& dc = hw.GetDepthControl();
 	const auto& sc = hw.GetStencilControl();
 	const auto& sm = hw.GetStencilMask();
+	const bool  has_stencil =
+	    z.stencil_info.format != Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid);
 	const bool depth_active =
 	    dc.z_enable || dc.z_write_enable || dc.depth_bounds_enable || rc.depth_clear_enable;
-	const bool stencil_active = dc.stencil_enable || rc.stencil_clear_enable;
+	const bool stencil_active = has_stencil && (dc.stencil_enable || rc.stencil_clear_enable);
 	if (!depth_active && !stencil_active) {
 		return;
 	}
@@ -139,8 +141,6 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 		}
 		return;
 	}
-	const bool has_stencil =
-	    z.stencil_info.format != Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid);
 	const bool has_htile = z.z_info.htile_acceleration;
 	const auto samples   = render_sample_count(z.z_info.num_samples);
 	if (samples == 0) {
@@ -150,16 +150,14 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 	    has_stencil, has_htile, z.stencil_info.htile_stencil_disabled);
 	const auto view = ResolveTargetViewInfo(z.depth_view.slice_start, z.depth_view.slice_max);
 	switch (view.type) {
-		case TargetViewType::Image2D: break;
-		case TargetViewType::Image2DArray:
-			DepthFatal("layered depth views are unsupported: base=%u count=%u", view.base_layer,
-			           view.layer_count);
+		case TargetViewType::Image2D:
+		case TargetViewType::Image2DArray: break;
 		case TargetViewType::Unsupported:
 			DepthFatal("invalid depth view: base=%u last=%u", z.depth_view.slice_start,
 			           z.depth_view.slice_max);
 	}
-	if ((stencil_active && !has_stencil) || rc.resummarize_enable || rc.copy_centroid ||
-	    rc.copy_sample != 0 || z.z_info.expclear_enabled || z.stencil_info.expclear_enabled ||
+	if (rc.resummarize_enable || rc.copy_centroid || rc.copy_sample != 0 ||
+	    z.z_info.expclear_enabled || z.stencil_info.expclear_enabled ||
 	    z.z_info.partially_resident || z.stencil_info.partially_resident ||
 	    z.z_info.max_mip_level != 0 || z.depth_view.current_mip_level != 0 ||
 	    z.depth_info.addr5_swizzle_mask != 0 || z.depth_info.array_mode != 0 ||
@@ -281,10 +279,10 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 	r.depth_min_bounds         = hw.GetDepthBoundsMin();
 	r.depth_max_bounds         = hw.GetDepthBoundsMax();
 
-	r.stencil_clear_enable = rc.stencil_clear_enable;
+	r.stencil_clear_enable = has_stencil && rc.stencil_clear_enable;
 	r.stencil_clear_value  = hw.GetStencilClearValue();
-	r.stencil_test_enable  = dc.stencil_enable;
-	if (dc.stencil_enable) {
+	r.stencil_test_enable  = has_stencil && dc.stencil_enable;
+	if (r.stencil_test_enable) {
 		if (dc.stencilfunc > static_cast<uint8_t>(vk::CompareOp::eAlways) ||
 		    (dc.backface_enable &&
 		     dc.stencilfunc_bf > static_cast<uint8_t>(vk::CompareOp::eAlways)) ||

src/graphics/host_gpu/renderer/depthRenderTarget.h

@@ -2,9 +2,9 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DEPTHRENDERTARGET_H_
 
 #include "common/assert.h"
-#include "graphics/host_gpu/renderer/imageView.h"
+#include "graphics/host_gpu/renderer/cache/textureCache.h"
+#include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
-#include "graphics/host_gpu/renderer/textureCache.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <cstdint>

src/graphics/host_gpu/renderer/gpuResourceManager.cpp

@@ -1,141 +0,0 @@
-#include "graphics/host_gpu/renderer/gpuResourceManager.h"
-
-#include "common/assert.h"
-#include "graphics/guest_gpu/command_processor/commandProcessor.h"
-#include "graphics/guest_gpu/graphicsRun.h"
-#include "graphics/host_gpu/renderer/commandScheduler.h"
-
-namespace Libs::Graphics {
-
-GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
-    : m_page_manager(FaultThunk, this),
-      m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex),
-      m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache, m_resource_mutex) {}
-
-GpuResourceManager::~GpuResourceManager() = default;
-
-bool GpuResourceManager::FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr,
-                                    uint64_t size, PageFaultPhase phase) noexcept {
-	return static_cast<GpuResourceManager*>(context)->InvalidateMemory(access, vaddr, size, phase);
-}
-
-bool GpuResourceManager::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
-                                          PageFaultPhase phase) noexcept {
-	// Let the authoritative image materialize first. A clean overlapping buffer marks a write
-	// fault CPU-dirty when it begins ownership transfer; doing that before image preflight would
-	// make the image appear to race a real CPU write. Completion and release retain buffer-first
-	// ordering so its pending fault is gone before TextureCache publishes the downloaded backing.
-	if (phase == PageFaultPhase::Invalidate) {
-		const bool image_handled  = m_texture_cache.InvalidateMemory(access, vaddr, size, phase);
-		const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase);
-		return buffer_handled || image_handled;
-	}
-	const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase);
-	const bool image_handled  = m_texture_cache.InvalidateMemory(access, vaddr, size, phase);
-	return buffer_handled || image_handled;
-}
-
-bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
-	if (!m_page_manager.IsMapped(fault_vaddr, 1)) {
-		return false;
-	}
-	if (CommandScheduler::InDeferredOperation()) {
-		EXIT("unsupported guest-memory fault from an asynchronous GPU completion, "
-		     "addr=0x%016" PRIx64 " access=%u\n",
-		     fault_vaddr, static_cast<uint32_t>(access));
-	}
-	bool       handled = false;
-	const auto resolve = [this, access, fault_vaddr, &handled](CommandProcessor& cp) {
-		cp.BeginReadbackTransaction();
-		(void)m_buffer_cache.SynchronizeBacking(fault_vaddr, 1);
-		{
-			ResourceMutex::FaultScope fault(m_resource_mutex);
-			handled = m_page_manager.HandleFault(access, fault_vaddr);
-		}
-		cp.EndReadbackTransaction();
-	};
-	if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
-		resolve(*cp);
-		return handled;
-	}
-	if (m_resource_mutex.IsOwnedByCurrentThread()) {
-		EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64
-		     " access=%u\n",
-		     fault_vaddr, static_cast<uint32_t>(access));
-	}
-	EXIT_IF(m_gpu == nullptr);
-	m_gpu->SendCommandSyncWithProcessor(resolve);
-	return handled;
-}
-
-void GpuResourceManager::PrepareHostWrite(uint64_t vaddr, uint64_t size) {
-	if (!m_page_manager.HasAnyMapping(vaddr, size)) {
-		return;
-	}
-	if (CommandScheduler::InDeferredOperation()) {
-		EXIT("unsupported host write from an asynchronous GPU completion, addr=0x%016" PRIx64
-		     " size=0x%016" PRIx64 "\n",
-		     vaddr, size);
-	}
-	const auto handle_range = [this, vaddr, size] {
-		if (!m_page_manager.HandleWriteRange(vaddr, size)) {
-			EXIT("failed to prepare host write, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
-			     vaddr, size);
-		}
-	};
-	const auto resolve = [this, &handle_range](CommandProcessor& cp) {
-		cp.BeginReadbackTransaction();
-		{
-			ResourceMutex::FaultScope fault(m_resource_mutex);
-			handle_range();
-		}
-		cp.EndReadbackTransaction();
-	};
-	if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
-		resolve(*cp);
-		return;
-	}
-	if (m_resource_mutex.IsOwnedByCurrentThread()) {
-		EXIT("unsupported host write from a pre-owned resource transaction, addr=0x%016" PRIx64
-		     " size=0x%016" PRIx64 "\n",
-		     vaddr, size);
-	}
-	EXIT_IF(m_gpu == nullptr);
-	m_gpu->SendCommandSyncWithProcessor(resolve);
-}
-
-bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
-	return m_page_manager.IsMapped(vaddr, size);
-}
-
-void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
-	m_page_manager.OnGpuMap(vaddr, size, access);
-}
-
-void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
-	if (!IsMapped(vaddr, size)) {
-		EXIT("cannot unmap an unmapped GPU resource range\n");
-	}
-	const auto unmap = [this, vaddr, size, access] {
-		m_texture_cache.UnmapMemory(vaddr, size);
-		m_buffer_cache.UnmapMemory(vaddr, size);
-		m_page_manager.OnGpuUnmap(vaddr, size, access);
-	};
-	if (m_gpu == nullptr) {
-		if (m_resource_mutex.IsOwnedByCurrentThread()) {
-			EXIT("cannot synchronously unmap from a resource transaction\n");
-		}
-		unmap();
-		return;
-	}
-	Gpu::SubmissionLock submissions(*m_gpu);
-	m_gpu->SendCommandSync(unmap);
-}
-
-void GpuResourceManager::RunGarbageCollector() {
-	m_texture_cache.ProcessDownloadImages();
-	m_texture_cache.RunGarbageCollector();
-	m_buffer_cache.RunGarbageCollector();
-}
-
-} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/image/blitHelper.cpp

@@ -1,11 +1,11 @@
-#include "graphics/host_gpu/renderer/blitHelper.h"
+#include "graphics/host_gpu/renderer/image/blitHelper.h"
 
 #include "common/assert.h"
 #include "gpu_blit_shaders/gpu_blit_color_to_ms_depth_spv.h"
 #include "gpu_blit_shaders/gpu_blit_fs_triangle_spv.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
-#include "graphics/host_gpu/renderer/image.h"
+#include "graphics/host_gpu/renderer/image/image.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
 
 #include <algorithm>
@@ -182,8 +182,8 @@ void BlitHelper::ReinterpretColorAsMsDepth(Image& source, Image& destination) {
 	auto  command        = command_buffer.Handle();
 	source.Transit(vk::ImageLayout::eShaderReadOnlyOptimal, vk::AccessFlagBits2::eShaderRead, {},
 	               command);
-	destination.Transit(ColorToMsDepthLayout,
-	                    vk::AccessFlagBits2::eDepthStencilAttachmentWrite, {}, command);
+	destination.Transit(ColorToMsDepthLayout, vk::AccessFlagBits2::eDepthStencilAttachmentWrite, {},
+	                    command);
 
 	vk::RenderingAttachmentInfo depth_attachment {};
 	depth_attachment.sType                   = vk::StructureType::eRenderingAttachmentInfo;

src/graphics/host_gpu/renderer/image/image.cpp

@@ -1,11 +1,11 @@
-#include "graphics/host_gpu/renderer/image.h"
+#include "graphics/host_gpu/renderer/image/image.h"
 
 #include "common/assert.h"
 #include "common/profiler.h"
+#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
-#include "graphics/host_gpu/renderer/imageView.h"
+#include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
-#include "graphics/host_gpu/renderer/streamBuffer.h"
 #include "kernel/memory.h"
 
 #include <algorithm>
@@ -105,6 +105,10 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
                                    std::optional<ImageSubresourceRange> range) {
 	auto& state              = backing.state;
 	auto& subresource_states = backing.subresource_states;
+	if (range && info.IsVolume()) {
+		range->base_layer  = 0;
+		range->layer_count = 1;
+	}
 
 	const bool partial =
 	    range && (range->base_level != 0 || range->level_count != info.resources.levels ||
@@ -191,8 +195,7 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
 }
 
 void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
-                    std::optional<ImageSubresourceRange> range,
-                    vk::CommandBuffer command_buffer) {
+                    std::optional<ImageSubresourceRange> range, vk::CommandBuffer command_buffer) {
 	const auto transfer_access =
 	    vk::AccessFlagBits2::eTransferRead | vk::AccessFlagBits2::eTransferWrite;
 	vk::PipelineStageFlags2 destination_stage {};
@@ -201,8 +204,8 @@ void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destina
 	}
 	if (!destination_access ||
 	    static_cast<bool>(destination_access & ~vk::AccessFlags2 {transfer_access})) {
-		destination_stage |= vk::PipelineStageFlagBits2::eAllGraphics |
-		                     vk::PipelineStageFlagBits2::eComputeShader;
+		destination_stage |=
+		    vk::PipelineStageFlagBits2::eAllGraphics | vk::PipelineStageFlagBits2::eComputeShader;
 	}
 	const auto barriers =
 	    GetBarriers(destination_layout, destination_access, destination_stage, range);
@@ -218,10 +221,9 @@ void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destina
 	command_buffer.pipelineBarrier2(dependency);
 }
 
-void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
-                   uint64_t offset, uint64_t size) {
-	EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr ||
-	        size == 0);
+void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
+                   uint64_t size) {
+	EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr || size == 0);
 	m_scheduler->EndRendering();
 	vk::BufferMemoryBarrier2 buffer_barrier {};
 	buffer_barrier.srcStageMask        = vk::PipelineStageFlagBits2::eAllCommands;
@@ -234,8 +236,7 @@ void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffe
 	buffer_barrier.offset              = offset;
 	buffer_barrier.size                = size;
 	const auto image_barriers =
-	    GetBarriers(vk::ImageLayout::eTransferDstOptimal,
-	                vk::AccessFlagBits2::eTransferWrite,
+	    GetBarriers(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
 	                vk::PipelineStageFlagBits2::eCopy, {});
 	vk::DependencyInfo dependency {};
 	dependency.dependencyFlags          = vk::DependencyFlagBits::eByRegion;
@@ -256,8 +257,7 @@ void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffe
 	dependency.pImageMemoryBarriers    = nullptr;
 	command.pipelineBarrier2(dependency);
 	Transit(vk::ImageLayout::eGeneral,
-	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
-	        command);
+	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
 }
 
 void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
@@ -276,8 +276,7 @@ void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buf
 	buffer_barrier.offset              = offset;
 	buffer_barrier.size                = size;
 	const auto image_barriers =
-	    GetBarriers(vk::ImageLayout::eTransferSrcOptimal,
-	                vk::AccessFlagBits2::eTransferRead,
+	    GetBarriers(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead,
 	                vk::PipelineStageFlagBits2::eCopy, {});
 	vk::DependencyInfo dependency {};
 	dependency.dependencyFlags          = vk::DependencyFlagBits::eByRegion;
@@ -299,8 +298,8 @@ void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buf
 	command.pipelineBarrier2(dependency);
 }
 
-std::pair<uint32_t, uint32_t>
-Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_t depth) {
+std::pair<uint32_t, uint32_t> Image::SanitizeCopyLayers(const Image& source,
+                                                        const Image& destination, uint32_t depth) {
 	const auto source_type        = source.backing.image_type;
 	const auto destination_type   = destination.backing.image_type;
 	uint32_t   source_layers      = source.backing.layers;
@@ -312,13 +311,10 @@ Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_
 		destination_layers = 1;
 	}
 	if (source_type == destination_type) {
-		source_layers = destination_layers =
-		    std::min(source_layers, destination_layers);
-	} else if (source_type == vk::ImageType::e2D &&
-	           destination_type == vk::ImageType::e3D) {
+		source_layers = destination_layers = std::min(source_layers, destination_layers);
+	} else if (source_type == vk::ImageType::e2D && destination_type == vk::ImageType::e3D) {
 		source_layers = depth;
-	} else if (source_type == vk::ImageType::e3D &&
-	           destination_type == vk::ImageType::e2D) {
+	} else if (source_type == vk::ImageType::e3D && destination_type == vk::ImageType::e2D) {
 		destination_layers = depth;
 	}
 	return {source_layers, destination_layers};
@@ -327,8 +323,7 @@ Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_
 void Image::CopyImage(Image& source) {
 	EXIT_IF(m_scheduler == nullptr || source.backing.samples != backing.samples);
 	m_scheduler->EndRendering();
-	const uint32_t levels =
-	    std::min(source.backing.mip_levels, backing.mip_levels);
+	const uint32_t levels     = std::min(source.backing.mip_levels, backing.mip_levels);
 	const uint32_t base_depth = backing.image_type == vk::ImageType::e3D
 	                                ? backing.extent.depth
 	                                : source.backing.extent.depth;
@@ -342,8 +337,7 @@ void Image::CopyImage(Image& source) {
 		const auto width  = std::max(source.backing.extent.width >> level, 1u);
 		const auto height = std::max(source.backing.extent.height >> level, 1u);
 		const auto depth  = std::max(base_depth >> level, 1u);
-		const auto [source_layers, destination_layers] =
-		    SanitizeCopyLayers(source, *this, depth);
+		const auto [source_layers, destination_layers] = SanitizeCopyLayers(source, *this, depth);
 		vk::ImageCopy copy {};
 		copy.srcSubresource = {source_aspect, level, 0, 1};
 		copy.dstSubresource = {destination_aspect, level, 0, 1};
@@ -351,8 +345,7 @@ void Image::CopyImage(Image& source) {
 			if (source.backing.image_type == vk::ImageType::e3D) {
 				copy.extent = {width, height, depth};
 			} else {
-				copy.srcSubresource.layerCount =
-				    std::min(source_layers, destination_layers);
+				copy.srcSubresource.layerCount = std::min(source_layers, destination_layers);
 				copy.dstSubresource.layerCount = copy.srcSubresource.layerCount;
 				copy.extent                    = {width, height, 1};
 			}
@@ -369,34 +362,30 @@ void Image::CopyImage(Image& source) {
 		return;
 	}
 	auto command = m_scheduler->Current().Handle();
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
-	               vk::AccessFlagBits2::eTransferRead, {}, command);
-	Transit(vk::ImageLayout::eTransferDstOptimal,
-	        vk::AccessFlagBits2::eTransferWrite, {}, command);
-	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
-	                  backing.image, vk::ImageLayout::eTransferDstOptimal,
-	                  static_cast<uint32_t>(copies.size()), copies.data());
-	Transit(vk::ImageLayout::eGeneral,
-	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
 	               command);
+	Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
+	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
+	                  vk::ImageLayout::eTransferDstOptimal, static_cast<uint32_t>(copies.size()),
+	                  copies.data());
+	Transit(vk::ImageLayout::eGeneral,
+	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
 }
 
 void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
                     const ImageSubresourceRange& destination_range) {
 	EXIT_IF(m_scheduler == nullptr || backing.samples != 1 ||
 	        source.backing.image_type != vk::ImageType::e2D ||
-	        backing.image_type != vk::ImageType::e2D ||
-	        source_range.level_count != 1 || destination_range.level_count != 1 ||
+	        backing.image_type != vk::ImageType::e2D || source_range.level_count != 1 ||
+	        destination_range.level_count != 1 ||
 	        source_range.base_level >= source.backing.mip_levels ||
 	        destination_range.base_level >= backing.mip_levels ||
 	        source_range.base_layer >= source.backing.layers ||
 	        destination_range.base_layer >= backing.layers);
-	const auto layers = std::min(
-	    {source_range.layer_count, destination_range.layer_count,
+	const auto layers       = std::min({source_range.layer_count, destination_range.layer_count,
 	                                    source.backing.layers - source_range.base_layer,
 	                                    backing.layers - destination_range.base_layer});
-	const auto source_width =
-	    std::max(source.backing.extent.width >> source_range.base_level, 1u);
+	const auto source_width = std::max(source.backing.extent.width >> source_range.base_level, 1u);
 	const auto source_height =
 	    std::max(source.backing.extent.height >> source_range.base_level, 1u);
 	const auto destination_width =
@@ -404,9 +393,8 @@ void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
 	const auto destination_height =
 	    std::max(backing.extent.height >> destination_range.base_level, 1u);
 	const bool copy = source.backing.samples == 1;
-	EXIT_IF(layers == 0 || info.extent.width > source_width ||
-	        info.extent.height > source_height || info.extent.width > destination_width ||
-	        info.extent.height > destination_height ||
+	EXIT_IF(layers == 0 || info.extent.width > source_width || info.extent.height > source_height ||
+	        info.extent.width > destination_width || info.extent.height > destination_height ||
 	        (copy ? !ImageViewOps::FormatsCompatible(source.backing.format, backing.format)
 	              : source.backing.format != backing.format));
 	auto resolved_source_range             = source_range;
@@ -417,25 +405,23 @@ void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
 
 	m_scheduler->EndRendering();
 	auto command = m_scheduler->Current().Handle();
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
-	               vk::AccessFlagBits2::eTransferRead, resolved_source_range, command);
-	Transit(vk::ImageLayout::eTransferDstOptimal,
-	        vk::AccessFlagBits2::eTransferWrite, resolved_destination_range, command);
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead,
+	               resolved_source_range, command);
+	Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
+	        resolved_destination_range, command);
 	if (copy) {
 		vk::ImageCopy region {};
-		region.srcSubresource = {vk::ImageAspectFlagBits::eColor,
-		                         resolved_source_range.base_level,
+		region.srcSubresource = {vk::ImageAspectFlagBits::eColor, resolved_source_range.base_level,
 		                         resolved_source_range.base_layer, layers};
 		region.dstSubresource = {vk::ImageAspectFlagBits::eColor,
 		                         resolved_destination_range.base_level,
 		                         resolved_destination_range.base_layer, layers};
 		region.extent         = resolve_extent;
-		command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
-		                  backing.image, vk::ImageLayout::eTransferDstOptimal, region);
+		command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
+		                  vk::ImageLayout::eTransferDstOptimal, region);
 	} else {
 		vk::ImageResolve region {};
-		region.srcSubresource = {vk::ImageAspectFlagBits::eColor,
-		                         resolved_source_range.base_level,
+		region.srcSubresource = {vk::ImageAspectFlagBits::eColor, resolved_source_range.base_level,
 		                         resolved_source_range.base_layer, layers};
 		region.dstSubresource = {vk::ImageAspectFlagBits::eColor,
 		                         resolved_destination_range.base_level,
@@ -454,11 +440,10 @@ uint32_t Image::CopyRows(uint64_t row_size, uint32_t rows, uint64_t capacity) no
 }
 
 void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
-	EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr ||
-	        source.backing.samples != 1 || backing.samples != 1);
+	EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr || source.backing.samples != 1 ||
+	        backing.samples != 1);
 	m_scheduler->EndRendering();
-	const uint32_t levels =
-	    std::min(source.backing.mip_levels, backing.mip_levels);
+	const uint32_t levels = std::min(source.backing.mip_levels, backing.mip_levels);
 	const auto     source_aspect =
 	    FullAspectMask(source.backing.format) & ~vk::ImageAspectFlagBits::eStencil;
 	const auto destination_aspect =
@@ -488,10 +473,9 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
 	dependency.bufferMemoryBarrierCount = 1;
 	dependency.pBufferMemoryBarriers    = &barrier;
 	auto command                        = m_scheduler->Current().Handle();
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
-	               vk::AccessFlagBits2::eTransferRead, {}, command);
-	Transit(vk::ImageLayout::eTransferDstOptimal,
-	        vk::AccessFlagBits2::eTransferWrite, {}, command);
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
+	               command);
+	Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
 	for (uint32_t level = 0; level < levels; level++) {
 		const auto width             = std::max(source.backing.extent.width >> level, 1u);
 		const auto height            = std::max(source.backing.extent.height >> level, 1u);
@@ -508,19 +492,16 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
 		const auto rows_per_copy = CopyRows(row_size, block_rows, buffer.Size());
 		EXIT_IF(slices == 0 || rows_per_copy == 0);
 		for (uint32_t slice = 0; slice < slices; slice++) {
-			for (uint32_t block_row = 0; block_row < block_rows;
-			     block_row += rows_per_copy) {
+			for (uint32_t block_row = 0; block_row < block_rows; block_row += rows_per_copy) {
 				const auto          copy_rows   = std::min(rows_per_copy, block_rows - block_row);
 				const auto          y           = block_row * source_block;
-				const auto copy_height =
-				    std::min(copy_rows * source_block, height - y);
+				const auto          copy_height = std::min(copy_rows * source_block, height - y);
 				const auto          copy_size   = row_size * copy_rows;
 				vk::BufferImageCopy source_copy {};
 				source_copy.imageSubresource = {
 				    source_aspect, level,
 				    source.backing.image_type == vk::ImageType::e3D ? 0u : slice, 1};
-				source_copy.imageOffset = {
-				    0, static_cast<int32_t>(y),
+				source_copy.imageOffset           = {0, static_cast<int32_t>(y),
 				                                     source.backing.image_type == vk::ImageType::e3D
 				                                         ? static_cast<int32_t>(slice)
 				                                         : 0};
@@ -530,28 +511,24 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
 				    destination_aspect, level,
 				    backing.image_type == vk::ImageType::e3D ? 0u : slice, 1};
 				destination_copy.imageOffset.z =
-				    backing.image_type == vk::ImageType::e3D
-				        ? static_cast<int32_t>(slice)
-				        : 0;
+				    backing.image_type == vk::ImageType::e3D ? static_cast<int32_t>(slice) : 0;
 				barrier.size          = copy_size;
 				barrier.srcAccessMask = vk::AccessFlagBits2::eTransferRead;
 				barrier.dstAccessMask = vk::AccessFlagBits2::eTransferWrite;
 				command.pipelineBarrier2(dependency);
 				command.copyImageToBuffer(source.backing.image,
-				                          vk::ImageLayout::eTransferSrcOptimal,
-				                          buffer.Handle(), source_copy);
+				                          vk::ImageLayout::eTransferSrcOptimal, buffer.Handle(),
+				                          source_copy);
 				barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
 				barrier.dstAccessMask = vk::AccessFlagBits2::eTransferRead;
 				command.pipelineBarrier2(dependency);
 				command.copyBufferToImage(buffer.Handle(), backing.image,
-				                          vk::ImageLayout::eTransferDstOptimal,
-				                          destination_copy);
+				                          vk::ImageLayout::eTransferDstOptimal, destination_copy);
 			}
 		}
 	}
 	Transit(vk::ImageLayout::eGeneral,
-	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
-	        command);
+	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
 }
 
 void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
@@ -561,10 +538,8 @@ void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
 	const auto width  = std::max(backing.extent.width >> mip, 1u);
 	const auto height = std::max(backing.extent.height >> mip, 1u);
 	const auto depth  = std::max(backing.extent.depth >> mip, 1u);
-	EXIT_IF(width != source.backing.extent.width ||
-	        height != source.backing.extent.height);
-	const auto [source_layers, destination_layers] =
-	    SanitizeCopyLayers(source, *this, depth);
+	EXIT_IF(width != source.backing.extent.width || height != source.backing.extent.height);
+	const auto [source_layers, destination_layers] = SanitizeCopyLayers(source, *this, depth);
 	const auto aspects                             = FullAspectMask(source.backing.format);
 	EXIT_IF(aspects != FullAspectMask(backing.format));
 	std::array<vk::ImageCopy, 2> copies {};
@@ -580,16 +555,13 @@ void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
 		copy.extent         = {width, height, depth};
 	}
 	auto command = m_scheduler->Current().Handle();
-	Transit(vk::ImageLayout::eTransferDstOptimal,
-	        vk::AccessFlagBits2::eTransferWrite, {}, command);
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
-	               vk::AccessFlagBits2::eTransferRead, {}, command);
-	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
-	                  backing.image, vk::ImageLayout::eTransferDstOptimal, copy_count,
-	                  copies.data());
-	Transit(vk::ImageLayout::eGeneral,
-	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
+	Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
 	               command);
+	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
+	                  vk::ImageLayout::eTransferDstOptimal, copy_count, copies.data());
+	Transit(vk::ImageLayout::eGeneral,
+	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
 }
 
 namespace ImageOps {
@@ -601,8 +573,7 @@ void Validate(const ImageInfo& info) {
 	if (info.pixel_format == vk::Format::eUndefined) {
 		const bool metadata_empty =
 		    info.metadata.range.address == 0 && info.metadata.range.size == 0 &&
-		    info.metadata.kind == ImageMetadataKind::None &&
-		    info.metadata.control == 0 &&
+		    info.metadata.kind == ImageMetadataKind::None && info.metadata.control == 0 &&
 		    info.metadata.compression == VideoOutCompression::Uncompressed &&
 		    !info.metadata.stencil_compressed;
 		if (info.data.Empty() || info.HasStencil() || !metadata_empty || info.extent.width == 0 ||
@@ -615,9 +586,9 @@ void Validate(const ImageInfo& info) {
 	}
 
 	if (info.extent.width == 0 || info.extent.height == 0 || info.extent.depth == 0 ||
-	    info.resources.levels == 0 ||
-	    info.resources.levels > info.mip_layout.size() || info.resources.layers == 0 ||
-	    info.samples == 0 || vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} ||
+	    info.resources.levels == 0 || info.resources.levels > info.mip_layout.size() ||
+	    info.resources.layers == 0 || info.samples == 0 ||
+	    vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} ||
 	    info.bytes_per_block == 0 || (info.data.address != 0 && info.pitch == 0)) {
 		EXIT("invalid image geometry or format\n");
 	}
@@ -688,11 +659,11 @@ uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element) {
 
 } // namespace ImageOps
 
-Image::Image(GraphicContext& graphics, CommandScheduler& scheduler,
-             const ImageInfo& image_info)
+Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageInfo& image_info)
     : info(image_info), m_graphics(&graphics), m_scheduler(&scheduler) {
 	KYTY_PROFILER_FUNCTION();
 	ImageOps::Validate(info);
+	m_cpu_dirty = !info.data.Empty();
 	if (info.pixel_format == vk::Format::eUndefined) {
 		return;
 	}

src/graphics/host_gpu/renderer/image/image.h

@@ -3,7 +3,7 @@
 
 #include "common/assert.h"
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/renderer/imageInfo.h"
+#include "graphics/host_gpu/renderer/image/imageInfo.h"
 
 #include <compare>
 #include <limits>
@@ -66,16 +66,16 @@ public:
 	[[nodiscard]] vk::ImageView FindView(const ImageViewInfo& view_info);
 	void                        AssociateDepth(ImageId image_id) { depth_id = image_id; }
 	using Barriers = std::vector<vk::ImageMemoryBarrier2>;
-	[[nodiscard]] Barriers
-	GetBarriers(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
+	[[nodiscard]] Barriers GetBarriers(vk::ImageLayout                      destination_layout,
+	                                   vk::AccessFlags2                     destination_access,
 	                                   vk::PipelineStageFlags2              destination_stage,
 	                                   std::optional<ImageSubresourceRange> range);
 	void Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
 	             std::optional<ImageSubresourceRange> range, vk::CommandBuffer command_buffer);
-	void Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
-	            uint64_t offset, uint64_t size);
-	void Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
-	              uint64_t offset, uint64_t size);
+	void Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
+	            uint64_t size);
+	void Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
+	              uint64_t size);
 	void CopyImage(Image& source);
 	void Resolve(Image& source, const ImageSubresourceRange& source_range,
 	             const ImageSubresourceRange& destination_range);
@@ -95,6 +95,11 @@ public:
 	[[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty || m_maybe_cpu_dirty; }
 	[[nodiscard]] bool IsDefinitelyCpuDirty() const { return m_cpu_dirty; }
 	[[nodiscard]] bool IsMaybeCpuDirty() const { return m_maybe_cpu_dirty; }
+	void               MarkMaybeCpuDirty() {
+		if (!m_cpu_dirty) {
+			m_maybe_cpu_dirty = true;
+		}
+	}
 	[[nodiscard]] bool NeedsMaybeCpuHash() const {
 		return m_maybe_cpu_dirty && !m_maybe_hash_valid;
 	}
@@ -132,7 +137,8 @@ public:
 	void               MarkBufferModified() noexcept { m_buffer_modified = true; }
 	void               ClearBufferModified() noexcept { m_buffer_modified = false; }
 
-	[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size, bool pages = false) const noexcept {
+	[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size,
+	                            bool pages = false) const noexcept {
 		return pages ? ImagePageRangesOverlap(info.data.address, info.data.size, address, size)
 		             : ImageRangeOverlaps(info.data.address, info.data.size, address, size);
 	}
@@ -142,6 +148,7 @@ public:
 	[[nodiscard]] bool SafeToDownload() const noexcept {
 		return IsGpuModified() && !IsBufferModified() && !IsCpuDirty();
 	}
+	[[nodiscard]] bool IsTracked() const noexcept { return track_addr != 0 && track_addr_end != 0; }
 	[[nodiscard]] uint64_t AccountedSize() const noexcept {
 		return backing.image == nullptr ? 0 : (info.data.size + 1023) & ~uint64_t {1023};
 	}
@@ -153,6 +160,9 @@ public:
 	ImageUsage       usage;
 	ImageBinding     binding;
 	bool             registered     = false;
+	mutable uint32_t query_epoch    = 0;
+	uint64_t         track_addr     = 0;
+	uint64_t         track_addr_end = 0;
 	ImageId          depth_id {};
 	uint64_t         tick_accessed_last = 0;
 	size_t           lru_id             = 0;

src/graphics/host_gpu/renderer/image/imageInfo.h

@@ -21,8 +21,7 @@ struct GuestRange {
 
 	[[nodiscard]] constexpr bool Empty() const noexcept { return address == 0 || size == 0; }
 	[[nodiscard]] constexpr bool Valid() const noexcept {
-		return !Empty() && address < TRACKER_ADDRESS_SIZE &&
-		       size <= TRACKER_ADDRESS_SIZE - address;
+		return !Empty() && address < TRACKER_ADDRESS_SIZE && size <= TRACKER_ADDRESS_SIZE - address;
 	}
 	[[nodiscard]] constexpr uint64_t End() const noexcept { return address + size; }
 	auto                             operator<=>(const GuestRange&) const = default;
@@ -352,8 +351,7 @@ inline bool ImageInfo::IsDepth() const noexcept {
 	}
 	const auto transfer_bytes = DepthAspectTransferBytes(info.pixel_format);
 	return transfer_bytes == info.bytes_per_block ||
-	       (info.bytes_per_block == sizeof(uint16_t) &&
-	        transfer_bytes == sizeof(uint32_t));
+	       (info.bytes_per_block == sizeof(uint16_t) && transfer_bytes == sizeof(uint32_t));
 }
 
 [[nodiscard]] inline VideoOutCompression
@@ -472,16 +470,11 @@ IsSupportedDisplayRenderTargetTileMode(uint32_t tile_mode) noexcept {
 	const auto unorm8 = [](uint32_t value) { return static_cast<float>(value & 0xffu) / 255.0f; };
 	const auto srgb8  = [](uint32_t value) {
 		const auto encoded = static_cast<float>(value & 0xffu) / 255.0f;
-		return encoded <= 0.04045f ? encoded / 12.92f
-		                           : std::pow((encoded + 0.055f) / 1.055f, 2.4f);
+		return encoded <= 0.04045f ? encoded / 12.92f : std::pow((encoded + 0.055f) / 1.055f, 2.4f);
 	};
 	switch (format) {
-		case vk::Format::eR32Uint:
-			next.uint32[0] = packed;
-			break;
-		case vk::Format::eR32Sint:
-			next.int32[0] = static_cast<int32_t>(packed);
-			break;
+		case vk::Format::eR32Uint: next.uint32[0] = packed; break;
+		case vk::Format::eR32Sint: next.int32[0] = static_cast<int32_t>(packed); break;
 		case vk::Format::eR8G8B8A8Srgb:
 			next.float32[0] = srgb8(packed);
 			next.float32[1] = srgb8(packed >> 8u);

src/graphics/host_gpu/renderer/image/imageView.cpp

@@ -1,8 +1,8 @@
-#include "graphics/host_gpu/renderer/imageView.h"
+#include "graphics/host_gpu/renderer/image/imageView.h"
 
 #include "common/assert.h"
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/renderer/image.h"
+#include "graphics/host_gpu/renderer/image/image.h"
 
 #include <mutex>
 
@@ -70,15 +70,14 @@ namespace {
 			}
 		case vk::ImageType::e3D:
 			switch (info.type) {
-				case vk::ImageViewType::e3D:
-					return info.base_layer == 0 && info.layer_count == 1;
+				case vk::ImageViewType::e3D: return info.base_layer == 0 && info.layer_count == 1;
 				case vk::ImageViewType::e2D:
-					return static_cast<bool>(
-					           image.flags & vk::ImageCreateFlagBits::e2DArrayCompatible) &&
+					return static_cast<bool>(image.flags &
+					                         vk::ImageCreateFlagBits::e2DArrayCompatible) &&
 					       info.level_count == 1 && info.layer_count == 1;
 				case vk::ImageViewType::e2DArray:
-					return static_cast<bool>(
-					           image.flags & vk::ImageCreateFlagBits::e2DArrayCompatible) &&
+					return static_cast<bool>(image.flags &
+					                         vk::ImageCreateFlagBits::e2DArrayCompatible) &&
 					       info.level_count == 1;
 				default: return false;
 			}
@@ -327,8 +326,7 @@ bool FormatsCompatible(vk::Format base, vk::Format view) noexcept {
 vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 	const auto& image      = backing;
 	auto        normalized = view_info;
-	const bool  is_storage =
-	    static_cast<bool>(normalized.usage & vk::ImageUsageFlagBits::eStorage);
+	const bool  is_storage = static_cast<bool>(normalized.usage & vk::ImageUsageFlagBits::eStorage);
 	normalized.aspect      = FullAspectMask(image.format);
 	if (normalized.aspect & vk::ImageAspectFlagBits::eDepth &&
 	    IsDepthViewFormat(normalized.format)) {
@@ -340,12 +338,11 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		normalized.format = image.format;
 		normalized.aspect = vk::ImageAspectFlagBits::eStencil;
 	}
-	normalized.usage =
-	    is_storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlags {};
+	normalized.usage = is_storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlags {};
 	const bool format_compatible = normalized.format != vk::Format::eUndefined &&
 	                               IsCompatibleViewFormat(image.format, normalized.format);
-	const bool slice_view = image.image_type == vk::ImageType::e3D &&
-	                        (normalized.type == vk::ImageViewType::e2D ||
+	const bool slice_view =
+	    image.image_type == vk::ImageType::e3D && (normalized.type == vk::ImageViewType::e2D ||
 	                                               normalized.type == vk::ImageViewType::e2DArray);
 	const bool levels_valid = normalized.level_count != 0 &&
 	                          normalized.base_level < image.mip_levels &&
@@ -353,15 +350,14 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 	const auto view_layers  = slice_view && levels_valid
 	                              ? std::max(image.extent.depth >> normalized.base_level, 1u)
 	                              : image.layers;
-	const bool ranges_valid = levels_valid &&
-	                          normalized.layer_count != 0 && normalized.base_layer < view_layers &&
+	const bool ranges_valid = levels_valid && normalized.layer_count != 0 &&
+	                          normalized.base_layer < view_layers &&
 	                          normalized.layer_count <= view_layers - normalized.base_layer;
 	const bool mapping_valid =
 	    IsComponentSwizzle(normalized.mapping.r) && IsComponentSwizzle(normalized.mapping.g) &&
 	    IsComponentSwizzle(normalized.mapping.b) && IsComponentSwizzle(normalized.mapping.a);
 	if (image.image == nullptr || !format_compatible || !ranges_valid || !mapping_valid ||
-	    !IsValidViewType(image, normalized) ||
-	    !IsValidAspect(image, normalized.aspect)) {
+	    !IsValidViewType(image, normalized) || !IsValidAspect(image, normalized.aspect)) {
 		EXIT("invalid image view: image_format=%d view_format=%d type=%d aspect=0x%x "
 		     "mip=%u+%u layer=%u+%u usage=0x%x image_levels=%u image_layers=%u\n",
 		     static_cast<int>(image.format), static_cast<int>(normalized.format),

src/graphics/host_gpu/renderer/image/imageView.h

@@ -2,8 +2,8 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEVIEW_H_
 
 #include "common/assert.h"
-#include "graphics/host_gpu/renderer/imageInfo.h"
-#include "graphics/shader/recompiler/ShaderIR.h"
+#include "graphics/host_gpu/renderer/image/imageInfo.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shader.h"
 
 namespace Libs::Graphics {
@@ -88,7 +88,9 @@ SelectSampledDepthView(vk::Format image_format, vk::Format view_format, uint32_t
 IsSupportedSampledDepthResource(const ShaderRecompiler::IR::ImageResource& resource) noexcept {
 	return resource.kind == ShaderRecompiler::IR::ResourceKind::Image &&
 	       (resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
-	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
+	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray ||
+	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa ||
+	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray) &&
 	       resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.read &&
 	       !resource.written && !resource.atomic;
 }
@@ -103,10 +105,7 @@ IsSupportedSampledDepthUintResource(const ShaderRecompiler::IR::ImageResource& r
 
 inline void ValidateStorageColorView(vk::Format image_format, vk::Format view_format,
                                      uint32_t swizzle) noexcept {
-	const auto srgb_view = SrgbStorageViewFormat(image_format);
-	const bool srgb_storage_view =
-	    srgb_view != vk::Format::eUndefined && view_format == srgb_view;
-	if ((image_format != view_format && !srgb_storage_view) ||
+	if (!ImageViewOps::FormatsCompatible(image_format, view_format) ||
 	    !IsValidImageSwizzle(swizzle)) {
 		UnsupportedColorView("storage", image_format, view_format, swizzle);
 	}
@@ -122,7 +121,10 @@ IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resou
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
 	       resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.written &&
-	       !resource.atomic && !resource.depth_compare;
+	       (!resource.atomic ||
+	        (resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint &&
+	         resource.read)) &&
+	       !resource.depth_compare;
 }
 
 inline void

src/graphics/host_gpu/renderer/image/textureCommon.cpp

@@ -1,9 +1,9 @@
-#include "graphics/host_gpu/objects/textureCommon.h"
+#include "graphics/host_gpu/renderer/image/textureCommon.h"
 
 #include "common/assert.h"
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/gpu_format.h"
-#include "graphics/host_gpu/renderer/tiler.h"
+#include "graphics/host_gpu/renderer/image/tiler.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <algorithm>
@@ -70,6 +70,10 @@ constexpr RenderTargetFormatMapping kRenderTargetFormats[] = {
      Prospero::ChannelType::kFloat,
      Prospero::ChannelOrder::kStandard,
      {vk::Format::eB10G11R11UfloatPack32, 4}},
+    {Prospero::ChannelLayout::k5_6_5,
+     Prospero::ChannelType::kUNorm,
+     Prospero::ChannelOrder::kStandard,
+     {vk::Format::eB5G6R5UnormPack16, 2}},
     {Prospero::ChannelLayout::k16,
      Prospero::ChannelType::kUNorm,
      Prospero::ChannelOrder::kStandard,
@@ -102,6 +106,10 @@ constexpr RenderTargetFormatMapping kRenderTargetFormats[] = {
      Prospero::ChannelType::kUNorm,
      Prospero::ChannelOrder::kStandard,
      {vk::Format::eR16G16B16A16Unorm, 8}},
+    {Prospero::ChannelLayout::k16_16_16_16,
+     Prospero::ChannelType::kUInt,
+     Prospero::ChannelOrder::kStandard,
+     {vk::Format::eR16G16B16A16Uint, 8}},
     {Prospero::ChannelLayout::k16_16_16_16,
      Prospero::ChannelType::kFloat,
      Prospero::ChannelOrder::kStandard,
@@ -393,10 +401,10 @@ TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64
 	return layout;
 }
 
-std::vector<vk::BufferImageCopy>
-TextureBuildImageCopies(const TextureUploadLayout& layout, uint32_t width, uint32_t height,
-                        uint32_t depth, uint64_t levels, bool array_texture,
-                        bool volume_texture) {
+std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayout& layout,
+                                                         uint32_t width, uint32_t height,
+                                                         uint32_t depth, uint64_t levels,
+                                                         bool array_texture, bool volume_texture) {
 	uint32_t mip_width  = width;
 	uint32_t mip_height = height;
 	uint32_t mip_pitch  = volume_texture && static_cast<Prospero::TileMode>(layout.tile) !=
@@ -414,10 +422,9 @@ TextureBuildImageCopies(const TextureUploadLayout& layout, uint32_t width, uint3
 		for (uint32_t z = 0; z < mip_depth; z++) {
 			const auto          slice_offset = z * layout.slice_stride;
 			vk::BufferImageCopy region {};
-			region.bufferOffset =
-			    layout.level_sizes[i].offset + slice_offset;
-			region.imageSubresource = {vk::ImageAspectFlagBits::eColor, i,
-			                           array_texture ? z : 0, 1};
+			region.bufferOffset     = layout.level_sizes[i].offset + slice_offset;
+			region.imageSubresource = {vk::ImageAspectFlagBits::eColor, i, array_texture ? z : 0,
+			                           1};
 			region.imageOffset.z    = volume_texture ? static_cast<int>(z) : 0;
 			region.imageExtent      = {mip_width, mip_height, 1};
 			const bool linear =
@@ -430,8 +437,7 @@ TextureBuildImageCopies(const TextureUploadLayout& layout, uint32_t width, uint3
 					return ((value + block - 1u) / block) * block;
 				};
 				const auto pitch       = align(mip_pitch, layout.texel_block);
-				region.bufferRowLength =
-				    pitch > align(mip_width, layout.texel_block) ? pitch : 0;
+				region.bufferRowLength = pitch > align(mip_width, layout.texel_block) ? pitch : 0;
 			}
 			regions.push_back(region);
 		}
@@ -476,11 +482,10 @@ static bool SetGpuTileSize(uint64_t offset, uint64_t length, uint64_t capacity,
 	return true;
 }
 
-bool TextureBuildGpuTileInfos(uint64_t size,
-                              const std::vector<vk::BufferImageCopy>& regions,
+bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferImageCopy>& regions,
                               const TextureUploadLayout& layout, uint32_t fmt, uint32_t depth,
                               uint64_t levels, std::vector<GpuTileInfo>& out_infos) {
-	if (size == 0 || levels == 0 || levels > 16 || depth == 0 ||
+	if (tiled_size == 0 || levels == 0 || levels > 16 || depth == 0 ||
 	    regions.size() != GetTextureRegionCount(depth, levels, layout.volume_texture) ||
 	    Prospero::IsFmaskTextureFormat(fmt)) {
 		return false;
@@ -518,9 +523,8 @@ bool TextureBuildGpuTileInfos(uint64_t size,
 			for (uint32_t z = 0; z < mip_depth; z += block.block_depth) {
 				const uint32_t copy_depth = std::min(block.block_depth, mip_depth - z);
 				const auto&    region     = regions[region_base + z];
-				const auto     pitch = region.bufferRowLength != 0
-				                           ? region.bufferRowLength
-				                           : region.imageExtent.width;
+				const auto     pitch =
+				    region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
 				const auto  logical_height = region.bufferImageHeight != 0
 				                                 ? region.bufferImageHeight
 				                                 : region.imageExtent.height;
@@ -534,22 +538,20 @@ bool TextureBuildGpuTileInfos(uint64_t size,
 				const uint64_t linear_span =
 				    static_cast<uint64_t>(copy_depth - 1u) * linear_stride +
 				    layout.level_sizes[level].size;
-				if (!SetGpuTileSize(info.linear_offset, linear_span, size, info.linear_size) ||
-				    !SetGpuTileSize(info.tiled_offset, volume.level_sizes[level], size,
+				if (!SetGpuTileSize(info.linear_offset, linear_span, UINT64_MAX,
+				                    info.linear_size) ||
+				    !SetGpuTileSize(info.tiled_offset, volume.level_sizes[level], tiled_size,
 				                    info.tiled_size)) {
 					return false;
 				}
 				info.linear_slice_stride = linear_stride;
-				info.width = std::max(
-				    (region.imageExtent.width + element.wide - 1u) / element.wide, 1u);
-				info.height = std::max(
-				    (logical_height + element.tall - 1u) / element.tall, 1u);
+				info.width =
+				    std::max((region.imageExtent.width + element.wide - 1u) / element.wide, 1u);
+				info.height = std::max((logical_height + element.tall - 1u) / element.tall, 1u);
 				info.depth  = copy_depth;
-				info.surface_z = block.block_depth == 1
-				                     ? static_cast<uint32_t>(region.imageOffset.z)
-				                     : 0;
-				info.pitch =
-				    std::max((pitch + element.wide - 1u) / element.wide, 1u);
+				info.surface_z =
+				    block.block_depth == 1 ? static_cast<uint32_t>(region.imageOffset.z) : 0;
+				info.pitch        = std::max((pitch + element.wide - 1u) / element.wide, 1u);
 				info.tail_x       = tail ? volume.tail_x[level] : 0;
 				info.tail_y       = tail ? volume.tail_y[level] : 0;
 				info.tail         = tail;
@@ -577,9 +579,8 @@ bool TextureBuildGpuTileInfos(uint64_t size,
 			const auto level_depth = GetTextureLevelDepth(depth, level, layout.volume_texture);
 			for (uint32_t z = 0; z < level_depth; z++) {
 				const auto& region = regions[region_index++];
-				const auto pitch = region.bufferRowLength != 0
-				                       ? region.bufferRowLength
-				                       : region.imageExtent.width;
+				const auto  pitch =
+				    region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
 				const auto  logical_height = region.bufferImageHeight != 0
 				                                 ? region.bufferImageHeight
 				                                 : region.imageExtent.height;
@@ -588,21 +589,20 @@ bool TextureBuildGpuTileInfos(uint64_t size,
 				info.bytes_per_element = block.bytes_per_element;
 				info.linear_offset     = region.bufferOffset;
 				info.tiled_offset      = TextureUploadSliceSourceOffset(layout, level, z);
-				if (!SetGpuTileSize(info.linear_offset, level_size.size, size, info.linear_size) ||
-				    !SetGpuTileSize(info.tiled_offset, GetLevelSrcSize(level_size), size,
+				if (!SetGpuTileSize(info.linear_offset, level_size.size, UINT64_MAX,
+				                    info.linear_size) ||
+				    !SetGpuTileSize(info.tiled_offset, GetLevelSrcSize(level_size), tiled_size,
 				                    info.tiled_size)) {
 					return false;
 				}
-				info.width = std::max(
-				    (region.imageExtent.width + element.wide - 1u) / element.wide, 1u);
-				info.height = std::max(
-				    (logical_height + element.tall - 1u) / element.tall, 1u);
+				info.width =
+				    std::max((region.imageExtent.width + element.wide - 1u) / element.wide, 1u);
+				info.height    = std::max((logical_height + element.tall - 1u) / element.tall, 1u);
 				info.surface_z = base_family == TileBlockFamily::RenderTarget64KB ||
 				                         base_family == TileBlockFamily::Depth64KB
 				                     ? region.imageSubresource.baseArrayLayer
 				                     : 0;
-				info.pitch =
-				    std::max((pitch + element.wide - 1u) / element.wide, 1u);
+				info.pitch     = std::max((pitch + element.wide - 1u) / element.wide, 1u);
 				info.tail      = tail;
 				info.tail_x    = tail ? level_size.x : 0;
 				info.tail_y    = tail ? level_size.y : 0;

src/graphics/host_gpu/renderer/image/textureCommon.h

@@ -1,5 +1,5 @@
-#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_TEXTURECOMMON_H_
-#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_TEXTURECOMMON_H_
+#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_
+#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_
 
 #include "common/abi.h"
 #include "common/common.h"
@@ -40,15 +40,14 @@ TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64
                                                uint64_t tile, uint64_t upload_size,
                                                bool allow_depth_tile, bool volume_texture,
                                                const char* owner);
-std::vector<vk::BufferImageCopy>
-TextureBuildImageCopies(const TextureUploadLayout& layout, uint32_t width, uint32_t height,
-                        uint32_t depth, uint64_t levels, bool array_texture,
-                        bool volume_texture);
-bool TextureBuildGpuTileInfos(uint64_t size,
-                              const std::vector<vk::BufferImageCopy>& regions,
+std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayout& layout,
+                                                         uint32_t width, uint32_t height,
+                                                         uint32_t depth, uint64_t levels,
+                                                         bool array_texture, bool volume_texture);
+bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferImageCopy>& regions,
                               const TextureUploadLayout& layout, uint32_t fmt, uint32_t depth,
                               uint64_t levels, std::vector<GpuTileInfo>& infos);
 
 } // namespace Libs::Graphics
 
-#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_TEXTURECOMMON_H_ */
+#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_ */

src/graphics/host_gpu/renderer/image/tiler.cpp

@@ -1,4 +1,4 @@
-#include "graphics/host_gpu/renderer/tiler.h"
+#include "graphics/host_gpu/renderer/image/tiler.h"
 
 #include "common/assert.h"
 #include "gpu_tiler_shaders/gpu_tiler_demote_d16_spv.h"
@@ -14,9 +14,9 @@
 #include "gpu_tiler_shaders/gpu_tiler_standard64_spv.h"
 #include "gpu_tiler_shaders/gpu_tiler_swap_bgra16_spv.h"
 #include "graphics/host_gpu/graphicContext.h"
+#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
-#include "graphics/host_gpu/renderer/image.h"
-#include "graphics/host_gpu/renderer/streamBuffer.h"
+#include "graphics/host_gpu/renderer/image/image.h"
 
 #include <algorithm>
 #include <array>

src/graphics/host_gpu/renderer/multiLevelPageTable.h

@@ -1,360 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_
-#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_
-
-#include "common/assert.h"
-
-#include <algorithm>
-#include <array>
-#include <cstddef>
-#include <cstdint>
-#include <list>
-#include <memory>
-#include <utility>
-#include <vector>
-
-namespace Libs::Graphics {
-
-// Sparse two-level lookup for texture-cache page ownership.
-// Entries are selected by guest page number; queries never allocate L1 buckets.
-template <typename EntryT, size_t PageBits = 20, size_t AddressSpaceBits = 40,
-          size_t FirstLevelBits = 10>
-class MultiLevelPageTable final {
-public:
-	using Entry = EntryT;
-
-	static_assert(AddressSpaceBits < 64);
-	static_assert(PageBits < AddressSpaceBits);
-	static_assert(FirstLevelBits > 0 && FirstLevelBits <= AddressSpaceBits - PageBits);
-
-	static constexpr size_t   kPageBits          = PageBits;
-	static constexpr size_t   kAddressSpaceBits  = AddressSpaceBits;
-	static constexpr size_t   kFirstLevelBits    = FirstLevelBits;
-	static constexpr size_t   kSecondLevelBits   = AddressSpaceBits - FirstLevelBits - PageBits;
-	static constexpr size_t   kFirstLevelEntries = size_t {1} << FirstLevelBits;
-	static constexpr size_t   kBucketEntries     = size_t {1} << kSecondLevelBits;
-	static constexpr size_t   kPageCount         = size_t {1} << (AddressSpaceBits - PageBits);
-	static constexpr uint64_t kAddressSpaceSize  = uint64_t {1} << AddressSpaceBits;
-
-	struct PageRange final {
-		size_t first          = 0;
-		size_t last_exclusive = 0;
-	};
-
-	MultiLevelPageTable(): m_first_level(kFirstLevelEntries) {}
-
-	[[nodiscard]] Entry* Find(size_t page) {
-		if (!IsValidPage(page)) {
-			return nullptr;
-		}
-		auto& bucket = m_first_level[FirstLevelIndex(page)];
-		return bucket == nullptr ? nullptr : &(*bucket)[SecondLevelIndex(page)];
-	}
-
-	[[nodiscard]] const Entry* Find(size_t page) const {
-		if (!IsValidPage(page)) {
-			return nullptr;
-		}
-		const auto& bucket = m_first_level[FirstLevelIndex(page)];
-		return bucket == nullptr ? nullptr : &(*bucket)[SecondLevelIndex(page)];
-	}
-
-	[[nodiscard]] Entry& GetOrCreate(size_t page) {
-		if (!IsValidPage(page)) {
-			EXIT("MultiLevelPageTable page is outside the guest address space");
-		}
-		auto& bucket = m_first_level[FirstLevelIndex(page)];
-		if (bucket == nullptr) {
-			bucket = std::make_unique<Bucket>();
-			++m_allocated_buckets;
-		}
-		return (*bucket)[SecondLevelIndex(page)];
-	}
-
-	[[nodiscard]] Entry& operator[](size_t page) { return GetOrCreate(page); }
-
-	[[nodiscard]] static constexpr bool IsValidPage(size_t page) { return page < kPageCount; }
-
-	// Returns the half-open page interval touched by a non-empty byte range.
-	// An end exactly at 2^AddressSpaceBits is valid; wrapping or crossing it is not.
-	[[nodiscard]] static constexpr bool TryGetPageRange(uint64_t address, uint64_t size,
-	                                                    PageRange& range) {
-		if (size == 0 || address >= kAddressSpaceSize || size > kAddressSpaceSize - address) {
-			return false;
-		}
-		const uint64_t end   = address + size;
-		range.first          = static_cast<size_t>(address >> PageBits);
-		range.last_exclusive = static_cast<size_t>(((end - 1) >> PageBits) + 1);
-		return true;
-	}
-
-	[[nodiscard]] size_t AllocatedBucketCount() const { return m_allocated_buckets; }
-
-private:
-	using Bucket = std::array<Entry, kBucketEntries>;
-
-	[[nodiscard]] static constexpr size_t FirstLevelIndex(size_t page) {
-		return page >> kSecondLevelBits;
-	}
-	[[nodiscard]] static constexpr size_t SecondLevelIndex(size_t page) {
-		return page & (kBucketEntries - 1);
-	}
-
-	std::vector<std::unique_ptr<Bucket>> m_first_level;
-	size_t                               m_allocated_buckets = 0;
-};
-
-// Removes only the requested owner. Other owners in the same page entry remain intact.
-template <typename Container, typename Value>
-[[nodiscard]] bool EraseExact(Container& owners, const Value& owner) {
-	const auto it = std::find(owners.begin(), owners.end(), owner);
-	if (it == owners.end()) {
-		return false;
-	}
-	owners.erase(it);
-	return true;
-}
-
-// Multi-range ownership with 1 MiB candidate buckets and precise 4 KiB
-// lifetime accounting. OwnerT only needs equality.
-template <typename OwnerT>
-class MultiRangePageOwnerIndex final {
-private:
-	struct Registration;
-	using MembershipList = std::vector<const Registration*>;
-
-public:
-	struct ByteRange final {
-		uint64_t address = 0;
-		uint64_t size    = 0;
-	};
-
-	using CoarseTable   = MultiLevelPageTable<MembershipList, 20, 40, 10>;
-	using TrackingTable = MultiLevelPageTable<MembershipList, 12, 40, 18>;
-
-	[[nodiscard]] bool Register(const OwnerT& owner, const std::vector<ByteRange>& ranges) {
-		if (FindRegistration(owner) != m_registrations.end()) {
-			return false;
-		}
-		auto normalized = Normalize(ranges);
-		if (normalized.empty()) {
-			return false;
-		}
-		m_registrations.push_back({owner, std::move(normalized)});
-		const Registration* registration = &m_registrations.back();
-		for (const size_t page: CollectPages<20>(registration->ranges)) {
-			m_coarse_pages[page].push_back(registration);
-		}
-		for (const size_t page: CollectPages<12>(registration->ranges)) {
-			m_tracking_pages[page].push_back(registration);
-		}
-		return true;
-	}
-
-	// No state changes if an expected membership is absent. Final releases are
-	// sorted and coalesced 4 KiB spans whose final owner disappeared.
-	[[nodiscard]] bool Unregister(const OwnerT& owner, std::vector<ByteRange>& final_releases) {
-		final_releases.clear();
-		auto registration = FindRegistration(owner);
-		if (registration == m_registrations.end()) {
-			return false;
-		}
-		const auto          coarse_pages     = CollectPages<20>(registration->ranges);
-		const auto          tracking_pages   = CollectPages<12>(registration->ranges);
-		const Registration* registration_ptr = &*registration;
-		if (!HasAllMemberships(m_coarse_pages, coarse_pages, registration_ptr) ||
-		    !HasAllMemberships(m_tracking_pages, tracking_pages, registration_ptr)) {
-			return false;
-		}
-		std::vector<size_t> final_pages;
-		for (const size_t page: coarse_pages) {
-			(void)EraseExact(*m_coarse_pages.Find(page), registration_ptr);
-		}
-		for (const size_t page: tracking_pages) {
-			auto& owners = *m_tracking_pages.Find(page);
-			if (owners.size() == 1) {
-				final_pages.push_back(page);
-			}
-			(void)EraseExact(owners, registration_ptr);
-		}
-		m_registrations.erase(registration);
-		final_releases = CoalesceTrackingPages(final_pages);
-		return true;
-	}
-
-	[[nodiscard]] std::vector<OwnerT> Query(uint64_t address, uint64_t size) const {
-		return Query(address, size, [](const OwnerT&) { return true; });
-	}
-
-	template <typename Predicate>
-	[[nodiscard]] std::vector<OwnerT> Query(uint64_t address, uint64_t size,
-	                                        Predicate&& predicate) const {
-		return QueryImpl(address, size, true, std::forward<Predicate>(predicate));
-	}
-
-	// Fault paths use page candidates: exact byte-disjoint owners sharing a
-	// touched 4 KiB page are intentionally retained.
-	[[nodiscard]] std::vector<OwnerT> QueryCandidates(uint64_t address, uint64_t size) const {
-		return QueryCandidates(address, size, [](const OwnerT&) { return true; });
-	}
-
-	template <typename Predicate>
-	[[nodiscard]] std::vector<OwnerT> QueryCandidates(uint64_t address, uint64_t size,
-	                                                  Predicate&& predicate) const {
-		return QueryImpl(address, size, false, std::forward<Predicate>(predicate));
-	}
-
-	[[nodiscard]] size_t CoarseMembershipCount(size_t page) const {
-		const auto* owners = m_coarse_pages.Find(page);
-		return owners == nullptr ? 0 : owners->size();
-	}
-	[[nodiscard]] size_t TrackingMembershipCount(size_t page) const {
-		const auto* owners = m_tracking_pages.Find(page);
-		return owners == nullptr ? 0 : owners->size();
-	}
-
-private:
-	template <typename Predicate>
-	[[nodiscard]] std::vector<OwnerT> QueryImpl(uint64_t address, uint64_t size, bool strict_bytes,
-	                                            Predicate&& predicate) const {
-		typename CoarseTable::PageRange   coarse_range {};
-		typename TrackingTable::PageRange tracking_range {};
-		if (!CoarseTable::TryGetPageRange(address, size, coarse_range) ||
-		    !TrackingTable::TryGetPageRange(address, size, tracking_range)) {
-			return {};
-		}
-		MembershipList candidates;
-		for (size_t page = coarse_range.first; page < coarse_range.last_exclusive; ++page) {
-			if (const auto* owners = m_coarse_pages.Find(page); owners != nullptr) {
-				AppendUnique(candidates, *owners);
-			}
-		}
-		std::vector<OwnerT> result;
-		for (const Registration* registration: candidates) {
-			if ((!strict_bytes || Overlaps(registration->ranges, address, size)) &&
-			    HasTrackingMembership(registration, tracking_range) &&
-			    predicate(registration->owner)) {
-				result.push_back(registration->owner);
-			}
-		}
-		return result;
-	}
-
-	struct Registration final {
-		OwnerT                 owner;
-		std::vector<ByteRange> ranges;
-	};
-	using RegistrationIterator      = typename std::list<Registration>::iterator;
-	using ConstRegistrationIterator = typename std::list<Registration>::const_iterator;
-
-	[[nodiscard]] RegistrationIterator FindRegistration(const OwnerT& owner) {
-		return std::find_if(m_registrations.begin(), m_registrations.end(),
-		                    [&](const Registration& item) { return item.owner == owner; });
-	}
-	[[nodiscard]] ConstRegistrationIterator FindRegistration(const OwnerT& owner) const {
-		return std::find_if(m_registrations.begin(), m_registrations.end(),
-		                    [&](const Registration& item) { return item.owner == owner; });
-	}
-
-	[[nodiscard]] static std::vector<ByteRange> Normalize(const std::vector<ByteRange>& ranges) {
-		std::vector<ByteRange> sorted;
-		for (const auto& range: ranges) {
-			typename CoarseTable::PageRange ignored {};
-			if (!CoarseTable::TryGetPageRange(range.address, range.size, ignored)) {
-				return {};
-			}
-			sorted.push_back(range);
-		}
-		std::sort(sorted.begin(), sorted.end(), [](const ByteRange& lhs, const ByteRange& rhs) {
-			return lhs.address < rhs.address;
-		});
-		std::vector<ByteRange> merged;
-		for (const auto& range: sorted) {
-			if (merged.empty() || range.address > merged.back().address + merged.back().size) {
-				merged.push_back(range);
-			} else {
-				const uint64_t end = std::max(merged.back().address + merged.back().size,
-				                              range.address + range.size);
-				merged.back().size = end - merged.back().address;
-			}
-		}
-		return merged;
-	}
-
-	template <size_t Bits>
-	[[nodiscard]] static std::vector<size_t> CollectPages(const std::vector<ByteRange>& ranges) {
-		std::vector<size_t> pages;
-		for (const auto& range: ranges) {
-			const size_t first = static_cast<size_t>(range.address >> Bits);
-			const size_t last  = static_cast<size_t>((range.address + range.size - 1) >> Bits);
-			for (size_t page = first; page <= last; ++page) {
-				pages.push_back(page);
-			}
-		}
-		std::sort(pages.begin(), pages.end());
-		pages.erase(std::unique(pages.begin(), pages.end()), pages.end());
-		return pages;
-	}
-
-	template <typename Table>
-	[[nodiscard]] static bool HasAllMemberships(const Table&               table,
-	                                            const std::vector<size_t>& pages,
-	                                            const Registration*        registration) {
-		for (const size_t page: pages) {
-			const auto* owners = table.Find(page);
-			if (owners == nullptr ||
-			    std::find(owners->begin(), owners->end(), registration) == owners->end()) {
-				return false;
-			}
-		}
-		return true;
-	}
-
-	[[nodiscard]] bool HasTrackingMembership(const Registration*                      registration,
-	                                         const typename TrackingTable::PageRange& range) const {
-		for (size_t page = range.first; page < range.last_exclusive; ++page) {
-			const auto* owners = m_tracking_pages.Find(page);
-			if (owners != nullptr &&
-			    std::find(owners->begin(), owners->end(), registration) != owners->end()) {
-				return true;
-			}
-		}
-		return false;
-	}
-
-	[[nodiscard]] static bool Overlaps(const std::vector<ByteRange>& ranges, uint64_t address,
-	                                   uint64_t size) {
-		const uint64_t end = address + size;
-		return std::any_of(ranges.begin(), ranges.end(), [&](const ByteRange& range) {
-			return range.address < end && address < range.address + range.size;
-		});
-	}
-	static void AppendUnique(MembershipList& destination, const MembershipList& source) {
-		for (const Registration* registration: source) {
-			if (std::find(destination.begin(), destination.end(), registration) ==
-			    destination.end()) {
-				destination.push_back(registration);
-			}
-		}
-	}
-	[[nodiscard]] static std::vector<ByteRange>
-	CoalesceTrackingPages(const std::vector<size_t>& pages) {
-		std::vector<ByteRange> result;
-		for (const size_t page: pages) {
-			const uint64_t address = static_cast<uint64_t>(page) << 12;
-			if (!result.empty() && result.back().address + result.back().size == address) {
-				result.back().size += uint64_t {1} << 12;
-			} else {
-				result.push_back({address, uint64_t {1} << 12});
-			}
-		}
-		return result;
-	}
-
-	CoarseTable             m_coarse_pages;
-	TrackingTable           m_tracking_pages;
-	std::list<Registration> m_registrations;
-};
-
-} // namespace Libs::Graphics
-
-#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_

src/graphics/host_gpu/renderer/pipeline/descriptorCache.cpp

@@ -1,8 +1,8 @@
-#include "graphics/host_gpu/renderer/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 
 #include "common/assert.h"
 #include "common/profiler.h"
-#include "graphics/shader/recompiler/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 
 #include <array>
 
@@ -26,11 +26,15 @@ bool IsSampledImage(BindingKind kind) {
 		case BindingKind::Sampled1DArray:
 		case BindingKind::Sampled2D:
 		case BindingKind::Sampled2DArray:
+		case BindingKind::Sampled2DMsaa:
+		case BindingKind::Sampled2DMsaaArray:
 		case BindingKind::Sampled3D:
 		case BindingKind::SampledUint1D:
 		case BindingKind::SampledUint1DArray:
 		case BindingKind::SampledUint2D:
 		case BindingKind::SampledUint2DArray:
+		case BindingKind::SampledUint2DMsaa:
+		case BindingKind::SampledUint2DMsaaArray:
 		case BindingKind::SampledUint3D: return true;
 		default: return false;
 	}

src/graphics/host_gpu/renderer/pipeline/descriptorCache.h

@@ -6,7 +6,7 @@
 #include "common/common.h"
 #include "common/threads.h"
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/renderer/textureCache.h"
+#include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/shader/shaderBindings.h"
 

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -1,4 +1,4 @@
-#include "graphics/host_gpu/renderer/descriptors.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
 
 #include "common/assert.h"
 #include "common/common.h"
@@ -14,18 +14,18 @@
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/hostMemory.h"
-#include "graphics/host_gpu/objects/textureCommon.h"
 #include "graphics/host_gpu/renderer/debug.h"
-#include "graphics/host_gpu/renderer/descriptorCache.h"
-#include "graphics/host_gpu/renderer/imageView.h"
+#include "graphics/host_gpu/renderer/image/imageView.h"
+#include "graphics/host_gpu/renderer/image/textureCommon.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
-#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
 #include "graphics/host_gpu/vma.h"
 #include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/recompiler/BindingLayout.h"
-#include "graphics/shader/recompiler/ResourceMaterialization.h"
-#include "graphics/shader/recompiler/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/BindingLayout.h"
+#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shader.h"
 
 #include <algorithm>
@@ -73,6 +73,11 @@ static Prospero::ImageType TextureBaseType(Prospero::ImageType type) {
 	}
 }
 
+static bool IsMultisampledTexture(Prospero::ImageType type) {
+	return type == Prospero::ImageType::kColor2DMsaa ||
+	       type == Prospero::ImageType::kColor2DMsaaArray;
+}
+
 static BufferView NativeStorageBuffer(RenderContext& context, CommandBuffer& command_buffer,
                                       const ShaderBufferResource&                 descriptor,
                                       const ShaderRecompiler::IR::BufferResource& resource,
@@ -159,6 +164,8 @@ static bool IsSupportedSampledColorResource(const ShaderRecompiler::IR::ImageRes
 		case ShaderRecompiler::Decoder::ImageDimension::Dim1DArray:
 		case ShaderRecompiler::Decoder::ImageDimension::Dim2D:
 		case ShaderRecompiler::Decoder::ImageDimension::Dim2DArray:
+		case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa:
+		case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray:
 			supported_dimension = true;
 			break;
 		default: break;
@@ -195,6 +202,22 @@ TargetTextureViewInfo ResolveTargetTextureView(const ShaderRecompiler::IR::Image
 			           ? TargetTextureViewInfo {vk::ImageViewType::e2DArray, base_layer,
 			                                    image_layers - base_layer}
 			           : TargetTextureViewInfo {};
+		case Prospero::ImageType::kColor2DMsaa:
+			return resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa &&
+			               base_layer == 0 && image_layers == 1
+			           ? TargetTextureViewInfo {vk::ImageViewType::e2D, 0, 1}
+			           : TargetTextureViewInfo {};
+		case Prospero::ImageType::kColor2DMsaaArray:
+			if (resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa &&
+			    base_layer == 0 && image_layers == 1) {
+				return {vk::ImageViewType::e2D, 0, 1};
+			}
+			return resource.dimension ==
+			                   ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray &&
+			               base_layer < image_layers
+			           ? TargetTextureViewInfo {vk::ImageViewType::e2DArray, base_layer,
+			                                    image_layers - base_layer}
+			           : TargetTextureViewInfo {};
 		default: return {};
 	}
 }
@@ -211,34 +234,57 @@ bool IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& r
 bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, const Image& image) {
 	const auto width        = static_cast<uint32_t>(descriptor.Width5()) + 1u;
 	const auto height       = static_cast<uint32_t>(descriptor.Height5()) + 1u;
-	const auto pitch  = TileGetTexturePitch(descriptor.Format(), width, 1, descriptor.TileMode());
 	const auto type         = static_cast<Prospero::ImageType>(descriptor.Type());
-	const bool supported_single_layer =
+	const bool multisampled = IsMultisampledTexture(type);
+	const auto samples      = multisampled ? 1u << descriptor.LastLevel() : 1u;
+	const auto pitch =
+	    multisampled ? TileGetDepthPitch(width, image.info.bytes_per_block, descriptor.LastLevel())
+	                 : TileGetTexturePitch(descriptor.Format(), width, 1, descriptor.TileMode());
+	const bool supported_2d    = type == Prospero::ImageType::kColor2D &&
 	                             image.info.resources.layers == 1 && descriptor.Depth() == 0 &&
-	    descriptor.BaseArray5() == 0 &&
-	    (type == Prospero::ImageType::kColor2D || type == Prospero::ImageType::kColor2DArray);
+	                             descriptor.BaseArray5() == 0;
+	const bool supported_array = type == Prospero::ImageType::kColor2DArray &&
+	                             descriptor.BaseArray5() <= descriptor.Depth() &&
+	                             descriptor.Depth() < image.info.resources.layers;
 	const bool supported_cube =
 	    type == Prospero::ImageType::kCube && width == height && image.info.resources.layers >= 6 &&
 	    image.info.resources.layers % 6u == 0 &&
 	    static_cast<uint32_t>(descriptor.Depth()) + 1u == image.info.resources.layers &&
 	    descriptor.BaseArray5() == 0;
+	const bool supported_msaa_2d    = type == Prospero::ImageType::kColor2DMsaa &&
+	                                  image.info.resources.layers == 1 && descriptor.Depth() == 0 &&
+	                                  descriptor.BaseArray5() == 0;
+	const bool supported_msaa_array = type == Prospero::ImageType::kColor2DMsaaArray &&
+	                                  descriptor.BaseArray5() <= descriptor.Depth() &&
+	                                  descriptor.Depth() < image.info.resources.layers;
+	const bool levels_ok =
+	    multisampled
+	        ? descriptor.BaseLevel() == 0 && descriptor.LastLevel() >= 1 &&
+	              descriptor.LastLevel() <= 3 && descriptor.MaxMip() == descriptor.LastLevel() &&
+	              image.info.resources.levels == 1 && image.info.samples == samples
+	        : descriptor.BaseLevel() == 0 && descriptor.LastLevel() == 0 &&
+	              descriptor.MaxMip() == 0 && image.info.samples == 1;
 	return image.info.IsDepth() && width == image.info.extent.width &&
-	       height == image.info.extent.height && (supported_single_layer || supported_cube) &&
-	       descriptor.BaseLevel() == 0 && descriptor.LastLevel() == 0 && descriptor.MaxMip() == 0 &&
-	       descriptor.MinLod() == 0 && descriptor.BaseArray5() == 0 &&
+	       height == image.info.extent.height &&
+	       (supported_2d || supported_array || supported_cube || supported_msaa_2d ||
+	        supported_msaa_array) &&
+	       levels_ok && descriptor.MinLod() == 0 &&
 	       descriptor.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) &&
-	       descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth() && pitch >= width &&
-	       pitch == image.info.pitch;
+	       descriptor.BCSwizzle() == 0 && (!descriptor.MsaaDepth() || multisampled) &&
+	       pitch >= width && pitch == image.info.pitch;
 }
 
 bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, const Image& image) {
 	constexpr uint32_t field1_reserved_mask = 0x200fff00u;
 	constexpr uint32_t field2_reserved_mask = 0xf0003000u;
-	constexpr uint32_t field3_common        = 0x01800000u;
-	constexpr uint32_t field5_expected      = 0x00700000u;
-	const uint32_t     field3_expected =
-	    (descriptor.Type() << 28u) | field3_common | descriptor.DstSelXYZW();
+	const uint32_t     field3_expected = descriptor.DstSelXYZW() |
+	                                     (static_cast<uint32_t>(descriptor.BaseLevel()) << 12u) |
+	                                     (static_cast<uint32_t>(descriptor.LastLevel()) << 16u) |
+	                                     (static_cast<uint32_t>(descriptor.TileMode()) << 20u) |
+	                                     (static_cast<uint32_t>(descriptor.Type()) << 28u);
 	const uint32_t     field4_expected = descriptor.Depth() | (descriptor.BaseArray5() << 16u);
+	const uint32_t     field5_expected =
+	    0x00700000u | (static_cast<uint32_t>(descriptor.MaxMip()) << 4u);
 	const bool common = (descriptor.fields[1] & field1_reserved_mask) == 0 &&
 	                    (descriptor.fields[2] & field2_reserved_mask) == 0 &&
 	                    descriptor.fields[3] == field3_expected &&
@@ -251,8 +297,9 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, co
 		return true;
 	}
 	constexpr uint32_t htile_control = 0x00280000u;
+	const uint32_t expected_control  = htile_control | (descriptor.MsaaDepth() ? (1u << 10u) : 0u);
 	const auto     metadata_addr     = descriptor.MetaAddr() << 8u;
-	return (descriptor.fields[6] & 0x00ffffffu) == htile_control && metadata_addr != 0 &&
+	return (descriptor.fields[6] & 0x00ffffffu) == expected_control && metadata_addr != 0 &&
 	       descriptor.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) &&
 	       image.info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) &&
 	       image.info.metadata.kind == ImageMetadataKind::Htile &&
@@ -318,8 +365,8 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	const bool valid_2d_slice =
 	    (is_color_2d && descriptor.Depth() == 0 && descriptor.BaseArray5() == 0) ||
 	    (is_color_2d_array && descriptor.BaseArray5() <= descriptor.Depth());
-	const bool is_2d = resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D &&
-	                   valid_2d_slice;
+	const bool is_2d =
+	    resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && valid_2d_slice;
 	const bool is_2d_array =
 	    resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray &&
 	    is_color_2d_array && descriptor.BaseArray5() <= descriptor.Depth();
@@ -333,17 +380,18 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	    !Prospero::IsFmaskTextureFormat(descriptor.Format()) && (is_2d || is_2d_array) &&
 	    TileGetBlockLayout(TileBlockFamily::Depth64KB, depth_bpe, depth_block);
 	const bool supported_standard_tile =
-	    tile == Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB) &&
-	    TileIsStandard4KBTextureSupported(descriptor.Format());
+	    (tile == Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB) &&
+	     TileIsStandard4KBTextureSupported(descriptor.Format())) ||
+	    (tile == Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB) &&
+	     TileIsStandard64KBTextureSupported(descriptor.Format()));
 	const bool supported_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kLinear) ||
 	                            tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) ||
 	                            supported_depth_tile || supported_standard_tile;
+	const auto swizzle        = descriptor.DstSelXYZW();
 	const bool supported_swizzle =
-	    IsValidImageSwizzle(descriptor.DstSelXYZW()) &&
-	    (descriptor.DstSelXYZW() == DstSel(4, 5, 6, 7) || !resource.read);
-	const bool supported_mip_view = descriptor.BaseLevel() == 0 || is_1d || is_2d;
+	    IsValidImageSwizzle(swizzle) &&
+	    (swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
 	return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
-	       supported_mip_view &&
 	       descriptor.BaseLevel() == descriptor.LastLevel() &&
 	       descriptor.LastLevel() <= descriptor.MaxMip() && descriptor.MinLod() == 0 &&
 	       supported_swizzle && descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth();
@@ -375,8 +423,14 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
 	const bool encoding_ok   = IsSupportedStorageTextureEncoding(descriptor);
 	const bool uint_resource =
 	    resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint;
-	const bool format_ok = Prospero::IsSupportedTextureFormat(format) &&
-	                       uint_resource == Prospero::IsUintTextureFormat(format);
+	const bool raw_sint_storage =
+	    format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32SInt) && uint_resource &&
+	    resource.written && !resource.read && !resource.atomic;
+	const bool format_ok =
+	    raw_sint_storage ||
+	    (Prospero::IsSupportedTextureFormat(format) &&
+	     uint_resource == Prospero::IsUintTextureFormat(format) &&
+	     (!resource.atomic || format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt)));
 	if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
 		return;
 	}
@@ -514,6 +568,7 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource&
 			view.layer_count = 1;
 			break;
 		case ShaderRecompiler::Decoder::ImageDimension::Dim2DArray:
+		case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray:
 			view.type       = vk::ImageViewType::e2DArray;
 			view.base_layer = descriptor.BaseArray5();
 			if (view.base_layer >= image_layers) {
@@ -522,6 +577,7 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource&
 			view.layer_count = image_layers - view.base_layer;
 			break;
 		case ShaderRecompiler::Decoder::ImageDimension::Dim2D:
+		case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa:
 			view.type       = vk::ImageViewType::e2D;
 			view.base_layer = descriptor.BaseArray5();
 			if (view.base_layer >= image_layers) {
@@ -558,19 +614,28 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	const auto base_level   = descriptor.BaseLevel();
 	const auto last_level   = descriptor.LastLevel();
 	const auto type         = TextureType(descriptor);
-	const bool multisampled =
-	    type == Prospero::ImageType::kColor2DMsaa || type == Prospero::ImageType::kColor2DMsaaArray;
+	const bool multisampled = IsMultisampledTexture(type);
 	const auto levels       = multisampled ? 1u : static_cast<uint32_t>(descriptor.MaxMip()) + 1u;
 	const auto tile         = descriptor.TileMode();
-	const bool msaa_tile  = tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget);
+	const bool depth_tile   = tile == Prospero::GpuEnumValue(Prospero::TileMode::kDepth);
+	const bool msaa_tile =
+	    depth_tile || tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget);
 	const bool msaa_array = type == Prospero::ImageType::kColor2DMsaaArray;
 	if ((!multisampled && (base_level > last_level || last_level >= levels)) ||
 	    (multisampled &&
 	     (base_level != 0 || last_level == 0 || last_level > 3 ||
-	      descriptor.MaxMip() != last_level || !msaa_tile || descriptor.MsaaDepth() ||
+	      descriptor.MaxMip() != last_level || !msaa_tile ||
+	      (descriptor.MsaaDepth() && !depth_tile) ||
 	      (!msaa_array && (descriptor.Depth() != 0 || descriptor.BaseArray5() != 0))))) {
-		EXIT("unsupported texture mip view: base=%u last=%u levels=%u\n", base_level, last_level,
-		     levels);
+		EXIT("unsupported texture mip view: base=%u last=%u levels=%u max=%u type=%u tile=%u "
+		     "kind=%u dimension=%u mip_mode=%u read=%d written=%d "
+		     "dwords=%08x,%08x,%08x,%08x,%08x,%08x,%08x,%08x\n",
+		     base_level, last_level, levels, descriptor.MaxMip(), descriptor.Type(), tile,
+		     static_cast<uint32_t>(resource.kind), static_cast<uint32_t>(resource.dimension),
+		     static_cast<uint32_t>(resource.mip_mode), resource.read, resource.written,
+		     descriptor.fields[0], descriptor.fields[1], descriptor.fields[2], descriptor.fields[3],
+		     descriptor.fields[4], descriptor.fields[5], descriptor.fields[6],
+		     descriptor.fields[7]);
 	}
 	const auto samples = multisampled ? 1u << last_level : 1u;
 	const auto view_levels =
@@ -597,7 +662,8 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	TileSizeAlign size {};
 	if (multisampled) {
 		const auto bytes = Prospero::NumBytesPerElement(format);
-		pitch            = TileGetRenderTargetPitch(width, bytes, last_level);
+		pitch            = depth_tile ? TileGetDepthPitch(width, bytes, last_level)
+		                              : TileGetRenderTargetPitch(width, bytes, last_level);
 		if (pitch == 0 || !TileGetRenderTargetSize(width, height, pitch, bytes, size, last_level) ||
 		    size.size > UINT32_MAX / image_layers) {
 			EXIT("unsupported multisample texture layout\n");
@@ -612,14 +678,15 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	                     (address & (static_cast<uint64_t>(size.align) - 1u)) != 0);
 	if (storage) {
 		ValidateStorageTexture(resource, descriptor, size.size);
-		m_context.GetBufferCache().ValidateGpuAccess(address, size.size, resource.read,
-		                                             resource.written);
 	}
 
 	const auto pixel_format = TextureGetFormat(format);
-	const auto              storage_view_format = SrgbStorageViewFormat(pixel_format);
-	const auto              view_format =
-	    storage && storage_view_format != vk::Format::eUndefined ? storage_view_format
+	const auto storage_view_format =
+	    storage && format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32SInt)
+	        ? vk::Format::eR32Uint
+	        : SrgbStorageViewFormat(pixel_format);
+	const auto              view_format = storage && storage_view_format != vk::Format::eUndefined
+	                                          ? storage_view_format
 	                                          : pixel_format;
 	const auto              block_bytes = Prospero::BlockCompressedBytesPerBlock(format);
 	TextureCache::ImageDesc desc {};

src/graphics/host_gpu/renderer/pipeline/descriptors.h

@@ -2,9 +2,9 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORS_H_
 
 #include "common/assert.h"
-#include "graphics/host_gpu/renderer/image.h"
+#include "graphics/host_gpu/renderer/image/image.h"
 #include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/recompiler/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shaderBindings.h"
 
 #include <cstdint>

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -1,4 +1,4 @@
-#include "graphics/host_gpu/renderer/pipelineCache.h"
+#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 
 #include "common/assert.h"
 #include "common/logging/log.h"
@@ -7,7 +7,7 @@
 #include "graphics/host_gpu/renderer/colorRenderTarget.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
-#include "graphics/host_gpu/renderer/imageView.h"
+#include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 
@@ -116,8 +116,8 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 		if (attachment_samples == 0) {
 			attachment_samples = depth.samples;
 		} else if (attachment_samples != depth.samples) {
-			EXIT("mixed color/depth sample counts are unsupported: %u and %u\n",
-			     attachment_samples, depth.samples);
+			EXIT("mixed color/depth sample counts are unsupported: %u and %u\n", attachment_samples,
+			     depth.samples);
 		}
 	}
 	EXIT_IF(attachment_samples == 0 ||
@@ -154,8 +154,9 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 	for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) {
 		static_params.color_mask[i] = color_mask[i];
 	}
-	static_params.cull_back  = mc.cull_back;
-	static_params.cull_front = mc.cull_front;
+	const bool rect_list     = topology == vk::PrimitiveTopology::ePatchList;
+	static_params.cull_back  = !rect_list && mc.cull_back;
+	static_params.cull_front = !rect_list && mc.cull_front;
 	static_params.face       = mc.face;
 
 	for (uint32_t i = 0; i < color_count; i++) {
@@ -203,9 +204,8 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 	LogPipelineTrace("CreatePipelineInternal begin", vs_id.hash0, vs_id.crc32, ps_id.hash0,
 	                 ps_id.crc32);
 	CreatePipelineInternal(m_graphics, m_descriptor_cache, *cached, rendering, vs_input_info,
-	                       vs_spirv, ps_input_info,
-	                       ps_spirv, static_params, vs_id.hash0, vs_id.crc32, ps_id.hash0,
-	                       ps_id.crc32, ps_active);
+	                       vs_spirv, ps_input_info, ps_spirv, static_params, vs_id.hash0,
+	                       vs_id.crc32, ps_id.hash0, ps_id.crc32, ps_active);
 	LogPipelineTrace("CreatePipelineInternal done", vs_id.hash0, vs_id.crc32, ps_id.hash0,
 	                 ps_id.crc32);
 

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -118,11 +118,12 @@ public:
 		ShaderId cs_shader_id;
 	};
 
-	GraphicsPipeline& CreateGraphicsPipeline(
-	    RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth,
+	GraphicsPipeline&
+	CreateGraphicsPipeline(RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth,
 	                       ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command,
-	    ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology, bool ps_active,
-	    std::span<const uint32_t> vs_spirv, std::span<const uint32_t> ps_spirv);
+	                       ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
+	                       bool ps_active, std::span<const uint32_t> vs_spirv,
+	                       std::span<const uint32_t> ps_spirv);
 	ComputePipeline& CreateComputePipeline(ShaderComputeInputInfo&      input_info,
 	                                       const HW::ComputeShaderInfo& cs_regs,
 	                                       std::span<const uint32_t>    cs_spirv);
@@ -211,16 +212,13 @@ private:
 
 void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0,
                       uint32_t ps_crc32);
-void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache,
-                            PipelineCache::GraphicsPipeline& pipeline,
-                            const PipelineRenderingState&   rendering,
-                            const ShaderVertexInputInfo&    vs_input_info,
-                            std::span<const uint32_t>       vs_shader,
-                            const ShaderPixelInputInfo*     ps_input_info,
-                            std::span<const uint32_t>       ps_shader,
-                            const PipelineStaticParameters& static_params, uint32_t vs_hash0,
-                            uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32,
-                            bool ps_active);
+void CreatePipelineInternal(
+    GraphicContext& graphics, DescriptorCache& descriptor_cache,
+    PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering,
+    const ShaderVertexInputInfo& vs_input_info, std::span<const uint32_t> vs_shader,
+    const ShaderPixelInputInfo* ps_input_info, std::span<const uint32_t> ps_shader,
+    const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32,
+    uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active);
 void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache,
                             PipelineCache::ComputePipeline& pipeline,
                             const ShaderComputeInputInfo&   input_info,

src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp

@@ -1,4 +1,4 @@
-#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 
 #include "common/assert.h"
 #include "graphics/shader/shader.h"

src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h

@@ -2,7 +2,7 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERRESOURCEBARRIER_H_
 
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/shader/recompiler/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 
 #include <vector>
 

src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp

@@ -1,6 +1,6 @@
-#include "graphics/host_gpu/renderer/shaderSubgroup.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 
-#include "graphics/shader/recompiler/SpirvEmitter.h"
+#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
 
 namespace Libs::Graphics {
 

src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.h

@@ -2,7 +2,7 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
 
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/shader/recompiler/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 
 namespace Libs::Graphics {
 

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -6,14 +6,15 @@
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/debug.h"
-#include "graphics/host_gpu/renderer/descriptorCache.h"
-#include "graphics/host_gpu/renderer/pipelineCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
-#include "graphics/host_gpu/renderer/shaderSubgroup.h"
 #include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/recompiler/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/rectListShader.h"
 #include "graphics/shader/shader.h"
 
 #include <algorithm>
@@ -386,8 +387,7 @@ static vk::BlendOp GetBlendOp(uint32_t op) {
 }
 
 static void CreateLayout(DescriptorCache&                   descriptor_cache,
-                         std::span<vk::DescriptorSetLayout> set_layouts,
-                         uint32_t& set_layouts_num,
+                         std::span<vk::DescriptorSetLayout> set_layouts, uint32_t& set_layouts_num,
                          std::span<vk::PushConstantRange>     push_constant_info,
                          uint32_t&                            push_constant_info_num,
                          const ShaderRecompiler::IR::Program& program,
@@ -412,8 +412,7 @@ static void CreateLayout(DescriptorCache& descriptor_cache,
 	}
 }
 
-static void ConfigureSubgroupSize(const GraphicContext&                                  graphics,
-                                  vk::ShaderStageFlagBits                                vk_stage,
+static void ConfigureSubgroupSize(const GraphicContext& graphics, vk::ShaderStageFlagBits vk_stage,
                                   const ShaderRecompiler::IR::Program&                   program,
                                   vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo& required,
                                   vk::PipelineShaderStageCreateInfo&                     stage) {
@@ -456,19 +455,20 @@ static void ConfigureSubgroupSize(const GraphicContext&
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache,
-                            PipelineCache::GraphicsPipeline& pipeline,
-                            const PipelineRenderingState&   rendering,
-                            const ShaderVertexInputInfo&    vs_input_info,
-                            std::span<const uint32_t>       vs_shader,
-                            const ShaderPixelInputInfo*     ps_input_info,
-                            std::span<const uint32_t>       ps_shader,
-                            const PipelineStaticParameters& static_params, uint32_t vs_hash0,
-                            uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32,
-                            bool ps_active) {
+void CreatePipelineInternal(
+    GraphicContext& graphics, DescriptorCache& descriptor_cache,
+    PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering,
+    const ShaderVertexInputInfo& vs_input_info, std::span<const uint32_t> vs_shader,
+    const ShaderPixelInputInfo* ps_input_info, std::span<const uint32_t> ps_shader,
+    const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32,
+    uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active) {
 	EXIT_IF(ps_active && ps_input_info == nullptr);
 
+	const bool rect_list = static_params.topology == vk::PrimitiveTopology::ePatchList;
+
 	vk::ShaderModule vert_shader_module         = nullptr;
+	vk::ShaderModule tess_control_shader_module = nullptr;
+	vk::ShaderModule tess_eval_shader_module    = nullptr;
 	vk::ShaderModule frag_shader_module         = nullptr;
 
 	vk::ShaderModuleCreateInfo create_info {};
@@ -496,8 +496,33 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 		}
 		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 	}
+	if (rect_list) {
+		const auto shaders =
+		    BuildRectListShaders(vs_input_info, ps_active ? ps_input_info : nullptr);
+		create_info.codeSize = shaders.control.size() * 4;
+		create_info.pCode    = shaders.control.data();
+		result =
+		    graphics.device.createShaderModule(&create_info, nullptr, &tess_control_shader_module);
+		if (graphics_debug_dump_enabled()) {
+			LOGF("PipelineTrace: vkCreateShaderModule RectList TCS done result=%s module=%p\n",
+			     VulkanToString(result).c_str(), static_cast<void*>(tess_control_shader_module));
+		}
+		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
+
+		create_info.codeSize = shaders.evaluation.size() * 4;
+		create_info.pCode    = shaders.evaluation.data();
+		result =
+		    graphics.device.createShaderModule(&create_info, nullptr, &tess_eval_shader_module);
+		if (graphics_debug_dump_enabled()) {
+			LOGF("PipelineTrace: vkCreateShaderModule RectList TES done result=%s module=%p\n",
+			     VulkanToString(result).c_str(), static_cast<void*>(tess_eval_shader_module));
+		}
+		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
+	}
 
 	EXIT_NOT_IMPLEMENTED(vert_shader_module == nullptr);
+	EXIT_NOT_IMPLEMENTED(
+	    rect_list && (tess_control_shader_module == nullptr || tess_eval_shader_module == nullptr));
 	EXIT_NOT_IMPLEMENTED(ps_active && frag_shader_module == nullptr);
 
 	vk::PipelineShaderStageCreateInfo                     vert_shader_stage_info {};
@@ -511,8 +536,7 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 	vert_shader_stage_info.pName               = "main";
 	vert_shader_stage_info.pSpecializationInfo = nullptr;
 	EXIT_IF(!vs_input_info.stage);
-	ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eVertex,
-	                      *vs_input_info.stage.program,
+	ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eVertex, *vs_input_info.stage.program,
 	                      vert_subgroup_size, vert_shader_stage_info);
 
 	vk::PipelineShaderStageCreateInfo                     frag_shader_stage_info {};
@@ -527,13 +551,32 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 	if (ps_active) {
 		EXIT_IF(!ps_input_info->stage);
 		ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eFragment,
-		                      *ps_input_info->stage.program,
-		                      frag_subgroup_size, frag_shader_stage_info);
+		                      *ps_input_info->stage.program, frag_subgroup_size,
+		                      frag_shader_stage_info);
 	}
 
-	vk::PipelineShaderStageCreateInfo shader_stages[]    = {vert_shader_stage_info,
-	                                                        frag_shader_stage_info};
-	const uint32_t                    shader_stage_count = ps_active ? 2u : 1u;
+	vk::PipelineShaderStageCreateInfo tess_control_shader_stage_info {};
+	tess_control_shader_stage_info.sType  = vk::StructureType::ePipelineShaderStageCreateInfo;
+	tess_control_shader_stage_info.stage  = vk::ShaderStageFlagBits::eTessellationControl;
+	tess_control_shader_stage_info.module = tess_control_shader_module;
+	tess_control_shader_stage_info.pName  = "main";
+
+	vk::PipelineShaderStageCreateInfo tess_eval_shader_stage_info {};
+	tess_eval_shader_stage_info.sType  = vk::StructureType::ePipelineShaderStageCreateInfo;
+	tess_eval_shader_stage_info.stage  = vk::ShaderStageFlagBits::eTessellationEvaluation;
+	tess_eval_shader_stage_info.module = tess_eval_shader_module;
+	tess_eval_shader_stage_info.pName  = "main";
+
+	vk::PipelineShaderStageCreateInfo shader_stages[4]   = {};
+	uint32_t                          shader_stage_count = 0;
+	shader_stages[shader_stage_count++]                  = vert_shader_stage_info;
+	if (rect_list) {
+		shader_stages[shader_stage_count++] = tess_control_shader_stage_info;
+		shader_stages[shader_stage_count++] = tess_eval_shader_stage_info;
+	}
+	if (ps_active) {
+		shader_stages[shader_stage_count++] = frag_shader_stage_info;
+	}
 
 	vk::VertexInputAttributeDescription input_attr[ShaderVertexInputInfo::RES_MAX];
 	vk::VertexInputBindingDescription   input_desc[ShaderVertexInputInfo::RES_MAX];
@@ -838,15 +881,13 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 
 	EXIT_IF(!vs_input_info.stage);
 	CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
-	             push_constant_info_num,
-	             *vs_input_info.stage.program, vk::ShaderStageFlagBits::eVertex,
-	             DescriptorCache::Stage::Vertex);
+	             push_constant_info_num, *vs_input_info.stage.program,
+	             vk::ShaderStageFlagBits::eVertex, DescriptorCache::Stage::Vertex);
 	if (ps_active) {
 		EXIT_IF(!ps_input_info->stage);
 		CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
-		             push_constant_info_num,
-		             *ps_input_info->stage.program, vk::ShaderStageFlagBits::eFragment,
-		             DescriptorCache::Stage::Pixel);
+		             push_constant_info_num, *ps_input_info->stage.program,
+		             vk::ShaderStageFlagBits::eFragment, DescriptorCache::Stage::Pixel);
 	}
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
@@ -937,7 +978,10 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 	pipeline_info.pStages                  = shader_stages;
 	pipeline_info.pVertexInputState        = &vertex_input_info;
 	pipeline_info.pInputAssemblyState      = &input_assembly;
-	pipeline_info.pTessellationState  = nullptr;
+	vk::PipelineTessellationStateCreateInfo tessellation_state {};
+	tessellation_state.sType              = vk::StructureType::ePipelineTessellationStateCreateInfo;
+	tessellation_state.patchControlPoints = 3;
+	pipeline_info.pTessellationState      = (rect_list ? &tessellation_state : nullptr);
 	pipeline_info.pViewportState          = &viewport_state;
 	pipeline_info.pRasterizationState     = &rasterizer;
 	pipeline_info.pMultisampleState       = &multisampling;
@@ -976,6 +1020,12 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 	if (frag_shader_module != nullptr) {
 		graphics.device.destroyShaderModule(frag_shader_module, nullptr);
 	}
+	if (tess_control_shader_module != nullptr) {
+		graphics.device.destroyShaderModule(tess_control_shader_module, nullptr);
+	}
+	if (tess_eval_shader_module != nullptr) {
+		graphics.device.destroyShaderModule(tess_eval_shader_module, nullptr);
+	}
 	graphics.device.destroyShaderModule(vert_shader_module, nullptr);
 }
 
@@ -1012,8 +1062,7 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 	comp_shader_stage_info.pName               = "main";
 	comp_shader_stage_info.pSpecializationInfo = nullptr;
 	EXIT_IF(!input_info.stage);
-	ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eCompute,
-	                      *input_info.stage.program,
+	ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eCompute, *input_info.stage.program,
 	                      comp_subgroup_size, comp_shader_stage_info);
 
 	vk::DescriptorSetLayout set_layouts[1]  = {};
@@ -1024,9 +1073,8 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 
 	EXIT_IF(!input_info.stage);
 	CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
-	             push_constant_info_num,
-	             *input_info.stage.program, vk::ShaderStageFlagBits::eCompute,
-	             DescriptorCache::Stage::Compute);
+	             push_constant_info_num, *input_info.stage.program,
+	             vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute);
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
 	pipeline_layout_info.sType                  = vk::StructureType::ePipelineLayoutCreateInfo;

src/graphics/host_gpu/renderer/render.h

@@ -4,7 +4,7 @@
 #include "common/abi.h"
 #include "common/assert.h"
 #include "common/common.h"
-#include "graphics/host_gpu/renderer/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -10,17 +10,17 @@
 #include "graphics/guest_gpu/graphicsRun.h"
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/renderer/descriptorCache.h"
-#include "graphics/host_gpu/renderer/descriptors.h"
-#include "graphics/host_gpu/renderer/imageInfo.h"
-#include "graphics/host_gpu/renderer/pipelineCache.h"
+#include "graphics/host_gpu/renderer/image/imageInfo.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
+#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
-#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
-#include "graphics/host_gpu/renderer/shaderSubgroup.h"
 #include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/recompiler/ResourceMaterialization.h"
-#include "graphics/shader/recompiler/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shader.h"
 #include "kernel/eventQueue.h"
 #include "kernel/pthread.h"

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -14,8 +14,7 @@ namespace Libs::Graphics {
 RenderContext::RenderContext(GraphicContext& graphics)
     : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
       m_descriptor_cache(graphics), m_pipeline_cache(graphics, m_descriptor_cache),
-      m_sampler_cache(graphics),
-      m_gpu_resources(graphics, m_command_scheduler) {
+      m_sampler_cache(graphics), m_gpu_resources(graphics, m_command_scheduler) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 }
 
@@ -99,8 +98,7 @@ void RenderContext::TriggerEopEvent(uint32_t context_id) {
 		    registration.eq, static_cast<uintptr_t>(registration.id),
 		    LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS,
 		    reinterpret_cast<void*>(static_cast<uintptr_t>(context_id)));
-		if (result == LibKernel::KERNEL_ERROR_EBADF ||
-		    result == LibKernel::KERNEL_ERROR_ENOENT) {
+		if (result == LibKernel::KERNEL_ERROR_EBADF || result == LibKernel::KERNEL_ERROR_ENOENT) {
 			DeleteEopEq(registration.eq, registration.id);
 			continue;
 		}

src/graphics/host_gpu/renderer/renderContext.h

@@ -5,13 +5,13 @@
 #include "common/assert.h"
 #include "common/common.h"
 #include "common/threads.h"
-#include "graphics/host_gpu/renderer/bufferCache.h"
+#include "graphics/host_gpu/renderer/cache/bufferCache.h"
+#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
+#include "graphics/host_gpu/renderer/cache/samplerCache.h"
+#include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
-#include "graphics/host_gpu/renderer/descriptorCache.h"
-#include "graphics/host_gpu/renderer/gpuResourceManager.h"
-#include "graphics/host_gpu/renderer/pipelineCache.h"
-#include "graphics/host_gpu/renderer/samplerCache.h"
-#include "graphics/host_gpu/renderer/textureCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "kernel/eventQueue.h"
 
 #include <memory>

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -2,7 +2,6 @@
 
 #include "common/assert.h"
 #include "common/common.h"
-#include "common/emulatorConfig.h"
 #include "common/file.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
@@ -16,17 +15,18 @@
 #include "graphics/host_gpu/renderer/colorRenderTarget.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
-#include "graphics/host_gpu/renderer/descriptorCache.h"
-#include "graphics/host_gpu/renderer/pipelineCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
-#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
-#include "graphics/host_gpu/renderer/shaderSubgroup.h"
 #include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/recompiler/ResourceMaterialization.h"
-#include "graphics/shader/recompiler/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shader.h"
 #include "kernel/eventQueue.h"
+#include "kernel/memory.h"
 #include "kernel/pthread.h"
 #include "libs/errno.h"
 
@@ -221,8 +221,7 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col
 	LogMrtState(draw_name, buffer, ps_input_info);
 }
 
-static void LogDrawInputState(const RenderCommandBuffer&   buffer,
-                               const RenderColorInfo&       color,
+static void LogDrawInputState(const RenderCommandBuffer& buffer, const RenderColorInfo& color,
                               const ShaderVertexInputInfo& vs_input_info,
                               uint32_t index_type_and_size, uint32_t index_count,
                               const void* index_addr) {
@@ -512,8 +511,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		auto& target = colors[i];
 		EXIT_IF(!target.image_id);
 		const auto old_image = cache.ResolveOwner(target.image_id);
-		if (old_image == nullptr ||
-		    (!old_image->registered && !old_image->info.data.Empty()) ||
+		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
 		    old_image->binding.needs_rebind) {
 			if (old_image != nullptr) {
 				old_image->binding = {};
@@ -531,8 +529,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 			     attachment_samples, target.samples);
 		}
 		const auto& view   = target.desc.view_info;
-		const auto layout =
-		    image.binding.is_bound ? vk::ImageLayout::eGeneral
+		const auto  layout = image.binding.is_bound ? vk::ImageLayout::eGeneral
 		                                            : vk::ImageLayout::eColorAttachmentOptimal;
 		image.Transit(layout,
 		              vk::AccessFlagBits2::eColorAttachmentRead |
@@ -561,8 +558,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		depth.depth_meta_clear_enable =
 		    depth.htile &&
 		    cache.IsMetaCleared(depth.htile_buffer_vaddr, depth.desc.view_info.base_layer);
-		depth.depth_load_clear_enable =
-		    depth.depth_clear_enable || depth.depth_meta_clear_enable;
+		depth.depth_load_clear_enable = depth.depth_clear_enable || depth.depth_meta_clear_enable;
 		if (depth.depth_meta_clear_enable &&
 		    !cache.TouchMeta(depth.htile_buffer_vaddr, depth.desc.view_info.base_layer, false)) {
 			EXIT("failed to consume HTile clear state\n");
@@ -572,8 +568,8 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		if (attachment_samples == 0) {
 			attachment_samples = depth.samples;
 		} else if (attachment_samples != depth.samples) {
-			EXIT("mixed color/depth sample counts are unsupported: %u and %u\n",
-			     attachment_samples, depth.samples);
+			EXIT("mixed color/depth sample counts are unsupported: %u and %u\n", attachment_samples,
+			     depth.samples);
 		}
 		const auto layout = depth_attachment_layout(depth);
 		const auto writes = depth.AttachmentWriteAspects();
@@ -595,11 +591,9 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		attachment.image_layout   = layout;
 		attachment.clear_value[0] = std::bit_cast<uint32_t>(depth.depth_clear_value);
 		attachment.clear_value[1] = depth.stencil_clear_value;
-		attachment.has_depth =
-		    static_cast<bool>(aspects & vk::ImageAspectFlagBits::eDepth);
+		attachment.has_depth      = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eDepth);
 		attachment.depth_clear    = depth.depth_load_clear_enable;
-		attachment.has_stencil =
-		    static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
+		attachment.has_stencil    = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
 		attachment.stencil_clear  = depth.stencil_clear_enable;
 	}
 	if (attachment_samples == 0 ||
@@ -656,8 +650,6 @@ static bool ConsumeMetadataColorOperation(const RenderCommandBuffer& buffer) {
 
 struct DrawEmitInfo {
 	bool     indexed       = false;
-	bool     draw_prim7_as_ngg = false;
-	uint32_t draw_vertex_count = 0;
 	int32_t  vertex_offset = 0;
 	uint32_t first_vertex  = 0;
 };
@@ -685,6 +677,85 @@ static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer
 	                           : buffer.num_records);
 }
 
+struct VertexBufferRange {
+	uint64_t      base_address  = 0;
+	uint64_t      requested_end = 0;
+	uint64_t      acquired_end  = 0;
+	BufferBinding binding;
+
+	[[nodiscard]] uint64_t RequestedSize() const { return requested_end - base_address; }
+};
+
+static std::vector<BufferBinding> AcquireVertexBuffers(RenderCommandBuffer&         buffer,
+                                                       const ShaderVertexInputInfo& vs_input_info) {
+	// Collect the non-empty guest vertex ranges.
+	std::vector<VertexBufferRange> ranges;
+	ranges.reserve(vs_input_info.buffers_num);
+	for (int i = 0; i < vs_input_info.buffers_num; i++) {
+		const auto& vertex = vs_input_info.buffers[i];
+		const auto  size   = VertexBufferDescriptorSize(vertex);
+		if (size == 0) {
+			continue;
+		}
+		if (vertex.addr == 0 || size > UINT64_MAX - vertex.addr) {
+			EXIT("invalid vertex buffer range: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+			     vertex.addr, size);
+		}
+		ranges.push_back({vertex.addr, vertex.addr + size});
+	}
+
+	std::ranges::sort(ranges, [](const VertexBufferRange& left, const VertexBufferRange& right) {
+		return left.base_address < right.base_address;
+	});
+
+	// Merge overlapping or touching ranges before acquiring host buffers.
+	std::vector<VertexBufferRange> merged_ranges;
+	merged_ranges.reserve(ranges.size());
+	for (const auto& range: ranges) {
+		if (!merged_ranges.empty() && merged_ranges.back().requested_end >= range.base_address) {
+			merged_ranges.back().requested_end =
+			    std::max(merged_ranges.back().requested_end, range.requested_end);
+			continue;
+		}
+		merged_ranges.push_back(range);
+	}
+
+	auto& cache = buffer.GetContext().GetBufferCache();
+	for (auto& range: merged_ranges) {
+		// PPSA20298
+		const auto size =
+		    Libs::LibKernel::Memory::ClampRangeSize(range.base_address, range.RequestedSize());
+		range.acquired_end = range.base_address + size;
+		range.binding      = cache.ObtainBuffer(buffer, range.base_address, size);
+	}
+
+	// Rebuild slot bindings, offsetting non-empty slots into their acquired merged range.
+	std::vector<BufferBinding> bindings;
+	bindings.reserve(vs_input_info.buffers_num);
+	for (int i = 0; i < vs_input_info.buffers_num; i++) {
+		const auto& vertex = vs_input_info.buffers[i];
+		const auto  size   = VertexBufferDescriptorSize(vertex);
+		if (size == 0) {
+			auto owner = cache.ObtainNullBuffer();
+			bindings.push_back({owner, owner->Handle(), 0});
+			continue;
+		}
+
+		const auto range = std::ranges::find_if(merged_ranges, [&](const VertexBufferRange& value) {
+			return vertex.addr >= value.base_address && vertex.addr < value.acquired_end;
+		});
+		if (range == merged_ranges.end()) {
+			EXIT("vertex buffer address is outside the acquired range: addr=0x%016" PRIx64 "\n",
+			     vertex.addr);
+		}
+
+		auto binding = range->binding;
+		binding.offset += vertex.addr - range->base_address;
+		bindings.push_back(std::move(binding));
+	}
+	return bindings;
+}
+
 static void SetDrawDebugPhase(RenderCommandBuffer& buffer, uint64_t submit_id,
                               const DrawCallInfo& draw, uint32_t phase) {
 	EXIT_IF(draw.name == nullptr);
@@ -693,7 +764,7 @@ static void SetDrawDebugPhase(RenderCommandBuffer& buffer, uint64_t submit_id,
 	                    draw.flags, draw.instance_count, draw.first_instance);
 }
 
-static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw, bool use_ngg_rectlist_draw,
+static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw,
                             vk::PrimitiveTopology& topology) {
 
 	topology = vk::PrimitiveTopology::ePointList;
@@ -717,8 +788,7 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw, bool use
 			topology = vk::PrimitiveTopology::eTriangleStrip;
 			break;
 		case Prospero::PrimitiveType::kRectList:
-			topology = (auto_draw && use_ngg_rectlist_draw ? vk::PrimitiveTopology::eTriangleStrip
-			                                               : vk::PrimitiveTopology::eTriangleList);
+			topology = vk::PrimitiveTopology::ePatchList;
 			break;
 		case Prospero::PrimitiveType::kRectListLegacy:
 			if (!auto_draw) {
@@ -823,37 +893,13 @@ static std::vector<BufferBinding> PrepareVertexBuffers(uint64_t
 	(void)submit_id;
 
 	LogDrawPhase(draw.name, "PrepareVertexBuffers");
-	std::vector<BufferBinding> bindings;
-	bindings.reserve(vs_input_info.buffers_num);
-	for (int i = 0; i < vs_input_info.buffers_num; i++) {
-		const auto& b    = vs_input_info.buffers[i];
-		const auto  size = VertexBufferDescriptorSize(b);
-		if (size == 0) {
-			auto owner = buffer.GetContext().GetBufferCache().ObtainNullBuffer();
-			bindings.push_back({owner, owner->Handle(), 0});
-		} else {
-			bindings.push_back(
-			    buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, b.addr, size));
-		}
-	}
-	return bindings;
+	return AcquireVertexBuffers(buffer, vs_input_info);
 }
 
 static void RebindVertexBuffers(RenderCommandBuffer&         buffer,
                                 const ShaderVertexInputInfo& vs_input_info,
                                 std::vector<BufferBinding>&  bindings) {
-	EXIT_IF(bindings.size() != static_cast<size_t>(vs_input_info.buffers_num));
-	for (int i = 0; i < vs_input_info.buffers_num; i++) {
-		const auto& vertex = vs_input_info.buffers[i];
-		const auto  size   = VertexBufferDescriptorSize(vertex);
-		if (size == 0) {
-			auto owner  = buffer.GetContext().GetBufferCache().ObtainNullBuffer();
-			bindings[i] = {owner, owner->Handle(), 0};
-		} else {
-			bindings[i] =
-			    buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, vertex.addr, size);
-		}
-	}
+	bindings = AcquireVertexBuffers(buffer, vs_input_info);
 }
 
 static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer&         buffer,
@@ -941,20 +987,6 @@ static void LogDrawStateIfNeeded(const RenderCommandBuffer& buffer, const DrawCa
 	// LogDrawTextureState(draw.name, state.color_info[0], state.ps_input_info);
 }
 
-static bool IsHostExpandedRectListDrawSupported(const ShaderVertexInputInfo& vs_input_info,
-                                                const DrawCallInfo&          draw,
-                                                const DrawEmitInfo&          emit) {
-	if (!emit.draw_prim7_as_ngg) {
-		return true;
-	}
-
-	if (vs_input_info.buffers_num != 0) {
-		return false;
-	}
-
-	return draw.index_count == 3 || draw.index_count == emit.draw_vertex_count;
-}
-
 static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_buffer,
                                const ShaderVertexInputInfo& vs_input_info, const DrawCallInfo& draw,
                                const DrawEmitInfo& emit) {
@@ -967,22 +999,12 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 		case Prospero::PrimitiveType::kTriList:
 		case Prospero::PrimitiveType::kTriFan:
 		case Prospero::PrimitiveType::kTriStrip:
-			if (emit.indexed) {
-				vk_buffer.drawIndexed(draw.index_count, draw.instance_count, 0, emit.vertex_offset,
-				                      draw.first_instance);
-			} else {
-				vk_buffer.draw(draw.index_count, draw.instance_count, emit.first_vertex,
-				               draw.first_instance);
-			}
-			break;
 		case Prospero::PrimitiveType::kRectList:
 			if (emit.indexed) {
 				vk_buffer.drawIndexed(draw.index_count, draw.instance_count, 0, emit.vertex_offset,
 				                      draw.first_instance);
 			} else {
-				EXIT_NOT_IMPLEMENTED(
-				    !IsHostExpandedRectListDrawSupported(vs_input_info, draw, emit));
-				vk_buffer.draw(emit.draw_vertex_count, draw.instance_count, emit.first_vertex,
+				vk_buffer.draw(draw.index_count, draw.instance_count, emit.first_vertex,
 				               draw.first_instance);
 			}
 			break;
@@ -1109,7 +1131,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 	                    reinterpret_cast<uint64_t>(index_addr));
 
 	Common::LockGuard lock(m_context.GetMutex());
-	if (index_count == 0) {
+	if (index_count == 0 || instance_count == 0) {
 		return;
 	}
 
@@ -1151,7 +1173,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 	hw_check(buffer);
 
 	vk::PrimitiveTopology topology = vk::PrimitiveTopology::ePointList;
-	if (!GetDrawTopology(ucfg, false, false, topology)) {
+	if (!GetDrawTopology(ucfg, false, topology)) {
 		return;
 	}
 
@@ -1179,10 +1201,6 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 
 	EXIT_NOT_IMPLEMENTED(flags != 0);
 	EXIT_NOT_IMPLEMENTED(type != 1);
-	if (instance_count == 0) {
-		instance_count = 1;
-	}
-
 	const DrawCallInfo    draw {"DrawIndex",    CommandBufferDebugOp::DrawIndex,
 	                            index_count,    flags,
 	                            instance_count, first_instance};
@@ -1228,8 +1246,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
-                              uint32_t index_count,
+void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_count,
                               uint32_t flags, uint32_t render_target_slice_offset,
                               uint32_t instance_count, uint32_t first_vertex,
                               uint32_t first_instance) {
@@ -1243,7 +1260,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
 	                    index_count, flags, first_vertex, instance_count, first_instance);
 
 	Common::LockGuard lock(m_context.GetMutex());
-	if (index_count == 0) {
+	if (index_count == 0 || instance_count == 0) {
 		return;
 	}
 
@@ -1281,34 +1298,26 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
 	hw_check(buffer);
 
 	EXIT_NOT_IMPLEMENTED(flags != 0);
-	if (instance_count == 0) {
-		instance_count = 1;
-	}
-
 	const DrawCallInfo draw {"DrawIndexAuto", CommandBufferDebugOp::DrawIndexAuto,
 	                         index_count,     flags,
 	                         instance_count,  first_instance};
 
 	DrawRenderState state {};
-	if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, false, state)) {
+	if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, false,
+	                            state)) {
 		ResetBindings();
 		return;
 	}
 
 	vk::PrimitiveTopology topology = vk::PrimitiveTopology::ePointList;
-	const bool            use_ngg_rectlist_draw = Config::NggRectlistDrawEnabled();
-
-	if (!GetDrawTopology(ucfg, true, use_ngg_rectlist_draw, topology)) {
+	if (!GetDrawTopology(ucfg, true, topology)) {
 		ResetBindings();
 		return;
 	}
-	const bool draw_prim7_as_ngg =
-	    (use_ngg_rectlist_draw &&
-	     ucfg.GetPrimType() == Prospero::GpuEnumValue(Prospero::PrimitiveType::kRectList));
-
 	RefreshShaders(buffer, draw, false, state);
 
-	if (draw_prim7_as_ngg && state.vs_input_info.buffers_num == 0 &&
+	const bool rect_list = topology == vk::PrimitiveTopology::ePatchList;
+	if (rect_list && state.vs_input_info.buffers_num == 0 &&
 	    state.vs_input_info.param_export_mask == 0 && state.ps_input_info.input_num != 0) {
 		if (graphics_debug_dump_enabled()) {
 			LOGF("DrawIndexAuto: skipping rect-list draw with no VS param exports and PS inputs: "
@@ -1325,12 +1334,9 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
 	                         Prospero::GpuEnumValue(Prospero::PrimitiveType::kRectListLegacy),
 	                     0, nullptr);
 
-	const uint32_t draw_vertex_count = (draw_prim7_as_ngg ? 4u : index_count);
 	const auto   vertex_offset = ResolveVertexOffset(ucfg.GetIndexOffset(), state.vs_input_info) +
 	                             static_cast<int32_t>(first_vertex);
 	DrawEmitInfo emit {};
-	emit.draw_prim7_as_ngg = draw_prim7_as_ngg;
-	emit.draw_vertex_count = draw_vertex_count;
 	emit.first_vertex = static_cast<uint32_t>(vertex_offset);
 
 	DrawIndexBufferSource index_source {};
@@ -1369,8 +1375,7 @@ bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer
 	cache.MarkGpuWritten(dst.image_id);
 	auto& source      = cache.GetImage(src.image_id);
 	auto& destination = cache.GetImage(dst.image_id);
-	destination.Resolve(source,
-	                    {src.base_mip_level, 1, src.base_array_layer, 1},
+	destination.Resolve(source, {src.base_mip_level, 1, src.base_array_layer, 1},
 	                    {dst.base_mip_level, 1, dst.base_array_layer, 1});
 	return true;
 }

src/graphics/host_gpu/renderer/sync.cpp

@@ -5,7 +5,7 @@
 #include "common/logging/log.h"
 #include "common/threads.h"
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/renderer/bufferCache.h"
+#include "graphics/host_gpu/renderer/cache/bufferCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/presentation/videoOut.h"
@@ -252,8 +252,8 @@ uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index, int f
 	for (;;) {
 		uint64_t   request_id = 0;
 		auto&      video_out  = buffer.GetContext().GetVideoOut();
-		const auto result = video_out.SubmitFlipFromGpu(
-		    buffer, handle, index, flip_mode, flip_arg, request_id);
+		const auto result =
+		    video_out.SubmitFlipFromGpu(buffer, handle, index, flip_mode, flip_arg, request_id);
 		if (result == OK) {
 			EXIT_IF(request_id == 0);
 			return request_id;

src/graphics/host_gpu/vma.cpp

@@ -122,8 +122,8 @@ uint64_t GraphicContext::GetDeviceMemoryUsage() const {
 	    physical_device_properties.deviceType == vk::PhysicalDeviceType::eDiscreteGpu;
 	uint64_t usage = 0;
 	for (uint32_t heap = 0; heap < physical_device_memory_properties.memoryHeapCount; heap++) {
-		const bool device_local = static_cast<bool>(
-		    physical_device_memory_properties.memoryHeaps[heap].flags &
+		const bool device_local =
+		    static_cast<bool>(physical_device_memory_properties.memoryHeaps[heap].flags &
 		                      vk::MemoryHeapFlagBits::eDeviceLocal);
 		if (!discrete || device_local) {
 			usage += budgets[heap].usage;
@@ -160,8 +160,7 @@ uint64_t GraphicContext::GetTotalMemoryBudget() const {
 	}
 	constexpr uint64_t system_reserve = 8ull * 1024 * 1024 * 1024;
 	const auto         available      = budget > usage ? budget - usage : uint64_t {0};
-	return std::max(local,
-	                available > system_reserve ? available - system_reserve : uint64_t {0});
+	return std::max(local, available > system_reserve ? available - system_reserve : uint64_t {0});
 }
 
 void GraphicContext::CreateBuffer(uint64_t size, VulkanBuffer& buffer) {

src/graphics/host_gpu/vulkanCommon.cpp

@@ -37,6 +37,7 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k16_16Float, vk::Format::eR16G16Sfloat},
     {Prospero::BufferFormat::k11_11_10Float, vk::Format::eB10G11R11UfloatPack32},
     {Prospero::BufferFormat::k10_10_10_2UNorm, vk::Format::eA2B10G10R10UnormPack32},
+    {Prospero::BufferFormat::k10_10_10_2UInt, vk::Format::eA2B10G10R10UintPack32},
     {Prospero::BufferFormat::k8_8_8_8UNorm, vk::Format::eR8G8B8A8Unorm},
     {Prospero::BufferFormat::k8_8_8_8SNorm, vk::Format::eR8G8B8A8Snorm},
     {Prospero::BufferFormat::k8_8_8_8UInt, vk::Format::eR8G8B8A8Uint},
@@ -55,6 +56,10 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k32_32_32_32UInt, vk::Format::eR32G32B32A32Uint},
     {Prospero::BufferFormat::k32_32_32_32SInt, vk::Format::eR32G32B32A32Sint},
     {Prospero::BufferFormat::k32_32_32_32Float, vk::Format::eR32G32B32A32Sfloat},
+    // Narrow-channel sRGB formats are optional in Vulkan. Keep a same-width fallback until
+    // sampler-aware sRGB emulation is available.
+    {Prospero::BufferFormat::k8Srgb, vk::Format::eR8Unorm},
+    {Prospero::BufferFormat::k8_8Srgb, vk::Format::eR8G8Unorm},
     {Prospero::BufferFormat::k8_8_8_8Srgb, vk::Format::eR8G8B8A8Srgb},
     {Prospero::BufferFormat::k9_9_9_5Float, vk::Format::eE5B9G9R9UfloatPack32},
     {Prospero::BufferFormat::k5_6_5UNorm, vk::Format::eB5G6R5UnormPack16},

src/graphics/presentation/renderDoc.cpp

@@ -19,12 +19,16 @@
 #include <windows.h>
 #undef min
 #undef max
+#else
+#include <dlfcn.h>
+
+// RenderDoc uses Windows-style names in its cross-platform API.
+#define __cdecl
+using HMODULE = void*;
 #endif
 
 namespace Libs::Graphics {
 
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-
 using RenderDocDevicePointer = void*;
 using RenderDocWindowHandle  = void*;
 
@@ -107,6 +111,8 @@ static RenderDocDevicePointer GetRenderDocDevicePointer(vk::Instance instance) {
 	return VulkanHandleToPointer(instance);
 }
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+
 static bool BindRenderDocApi(HMODULE module) {
 	auto* get_api = reinterpret_cast<pRENDERDOC_GetAPI>(GetProcAddress(module, "RENDERDOC_GetAPI"));
 	if (get_api == nullptr) {
@@ -147,10 +153,84 @@ static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) {
 	return info.info.win.window;
 }
 
+#else
+
+static bool BindRenderDocApi(HMODULE module) {
+	auto* get_api = reinterpret_cast<pRENDERDOC_GetAPI>(::dlsym(module, "RENDERDOC_GetAPI"));
+	if (get_api == nullptr) {
+		return false;
+	}
+
+	void* api = nullptr;
+	if (get_api(eRENDERDOC_API_Version_1_4_2, &api) == 0 || api == nullptr) {
+		return false;
+	}
+
+	g_module = module;
+	g_api    = static_cast<RenderDocApi*>(api);
+
+	g_api->SetCaptureFilePathTemplate("_RenderDoc/kyty");
+	static RenderDocInputButton capture_keys[] = {eRENDERDOC_Key_F1};
+	g_api->SetCaptureKeys(capture_keys, 1);
+	g_api->UnloadCrashHandler();
+
+	Dl_info info {};
+	if (::dladdr(reinterpret_cast<void*>(get_api), &info) != 0 && info.dli_fname != nullptr) {
+		LOGF("RenderDoc: bound API from %s\n", info.dli_fname);
+	} else {
+		LOGF("RenderDoc: bound API\n");
+	}
+	return true;
+}
+
+static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) {
+	if (window == nullptr) {
+		return nullptr;
+	}
+
+	SDL_SysWMinfo info {};
+	SDL_VERSION(&info.version);
+
+	if (SDL_GetWindowWMInfo(window, &info) != SDL_TRUE) {
+		return nullptr;
+	}
+
+#if defined(SDL_VIDEO_DRIVER_X11)
+	if (info.subsystem == SDL_SYSWM_X11) {
+		// RenderDoc takes the raw xlib Window id in the pointer slot, not a Display*.
+		return reinterpret_cast<RenderDocWindowHandle>(
+		    static_cast<uintptr_t>(info.info.x11.window));
+	}
+#endif
+
+	// Wayland capture works without an active-window handle.
+	static std::atomic_bool logged = false;
+	if (!logged.exchange(true)) {
+		LOGF("RenderDoc: no native window handle for SDL subsystem %d (Wayland?); the in-app "
+		     "overlay is unavailable, but --rd captures still work\n",
+		     static_cast<int>(info.subsystem));
+	}
+	return nullptr;
+}
+
+#endif
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+
 static bool IsAvailable() {
 	return g_api != nullptr && g_device != nullptr && g_window != nullptr;
 }
 
+#else
+
+static bool IsAvailable() {
+	return g_api != nullptr && g_device != nullptr;
+}
+
+#endif
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+
 void RenderDocInit() {
 	bool expected = false;
 	if (!g_init_done.compare_exchange_strong(expected, true)) {
@@ -187,6 +267,33 @@ void RenderDocInit() {
 	}
 }
 
+#else
+
+void RenderDocInit() {
+	bool expected = false;
+	if (!g_init_done.compare_exchange_strong(expected, true)) {
+		return;
+	}
+
+	// Prefer an injected RenderDoc instance.
+	auto* module = ::dlopen("librenderdoc.so", RTLD_NOW | RTLD_NOLOAD);
+	if (module == nullptr) {
+		module = ::dlopen("librenderdoc.so", RTLD_NOW);
+	}
+	if (module == nullptr) {
+		LOGF("RenderDoc: librenderdoc.so was not found; in-app capture disabled\n");
+		return;
+	}
+
+	if (!BindRenderDocApi(module)) {
+		LOGF("RenderDoc: API 1.4.2 is not available; in-app capture disabled\n");
+		::dlclose(module);
+		return;
+	}
+}
+
+#endif
+
 void RenderDocSetActiveWindow(vk::Instance instance, SDL_Window* window) {
 	if (g_api == nullptr) {
 		return;
@@ -274,13 +381,4 @@ void RenderDocOnPresent() {
 	}
 }
 
-#else
-
-void RenderDocInit() {}
-void RenderDocSetActiveWindow(vk::Instance /*instance*/, SDL_Window* /*window*/) {}
-void RenderDocRequestCapture() {}
-void RenderDocOnPresent() {}
-
-#endif
-
 } // namespace Libs::Graphics

src/graphics/presentation/videoOut.cpp

@@ -12,7 +12,7 @@
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/graphicsRun.h"
 #include "graphics/guest_gpu/tile.h"
-#include "graphics/host_gpu/renderer/imageInfo.h"
+#include "graphics/host_gpu/renderer/image/imageInfo.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/presentation/presenter.h"
@@ -359,9 +359,9 @@ static void TriggerVideoOutEvents(VideoOutConfig& video_out, VideoOutEventKind k
 		if (!registration || registration->generation != video_out.generation) {
 			continue;
 		}
-		const auto result = EventQueue::KernelTriggerEvent(
-		    registration->handle, VideoOutEventId(kind), EventQueue::KERNEL_EVFILT_VIDEO_OUT,
-		    trigger_data);
+		const auto result =
+		    EventQueue::KernelTriggerEvent(registration->handle, VideoOutEventId(kind),
+		                                   EventQueue::KERNEL_EVFILT_VIDEO_OUT, trigger_data);
 		EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_EBADF &&
 		                     result != LibKernel::KERNEL_ERROR_ENOENT);
 	}
@@ -372,9 +372,8 @@ static void DeleteVideoOutEvents(const VideoOutEventQueues& queues, VideoOutEven
 		if (!registration) {
 			continue;
 		}
-		const auto result =
-		    EventQueue::KernelDeleteEvent(registration->handle, VideoOutEventId(kind),
-		                                  EventQueue::KERNEL_EVFILT_VIDEO_OUT);
+		const auto result = EventQueue::KernelDeleteEvent(
+		    registration->handle, VideoOutEventId(kind), EventQueue::KERNEL_EVFILT_VIDEO_OUT);
 		EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_EBADF &&
 		                     result != LibKernel::KERNEL_ERROR_ENOENT);
 	}
@@ -425,17 +424,15 @@ static int RegisterVideoOutEvent(int handle, EventQueue::KernelEqueue eq, VideoO
 	bool                         add_queue = false;
 	{
 		Common::LockGuard event_lock(event_state->mutex);
-		const auto existing = std::find_if(queues.begin(), queues.end(), [&](const auto& candidate) {
+		const auto        existing =
+		    std::find_if(queues.begin(), queues.end(), [&](const auto& candidate) {
 			    return candidate->handle == eq && candidate->generation == generation;
 		    });
 		if (existing != queues.end()) {
 			registration = *existing;
 		} else {
-			registration = std::make_shared<VideoOutEventRegistration>(
-			    VideoOutEventRegistration {.handle     = eq,
-			                               .state      = event_state,
-			                               .generation = generation,
-			                               .kind       = kind});
+			registration = std::make_shared<VideoOutEventRegistration>(VideoOutEventRegistration {
+			    .handle = eq, .state = event_state, .generation = generation, .kind = kind});
 			queues.push_back(registration);
 			add_queue = true;
 		}
@@ -814,8 +811,8 @@ void VideoOutDriver::Impl::PresentThread(std::stop_token token) {
 				m_presenter.Present(*frame, true);
 			}
 			const auto frame_end = Common::Timer::QueryPerformanceCounter();
-			total_wait += static_cast<int64_t>(period) -
-			              static_cast<int64_t>(frame_end - frame_begin);
+			total_wait +=
+			    static_cast<int64_t>(period) - static_cast<int64_t>(frame_end - frame_begin);
 			continue;
 		}
 
@@ -841,8 +838,7 @@ void VideoOutDriver::Impl::PresentThread(std::stop_token token) {
 		VblankEnd();
 
 		const auto frame_end = Common::Timer::QueryPerformanceCounter();
-		total_wait += static_cast<int64_t>(period) -
-		              static_cast<int64_t>(frame_end - frame_begin);
+		total_wait += static_cast<int64_t>(period) - static_cast<int64_t>(frame_end - frame_begin);
 	}
 }
 
@@ -1000,8 +996,8 @@ void FlipQueue::Prepare(uint64_t request_id, Graphics::CommandBuffer& buffer) {
 	}
 	Graphics::Presenter::Frame* frame = nullptr;
 	if (special) {
-		frame = &m_presenter.PrepareBlankFrame(width, height,
-		                                       index == VIDEO_OUT_BUFFER_INDEX_BLACK, &buffer);
+		frame = &m_presenter.PrepareBlankFrame(width, height, index == VIDEO_OUT_BUFFER_INDEX_BLACK,
+		                                       &buffer);
 	} else {
 		frame = &m_presenter.PrepareFrame(buffer, source_info);
 	}

src/graphics/presentation/window/swapchain.cpp

@@ -206,21 +206,18 @@ private:
 	vk::Format                                     m_format     = vk::Format::eUndefined;
 };
 
-void Presenter::Frame::Configure(GraphicContext& graphics, vk::Extent2D extent,
-                                 vk::Format format) {
+void Presenter::Frame::Configure(GraphicContext& graphics, vk::Extent2D extent, vk::Format format) {
 	if (extent.width == 0 || extent.height == 0 || format == vk::Format::eUndefined) {
 		EXIT("unsupported prepared frame, extent=%ux%u format=%d\n", extent.width, extent.height,
 		     static_cast<int>(format));
 	}
 	const auto features = graphics.GetFormatProperties(format).optimalTilingFeatures;
-	const auto required = vk::FormatFeatureFlagBits::eBlitSrc |
-	                      vk::FormatFeatureFlagBits::eSampledImageFilterLinear |
-	                      vk::FormatFeatureFlagBits::eTransferSrc |
-	                      vk::FormatFeatureFlagBits::eTransferDst;
+	const auto required =
+	    vk::FormatFeatureFlagBits::eBlitSrc | vk::FormatFeatureFlagBits::eSampledImageFilterLinear |
+	    vk::FormatFeatureFlagBits::eTransferSrc | vk::FormatFeatureFlagBits::eTransferDst;
 	if ((features & required) != required) {
 		EXIT("prepared presentation format lacks optimal blit support: format=%d features=0x%x\n",
-		     static_cast<int>(format),
-		     static_cast<vk::FormatFeatureFlags::MaskType>(features));
+		     static_cast<int>(format), static_cast<vk::FormatFeatureFlags::MaskType>(features));
 	}
 
 	auto&      dst        = image;
@@ -267,8 +264,7 @@ void Presenter::Frame::Transit(vk::CommandBuffer command, vk::ImageLayout layout
 	                            ? vk::PipelineStageFlagBits2::eTransfer
 	                            : vk::PipelineStageFlagBits2::eAllCommands;
 	constexpr auto writes = vk::AccessFlagBits2::eTransferWrite |
-	                        vk::AccessFlagBits2::eShaderWrite |
-	                        vk::AccessFlagBits2::eMemoryWrite;
+	                        vk::AccessFlagBits2::eShaderWrite | vk::AccessFlagBits2::eMemoryWrite;
 	if (image.state.layout == layout && image.state.access_mask == access &&
 	    !static_cast<bool>(image.state.access_mask & writes)) {
 		return;
@@ -299,35 +295,27 @@ void Presenter::Frame::Transit(vk::CommandBuffer command, vk::ImageLayout layout
 void Presenter::Frame::CopyFrom(CommandBuffer& command_buffer, Image& source) {
 	command_buffer.EndRendering();
 	auto command = command_buffer.Handle();
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
-	               vk::AccessFlagBits2::eTransferRead, {}, command);
-	Transit(command, vk::ImageLayout::eTransferDstOptimal,
-	        vk::AccessFlagBits2::eTransferWrite);
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
+	               command);
+	Transit(command, vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite);
 	vk::ImageCopy copy {};
-	copy.srcSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0,
-	                       source.backing.layers};
+	copy.srcSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, source.backing.layers};
 	copy.dstSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, image.layers};
 	copy.extent         = {std::min(source.backing.extent.width, image.extent.width),
 	                       std::min(source.backing.extent.height, image.extent.height), 1};
 	EXIT_IF(copy.srcSubresource.layerCount != copy.dstSubresource.layerCount);
-	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
-	                  image.image, vk::ImageLayout::eTransferDstOptimal, copy);
-	Transit(command, vk::ImageLayout::eTransferSrcOptimal,
-	        vk::AccessFlagBits2::eTransferRead);
+	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, image.image,
+	                  vk::ImageLayout::eTransferDstOptimal, copy);
+	Transit(command, vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead);
 }
 
-void Presenter::Frame::Clear(CommandBuffer& command_buffer,
-                             const vk::ClearColorValue& color) {
+void Presenter::Frame::Clear(CommandBuffer& command_buffer, const vk::ClearColorValue& color) {
 	command_buffer.EndRendering();
 	auto command = command_buffer.Handle();
-	Transit(command, vk::ImageLayout::eTransferDstOptimal,
-	        vk::AccessFlagBits2::eTransferWrite);
-	const vk::ImageSubresourceRange range {
-	    vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
-	command.clearColorImage(image.image, vk::ImageLayout::eTransferDstOptimal, &color, 1,
-	                        &range);
-	Transit(command, vk::ImageLayout::eTransferSrcOptimal,
-	        vk::AccessFlagBits2::eTransferRead);
+	Transit(command, vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite);
+	const vk::ImageSubresourceRange range {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
+	command.clearColorImage(image.image, vk::ImageLayout::eTransferDstOptimal, &color, 1, &range);
+	Transit(command, vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead);
 }
 
 class Swapchain final {
@@ -441,16 +429,12 @@ void Swapchain::Create() {
 	        ? vk::CompositeAlphaFlagBitsKHR::eOpaque
 	        : vk::CompositeAlphaFlagBitsKHR::eInherit;
 
-	vk::SurfaceFormatKHR format {vk::Format::eR8G8B8A8Unorm,
-	                             vk::ColorSpaceKHR::eSrgbNonlinear};
-	if (surface.formats.size() != 1 ||
-	    surface.formats.front().format != vk::Format::eUndefined) {
+	vk::SurfaceFormatKHR format {vk::Format::eR8G8B8A8Unorm, vk::ColorSpaceKHR::eSrgbNonlinear};
+	if (surface.formats.size() != 1 || surface.formats.front().format != vk::Format::eUndefined) {
 		const auto it = std::find_if(surface.formats.begin(), surface.formats.end(),
 		                             [](const vk::SurfaceFormatKHR& candidate) {
-			                             return candidate.format ==
-			                                        vk::Format::eB8G8R8A8Unorm ||
-			                                    candidate.format ==
-			                                        vk::Format::eR8G8B8A8Unorm;
+			                             return candidate.format == vk::Format::eB8G8R8A8Unorm ||
+			                                    candidate.format == vk::Format::eR8G8B8A8Unorm;
 		                             });
 		if (it == surface.formats.end()) {
 			EXIT("no supported UNORM swapchain format\n");
@@ -458,8 +442,7 @@ void Swapchain::Create() {
 		format = *it;
 	}
 	m_format                      = format.format;
-	const auto swapchain_features =
-	    graphics.GetFormatProperties(m_format).optimalTilingFeatures;
+	const auto swapchain_features = graphics.GetFormatProperties(m_format).optimalTilingFeatures;
 	if (!static_cast<bool>(swapchain_features & vk::FormatFeatureFlagBits::eBlitDst)) {
 		EXIT("swapchain format cannot be a blit destination: format=%d\n",
 		     static_cast<int>(m_format));
@@ -503,8 +486,7 @@ void Swapchain::Create() {
 		view.subresourceRange.baseMipLevel   = 0;
 		view.subresourceRange.layerCount     = 1;
 		view.subresourceRange.levelCount     = 1;
-		RequireVulkanSuccess(
-		    graphics.device.createImageView(&view, nullptr, &m_image_views[i]),
+		RequireVulkanSuccess(graphics.device.createImageView(&view, nullptr, &m_image_views[i]),
 		                     "vkCreateImageView");
 		EXIT_IF(m_image_views[i] == nullptr);
 	}
@@ -683,10 +665,9 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, VulkanImage& sourc
 	to_present.subresourceRange.levelCount     = 1;
 	to_present.subresourceRange.baseArrayLayer = 0;
 	to_present.subresourceRange.layerCount     = 1;
-	vk_command.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
-	                           vk::PipelineStageFlagBits::eAllCommands,
-	                           vk::DependencyFlagBits::eByRegion, 0,
-	                           nullptr, 0, nullptr, 1, &to_present);
+	vk_command.pipelineBarrier(
+	    vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eAllCommands,
+	    vk::DependencyFlagBits::eByRegion, 0, nullptr, 0, nullptr, 1, &to_present);
 	command.End();
 }
 
@@ -752,8 +733,7 @@ Presenter::Frame& Presenter::PrepareFrame(CommandBuffer& buffer, const ImageInfo
 		default: break;
 	}
 	frame->Configure(m_impl->window.graphic_ctx,
-	                 {image.backing.extent.width, image.backing.extent.height},
-	                 frame_format);
+	                 {image.backing.extent.width, image.backing.extent.height}, frame_format);
 	frame->CopyFrom(buffer, image);
 	return *frame;
 }
@@ -772,8 +752,7 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
 		frame->Clear(*producer, clear);
 	} else {
 		if (frame->present_commands == nullptr) {
-			frame->present_commands =
-			    std::make_unique<CommandBuffer>(m_impl->present_scheduler);
+			frame->present_commands = std::make_unique<CommandBuffer>(m_impl->present_scheduler);
 		}
 		auto& command = *frame->present_commands;
 		command.WaitForFenceAndReset();
@@ -830,8 +809,7 @@ void Presenter::Present(Frame& frame, bool reuse) {
 			continue;
 		}
 		if (frame.present_commands == nullptr) {
-			frame.present_commands =
-			    std::make_unique<CommandBuffer>(m_impl->present_scheduler);
+			frame.present_commands = std::make_unique<CommandBuffer>(m_impl->present_scheduler);
 		}
 		{
 			Common::LockGuard render_lock(m_impl->renderer.GetMutex());

src/graphics/presentation/window/vulkanWindow.cpp

@@ -32,11 +32,11 @@
 #include "graphics/host_gpu/vma.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/presentation/presenter.h"
-#include "kernel/memory.h"
 #include "graphics/presentation/renderDoc.h"
 #include "graphics/presentation/videoOut.h"
 #include "graphics/presentation/window.h"
 #include "graphics/presentation/window/windowInternal.h"
+#include "kernel/memory.h"
 #include "libs/controller.h"
 #include "loader/systemContent.h"
 
@@ -909,10 +909,9 @@ void WindowContext::CreateVulkan() {
 	}
 	surface = native_surface;
 
-	std::vector<const char*> device_extensions = {VK_KHR_SWAPCHAIN_EXTENSION_NAME,
-	                                              VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME,
-	                                              VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME,
-	                                              "VK_KHR_maintenance1"};
+	std::vector<const char*> device_extensions = {
+	    VK_KHR_SWAPCHAIN_EXTENSION_NAME, VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME,
+	    VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, "VK_KHR_maintenance1"};
 
 #if defined(__APPLE__)
 	// MoltenVK lacks VK_EXT_depth_clip_enable and VK_EXT_color_write_enable; the renderer
@@ -932,8 +931,8 @@ void WindowContext::CreateVulkan() {
 
 	uint32_t queue_family = static_cast<uint32_t>(-1);
 
-	VulkanFindPhysicalDevice(graphic_ctx.instance, surface, device_extensions,
-	                         surface_capabilities, graphic_ctx.physical_device, queue_family);
+	VulkanFindPhysicalDevice(graphic_ctx.instance, surface, device_extensions, surface_capabilities,
+	                         graphic_ctx.physical_device, queue_family);
 
 	if (graphic_ctx.physical_device == nullptr) {
 		EXIT("Could not find suitable device");
@@ -949,9 +948,8 @@ void WindowContext::CreateVulkan() {
 		auto available_extensions = EnumerateVulkan<vk::ExtensionProperties>(
 		    "vkEnumerateDeviceExtensionProperties",
 		    [&](uint32_t* count, vk::ExtensionProperties* values) {
-			    return graphic_ctx.physical_device.enumerateDeviceExtensionProperties(nullptr,
-			                                                                           count,
-			                                                                           values);
+			    return graphic_ctx.physical_device.enumerateDeviceExtensionProperties(
+			        nullptr, count, values);
 		    });
 
 		if (HasExtension(available_extensions, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) {

src/graphics/presentation/window/window.cpp

@@ -1,7 +1,5 @@
 #include "graphics/presentation/window.h"
 
-#include <cstdlib>
-
 #include "SDL.h"
 #include "SDL_error.h"
 #include "SDL_events.h"
@@ -40,6 +38,7 @@
 
 #include <algorithm>
 #include <cstdio>
+#include <cstdlib>
 #include <cstring>
 #include <memory>
 #include <string>
@@ -251,9 +250,7 @@ static void GameEventKeyboard(WindowLoopState& game, const EventKeyboard& key) {
 	if (key.down) {
 		switch (key.key_code) {
 			case SDLK_ESCAPE: game.need_exit = true; break;
-			case SDLK_SPACE:
-				SetPause(game, !game.paused.load(std::memory_order_acquire));
-				break;
+			case SDLK_SPACE: SetPause(game, !game.paused.load(std::memory_order_acquire)); break;
 			case SDLK_F1:
 				if (!key.repeat) {
 					RenderDocRequestCapture();
@@ -390,7 +387,9 @@ void WindowContext::Resize(uint32_t new_width, uint32_t new_height) {
 void WindowContext::ProcessWindowEvent(const SDL_WindowEvent& event) {
 	const auto& window_event = event;
 	switch (window_event.event) {
-		case SDL_WINDOWEVENT_SHOWN: LOGF("Window %" PRIu32 " shown\n", window_event.windowID); break;
+		case SDL_WINDOWEVENT_SHOWN:
+			LOGF("Window %" PRIu32 " shown\n", window_event.windowID);
+			break;
 
 		case SDL_WINDOWEVENT_HIDDEN:
 			LOGF("Window %" PRIu32 " hidden\n", window_event.windowID);
@@ -401,13 +400,13 @@ void WindowContext::ProcessWindowEvent(const SDL_WindowEvent& event) {
 			break;
 
 		case SDL_WINDOWEVENT_MOVED:
-			LOGF("Window %" PRIu32 " moved to %" PRId32 ",%" PRId32 "\n",
-			     window_event.windowID, window_event.data1, window_event.data2);
+			LOGF("Window %" PRIu32 " moved to %" PRId32 ",%" PRId32 "\n", window_event.windowID,
+			     window_event.data1, window_event.data2);
 			break;
 
 		case SDL_WINDOWEVENT_RESIZED:
-			LOGF("Window %" PRIu32 " resized to %" PRId32 "x%" PRId32 "\n",
-			     window_event.windowID, window_event.data1, window_event.data2);
+			LOGF("Window %" PRIu32 " resized to %" PRId32 "x%" PRId32 "\n", window_event.windowID,
+			     window_event.data1, window_event.data2);
 
 			LOGF("m: %d\n", static_cast<int>(SDL_ThreadID()));
 			Resize(window_event.data1, window_event.data2);
@@ -807,8 +806,7 @@ static void WindowCreate(WindowContext& context) {
 		window_flags |= static_cast<uint32_t>(SDL_WINDOW_BORDERLESS);
 	}
 #endif
-	context.window =
-	    SDL_CreateWindow(KYTY_SDL_WINDOW_CAPTION, KYTY_SDL_WINDOWPOS_CENTERED,
+	context.window = SDL_CreateWindow(KYTY_SDL_WINDOW_CAPTION, KYTY_SDL_WINDOWPOS_CENTERED,
 	                                  KYTY_SDL_WINDOWPOS_CENTERED, width, height, window_flags);
 
 	context.window_hidden = true;
@@ -950,11 +948,11 @@ void WindowContext::UpdateTitle() {
 		fps_frames  = 0;
 	}
 
-	auto fps = fmt::format("{}{}{}{}{}{}[{}] [{}], frame: {}, fps: {:f}", (has_title ? title : ""),
+	auto fps =
+	    fmt::format("{}{}{}{}{}{}[{}] [{}], frame: {}, fps: {:f}", (has_title ? title : ""),
 	                (has_title ? ", " : ""), (has_title_id ? title_id : ""),
 	                (has_title_id ? ", " : ""), (has_app_ver ? app_ver : ""),
-	                       (has_app_ver ? " " : ""), device_name, processor_name,
-	                       frame_num, current_fps);
+	                (has_app_ver ? " " : ""), device_name, processor_name, frame_num, current_fps);
 
 #if defined(__APPLE__)
 	// AppKit traps on title changes off the main thread; fire-and-forget keeps present pacing.

src/graphics/presentation/window/windowInternal.h

@@ -38,8 +38,7 @@ struct WindowContext {
 	~WindowContext();
 	KYTY_CLASS_NO_COPY(WindowContext);
 
-	[[nodiscard]] static vk::PhysicalDeviceVulkan13Features
-	RequiredVulkan13Features() noexcept;
+	[[nodiscard]] static vk::PhysicalDeviceVulkan13Features RequiredVulkan13Features() noexcept;
 	void                                                    CreateVulkan();
 	void                                                    RecreateSurface();
 	void                                                    RefreshSurfaceCapabilities();

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -2,17 +2,18 @@
 
 #include "common/assert.h"
 #include "common/logging/log.h"
-#include "graphics/shader/recompiler/BindingLayout.h"
-#include "graphics/shader/recompiler/ResourceMaterialization.h"
-#include "graphics/shader/recompiler/ResourceTracking.h"
-#include "graphics/shader/recompiler/ScalarProvenance.h"
-#include "graphics/shader/recompiler/ShaderCFG.h"
-#include "graphics/shader/recompiler/ShaderDecoder.h"
-#include "graphics/shader/recompiler/ShaderIR.h"
-#include "graphics/shader/recompiler/ShaderInfoCollection.h"
-#include "graphics/shader/recompiler/SpirvEmitter.h"
-#include "graphics/shader/recompiler/SrtPatcher.h"
-#include "graphics/shader/recompiler/SrtWalker.h"
+#include "graphics/shader/recompiler/cfg/ShaderCFG.h"
+#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
+#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
+#include "graphics/shader/recompiler/ir/BindingLayout.h"
+#include "graphics/shader/recompiler/ir/ReadLaneElimination.h"
+#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/ResourceTracking.h"
+#include "graphics/shader/recompiler/ir/ScalarProvenance.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/ShaderInfoCollection.h"
+#include "graphics/shader/recompiler/ir/SrtPatcher.h"
+#include "graphics/shader/recompiler/ir/SrtWalker.h"
 
 #include <algorithm>
 #include <array>
@@ -838,6 +839,11 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	if (!IR::AllocateBindings(ir, layout_options, error)) {
 		return false;
 	}
+	const auto read_lane_stats = IR::EliminateReadLane(ir);
+	if (read_lane_stats.rewritten_reads != 0) {
+		LOGF("%s read-lane elimination: reads=%" PRIu32 " shadow_writes=%" PRIu32 "\n",
+		     GetDumpLabel(options), read_lane_stats.rewritten_reads, read_lane_stats.shadow_writes);
+	}
 	std::string ir_dump;
 	if (options.dump_ir) {
 		ir_dump = MakeIrDump(cfg, ir);

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -3,7 +3,7 @@
 
 #include "common/common.h"
 #include "common/stringUtils.h"
-#include "graphics/shader/recompiler/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 #include "graphics/shader/shader.h"
 
 #include <optional>