Compare commits

...
Author SHA1 Message Date
nmzik b7888046ba Fix macOS guest address-space reservation 2026-07-31 02:48:09 +02:00
nmzik 4532883b83 add unknwon flag 2026-07-31 02:33:00 +02:00
nmzik e87ae56544 Rework guest memory tracking 2026-07-31 02:14:17 +02:00
ecb48f90bb Emulate SHA-NI and fix SSE4a EXTRQ/INSERTQ register form (#126)
* Emulate SHA-NI on illegal instruction faults

* Fix SSE4a EXTRQ/INSERTQ register form

* Fix SHA-NI memory operand emulation

* Revert "Fix SSE4a EXTRQ/INSERTQ register form"

This reverts commit ea2b54a4d0.

---------

Co-authored-by: neobugs1 <neobugs1@users.noreply.github.com>
Co-authored-by: nmzik <Nmzik@mail.ru>
2026-07-30 16:21:36 +02:00
nmzik 77aa28b27c update README 2026-07-30 05:16:37 +02:00
nmzikandGitHub d04938c88c Embedded fetch shader: Fix overlapping buffer loads (#133)
Fix overlapping buffer loads. Fixes many games
2026-07-30 05:08:38 +02:00
nmzikandGitHub 85622befb8 Fix fabricated HTTP2 success (#129)
@StefanosCosta Thanks!
2026-07-30 00:48:08 +02:00
nmzik 3965d41d36 texture_cache: fix exact-match reuse across different tile modes 2026-07-30 00:10:40 +02:00
nmzik c508c4a9c0 shader_recompiler: allow GDS append/consume offsets 2026-07-30 00:10:40 +02:00
ClaxtenandGitHub e91dd39cb0 Drop redundant PROT_NONE tracking in reserve paths for Linux (#122)
src: platform: Linux: Drop redundant PROT_NONE tracking in reserve paths

* Some UE4 games, such as The Pathless, reserve a 512 GiB virtual address range during libc startup.
  Tracking every 4 KiB page causes a long delay and is unnecessary since the range is already PROT_NONE,
  and untracked pages are treated as NoAccess.

Signed-off-by: Claxten <claxten10@gmail.com>
2026-07-30 00:00:21 +02:00
nmzik cc76827e63 Fix vertex buffer ranges crossing memory mappings 2026-07-29 21:05:24 +02:00
nmzik 832bc84100 fix(shader): stabilize scalar provenance phis in cyclic CFGs 2026-07-29 21:05:24 +02:00
nmzik 65a0f0baa7 NpManager ABI 2026-07-29 21:05:24 +02:00
nmzik b9ae2537ef renderer: broaden compatibility 2026-07-29 18:47:26 +02:00
nmzik 0b9edaa721 graphics: broaden storage image atomic compatibility 2026-07-29 18:47:24 +02:00
nmzik 8a244677d7 fix(renderer): resolve delayed GPU page faults through buffer and texture caches 2026-07-29 18:47:19 +02:00
nmzikandGitHub f6e01e5403 Optimize bulk memory invalidation (#124)
Build and Release KytyPS5 / Build KytyPS5 (Windows) (push) Canceled after 0s
Build and Release KytyPS5 / Build KytyPS5 (macOS) (push) Canceled after 0s
Build and Release KytyPS5 / Build KytyPS5 (Linux) (push) Canceled after 0s
Build and Release KytyPS5 / Release KytyPS5 (push) Canceled after 0s
* Per page -> per range search (optimization)
2026-07-29 05:09:36 +02:00
nmzikandGitHub 861729fc6c Optimize texture cache tracking (#123)
Optimize texture cache page tracking
2026-07-29 03:37:09 +02:00
nmzik 687ce025c6 KernelOpen: minor fix 2026-07-29 00:31:09 +02:00
nmzik a21d1aaa47 fix 2026-07-29 00:31:09 +02:00
nmzik ec11f31aa6 fix graphical bug (PPSA17221) 2026-07-29 00:31:08 +02:00
nmzik aeceaff028 new ABIs + one stub 2026-07-29 00:31:08 +02:00
nmzik e76f2d1af8 new ABIs 2026-07-29 00:31:08 +02:00
nmzik 64845e2294 new ABI, broaden support 2026-07-29 00:31:08 +02:00
nmzik 4b9030bd3d ABI fixes: libNet, libSaveData (rewrite the legacy Kyty implementation) 2026-07-29 00:30:24 +02:00
nmzik a6b61d7aa0 refactor(graphics): split renderer and recompiler into focused modules 2026-07-29 00:30:24 +02:00
Stefanos CostaandGitHub 2b9cba457e Linux: port the emulator to a working state on Linux (#117)
Linux: port the emulator to a working state
2026-07-29 00:25:58 +02:00
160 changed files with 8259 additions and 4307 deletions
-82
View File
@@ -1,82 +0,0 @@
name: Build KytyPS5 (Linux)
on:
workflow_dispatch:
push:
branches: [ main, master ]
pull_request:
jobs:
build:
runs-on: ubuntu-24.04
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
submodules: recursive
- name: Install build dependencies
shell: bash
run: |
sudo apt-get update
sudo apt-get install --no-install-recommends --yes \
clang \
glslang-tools \
libgl1-mesa-dev \
libx11-dev \
libxcursor-dev \
libxext-dev \
libxfixes-dev \
libxi-dev \
libxrandr-dev \
libxss-dev \
lld \
ninja-build
- name: Install Qt 6.10.3
uses: jurplel/install-qt-action@v4
with:
version: "6.10.3"
host: linux
target: desktop
arch: linux_gcc_64
cache: true
- name: Verify toolchain
shell: bash
run: |
git --version
cmake --version
ninja --version
clang++ --version
ld.lld --version
glslangValidator --version
echo "Qt6_DIR=$Qt6_DIR"
- name: Configure
shell: bash
run: |
cmake -S src -B _Build/linux \
-G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER=clang++ \
-DCMAKE_PREFIX_PATH="$Qt6_DIR"
- name: Build
shell: bash
run: |
cmake --build _Build/linux --target launcher --parallel
- name: Install
shell: bash
run: |
cmake --install _Build/linux --prefix _Build/linux/install
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: KytyPS5-Linux
path: _Build/linux/install/**
if-no-files-found: error
+153 -5
View File
@@ -83,7 +83,12 @@ jobs:
- name: Build
shell: cmd
run: |
cmake --build _Build/windows --target launcher --parallel
cmake --build _Build/windows --target launcher virtual_memory_allocation_tests --parallel
- name: Test
shell: cmd
run: |
ctest --test-dir _Build/windows --output-on-failure -R "^virtual_memory_allocation$"
- name: Install
shell: cmd
@@ -153,7 +158,15 @@ jobs:
- name: Build
shell: bash
run: |
cmake --build _Build/macos --target launcher --parallel
cmake --build _Build/macos \
--target launcher virtual_memory_allocation_tests \
--parallel
- name: Test
shell: bash
run: |
ctest --test-dir _Build/macos --output-on-failure \
-R '^virtual_memory_allocation$'
- name: Install
shell: bash
@@ -198,10 +211,138 @@ jobs:
path: _Build/macos/install/**
if-no-files-found: error
linux:
name: Build KytyPS5 (Linux)
runs-on: ubuntu-24.04
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
submodules: recursive
- name: Install build dependencies
shell: bash
run: |
sudo apt-get update
sudo apt-get install --no-install-recommends --yes \
clang \
glslang-tools \
libasound2-dev \
libdbus-1-dev \
libgl1-mesa-dev \
libpulse-dev \
libudev-dev \
libwayland-dev \
libx11-dev \
libxcursor-dev \
libxext-dev \
libxfixes-dev \
libxi-dev \
libxkbcommon-dev \
libxrandr-dev \
libxss-dev \
lld \
ninja-build \
wayland-protocols
- name: Install Qt 6.10.3
uses: jurplel/install-qt-action@v4
with:
version: "6.10.3"
host: linux
target: desktop
arch: linux_gcc_64
cache: true
- name: Verify toolchain
shell: bash
run: |
git --version
cmake --version
ninja --version
clang++ --version
ld.lld --version
glslangValidator --version
echo "Qt6_DIR=$Qt6_DIR"
- name: Configure
shell: bash
run: |
mkdir -p _Build
cmake -S src -B _Build/linux \
-G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER=clang++ \
-DCMAKE_PREFIX_PATH="$Qt6_DIR" 2>&1 | tee _Build/configure.log
exit "${PIPESTATUS[0]}"
- name: Verify SDL2 backends
shell: bash
run: |
status=0
for feature in SDL_ALSA SDL_PULSEAUDIO SDL_WAYLAND SDL_X11 SDL_LIBUDEV SDL_DBUS; do
if grep -qE "^-- ${feature} +\\(Wanted: ON\\): ON" _Build/configure.log; then
echo "ok ${feature}"
else
echo "FAIL ${feature} is not enabled"
status=1
fi
done
exit "$status"
- name: Build
shell: bash
run: |
cmake --build _Build/linux \
--target launcher page_manager_tests memory_tracker_tests \
virtual_memory_allocation_tests \
--parallel
- name: Test
shell: bash
run: |
ctest --test-dir _Build/linux --output-on-failure \
-R '^(page_manager|memory_tracker|virtual_memory_allocation)$'
- name: Install
shell: bash
run: |
cmake --install _Build/linux --prefix _Build/linux/install
- name: Verify artifacts
shell: bash
run: |
file _Build/linux/install/launcher
file _Build/linux/install/kyty_emulator
file _Build/linux/install/kyty_emulator | grep -q "ELF 64-bit LSB .*x86-64"
ldd _Build/linux/install/kyty_emulator > /dev/null
readelf -d _Build/linux/install/launcher | grep -q 'RPATH.*\$ORIGIN/lib'
while IFS= read -r -d '' binary; do
while read -r dependency; do
test -e "_Build/linux/install/lib/$dependency"
done < <(
readelf -d "$binary" |
sed -n 's/.*Shared library: \[\(libQt6[^]]*\|libicu[^]]*\)\].*/\1/p'
)
done < <(
find _Build/linux/install/launcher _Build/linux/install/plugins \
-type f \( -name launcher -o -name '*.so' \) -print0
)
_Build/linux/install/kyty_emulator --help > /dev/null
- name: Upload Linux artifacts
uses: actions/upload-artifact@v4
with:
name: KytyPS5-Linux-x86_64
path: _Build/linux/install/**
if-no-files-found: error
release:
name: Release KytyPS5
if: github.event_name == 'push'
needs: [windows, macos]
if: github.event_name == 'push' && github.repository == 'KytyPS5/KytyPS5'
needs: [windows, macos, linux]
runs-on: ubuntu-latest
permissions:
contents: write
@@ -222,12 +363,16 @@ jobs:
run: |
windows_dir="artifacts/KytyPS5-Windows-x64"
macos_dir="artifacts/KytyPS5-macOS-x86_64"
linux_dir="artifacts/KytyPS5-Linux-x86_64"
test -d "$windows_dir"
test -d "$macos_dir"
test -d "$linux_dir"
chmod a+x \
"$macos_dir/launcher" \
"$macos_dir/kyty_emulator" \
"$macos_dir/libMoltenVK.dylib"
"$macos_dir/libMoltenVK.dylib" \
"$linux_dir/launcher" \
"$linux_dir/kyty_emulator"
(
cd "$windows_dir"
zip -r "$GITHUB_WORKSPACE/$RELEASE_NAME-Windows-x64.zip" .
@@ -236,6 +381,8 @@ jobs:
cd "$macos_dir"
zip -r "$GITHUB_WORKSPACE/$RELEASE_NAME-macOS-x86_64.zip" .
)
tar -C "$linux_dir" -czf \
"$GITHUB_WORKSPACE/$RELEASE_NAME-Linux-x86_64.tar.gz" .
- name: Create release
shell: bash
@@ -245,6 +392,7 @@ jobs:
assets=(
"$RELEASE_NAME-Windows-x64.zip"
"$RELEASE_NAME-macOS-x86_64.zip"
"$RELEASE_NAME-Linux-x86_64.tar.gz"
)
if gh release view "$RELEASE_NAME" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then
gh release upload "$RELEASE_NAME" "${assets[@]}" \
+71 -12
View File
@@ -1,14 +1,14 @@
# KytyPS5
[![Build KytyPS5 (Windows)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28Windows%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
[![Build KytyPS5 (Linux)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build-linux.yml/badge.svg)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build-linux.yml)
[![Build KytyPS5 (Linux)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28Linux%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
[![Build KytyPS5 (macOS)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28macOS%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
[![Platform](https://img.shields.io/badge/platform-Windows%20x64%20%7C%20macOS%20x86__64-0078D4.svg)](#system-requirements)
[![Platform](https://img.shields.io/badge/platform-Windows%20x64%20%7C%20Linux%20x64%20%7C%20macOS%20x86__64-0078D4.svg)](#system-requirements)
[![Status](https://img.shields.io/badge/status-early%20development-orange.svg)](#current-status)
[![License](https://img.shields.io/badge/license-GPL--2.0-blue.svg)](LICENSE)
KytyPS5 is a free and open-source PlayStation 5 emulator written in C++ for Windows, with
experimental macOS support. It is based on a heavily modified version of
KytyPS5 is a free and open-source PlayStation 5 emulator written in C++ for Windows and Linux,
with experimental macOS support. It is based on a heavily modified version of
[Kyty](https://github.com/InoriRus/Kyty). The project is in an early stage of development, so
compatibility is limited and behavior may change significantly between builds.
@@ -24,7 +24,8 @@ KytyPS5 can boot 2D games and a selection of 3D games, including titles built wi
Development is focused on compatibility and boot reliability.
Linux support is planned. Windows remains the primary supported platform.
Windows is the primary platform and receives the most testing. Linux builds and runs; see
[Building on Linux](#building-on-linux).
macOS support is experimental. Compatibility with the same games on Windows and macOS has not yet
been tested.
@@ -49,7 +50,7 @@ graphical glitches, low compatibility, and poor performance.
</tr>
<tr>
<td align="center">
<strong>Minecraft Legends</strong><br>
<strong>Neptunia ReVerse</strong><br>
<img src="docs/screenshots/ps5-04.png" width="300" alt="Minecraft Legends running in KytyPS5">
</td>
<td align="center">
@@ -57,15 +58,28 @@ graphical glitches, low compatibility, and poor performance.
<img src="docs/screenshots/ps5-05.png" width="300" alt="SILENT HILL: The Short Message running in KytyPS5">
</td>
</tr>
<tr>
<td align="center">
<strong>Hellboy</strong><br>
<img src="docs/screenshots/ps5-02.png" width="300" alt="Disgaea 6 running in KytyPS5">
</td>
<td align="center">
<strong>Paleo Pines</strong><br>
<img src="docs/screenshots/ps5-06.png" width="300" alt="Dreaming Sarah running in KytyPS5">
</td>
</tr>
</table>
<p align="center"><em>And many more...</em></p>
## Contributing
Testing games and submitting detailed bug reports are useful ways to contribute. Search existing
issues first, then use the **Game Emulation Bug Report** template and attach the complete log file.
Code contributions should be focused, build successfully on Windows, and include relevant tests
where practical. Because KytyPS5 is still evolving quickly, consider opening an issue before
Code contributions should be focused, build successfully on the platforms they touch, and include
relevant tests where practical. Windows is the primary target, so a change that alters shared code
should not regress it; changes confined to a platform's own code paths only need to build there. Because KytyPS5 is still evolving quickly, consider opening an issue before
starting a large change.
### Formatting
@@ -100,11 +114,11 @@ the Vulkan/SPIR-V validation rules.
### System requirements
- Windows 10 version 1803
- Windows 10 version 1803, or a current Linux distribution
- A 64-bit x86 processor
- A Vulkan 1.3-capable GPU with current drivers
### Build requirements
### Build requirements (Windows)
- Git
- CMake 3.12 or newer
@@ -138,11 +152,48 @@ cmake --install _Build/windows --prefix _Build/windows/install
The finished application and its runtime dependencies will be placed in
`_Build/windows/install`.
### Building on Linux
Install the toolchain and the libraries the bundled SDL2 needs. Without the audio, Wayland and
udev development packages SDL2 quietly configures itself without those backends, and the resulting
build has no working sound and no gamepad hotplug:
```bash
sudo apt-get install --no-install-recommends \
clang lld ninja-build cmake git glslang-tools \
libgl1-mesa-dev libx11-dev libxcursor-dev libxext-dev libxfixes-dev \
libxi-dev libxrandr-dev libxss-dev libxkbcommon-dev \
libasound2-dev libpulse-dev libudev-dev libdbus-1-dev libwayland-dev wayland-protocols
```
Qt 6 (Concurrent, Network, Widgets) is also required — either the distribution packages
(`qt6-base-dev`) or an official Qt installation.
```bash
git submodule update --init --recursive
cmake -S src -B _Build/linux -G Ninja -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
-DCMAKE_PREFIX_PATH="$Qt6_DIR"
cmake --build _Build/linux --target launcher --parallel
cmake --install _Build/linux --prefix _Build/linux/install
```
The install step copies the Qt libraries and plugins next to the binaries, so
`_Build/linux/install` runs without a matching system Qt.
As on Windows, the MSVC compiler is not used; Clang is required. `cl.exe` is rejected at configure
time.
Note that the CMake source root is `src`, not the repository root.
### Visual Studio Code
A ready-made Visual Studio Code setup is included in [`.vscode`](.vscode). It configures CMake
Tools to build the project with Ninja and `clang-cl` and provides launch profiles for both
`launcher.exe` and `kyty_emulator.exe`.
`launcher.exe` and `kyty_emulator.exe`. It is Windows-only: VS Code settings cannot select a
compiler per platform, so on Linux configure from the command line as shown above.
Before using it:
@@ -164,6 +215,10 @@ To use the graphical launcher:
.\_Build\windows\install\launcher.exe
```
```bash
./_Build/linux/install/launcher
```
On first launch, add one or more game folders in the global settings. The launcher searches those
folders recursively for game directories containing `eboot.bin`. Select a detected game and run it
from the game list.
@@ -174,7 +229,11 @@ The emulator can also be started directly with a legally obtained game directory
.\_Build\windows\install\kyty_emulator.exe --game "D:\Games\ExampleGame"
```
Run `kyty_emulator.exe --help` to see the available graphics, logging, validation, profiling, and
```bash
./_Build/linux/install/kyty_emulator --game "/games/ExampleGame"
```
Run `kyty_emulator --help` to see the available graphics, logging, validation, profiling, and
debugging options.
### AI Use
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.7 MiB

After

Width:  |  Height:  |  Size: 2.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.4 MiB

+80 -27
View File
@@ -158,16 +158,24 @@ file(GLOB kyty_emulator_src CONFIGURE_DEPENDS
graphics/host_gpu/*.h
graphics/host_gpu/renderer/*.cpp
graphics/host_gpu/renderer/*.h
graphics/host_gpu/renderer/cache/*.cpp
graphics/host_gpu/renderer/cache/*.h
graphics/host_gpu/renderer/image/*.cpp
graphics/host_gpu/renderer/image/*.h
graphics/host_gpu/renderer/pipeline/*.cpp
graphics/host_gpu/renderer/pipeline/*.h
graphics/shader/*.cpp
graphics/shader/*.h
graphics/shader/recompiler/*.cpp
graphics/shader/recompiler/*.h
graphics/shader/recompiler/shaderIR/*.cpp
graphics/shader/recompiler/shaderIR/*.h
graphics/shader/recompiler/spirvEmitter/*.cpp
graphics/shader/recompiler/spirvEmitter/*.h
graphics/host_gpu/objects/*.cpp
graphics/host_gpu/objects/*.h
graphics/shader/recompiler/cfg/*.cpp
graphics/shader/recompiler/cfg/*.h
graphics/shader/recompiler/decompiler/*.cpp
graphics/shader/recompiler/decompiler/*.h
graphics/shader/recompiler/emitter/*.cpp
graphics/shader/recompiler/emitter/*.h
graphics/shader/recompiler/ir/*.cpp
graphics/shader/recompiler/ir/*.h
graphics/presentation/*.cpp
graphics/presentation/*.h
graphics/presentation/window/*.cpp
@@ -251,6 +259,26 @@ endif()
set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL2-static xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9)
# Linux system libraries required by the static FFmpeg archive.
if(LINUX)
find_package(Threads REQUIRED)
list(APPEND kyty_emulator_link_libraries m ${CMAKE_DL_LIBS} Threads::Threads)
# Optional FFmpeg dependencies.
find_package(ZLIB)
if(ZLIB_FOUND)
list(APPEND kyty_emulator_link_libraries ZLIB::ZLIB)
endif()
find_library(KYTY_BZ2_LIBRARY bz2)
if(KYTY_BZ2_LIBRARY)
list(APPEND kyty_emulator_link_libraries ${KYTY_BZ2_LIBRARY})
endif()
find_library(KYTY_LZMA_LIBRARY lzma)
if(KYTY_LZMA_LIBRARY)
list(APPEND kyty_emulator_link_libraries ${KYTY_LZMA_LIBRARY})
endif()
endif()
set(inc_headers
${CMAKE_CURRENT_SOURCE_DIR}
${KYTY_THIRD_PARTY_DIR}/SDL2/include
@@ -286,13 +314,23 @@ function(add_kyty_full_emulator_test target source)
endif()
endfunction()
function(configure_macos_guest_address_space target)
if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
(NOT CMAKE_OSX_ARCHITECTURES AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$")))
target_sources(${target} PRIVATE kernel/macosGuestAddressSpace.cpp)
target_compile_definitions(${target} PRIVATE KYTY_LINKED_GUEST_ADDRESS_SPACE=1)
target_link_options(${target} PRIVATE
-Wl,-ld_classic,-no_pie,-no_fixup_chains,-no_huge,-pagezero_size,0x40000,-segaddr,SYSTEM_MANAGED,0x40000,-segaddr,SYSTEM_RESERVED,0x7ffffc000,-segaddr,USER_AREA,0x7000000000,-image_base,0x700000000000)
endif()
endfunction()
add_kyty_full_emulator_test(shader_cfg_tests ../tests/shaderCfgTests.cpp)
add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL
../tests/ScalarProvenanceTests.cpp
graphics/host_gpu/hostMemory.cpp
graphics/shader/recompiler/ScalarProvenance.cpp
graphics/shader/recompiler/SrtWalker.cpp
graphics/shader/recompiler/ir/ScalarProvenance.cpp
graphics/shader/recompiler/ir/SrtWalker.cpp
)
target_link_libraries(scalar_provenance_tests fmt::fmt)
target_include_directories(scalar_provenance_tests PRIVATE ${inc_headers})
@@ -310,7 +348,6 @@ add_executable(memory_tracker_tests EXCLUDE_FROM_ALL
)
target_link_libraries(memory_tracker_tests fmt::fmt common)
target_include_directories(memory_tracker_tests PRIVATE ${inc_headers})
target_compile_definitions(memory_tracker_tests PRIVATE KYTY_MEMORY_TRACKER_TESTS=1)
add_executable(shader_vertex_metadata_tests EXCLUDE_FROM_ALL
../tests/ShaderVertexMetadataTests.cpp
@@ -324,9 +361,9 @@ add_executable(shader_stage_runtime_tests EXCLUDE_FROM_ALL
graphics/guest_gpu/gpu_format.cpp
graphics/host_gpu/hostMemory.cpp
graphics/shader/shaderStageRuntime.cpp
graphics/shader/recompiler/ResourceMaterialization.cpp
graphics/shader/recompiler/ScalarProvenance.cpp
graphics/shader/recompiler/SrtWalker.cpp
graphics/shader/recompiler/ir/ResourceMaterialization.cpp
graphics/shader/recompiler/ir/ScalarProvenance.cpp
graphics/shader/recompiler/ir/SrtWalker.cpp
)
target_link_libraries(shader_stage_runtime_tests fmt::fmt)
target_include_directories(shader_stage_runtime_tests PRIVATE ${inc_headers})
@@ -335,20 +372,20 @@ add_executable(resource_tracking_tests EXCLUDE_FROM_ALL
../tests/ResourceTrackingTests.cpp
graphics/guest_gpu/gpu_format.cpp
graphics/host_gpu/hostMemory.cpp
graphics/shader/recompiler/ScalarProvenance.cpp
graphics/shader/recompiler/SrtWalker.cpp
graphics/shader/recompiler/SrtPatcher.cpp
graphics/shader/recompiler/ResourceTracking.cpp
graphics/shader/recompiler/ResourceMaterialization.cpp
graphics/shader/recompiler/ShaderInfoCollection.cpp
graphics/shader/recompiler/BindingLayout.cpp
graphics/shader/recompiler/ir/ScalarProvenance.cpp
graphics/shader/recompiler/ir/SrtWalker.cpp
graphics/shader/recompiler/ir/SrtPatcher.cpp
graphics/shader/recompiler/ir/ResourceTracking.cpp
graphics/shader/recompiler/ir/ResourceMaterialization.cpp
graphics/shader/recompiler/ir/ShaderInfoCollection.cpp
graphics/shader/recompiler/ir/BindingLayout.cpp
)
target_link_libraries(resource_tracking_tests fmt::fmt)
target_include_directories(resource_tracking_tests PRIVATE ${inc_headers})
add_executable(resource_mutex_tests EXCLUDE_FROM_ALL
../tests/ResourceMutexTests.cpp
graphics/host_gpu/renderer/resourceMutex.cpp
graphics/host_gpu/renderer/cache/resourceMutex.cpp
)
target_link_libraries(resource_mutex_tests common)
target_include_directories(resource_mutex_tests PRIVATE ${inc_headers})
@@ -393,6 +430,15 @@ target_sources(shader_recompiler_compute_tests PRIVATE
add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp)
target_compile_definitions(virtual_memory_allocation_tests PRIVATE
KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1)
configure_macos_guest_address_space(virtual_memory_allocation_tests)
# These tests use exceptions.
if(NOT KYTY_CLANG_CL)
foreach(kyty_exception_test scalar_provenance_tests resource_tracking_tests
virtual_memory_allocation_tests)
target_compile_options(${kyty_exception_test} PRIVATE -fexceptions)
endforeach()
endif()
if(BUILD_TESTING)
add_test(NAME image_page_table COMMAND $<TARGET_FILE:image_page_table_tests>)
@@ -401,6 +447,8 @@ if(BUILD_TESTING)
add_test(NAME resource_mutex COMMAND $<TARGET_FILE:resource_mutex_tests>)
add_test(NAME event_queue_lifetime COMMAND $<TARGET_FILE:event_queue_lifetime_tests>)
add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
add_test(NAME virtual_memory_allocation
COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
add_test(NAME command_scheduler_timeline
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --scheduler-only)
add_test(NAME stream_buffer_ring
@@ -410,11 +458,6 @@ if(BUILD_TESTING)
add_test(NAME gpu_tiler
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --gpu-tiler-only)
if(WIN32)
add_test(NAME texture_cache_image_overlap
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --image-overlap-only)
add_test(NAME texture_cache_htile_clear
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --htile-clear-only)
add_test(NAME texture_cache_layered_image
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --layered-image-only)
add_test(NAME texture_cache_image_views
@@ -423,15 +466,23 @@ if(BUILD_TESTING)
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --storage-sampled-only)
add_test(NAME texture_cache_depth_readback
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --depth-readback-only)
add_test(NAME buffer_cache_ranges
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-range-only)
add_test(NAME buffer_cache_dirty_gc
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-gc-only)
if(WIN32)
# These tests still depend on the Windows multisample-depth path.
add_test(NAME texture_cache_image_overlap
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --image-overlap-only)
add_test(NAME texture_cache_htile_clear
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --htile-clear-only)
add_test(NAME buffer_cache_ranges
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-range-only)
endif()
endif()
add_executable(kyty_emulator main.cpp ${kyty_emulator_src})
configure_macos_guest_address_space(kyty_emulator)
target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries})
if (WIN32)
@@ -446,6 +497,8 @@ endif()
if (CLANG AND NOT KYTY_CLANG_CL)
target_link_libraries(kyty_emulator pthread)
endif()
# dlopen/dlsym/dladdr for RenderDoc.
target_link_libraries(kyty_emulator ${CMAKE_DL_LIBS})
target_include_directories(kyty_emulator PRIVATE ${inc_headers})
clang_tidy_check(kyty_emulator "" "${check_headers}" "${inc_headers}")
+111 -28
View File
@@ -9,6 +9,11 @@
#elif defined(__APPLE__)
#include <csignal>
#include <sys/ucontext.h>
#else
#include <csignal>
#include <initializer_list>
#include <ucontext.h> // IWYU pragma: keep
#include <unistd.h>
#endif
// IWYU pragma: no_include <errhandlingapi.h>
@@ -19,7 +24,7 @@
namespace Common::HostException {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#if !defined(__APPLE__)
static std::atomic<Handler> g_handler {nullptr};
static std::atomic_uint32_t g_install_state {0};
@@ -33,7 +38,9 @@ static_assert(decltype(g_install_state)::is_always_lock_free);
std::fputs(reason != nullptr ? reason : "unspecified", stderr);
std::fputc('\n', stderr);
std::fflush(stderr);
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
TerminateProcess(GetCurrentProcess(), static_cast<UINT>(EXCEPTION_NONCONTINUABLE_EXCEPTION));
#endif
std::_Exit(321);
}
@@ -51,6 +58,21 @@ public:
KYTY_CLASS_NO_COPY(FilterScope);
};
static Handler LoadInstalledHandler() noexcept {
if (g_install_state.load(std::memory_order_acquire) == 0) {
FailFast("host exception handler is not installed");
}
const auto handler = g_handler.load(std::memory_order_acquire);
if (handler == nullptr) {
FailFast("host exception callback is null");
}
return handler;
}
#endif
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
FilterScope filter_scope;
@@ -109,14 +131,7 @@ static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
info.r14 = exception->ContextRecord->R14;
info.r15 = exception->ContextRecord->R15;
if (g_install_state.load(std::memory_order_acquire) == 0) {
FailFast("host exception handler is not installed");
}
const auto handler = g_handler.load(std::memory_order_acquire);
if (handler == nullptr) {
FailFast("host exception callback is null");
}
const auto handler = LoadInstalledHandler();
return handler(info) ? EXCEPTION_CONTINUE_EXECUTION : EXCEPTION_CONTINUE_SEARCH;
}
@@ -213,10 +228,79 @@ static void SignalHandler(int sig, siginfo_t* si, void* uctx) {
sigaction(sig, &dfl, nullptr);
}
#else
// x86-64 page-fault error bits.
constexpr uint64_t PAGE_FAULT_ERROR_WRITE = 0x02;
constexpr uint64_t PAGE_FAULT_ERROR_INSTRUCTION = 0x10;
// Let the kernel handle an unresolved fault on retry.
static void ChainToDefault(int signal_number) noexcept {
struct sigaction restore {};
restore.sa_handler = SIG_DFL;
sigemptyset(&restore.sa_mask);
restore.sa_flags = 0;
::sigaction(signal_number, &restore, nullptr);
}
static void SignalHandler(int signal_number, siginfo_t* signal_info, void* native_context) {
FilterScope filter_scope;
auto* context = static_cast<ucontext_t*>(native_context);
auto* gregs = context->uc_mcontext.gregs;
ExceptionInfo info {};
info.exception_address = static_cast<uint64_t>(gregs[REG_RIP]);
info.native_code = static_cast<uint32_t>(signal_number);
info.native_context = context;
if (signal_number == SIGSEGV || signal_number == SIGBUS) {
info.type = ExceptionType::AccessViolation;
const auto error_code = static_cast<uint64_t>(gregs[REG_ERR]);
if ((error_code & PAGE_FAULT_ERROR_INSTRUCTION) != 0) {
info.access_violation_type = AccessViolationType::Execute;
} else if ((error_code & PAGE_FAULT_ERROR_WRITE) != 0) {
info.access_violation_type = AccessViolationType::Write;
} else {
info.access_violation_type = AccessViolationType::Read;
}
info.access_violation_vaddr = reinterpret_cast<uint64_t>(signal_info->si_addr);
} else if (signal_number == SIGILL) {
info.type = ExceptionType::IllegalInstruction;
} else {
ChainToDefault(signal_number);
return;
}
info.rax = static_cast<uint64_t>(gregs[REG_RAX]);
info.rbx = static_cast<uint64_t>(gregs[REG_RBX]);
info.rcx = static_cast<uint64_t>(gregs[REG_RCX]);
info.rdx = static_cast<uint64_t>(gregs[REG_RDX]);
info.rsi = static_cast<uint64_t>(gregs[REG_RSI]);
info.rdi = static_cast<uint64_t>(gregs[REG_RDI]);
info.rbp = static_cast<uint64_t>(gregs[REG_RBP]);
info.rsp = static_cast<uint64_t>(gregs[REG_RSP]);
info.r8 = static_cast<uint64_t>(gregs[REG_R8]);
info.r9 = static_cast<uint64_t>(gregs[REG_R9]);
info.r10 = static_cast<uint64_t>(gregs[REG_R10]);
info.r11 = static_cast<uint64_t>(gregs[REG_R11]);
info.r12 = static_cast<uint64_t>(gregs[REG_R12]);
info.r13 = static_cast<uint64_t>(gregs[REG_R13]);
info.r14 = static_cast<uint64_t>(gregs[REG_R14]);
info.r15 = static_cast<uint64_t>(gregs[REG_R15]);
const auto handler = LoadInstalledHandler();
if (handler(info)) {
return;
}
ChainToDefault(signal_number);
}
#endif
bool InstallHandler(Handler handler) {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (handler == nullptr) {
return false;
}
@@ -228,27 +312,14 @@ bool InstallHandler(Handler handler) {
g_handler.store(handler, std::memory_order_release);
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (AddVectoredExceptionHandler(1, ExceptionFilter) == nullptr) {
g_handler.store(nullptr, std::memory_order_release);
g_install_state.store(0, std::memory_order_release);
printf("AddVectoredExceptionHandler() failed\n");
return false;
}
g_install_state.store(2, std::memory_order_release);
return true;
#elif defined(__APPLE__)
if (handler == nullptr) {
return false;
}
uint32_t expected_state = 0;
if (!g_install_state.compare_exchange_strong(expected_state, 1, std::memory_order_acq_rel)) {
return expected_state == 2 && g_handler.load(std::memory_order_acquire) == handler;
}
g_handler.store(handler, std::memory_order_release);
struct sigaction sa {};
sa.sa_sigaction = SignalHandler;
sa.sa_flags = SA_SIGINFO;
@@ -264,13 +335,25 @@ bool InstallHandler(Handler handler) {
printf("sigaction() failed to install the host fault handler\n");
return false;
}
#else
struct sigaction action {};
action.sa_sigaction = SignalHandler;
sigemptyset(&action.sa_mask);
// Fault resolution needs the normal thread stack.
action.sa_flags = SA_SIGINFO | SA_RESTART;
for (const int signal_number: {SIGSEGV, SIGBUS, SIGILL}) {
if (::sigaction(signal_number, &action, nullptr) != 0) {
g_handler.store(nullptr, std::memory_order_release);
g_install_state.store(0, std::memory_order_release);
printf("sigaction(%d) failed\n", signal_number);
return false;
}
}
#endif
g_install_state.store(2, std::memory_order_release);
return true;
#else
(void)handler;
return false;
#endif
}
} // namespace Common::HostException
+4
View File
@@ -23,6 +23,10 @@ struct sys_dbg_stack_info_t {
size_t commited_size;
size_t total_size;
size_t code_size;
// Full stack reservation reported by pthread.
uintptr_t reserved_addr;
size_t reserved_size;
#endif
};
+75 -3
View File
@@ -8,6 +8,8 @@
#include <cstdlib>
#include <cstring>
#include <execinfo.h>
#include <pthread.h>
#include <sys/param.h>
#include <sys/types.h>
#include <unistd.h>
@@ -15,8 +17,48 @@
#include <libgen.h> // POSIX basename() lives here on macOS, not in <cstring>
#endif
void SysStackWalk(void** /*stack*/, int* depth) {
// Avoid unwinding a guest-owned stack.
static bool OnOwnStack() {
const char* probe = reinterpret_cast<const char*>(&probe);
pthread_attr_t attr {};
#if defined(__APPLE__)
const auto* top = static_cast<const char*>(pthread_get_stackaddr_np(pthread_self()));
const auto size = pthread_get_stacksize_np(pthread_self());
(void)attr;
return top != nullptr && size != 0 && probe < top && probe >= top - size;
#else
if (pthread_getattr_np(pthread_self(), &attr) != 0) {
return false;
}
void* base = nullptr;
size_t size = 0;
const bool ok =
pthread_attr_getstack(&attr, &base, &size) == 0 && base != nullptr && size != 0;
pthread_attr_destroy(&attr);
if (!ok) {
return false;
}
const auto* low = static_cast<const char*>(base);
return probe >= low && probe < low + size;
#endif
}
void SysStackWalk(void** stack, int* depth) {
if (stack == nullptr || depth == nullptr || *depth <= 0) {
if (depth != nullptr) {
*depth = 0;
}
return;
}
if (!OnOwnStack()) {
*depth = 0;
return;
}
const int n = ::backtrace(stack, *depth);
*depth = (n < 0 ? 0 : n);
}
void SysStackUsagePrint(sys_dbg_stack_info_t& stack) {
@@ -33,6 +75,33 @@ void SysStackUsage(sys_dbg_stack_info_t& s) {
[[maybe_unused]] int result = 0;
memset(&s, 0, sizeof(sys_dbg_stack_info_t));
// Record the reservation before the Linux /proc walk.
{
pthread_attr_t self_attr {};
#if defined(__APPLE__)
void* stack_top = pthread_get_stackaddr_np(pthread_self());
const size_t stack_size = pthread_get_stacksize_np(pthread_self());
if (stack_top != nullptr && stack_size != 0) {
s.reserved_addr = reinterpret_cast<uintptr_t>(stack_top) - stack_size;
s.reserved_size = stack_size;
}
(void)self_attr;
#else
if (pthread_getattr_np(pthread_self(), &self_attr) == 0) {
void* stack_base = nullptr;
size_t stack_size = 0;
if (pthread_attr_getstack(&self_attr, &stack_base, &stack_size) == 0 &&
stack_base != nullptr && stack_size != 0) {
s.reserved_addr = reinterpret_cast<uintptr_t>(stack_base);
s.reserved_size = stack_size;
}
pthread_attr_destroy(&self_attr);
}
#endif
}
char str[1024];
char str2[1024];
result = sprintf(str, "/proc/%d/exe", static_cast<int>(pid));
@@ -46,8 +115,6 @@ void SysStackUsage(sys_dbg_stack_info_t& s) {
result = sprintf(str, "/proc/%d/maps", static_cast<int>(pid));
memset(&s, 0, sizeof(sys_dbg_stack_info_t));
FILE* f = fopen(str, "r");
if (f == nullptr) {
@@ -121,6 +188,11 @@ void SysStackUsage(sys_dbg_stack_info_t& s) {
}
result = fclose(f);
if (s.reserved_addr == 0) {
s.reserved_addr = s.addr;
s.reserved_size = s.total_size;
}
}
#endif
+187 -39
View File
@@ -12,7 +12,10 @@
#include <cerrno>
#include <cstdlib>
#include <dirent.h>
#include <fcntl.h>
#include <filesystem>
#include <sys/stat.h>
#include <system_error>
#include <unistd.h>
#include <utime.h>
@@ -40,10 +43,43 @@ struct sys_file_t {
};
};
// Darwin uses BSD timestamp member names.
#if defined(__APPLE__)
#define KYTY_STAT_ATIME_NS(st) ((st).st_atimespec.tv_nsec)
#define KYTY_STAT_MTIME_NS(st) ((st).st_mtimespec.tv_nsec)
#else
#define KYTY_STAT_ATIME_NS(st) ((st).st_atim.tv_nsec)
#define KYTY_STAT_MTIME_NS(st) ((st).st_mtim.tv_nsec)
#endif
static std::filesystem::path get_internal_name(const std::filesystem::path& name) {
return name.is_absolute() ? name : (std::filesystem::path(".") / name);
}
// Pass access-pattern hints to the host.
static void apply_cache_hint(FILE* f, sys_file_cache_type_t cache_type) {
if (f == nullptr) {
return;
}
#if !defined(__APPLE__)
int advice = POSIX_FADV_NORMAL;
switch (cache_type) {
case SYS_FILE_CACHE_RANDOM_ACCESS: advice = POSIX_FADV_RANDOM; break;
case SYS_FILE_CACHE_SEQUENTIAL_SCAN: advice = POSIX_FADV_SEQUENTIAL; break;
case SYS_FILE_CACHE_AUTO:
default: return;
}
::posix_fadvise(fileno(f), 0, 0, advice);
#else
if (cache_type == SYS_FILE_CACHE_SEQUENTIAL_SCAN) {
::fcntl(fileno(f), F_RDAHEAD, 1);
} else if (cache_type == SYS_FILE_CACHE_RANDOM_ACCESS) {
::fcntl(fileno(f), F_RDAHEAD, 0);
}
#endif
}
void SysFileRead(void* data, uint32_t size, sys_file_t& f, uint32_t* bytes_read) {
if (f.type == SYS_FILE_FILE) {
size_t w = fread(data, 1, size, f.f);
@@ -137,7 +173,7 @@ sys_file_t* SysFileCreate(const std::filesystem::path& file_name) {
}
sys_file_t* SysFileOpenR(const std::filesystem::path& file_name,
sys_file_cache_type_t /*cache_type*/) {
sys_file_cache_type_t cache_type) {
auto* ret = new sys_file_t;
ret->type = SYS_FILE_FILE;
@@ -151,6 +187,8 @@ sys_file_t* SysFileOpenR(const std::filesystem::path& file_name,
ret->type = SYS_FILE_ERROR;
}
apply_cache_hint(f, cache_type);
ret->f = f;
return ret;
@@ -181,7 +219,7 @@ sys_file_t* SysFileCreate() {
}
sys_file_t* SysFileOpenW(const std::filesystem::path& file_name,
sys_file_cache_type_t /*cache_type*/) {
sys_file_cache_type_t cache_type) {
auto* ret = new sys_file_t;
auto real_name = get_internal_name(file_name);
@@ -195,13 +233,15 @@ sys_file_t* SysFileOpenW(const std::filesystem::path& file_name,
ret->type = SYS_FILE_FILE;
}
apply_cache_hint(f, cache_type);
ret->f = f;
return ret;
}
sys_file_t* SysFileOpenRw(const std::filesystem::path& file_name,
sys_file_cache_type_t /*cache_type*/) {
sys_file_cache_type_t cache_type) {
auto* ret = new sys_file_t;
auto real_name = get_internal_name(file_name);
@@ -215,6 +255,8 @@ sys_file_t* SysFileOpenRw(const std::filesystem::path& file_name,
ret->type = SYS_FILE_FILE;
}
apply_cache_hint(f, cache_type);
ret->f = f;
return ret;
@@ -240,11 +282,17 @@ uint64_t SysFileSize(sys_file_t& f) {
[[maybe_unused]] int result = 0;
if (f.type == SYS_FILE_FILE) {
uint32_t pos = ftell(f.f);
result = fseek(f.f, 0, SEEK_END);
uint32_t size = ftell(f.f);
result = fseek(f.f, pos, SEEK_SET);
return size;
// Preserve sizes above 4 GiB.
const off_t pos = ftello(f.f);
if (pos < 0) {
return 0;
}
if (fseeko(f.f, 0, SEEK_END) != 0) {
return 0;
}
const off_t size = ftello(f.f);
result = fseeko(f.f, pos, SEEK_SET);
return (size < 0 ? 0 : static_cast<uint64_t>(size));
}
if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) {
@@ -261,8 +309,18 @@ uint64_t SysFileSize(const std::filesystem::path& file_name) {
return size;
}
bool SysFileTruncate(sys_file_t& /*f*/, uint64_t /*size*/) {
return false;
bool SysFileTruncate(sys_file_t& f, uint64_t size) {
bool ok = false;
if (f.type == SYS_FILE_FILE) {
// Flush before resizing and restore the caller's position.
const auto position = ftell(f.f);
fflush(f.f);
ok = (ftruncate(fileno(f.f), static_cast<off_t>(size)) == 0);
if (position >= 0) {
fseek(f.f, position, SEEK_SET);
}
}
return ok;
}
bool SysFileUnlink(sys_file_t& /*f*/, const std::filesystem::path& name) {
@@ -383,6 +441,7 @@ SysFileTimeStruct SysFileGetLastAccessTimeUtc(const std::filesystem::path& name)
} else {
r.is_invalid = false;
r.time = s.st_atime;
r.nanos = KYTY_STAT_ATIME_NS(s);
}
return r;
@@ -401,6 +460,7 @@ SysFileTimeStruct SysFileGetLastWriteTimeUtc(const std::filesystem::path& name)
} else {
r.is_invalid = false;
r.time = s.st_mtime;
r.nanos = KYTY_STAT_MTIME_NS(s);
}
return r;
@@ -420,13 +480,36 @@ void SysFileGetLastAccessAndWriteTimeUtc(const std::filesystem::path& name, SysF
a.is_invalid = false;
w.is_invalid = false;
a.time = s.st_atime;
a.nanos = KYTY_STAT_ATIME_NS(s);
w.time = s.st_mtime;
w.nanos = KYTY_STAT_MTIME_NS(s);
}
}
void SysFileGetLastAccessAndWriteTimeUtc(sys_file_t& /*f*/, SysFileTimeStruct& /*a*/,
SysFileTimeStruct& /*w*/) {
EXIT("not implemented\n");
void SysFileGetLastAccessAndWriteTimeUtc(sys_file_t& f, SysFileTimeStruct& a,
SysFileTimeStruct& w) {
if (f.type == SYS_FILE_FILE) {
struct stat s {};
const bool ok = (0 == fstat(fileno(f.f), &s));
a.is_invalid = w.is_invalid = !ok;
if (ok) {
a.time = s.st_atime;
a.nanos = KYTY_STAT_ATIME_NS(s);
w.time = s.st_mtime;
w.nanos = KYTY_STAT_MTIME_NS(s);
}
} else if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) {
// Memory-backed files use the current time.
SysTimeStruct t {};
SysGetSystemTimeUtc(t);
SysSystemToFileTimeUtc(t, a);
SysSystemToFileTimeUtc(t, w);
} else {
a.is_invalid = w.is_invalid = true;
}
}
bool SysFileSetLastAccessTimeUtc(const std::filesystem::path& name, SysFileTimeStruct& access) {
@@ -532,45 +615,110 @@ bool SysFileSetLastAccessAndWriteTimeUtc(const std::filesystem::path& name,
// }
}
void SysFileFindFiles(const std::filesystem::path& /*path*/,
std::vector<sys_file_find_t>& /*out*/) {
EXIT("not implemented\n");
}
// Recursively collect regular files.
void SysFileFindFiles(const std::filesystem::path& path, std::vector<sys_file_find_t>& out) {
auto real_path = get_internal_name(path);
void SysFileGetDents(const std::filesystem::path& path, std::vector<sys_dir_entry_t>& out) {
DIR* dir = opendir(path.c_str());
DIR* dir = opendir(real_path.string().c_str());
if (dir == nullptr) {
return;
}
for (dirent* entry = readdir(dir); entry != nullptr; entry = readdir(dir)) {
sys_dir_entry_t r {};
r.name = entry->d_name;
if (entry->d_type == DT_REG) {
r.is_file = true;
} else if (entry->d_type == DT_DIR) {
r.is_file = false;
} else {
// DT_UNKNOWN / symlink: resolve with stat.
struct stat st {};
r.is_file = (stat((path / entry->d_name).c_str(), &st) == 0) ? S_ISREG(st.st_mode) : true;
for (const dirent* entry = readdir(dir); entry != nullptr; entry = readdir(dir)) {
const std::string file_name(entry->d_name);
if (file_name == "." || file_name == "..") {
continue;
}
out.push_back(std::move(r));
auto child = real_path / file_name;
struct stat s {};
// lstat, so a symlink is never followed into a cycle during the recursive walk.
if (0 != lstat(child.string().c_str(), &s)) {
continue;
}
if (S_ISDIR(s.st_mode)) {
SysFileFindFiles(child, out);
} else if (S_ISREG(s.st_mode)) {
sys_file_find_t r {};
r.path_with_name = child;
r.size = static_cast<uint64_t>(s.st_size);
r.last_access_time.is_invalid = false;
r.last_access_time.time = s.st_atime;
r.last_access_time.nanos = KYTY_STAT_ATIME_NS(s);
r.last_write_time.is_invalid = false;
r.last_write_time.time = s.st_mtime;
r.last_write_time.nanos = KYTY_STAT_MTIME_NS(s);
out.push_back(r);
}
}
closedir(dir);
}
bool SysFileCopyFile(const std::filesystem::path& /*src*/, const std::filesystem::path& /*dst*/) {
EXIT("not implemented\n");
return false;
// Keep "." and ".." to match FindFirstFileW.
void SysFileGetDents(const std::filesystem::path& path, std::vector<sys_dir_entry_t>& out) {
auto real_path = get_internal_name(path);
DIR* dir = opendir(real_path.string().c_str());
if (dir == nullptr) {
return;
}
for (const dirent* entry = readdir(dir); entry != nullptr; entry = readdir(dir)) {
sys_dir_entry_t r {};
r.name = entry->d_name;
if (entry->d_type == DT_UNKNOWN) {
// Some filesystems do not populate d_type.
struct stat s {};
r.is_file = 0 == lstat((real_path / r.name).string().c_str(), &s) && S_ISREG(s.st_mode);
} else {
r.is_file = entry->d_type != DT_DIR;
}
out.push_back(r);
}
closedir(dir);
}
bool SysFileMoveFile(const std::filesystem::path& /*src*/, const std::filesystem::path& /*dst*/) {
EXIT("not implemented\n");
return false;
bool SysFileCopyFile(const std::filesystem::path& src, const std::filesystem::path& dst) {
std::error_code error;
return std::filesystem::copy_file(get_internal_name(src), get_internal_name(dst),
std::filesystem::copy_options::overwrite_existing, error) &&
!error;
}
void SysFileRemoveReadonly(const std::filesystem::path& /*name*/) {
EXIT("not implemented\n");
bool SysFileMoveFile(const std::filesystem::path& src, const std::filesystem::path& dst) {
auto real_src = get_internal_name(src);
auto real_dst = get_internal_name(dst);
// Match MoveFileW: fail when the destination exists.
std::error_code error;
if (std::filesystem::exists(real_dst, error)) {
return false;
}
return 0 == rename(real_src.string().c_str(), real_dst.string().c_str());
}
void SysFileRemoveReadonly(const std::filesystem::path& name) {
auto real_name = get_internal_name(name);
auto real_name_str = real_name.string();
struct stat s {};
if (0 != stat(real_name_str.c_str(), &s)) {
return;
}
chmod(real_name_str.c_str(), s.st_mode | S_IWUSR);
}
#endif
+125 -31
View File
@@ -8,9 +8,11 @@
#include "common/platform/sysVirtual.h"
#include "common/virtualMemory.h"
#include <atomic>
#include <map>
#include <pthread.h>
#include <sys/mman.h>
#include <unistd.h>
#if defined(__APPLE__)
#include <mach/mach.h>
@@ -81,6 +83,70 @@ static VirtualMemory::Mode get_protection_flag(int mode) {
}
}
// Keep automatic mappings inside the guest and GPU-addressable low window.
#ifdef KYTY_FIXED_NOREPLACE
static constexpr uintptr_t LOW_ARENA_LIMIT = 0x000000FC00000000ULL; // libc mspace window ceiling
static constexpr uintptr_t LOW_ARENA_FLOOR = 0x000000A000000000ULL; // 640 GiB
static constexpr uintptr_t LOW_ARENA_GRAIN = 0x0000000000010000ULL; // 64 KiB
static_assert(LOW_ARENA_LIMIT <= 0x0000010000000000ULL,
"arena must stay inside the GPU page tracker's 1<<40 window");
static_assert(LOW_ARENA_FLOOR < LOW_ARENA_LIMIT, "arena floor must sit below its ceiling");
static std::atomic<uintptr_t> g_low_arena_next {LOW_ARENA_LIMIT};
#endif
// Caller holds g_virtual_mutex.
static void record_alloc(uintptr_t addr, size_t size) {
auto next = g_allocs->upper_bound(addr);
if (next != g_allocs->begin()) {
auto it = std::prev(next);
const auto alloc_addr = it->first;
const auto alloc_end = alloc_addr + it->second;
if (alloc_addr <= addr && addr + size <= alloc_end) {
g_allocs->erase(it);
if (alloc_addr < addr) {
(*g_allocs)[alloc_addr] = addr - alloc_addr;
}
if (addr + size < alloc_end) {
(*g_allocs)[addr + size] = alloc_end - (addr + size);
}
}
}
(*g_allocs)[addr] = size;
}
#ifdef KYTY_FIXED_NOREPLACE
static uintptr_t align_up_to(uintptr_t addr, uint64_t alignment) {
return (addr + alignment - 1) & ~(alignment - 1);
}
#endif
// Freed arena addresses are not reused while GPU caches remain keyed by address.
static void* map_anonymous(uintptr_t addr, size_t size, int protect, int flags) {
if (addr != 0) {
return mmap(reinterpret_cast<void*>(addr), size, protect, flags, -1, 0); // NOLINT
}
#ifdef KYTY_FIXED_NOREPLACE
const auto step = align_up_to(size, LOW_ARENA_GRAIN);
for (int attempt = 0; attempt < 256; attempt++) {
const auto top = g_low_arena_next.fetch_sub(step, std::memory_order_relaxed);
if (top < step || top - step < LOW_ARENA_FLOOR) {
break;
}
const auto hint = (top - step) & ~(LOW_ARENA_GRAIN - 1);
void* ptr = mmap(reinterpret_cast<void*>(hint), size, protect,
flags | MAP_FIXED_NOREPLACE, -1, 0); // NOLINT
if (ptr != MAP_FAILED) {
return ptr;
}
}
#endif
return mmap(nullptr, size, protect, flags, -1, 0); // NOLINT
}
uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
EXIT_IF(g_allocs == nullptr);
@@ -88,14 +154,13 @@ uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mo
int protect = get_protection_flag(mode);
void* ptr =
mmap(reinterpret_cast<void*>(addr), size, protect, MAP_PRIVATE | MAP_ANON, -1, 0); // NOLINT
void* ptr = map_anonymous(addr, size, protect, MAP_PRIVATE | MAP_ANON);
auto ret_addr = reinterpret_cast<uintptr_t>(ptr);
if (ptr != MAP_FAILED) {
pthread_mutex_lock(&g_virtual_mutex);
(*g_allocs)[ret_addr] = size;
record_alloc(ret_addr, size);
uintptr_t page_start = ret_addr >> 12u;
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
for (uintptr_t page = page_start; page <= page_end; page++) {
@@ -122,16 +187,15 @@ uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::
auto addr = static_cast<uintptr_t>(address);
int protect = get_protection_flag(mode);
void* ptr =
mmap(reinterpret_cast<void*>(addr), size, protect, MAP_PRIVATE | MAP_ANON, -1, 0); // NOLINT
void* ptr = map_anonymous(addr, size, protect, MAP_PRIVATE | MAP_ANON);
auto ret_addr = reinterpret_cast<uintptr_t>(ptr);
if (ptr != MAP_FAILED && ((ret_addr & (alignment - 1)) != 0)) {
munmap(ptr, size);
ptr = mmap(reinterpret_cast<void*>(addr), size + alignment, protect,
MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); // NOLINT
ptr = map_anonymous(addr, size + alignment, protect,
MAP_PRIVATE | MAP_ANON | MAP_NORESERVE);
ret_addr = reinterpret_cast<uintptr_t>(ptr);
if (ptr != MAP_FAILED) {
#if defined(__APPLE__)
@@ -186,7 +250,7 @@ uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::
}
pthread_mutex_lock(&g_virtual_mutex);
(*g_allocs)[ret_addr] = size;
record_alloc(ret_addr, size);
uintptr_t page_start = ret_addr >> 12u;
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
for (uintptr_t page = page_start; page <= page_end; page++) {
@@ -272,7 +336,7 @@ bool SysVirtualAllocFixed(uint64_t address, uint64_t size, VirtualMemory::Mode m
if (ptr != MAP_FAILED) {
pthread_mutex_lock(&g_virtual_mutex);
(*g_allocs)[ret_addr] = size;
record_alloc(ret_addr, size);
uintptr_t page_start = ret_addr >> 12u;
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
for (uintptr_t page = page_start; page <= page_end; page++) {
@@ -303,16 +367,15 @@ uint64_t SysVirtualReserveAligned(uint64_t address, uint64_t size, uint64_t alig
auto addr = static_cast<uintptr_t>(address);
void* ptr = mmap(reinterpret_cast<void*>(addr), size, PROT_NONE,
MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); // NOLINT
void* ptr = map_anonymous(addr, size, PROT_NONE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE);
auto ret_addr = reinterpret_cast<uintptr_t>(ptr);
if (ptr != MAP_FAILED && ((ret_addr & (alignment - 1)) != 0)) {
munmap(ptr, size);
ptr = mmap(reinterpret_cast<void*>(addr), size + alignment, PROT_NONE,
MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); // NOLINT
ptr = map_anonymous(addr, size + alignment, PROT_NONE,
MAP_PRIVATE | MAP_ANON | MAP_NORESERVE);
ret_addr = reinterpret_cast<uintptr_t>(ptr);
if (ptr != MAP_FAILED) {
#if defined(__APPLE__)
@@ -368,12 +431,7 @@ uint64_t SysVirtualReserveAligned(uint64_t address, uint64_t size, uint64_t alig
}
pthread_mutex_lock(&g_virtual_mutex);
(*g_allocs)[ret_addr] = size;
uintptr_t page_start = ret_addr >> 12u;
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
for (uintptr_t page = page_start; page <= page_end; page++) {
(*g_protects)[page] = PROT_NONE;
}
record_alloc(ret_addr, size);
pthread_mutex_unlock(&g_virtual_mutex);
return ret_addr;
@@ -406,12 +464,7 @@ bool SysVirtualReserveFixed(uint64_t address, uint64_t size) {
if (ptr != MAP_FAILED) {
pthread_mutex_lock(&g_virtual_mutex);
(*g_allocs)[ret_addr] = size;
uintptr_t page_start = ret_addr >> 12u;
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
for (uintptr_t page = page_start; page <= page_end; page++) {
(*g_protects)[page] = PROT_NONE;
}
record_alloc(ret_addr, size);
pthread_mutex_unlock(&g_virtual_mutex);
return true;
@@ -421,7 +474,29 @@ bool SysVirtualReserveFixed(uint64_t address, uint64_t size) {
}
bool SysVirtualDecommit(uint64_t address, uint64_t size) {
return SysVirtualProtect(address, size, VirtualMemory::Mode::NoAccess);
// Drop physical pages while preserving the reservation.
if (!SysVirtualProtect(address, size, VirtualMemory::Mode::NoAccess)) {
return false;
}
if (size != 0) {
#if defined(__APPLE__)
constexpr int RECLAIM_ADVICE = MADV_FREE;
#else
constexpr int RECLAIM_ADVICE = MADV_DONTNEED;
#endif
const auto page_size = static_cast<uintptr_t>(sysconf(_SC_PAGESIZE));
if (page_size != 0) {
// Do not discard pages outside the requested range.
const auto begin = (static_cast<uintptr_t>(address) + page_size - 1) & ~(page_size - 1);
const auto end = (static_cast<uintptr_t>(address) + size) & ~(page_size - 1);
if (end > begin) {
::madvise(reinterpret_cast<void*>(begin), end - begin, RECLAIM_ADVICE);
}
}
}
return true;
}
bool SysVirtualFree(uint64_t address) {
@@ -473,15 +548,34 @@ bool SysVirtualFreeRange(uint64_t address, uint64_t size) {
pthread_mutex_unlock(&g_virtual_mutex);
return false;
}
auto it = std::prev(next);
const auto alloc_addr = it->first;
const auto alloc_end = alloc_addr + it->second;
if (addr < alloc_addr || end > alloc_end || munmap(reinterpret_cast<void*>(addr), size) != 0) {
// A reservation may have been split into several adjacent records.
auto first = std::prev(next);
const auto alloc_addr = first->first;
if (addr < alloc_addr || alloc_addr + first->second <= addr) {
pthread_mutex_unlock(&g_virtual_mutex);
return false;
}
g_allocs->erase(it);
auto last = first;
uintptr_t cursor = alloc_addr + first->second;
while (cursor < end) {
auto following = std::next(last);
if (following == g_allocs->end() || following->first != cursor) {
pthread_mutex_unlock(&g_virtual_mutex);
return false;
}
last = following;
cursor = following->first + following->second;
}
const auto alloc_end = cursor;
if (munmap(reinterpret_cast<void*>(addr), size) != 0) {
pthread_mutex_unlock(&g_virtual_mutex);
return false;
}
g_allocs->erase(first, std::next(last));
if (alloc_addr < addr) {
(*g_allocs)[alloc_addr] = addr - alloc_addr;
}
+11 -7
View File
@@ -27,7 +27,9 @@ struct SysFileTimeStruct {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
FILETIME time;
#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
// Nanoseconds preserve sub-second file timestamps.
time_t time;
long nanos;
#endif
bool is_invalid;
};
@@ -139,7 +141,7 @@ inline void SysFileToSystemTimeUtc(const SysFileTimeStruct& f, SysTimeStruct& t)
t.Hour = i.tm_hour;
t.Minute = i.tm_min;
t.Second = (i.tm_sec == 60 ? 59 : i.tm_sec);
t.Milliseconds = 0;
t.Milliseconds = static_cast<uint16_t>((f.nanos / 1000000) % 1000);
}
inline void SysTimeTToSystem(time_t t, SysTimeStruct& s) {
@@ -168,10 +170,11 @@ inline void SysSystemToFileTimeUtc(const SysTimeStruct& f, SysFileTimeStruct& t)
// Retrieves the current local date and time.
inline void SysGetSystemTime(SysTimeStruct& t) {
time_t st {};
// Preserve millisecond precision.
timespec now {};
struct tm i {};
if (time(&st) == static_cast<time_t>(-1) || localtime_r(&st, &i) == nullptr) {
if (clock_gettime(CLOCK_REALTIME, &now) != 0 || localtime_r(&now.tv_sec, &i) == nullptr) {
t.is_invalid = true;
return;
}
@@ -183,15 +186,16 @@ inline void SysGetSystemTime(SysTimeStruct& t) {
t.Hour = i.tm_hour;
t.Minute = i.tm_min;
t.Second = (i.tm_sec == 60 ? 59 : i.tm_sec);
t.Milliseconds = 0;
t.Milliseconds = static_cast<uint16_t>((now.tv_nsec / 1000000) % 1000);
}
// Retrieves the current system date and time in Coordinated Universal Time (UTC).
inline void SysGetSystemTimeUtc(SysTimeStruct& t) {
time_t st {};
// Preserve millisecond precision.
timespec now {};
struct tm i {};
if (time(&st) == static_cast<time_t>(-1) || gmtime_r(&st, &i) == nullptr) {
if (clock_gettime(CLOCK_REALTIME, &now) != 0 || gmtime_r(&now.tv_sec, &i) == nullptr) {
t.is_invalid = true;
return;
}
@@ -203,7 +207,7 @@ inline void SysGetSystemTimeUtc(SysTimeStruct& t) {
t.Hour = i.tm_hour;
t.Minute = i.tm_min;
t.Second = (i.tm_sec == 60 ? 59 : i.tm_sec);
t.Milliseconds = 0;
t.Milliseconds = static_cast<uint16_t>((now.tv_nsec / 1000000) % 1000);
}
inline void SysQueryPerformanceFrequency(uint64_t* freq) {
+47
View File
@@ -7,6 +7,7 @@
#include <atomic>
#include <chrono> // IWYU pragma: keep
#include <condition_variable> // IWYU pragma: keep
#include <cerrno>
#include <mutex>
#include <vector>
@@ -14,6 +15,12 @@
#define KYTY_WIN_CS
#endif
// macOS has no clock_nanosleep.
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS && !defined(__APPLE__)
#define KYTY_POSIX_HIGH_RES_SLEEP
#include <ctime>
#endif
#include <sstream>
#include <string>
#include <thread>
@@ -121,6 +128,42 @@ static SleepConditionVariableCS_func_t ResolveSleepConditionVariableCS() {
#endif
#ifdef KYTY_POSIX_HIGH_RES_SLEEP
// Spin for very short waits; use an absolute deadline for longer waits.
static void SleepHighResolutionNanos(uint64_t nanos) {
if (nanos == 0) {
return;
}
constexpr uint64_t NANOS_PER_SEC = 1000000000;
constexpr uint64_t SPIN_LIMIT_NS = 50000; // below this a context switch dominates
timespec deadline {};
if (clock_gettime(CLOCK_MONOTONIC, &deadline) != 0) {
std::this_thread::sleep_for(std::chrono::nanoseconds(nanos));
return;
}
auto target_nsec = static_cast<uint64_t>(deadline.tv_nsec) + nanos;
deadline.tv_sec += static_cast<time_t>(target_nsec / NANOS_PER_SEC);
deadline.tv_nsec = static_cast<long>(target_nsec % NANOS_PER_SEC);
if (nanos <= SPIN_LIMIT_NS) {
timespec now {};
do {
if (clock_gettime(CLOCK_MONOTONIC, &now) != 0) {
return;
}
} while (now.tv_sec < deadline.tv_sec ||
(now.tv_sec == deadline.tv_sec && now.tv_nsec < deadline.tv_nsec));
return;
}
while (clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &deadline, nullptr) == EINTR) {
}
}
#endif
namespace Common {
using thread_id_t = std::thread::id;
@@ -249,6 +292,8 @@ void Thread::Sleep(uint32_t millis) {
void Thread::SleepMicro(uint32_t micros) {
#ifdef KYTY_WIN_CS
SleepHighResolution100ns(static_cast<uint64_t>(micros) * 10);
#elif defined(KYTY_POSIX_HIGH_RES_SLEEP)
SleepHighResolutionNanos(static_cast<uint64_t>(micros) * 1000);
#else
std::this_thread::sleep_for(std::chrono::microseconds(micros));
#endif
@@ -257,6 +302,8 @@ void Thread::SleepMicro(uint32_t micros) {
void Thread::SleepNano(uint64_t nanos) {
#ifdef KYTY_WIN_CS
SleepHighResolution100ns((nanos + 99) / 100);
#elif defined(KYTY_POSIX_HIGH_RES_SLEEP)
SleepHighResolutionNanos(nanos);
#else
std::this_thread::sleep_for(std::chrono::nanoseconds(nanos));
#endif
-19
View File
@@ -58,25 +58,6 @@ bool FlushInstructionCache(uint64_t address, uint64_t size) {
return SysVirtualFlushInstructionCache(address, size);
}
bool PatchReplace(uint64_t vaddr, uint64_t value) {
Mode old_mode {};
Protect(vaddr, 8, Mode::ReadWrite, &old_mode);
auto* ptr = reinterpret_cast<uint64_t*>(vaddr);
bool ret = (*ptr != value);
*ptr = value;
Protect(vaddr, 8, old_mode);
if (IsExecute(old_mode)) {
FlushInstructionCache(vaddr, 8);
}
return ret;
}
} // namespace VirtualMemory
} // namespace Common
-1
View File
@@ -37,7 +37,6 @@ bool Free(uint64_t address);
bool FreeRange(uint64_t address, uint64_t size);
bool Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode = nullptr);
bool FlushInstructionCache(uint64_t address, uint64_t size);
bool PatchReplace(uint64_t vaddr, uint64_t value);
} // namespace VirtualMemory
+13 -12
View File
@@ -105,7 +105,7 @@ static void ClearDebugTextureFolder() {
}
}
static void Init(const Config::ConfigOptions& cfg) {
static void Init(const Config::ConfigOptions& cfg, const std::filesystem::path& param_json) {
EXIT_IF(!Common::Thread::IsMainThread());
auto* slist = Common::SubsystemsList::Instance();
@@ -127,12 +127,21 @@ static void Init(const Config::ConfigOptions& cfg) {
slist->InitAll(true);
Config::Load(cfg);
slist->Add(log, {core, config});
slist->InitAll(true);
if (Common::File::IsFileExisting(param_json)) {
Loader::SystemContentLoadParamSfo(param_json);
if (const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
flexible_memory_size != 0) {
Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
}
}
slist->Add(audio, {core, log, pthread, memory});
slist->Add(controller, {core, log, config});
slist->Add(file_system, {core, log, pthread});
slist->Add(graphics, {core, log, pthread, memory, config, profiler, controller});
slist->Add(log, {core, config});
slist->Add(memory, {core, log});
slist->Add(network, {core, log, pthread});
slist->Add(profiler, {core, config});
@@ -180,7 +189,8 @@ void Run(const RunOptions& options) {
EXIT("ELF is required\n");
}
Init(options.config);
const auto param_json = options.app0_dir / "sce_sys" / "param.json";
Init(options.config, param_json);
ClearDebugTextureFolder();
@@ -192,15 +202,6 @@ void Run(const RunOptions& options) {
Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/app0");
Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/hostapp");
auto param_json = options.app0_dir / "sce_sys" / "param.json";
if (Common::File::IsFileExisting(param_json)) {
Loader::SystemContentLoadParamSfo(param_json);
if (auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
flexible_memory_size != 0) {
Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
}
}
MountSandboxDirs();
auto* rt = Common::Singleton<Loader::RuntimeLinker>::Instance();
@@ -65,10 +65,23 @@ constexpr uint32_t GcrKnownMask = GcrGl2MetadataInvalidate | GcrGl0V
GcrGl2Writeback | GcrOrder012 | GcrOrder210;
constexpr uint32_t RegisterSelectorMask = 0x70000000u;
uint32_t NormalizeRegisterOffset(uint32_t raw_offset) {
return (raw_offset & ~RegisterSelectorMask);
constexpr uint32_t NormalizeRegisterOffset(uint32_t raw_offset) {
return raw_offset & ~RegisterSelectorMask;
}
// Indirect Cx descriptors retain their selector. Selector 1 offsets 0..31 address the
// SPI_PS_INPUT_CNTL register bank; ordinary context-register offsets remain unchanged.
constexpr uint32_t DecodeIndirectCxRegisterOffset(uint32_t raw_offset) {
const auto offset = NormalizeRegisterOffset(raw_offset);
return (raw_offset & RegisterSelectorMask) == Pm4::CX_PS_SHADER_USAGE_BASE && offset < 32u
? Pm4::SPI_PS_INPUT_CNTL_0 + offset
: offset;
}
static_assert(DecodeIndirectCxRegisterOffset(Pm4::CX_PS_SHADER_USAGE_BASE + 2u) ==
Pm4::SPI_PS_INPUT_CNTL_0 + 2u);
static_assert(DecodeIndirectCxRegisterOffset(Pm4::DB_Z_INFO) == Pm4::DB_Z_INFO);
bool ReleaseMemGcrNeedsBarrier(uint32_t eop_event_type, uint32_t gcr_cntl) {
return eop_event_type != 0x28u ||
(gcr_cntl & (GcrGl2MetadataInvalidate | GcrGl0VectorInvalidate | GcrGl1Invalidate |
@@ -2321,14 +2334,18 @@ KYTY_CP_OP_PARSER(CpOpIndirectCxRegs) {
EXIT("indirect CX registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw);
}
for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) {
auto cmd_offset = indirect_buffer[0];
// Keep the encoded offset for packet control values, and use the decoded offset only
// for register dispatch.
auto raw_cmd_offset = indirect_buffer[0];
auto cmd_offset = DecodeIndirectCxRegisterOffset(raw_cmd_offset);
auto value = indirect_buffer[1];
if (HwCtxTrySetFakeRegister(cmd_offset, value)) {
continue;
}
if (cmd_offset == 0xffffffffu) {
// The sentinel is an encoded descriptor value and must be checked before normalization.
if (raw_cmd_offset == 0xffffffffu) {
static bool logged = false;
if (!logged) {
LOGF("\t temporary: skipping indirect CX sentinel pair offset = 0xffffffff, value "
+3
View File
@@ -385,6 +385,9 @@ constexpr uint32_t SPI_SHADER_POS_FORMAT = 0x1C3;
constexpr uint32_t SPI_SHADER_Z_FORMAT = 0x1C4;
constexpr uint32_t SPI_SHADER_COL_FORMAT = 0x1C5;
// Indirect Cx descriptor selector for the 32-entry PS input-control register bank.
constexpr uint32_t CX_PS_SHADER_USAGE_BASE = 0x10000000u;
constexpr uint32_t CB_BLEND0_CONTROL = 0x1E0;
constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_SHIFT = 0;
constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_MASK = 0x1F;
-33
View File
@@ -4,16 +4,6 @@
namespace Libs::Graphics {
#if defined(KYTY_MEMORY_TRACKER_TESTS)
namespace {
std::atomic<MemoryTracker::UnmapContentionHook> g_unmap_contention_hook {nullptr};
}
void MemoryTracker::SetUnmapContentionHook(UnmapContentionHook hook) noexcept {
g_unmap_contention_hook.store(hook, std::memory_order_release);
}
#endif
static_assert(std::atomic<void*>::is_always_lock_free);
MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode)
@@ -94,7 +84,6 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
return manager->IsModified<DirtySource::Cpu>(offset, bytes);
@@ -104,7 +93,6 @@ bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
return Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
return manager->IsModified<DirtySource::Gpu>(offset, bytes);
@@ -114,7 +102,6 @@ bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
const auto changed =
@@ -126,7 +113,6 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
const auto changed =
@@ -138,7 +124,6 @@ void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
if (!manager->IsFullyModified<DirtySource::Gpu>(offset, bytes)) {
@@ -151,8 +136,6 @@ void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
}
void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
RequireMapped(vaddr, size);
std::vector<RegionManager*> managers;
managers.reserve((vaddr % TRACKER_REGION_SIZE + size + TRACKER_REGION_SIZE - 1) /
TRACKER_REGION_SIZE);
@@ -185,22 +168,6 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
UntrackMemoryLocked(vaddr, size);
}
void MemoryTracker::UnmapMemory(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::unique_lock access(m_access_mutex, std::try_to_lock);
if (!access.owns_lock()) {
#if defined(KYTY_MEMORY_TRACKER_TESTS)
if (const auto hook = g_unmap_contention_hook.load(std::memory_order_acquire);
hook != nullptr) {
hook();
}
#endif
access.lock();
}
UntrackMemoryLocked(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size);
}
bool MemoryTracker::InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept {
switch (phase) {
case PageFaultPhase::Release: return true;
+41 -16
View File
@@ -30,7 +30,6 @@ public:
void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
void UntrackMemory(uint64_t vaddr, uint64_t size);
void UnmapMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] CpuFaultAction
BeginCpuFault(uint64_t vaddr, uint64_t size,
PageFaultAccess access = PageFaultAccess::Write) noexcept;
@@ -38,6 +37,47 @@ public:
bool downloaded) noexcept;
[[nodiscard]] bool InvalidateRegion(uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept;
template <typename Flush>
void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) {
static_assert(std::is_invocable_v<Flush&>);
CheckNotInUploadCallback();
ValidateRange(vaddr, size);
const auto update_cpu_state = [this, vaddr, size] {
std::lock_guard access(m_access_mutex);
std::vector<RegionManager*> managers;
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
managers.push_back(manager);
});
std::vector<std::unique_lock<TrackingSpinLock>> locks;
locks.reserve(managers.size());
for (auto* manager: managers) {
locks.emplace_back(manager->lock);
}
const bool gpu_modified = Iterate<false>(
vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
return manager->IsModified<DirtySource::Gpu>(offset, bytes);
});
if (gpu_modified) {
return true;
}
Iterate<false>(vaddr, size,
[](RegionManager* manager, uint64_t offset, uint64_t bytes) {
const auto changed = manager->ChangeState<DirtySource::Cpu, true>(
manager->GetCpuAddr() + offset, bytes);
manager->ApplyProtection(changed, false);
});
return false;
};
if (!update_cpu_state()) {
return;
}
std::forward<Flush>(on_flush)();
if (update_cpu_state()) {
EXIT("memory invalidation retained GPU-owned pages\n");
}
}
[[nodiscard]] bool InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr,
uint64_t size, PageFaultPhase phase) noexcept;
void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
@@ -51,7 +91,6 @@ public:
static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
std::vector<RegionManager*> managers;
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
managers.push_back(manager);
@@ -91,11 +130,6 @@ public:
vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward<Func>(func));
}
#if defined(KYTY_MEMORY_TRACKER_TESTS)
using UnmapContentionHook = void (*)() noexcept;
static void SetUnmapContentionHook(UnmapContentionHook hook) noexcept;
#endif
template <typename RangeFunc, typename UploadFunc>
void ForEachUploadRange(uint64_t vaddr, uint64_t size, bool is_written, RangeFunc&& range_func,
UploadFunc&& upload_func) {
@@ -103,7 +137,6 @@ public:
static_assert(std::is_nothrow_invocable_v<UploadFunc&>);
CheckNotInUploadCallback();
std::unique_lock access(m_access_mutex);
RequireMapped(vaddr, size);
Iterate<true>(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {});
const auto* previous_upload_owner = std::exchange(s_upload_owner, this);
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
@@ -170,14 +203,6 @@ private:
static void ValidateRange(uint64_t vaddr, uint64_t size);
void UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
void RequireMapped(uint64_t vaddr, uint64_t size) const {
ValidateRange(vaddr, size);
if (!m_page_manager.IsMapped(vaddr, size)) {
EXIT("memory tracker range [0x%llx, 0x%llx) is not mapped\n",
static_cast<unsigned long long>(vaddr),
static_cast<unsigned long long>(vaddr + size));
}
}
RegionManager* GetOrCreateRegion(uint64_t index);
std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;
+207 -299
View File
@@ -1,7 +1,9 @@
#include "graphics/host_gpu/pageManager.h"
#include "graphics/host_gpu/regionDefinitions.h"
#include "kernel/memory.h"
#include <algorithm>
#include <array>
#include <atomic>
#include <cstdarg>
@@ -20,11 +22,14 @@
#undef min
#undef max
#elif defined(__APPLE__)
#include <mach/mach.h>
#include <mach/mach_vm.h>
#include <pthread.h>
#include <sys/mman.h>
#include <unistd.h>
#else
#include <execinfo.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <unistd.h>
#endif
namespace Libs::Graphics {
@@ -48,42 +53,8 @@ constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE;
constexpr uint32_t NO_ACCESS_PROTECTION = PAGE_NOACCESS;
constexpr uint32_t READ_ONLY_PROTECTION = PAGE_READONLY;
constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
#if defined(__APPLE__)
// Map the tracker's Win32-style protection tags to POSIX mprotect flags.
static int PageProtToPosix(uint32_t protection) {
switch (protection) {
case PAGE_NOACCESS: return PROT_NONE;
case PAGE_READONLY: return PROT_READ;
case PAGE_READWRITE: return PROT_READ | PROT_WRITE;
default: return PROT_NONE;
}
}
// Query the current protection of the page containing vaddr via the Mach VM map and
// collapse it to the tracker's read/write tags (execute is irrelevant to write tracking).
static uint32_t MachQueryPageProt(uint64_t vaddr) {
auto region_addr = static_cast<mach_vm_address_t>(vaddr);
mach_vm_size_t region_size = 0;
vm_region_basic_info_data_64_t info {};
mach_msg_type_number_t count = VM_REGION_BASIC_INFO_COUNT_64;
mach_port_t object_name = MACH_PORT_NULL;
kern_return_t kr =
mach_vm_region(mach_task_self(), &region_addr, &region_size, VM_REGION_BASIC_INFO_64,
reinterpret_cast<vm_region_info_t>(&info), &count, &object_name);
if (kr != KERN_SUCCESS || region_addr > vaddr) {
return PAGE_NOACCESS; // no region covering vaddr
}
if ((info.protection & VM_PROT_WRITE) != 0) {
return PAGE_READWRITE;
}
if ((info.protection & VM_PROT_READ) != 0) {
return PAGE_READONLY;
}
return PAGE_NOACCESS;
}
#endif
// Zero is the unknown protection sentinel.
constexpr uint32_t UNKNOWN_PROTECTION = 0;
thread_local bool g_in_fault_resolution = false;
@@ -101,6 +72,10 @@ thread_local bool g_in_fault_resolution = false;
std::fprintf(stderr, " frame[%u]=0x%016" PRIxPTR " image_rva=0x%016" PRIxPTR "\n", i,
address, address >= image_base ? address - image_base : 0);
}
#elif !defined(__APPLE__)
void* frames[16] {};
const int frame_count = ::backtrace(frames, static_cast<int>(std::size(frames)));
::backtrace_symbols_fd(frames, frame_count, STDERR_FILENO);
#endif
std::fflush(stderr);
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
@@ -120,13 +95,31 @@ thread_local bool g_in_fault_resolution = false;
std::_Exit(322);
}
Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
switch (protection) {
case NO_ACCESS_PROTECTION: return Common::VirtualMemory::Mode::NoAccess;
case READ_ONLY_PROTECTION: return Common::VirtualMemory::Mode::Read;
case READ_WRITE_PROTECTION: return Common::VirtualMemory::Mode::ReadWrite;
default: Fatal("unmappable protection 0x%08" PRIx32, protection);
}
}
uint32_t CurrentThread() noexcept {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
return GetCurrentThreadId();
#elif defined(__APPLE__)
return static_cast<uint32_t>(pthread_mach_thread_np(pthread_self()));
#elif defined(__linux__)
static thread_local const uint32_t tid = [] {
const auto raw = static_cast<uint32_t>(::syscall(SYS_gettid));
if (raw == 0) {
FailFast("gettid returned the reserved zero owner token");
}
return raw;
}();
return tid;
#else
FailFast();
FailFast("page tracking thread identity is unsupported on this platform");
#endif
}
@@ -164,13 +157,12 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
struct PageManager::Impl {
struct PageState {
std::atomic_flag lock = ATOMIC_FLAG_INIT;
uint32_t mappings = 0;
uint32_t gpu_read_mappings = 0;
uint32_t gpu_write_mappings = 0;
uint32_t write_watchers = 0;
uint32_t access_watchers = 0;
uint32_t original_protection = 0;
uint32_t backing_writer = 0;
// Shadow the protection applied through Protect().
uint32_t current_protection = UNKNOWN_PROTECTION;
bool resolving = false;
bool resolving_read_write = false;
bool late_read_pending = false;
@@ -181,9 +173,29 @@ struct PageManager::Impl {
std::array<PageState, REGION_PAGES> pages;
};
class PageRangeGuard final {
public:
explicit PageRangeGuard(std::span<PageState*> pages): m_pages(pages) {
for (auto* page: m_pages) {
while (page->lock.test_and_set(std::memory_order_acquire)) {
std::atomic_signal_fence(std::memory_order_seq_cst);
}
}
}
~PageRangeGuard() {
for (auto it = m_pages.rbegin(); it != m_pages.rend(); ++it) {
(*it)->lock.clear(std::memory_order_release);
}
}
KYTY_CLASS_NO_COPY(PageRangeGuard);
private:
std::span<PageState*> m_pages;
};
Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) {
if (fault_handler == nullptr) {
Fatal("null fault handler");
Fatal("null page-manager fault callback");
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
SYSTEM_INFO info {};
@@ -195,11 +207,13 @@ struct PageManager::Impl {
#elif defined(__APPLE__)
// Under Rosetta the host page size is 4 KB, matching TRACKER_PAGE_SIZE.
if (static_cast<uint64_t>(getpagesize()) != PAGE_SIZE) {
Fatal("unsupported host page size 0x%08" PRIx32,
static_cast<uint32_t>(getpagesize()));
Fatal("unsupported host page size 0x%08" PRIx32, static_cast<uint32_t>(getpagesize()));
}
#else
Fatal("page-fault invalidation is not implemented on this platform");
const auto host_page_size = ::sysconf(_SC_PAGESIZE);
if (host_page_size < 0 || static_cast<uint64_t>(host_page_size) != PAGE_SIZE) {
Fatal("unsupported host page size %ld", static_cast<long>(host_page_size));
}
#endif
regions = std::make_unique<std::atomic<Region*>[]>(REGION_COUNT);
for (uint64_t i = 0; i < REGION_COUNT; i++) {
@@ -211,9 +225,8 @@ struct PageManager::Impl {
for (const auto& region: region_storage) {
for (auto& page: region->pages) {
SpinGuard lock(page.lock);
if (page.mappings != 0 || page.gpu_read_mappings != 0 ||
page.gpu_write_mappings != 0 || page.write_watchers != 0 ||
page.access_watchers != 0 || page.backing_writer != 0 || page.resolving) {
if (page.write_watchers != 0 || page.access_watchers != 0 ||
page.backing_writer != 0 || page.resolving) {
FailFast("PageManager destroyed with live page state");
}
}
@@ -260,98 +273,65 @@ struct PageManager::Impl {
if (old_protection == NO_ACCESS_PROTECTION && new_protection != NO_ACCESS_PROTECTION) {
page.late_read_pending = true;
}
if ((old_protection == NO_ACCESS_PROTECTION ||
old_protection == READ_ONLY_PROTECTION) &&
if ((old_protection == NO_ACCESS_PROTECTION || old_protection == READ_ONLY_PROTECTION) &&
new_protection == READ_WRITE_PROTECTION) {
page.late_write_pending = true;
}
}
static uint32_t QueryProtection(uint64_t vaddr) {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
MEMORY_BASIC_INFORMATION info {};
if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
sizeof(info)) == 0 ||
info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) {
Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32
", protection=0x%08" PRIx32 ")",
vaddr, static_cast<uint32_t>(info.State), static_cast<uint32_t>(info.Protect));
static void InitializeProtection(std::span<PageState*> pages) {
for (auto* page: pages) {
page->original_protection = READ_WRITE_PROTECTION;
page->current_protection = READ_WRITE_PROTECTION;
}
return info.Protect;
#elif defined(__APPLE__)
const uint32_t protection = MachQueryPageProt(vaddr);
if (protection != PAGE_READWRITE) {
Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (protection=0x%08" PRIx32
")",
vaddr, protection);
}
return protection;
#else
(void)vaddr;
Fatal("page query is unsupported on this platform");
#endif
}
static bool AllowsAccess(uint64_t vaddr, PageFaultAccess access) noexcept {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
MEMORY_BASIC_INFORMATION info {};
if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
sizeof(info)) == 0 ||
info.State != MEM_COMMIT) {
return false;
}
static bool AllowsAccess(const PageState& page, [[maybe_unused]] uint64_t vaddr,
PageFaultAccess access) noexcept {
switch (access) {
case PageFaultAccess::Read:
return info.Protect == PAGE_READONLY || info.Protect == PAGE_READWRITE;
case PageFaultAccess::Write: return info.Protect == PAGE_READWRITE;
return page.current_protection == READ_ONLY_PROTECTION ||
page.current_protection == READ_WRITE_PROTECTION;
case PageFaultAccess::Write: return page.current_protection == READ_WRITE_PROTECTION;
default: return false;
}
#elif defined(__APPLE__)
const uint32_t protection = MachQueryPageProt(vaddr);
switch (access) {
case PageFaultAccess::Read:
return protection == PAGE_READONLY || protection == PAGE_READWRITE;
case PageFaultAccess::Write: return protection == PAGE_READWRITE;
default: return false;
}
#else
(void)vaddr;
return false;
#endif
}
static void Protect(uint64_t vaddr, uint32_t protection, uint32_t expected_old,
bool fault_path) noexcept {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
DWORD old_protection = 0;
if (VirtualProtect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), PAGE_SIZE,
protection, &old_protection) == 0 ||
old_protection != expected_old) {
void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
std::span<const uint32_t> expected_old, bool fault_path) noexcept {
const auto size = pages.size() * PAGE_SIZE;
if (pages.size() != expected_old.size()) {
FailFast("protection range state size mismatch");
}
for (size_t i = 0; i < pages.size(); i++) {
const auto actual = pages[i]->current_protection;
if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) {
if (fault_path) {
FailFast("VirtualProtect fault transition did not match expected protection");
FailFast("mprotect fault transition did not match expected protection");
}
Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
vaddr, static_cast<uint32_t>(old_protection), expected_old, protection);
vaddr + i * PAGE_SIZE, actual, expected_old[i], protection);
}
#elif defined(__APPLE__)
// mprotect cannot report the previous protection, so the expected_old comparison
// is dropped; the tracker is the sole mutator of these pages and drives the
// transition from its own shadow state.
(void)expected_old;
if (mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), PAGE_SIZE,
PageProtToPosix(protection)) != 0) {
}
if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
ToMemoryMode(protection))) {
if (fault_path) {
FailFast("mprotect fault transition failed");
FailFast("address-space fault protection transition failed");
}
Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, protection);
Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
protection);
}
#else
(void)vaddr;
(void)protection;
(void)fault_path;
FailFast("page protection is unsupported on this platform");
#endif
for (auto* page: pages) {
page->current_protection = protection;
}
}
void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old,
bool fault_path) noexcept {
PageState* pages[] = {&page};
uint32_t expected[] = {expected_old};
ProtectRange(pages, vaddr, protection, expected, fault_path);
}
std::unique_ptr<std::atomic<Region*>[]> regions;
@@ -388,107 +368,120 @@ bool PageManager::IsTracked(uint64_t vaddr) const noexcept {
return page.write_watchers != 0 || page.access_watchers != 0;
}
bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
return false;
}
const auto end = PageStart(vaddr + size - 1) + PAGE_SIZE;
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
auto* region = m_impl->FindRegion(page_vaddr);
if (region == nullptr) {
return false;
}
auto& page = m_impl->GetPage(*region, page_vaddr);
SpinGuard lock(page.lock);
if (page.mappings == 0) {
return false;
}
}
return true;
}
bool PageManager::HasAnyMapping(uint64_t vaddr, uint64_t size) const noexcept {
if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE ||
size > ADDRESS_SIZE - vaddr) {
return false;
}
const auto end = PageEnd(vaddr, size);
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
auto* region = m_impl->FindRegion(page_vaddr);
if (region == nullptr) {
continue;
}
auto& page = m_impl->GetPage(*region, page_vaddr);
SpinGuard lock(page.lock);
if (page.mappings != 0) {
return true;
}
}
return false;
}
bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept {
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
FailFast("HasGpuAccess received an invalid GPU access mode");
}
const bool need_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
const bool need_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
return false;
}
const auto end = PageEnd(vaddr, size);
for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
auto* region = m_impl->FindRegion(addr);
if (region == nullptr) {
return false;
}
auto& page = m_impl->GetPage(*region, addr);
SpinGuard lock(page.lock);
if ((need_read && page.gpu_read_mappings == 0) ||
(need_write && page.gpu_write_mappings == 0)) {
return false;
}
}
return true;
}
void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
PageWatchMode mode) {
if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
Fatal("invalid watcher mode");
}
const auto begin = PageStart(vaddr);
const auto end = PageEnd(vaddr, size);
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
for (auto chunk_begin = begin; chunk_begin < end;) {
const auto chunk_end = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE);
auto* region =
track ? m_impl->GetOrCreateRegion(page_vaddr) : m_impl->FindRegion(page_vaddr);
track ? m_impl->GetOrCreateRegion(chunk_begin) : m_impl->FindRegion(chunk_begin);
if (region == nullptr) {
Fatal("untracking unknown page 0x%016" PRIx64, page_vaddr);
Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin);
}
auto& page = m_impl->GetPage(*region, page_vaddr);
SpinGuard lock(page.lock);
const auto page_count = static_cast<size_t>((chunk_end - chunk_begin) / PAGE_SIZE);
std::vector<Impl::PageState*> pages;
pages.reserve(page_count);
for (auto address = chunk_begin; address < chunk_end; address += PAGE_SIZE) {
pages.push_back(&m_impl->GetPage(*region, address));
}
Impl::PageRangeGuard lock(pages);
std::vector<uint8_t> first_watchers(page_count);
for (size_t i = 0; i < page_count; i++) {
auto& page = *pages[i];
const auto address = chunk_begin + i * PAGE_SIZE;
if (page.resolving && track) {
FailFast("new page watcher raced active fault resolution");
}
if (page.mappings == 0) {
Fatal("watching unmapped page 0x%016" PRIx64, page_vaddr);
}
auto& watchers =
(mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
if (track) {
if (watchers == std::numeric_limits<uint32_t>::max()) {
Fatal("watcher overflow at 0x%016" PRIx64, page_vaddr);
Fatal("watcher overflow at 0x%016" PRIx64, address);
}
const bool first_watcher = page.write_watchers == 0 && page.access_watchers == 0;
if (first_watcher) {
page.original_protection = Impl::QueryProtection(page_vaddr);
first_watchers[i] = page.write_watchers == 0 && page.access_watchers == 0;
} else {
if (watchers == 0) {
Fatal("watcher underflow at 0x%016" PRIx64, address);
}
if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) {
Fatal("backing write ownership changed at 0x%016" PRIx64, address);
}
}
}
if (track) {
for (size_t first = 0; first < page_count;) {
while (first < page_count && first_watchers[first] == 0) {
first++;
}
auto last = first;
while (last < page_count && first_watchers[last] != 0) {
last++;
}
if (first != last) {
Impl::InitializeProtection(std::span {pages}.subspan(first, last - first));
}
first = last;
}
}
std::vector<uint32_t> old_protections(page_count);
std::vector<uint32_t> new_protections(page_count);
std::vector<uint8_t> transitions(page_count);
for (size_t i = 0; i < page_count; i++) {
auto& page = *pages[i];
auto& watchers =
(mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
const auto old_protection = Impl::WatcherProtection(page);
if (track) {
watchers++;
const auto new_protection = Impl::WatcherProtection(page);
if (new_protection != old_protection) {
Impl::Protect(page_vaddr, new_protection, old_protection, false);
} else {
watchers--;
}
switch (new_protection) {
const auto new_protection = Impl::WatcherProtection(page);
old_protections[i] = old_protection;
new_protections[i] = new_protection;
if (new_protection != old_protection && (track || page.backing_writer == 0)) {
transitions[i] = 1;
}
}
for (size_t first = 0; first < page_count;) {
while (first < page_count && transitions[first] == 0) {
first++;
}
if (first == page_count) {
break;
}
const auto protection = new_protections[first];
auto current = first + 1;
auto last = current;
for (; current < page_count && new_protections[current] == protection; current++) {
if (old_protections[current] != new_protections[current] &&
transitions[current] == 0) {
break;
}
if (transitions[current] != 0) {
last = current + 1;
}
}
m_impl->ProtectRange(std::span {pages}.subspan(first, last - first),
chunk_begin + first * PAGE_SIZE, protection,
std::span {old_protections}.subspan(first, last - first), false);
first = current;
}
for (size_t i = 0; i < page_count; i++) {
auto& page = *pages[i];
const auto protection = new_protections[i];
if (track) {
switch (protection) {
case NO_ACCESS_PROTECTION:
page.late_read_pending = false;
page.late_write_pending = false;
@@ -496,87 +489,20 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
case READ_ONLY_PROTECTION: page.late_write_pending = false; break;
default: break;
}
} else {
if (watchers == 0) {
Fatal("watcher underflow at 0x%016" PRIx64, page_vaddr);
}
if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) {
Fatal("backing write ownership changed at 0x%016" PRIx64, page_vaddr);
}
const auto old_protection = Impl::WatcherProtection(page);
watchers--;
const auto new_protection = Impl::WatcherProtection(page);
if (page.backing_writer == 0 && new_protection != old_protection) {
Impl::Protect(page_vaddr, new_protection, old_protection, false);
}
if (page.backing_writer == 0) {
Impl::PublishDelayedFaults(page, old_protection, new_protection);
}
if (page.backing_writer == 0 && page.write_watchers == 0 && page.access_watchers == 0) {
} else if (page.backing_writer == 0) {
Impl::PublishDelayedFaults(page, old_protections[i], protection);
if (page.write_watchers == 0 && page.access_watchers == 0) {
page.original_protection = 0;
}
}
}
}
void PageManager::OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access) {
if (g_in_fault_resolution) {
FailFast("GPU mapping changed during fault resolution");
}
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
FailFast("GPU map received an invalid access mode");
}
const bool gpu_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
const auto end = PageEnd(vaddr, size);
for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
auto& page = m_impl->GetPage(*m_impl->GetOrCreateRegion(addr), addr);
SpinGuard lock(page.lock);
if (page.resolving || page.mappings == std::numeric_limits<uint32_t>::max() ||
(gpu_read && page.gpu_read_mappings == std::numeric_limits<uint32_t>::max()) ||
(gpu_write && page.gpu_write_mappings == std::numeric_limits<uint32_t>::max())) {
Fatal("invalid map state at 0x%016" PRIx64, addr);
}
page.mappings++;
page.gpu_read_mappings += gpu_read ? 1u : 0u;
page.gpu_write_mappings += gpu_write ? 1u : 0u;
chunk_begin = chunk_end;
}
}
void PageManager::OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access) {
if (g_in_fault_resolution) {
FailFast("GPU unmapping changed during fault resolution");
}
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
FailFast("GPU unmap received an invalid access mode");
}
const bool gpu_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
const auto end = PageEnd(vaddr, size);
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
auto* region = m_impl->FindRegion(page_vaddr);
if (region == nullptr) {
Fatal("unmapping unknown page 0x%016" PRIx64, page_vaddr);
}
auto& page = m_impl->GetPage(*region, page_vaddr);
SpinGuard lock(page.lock);
if (page.resolving || page.mappings == 0 || (gpu_read && page.gpu_read_mappings == 0) ||
(gpu_write && page.gpu_write_mappings == 0) ||
(page.mappings == 1 && (page.write_watchers != 0 || page.access_watchers != 0))) {
Fatal("invalid unmap state at 0x%016" PRIx64, page_vaddr);
}
page.mappings--;
page.gpu_read_mappings -= gpu_read ? 1u : 0u;
page.gpu_write_mappings -= gpu_write ? 1u : 0u;
if (page.mappings == 0) {
if (page.gpu_read_mappings != 0 || page.gpu_write_mappings != 0) {
FailFast("GPU unmap left nonzero GPU mapping counts");
}
page.late_read_pending = false;
page.late_write_pending = false;
}
}
}
void PageManager::OnGpuMap(uint64_t, uint64_t) {}
void PageManager::OnGpuUnmap(uint64_t, uint64_t) {}
PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr,
uint64_t size) noexcept
@@ -632,8 +558,7 @@ void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
}
auto& page = m_impl->GetPage(*region, address);
SpinGuard lock(page.lock);
if (page.mappings == 0 || page.resolving || page.backing_writer != 0 ||
page.access_watchers == 0) {
if (page.resolving || page.backing_writer != 0 || page.access_watchers == 0) {
Fatal("backing write races page resolution at 0x%016" PRIx64, address);
}
page.resolving = true;
@@ -661,7 +586,7 @@ void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
const auto old_protection = NO_ACCESS_PROTECTION;
const auto new_protection = Impl::WatcherProtection(page);
if (new_protection != old_protection) {
Impl::Protect(address, new_protection, old_protection, false);
m_impl->Protect(page, address, new_protection, old_protection, false);
}
Impl::PublishDelayedFaults(page, old_protection, new_protection);
if (page.write_watchers == 0 && page.access_watchers == 0) {
@@ -686,12 +611,12 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex
while (true) {
SpinGuard lock(page.lock);
if (access == PageFaultAccess::Read && page.late_read_pending &&
Impl::AllowsAccess(fault_vaddr, access)) {
Impl::AllowsAccess(page, fault_vaddr, access)) {
page.late_read_pending = false;
return true;
}
if (access == PageFaultAccess::Write && page.late_write_pending &&
Impl::AllowsAccess(fault_vaddr, access)) {
Impl::AllowsAccess(page, fault_vaddr, access)) {
page.late_write_pending = false;
return true;
}
@@ -713,7 +638,7 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex
}
bool& pending = (access == PageFaultAccess::Read ? page.late_read_pending
: page.late_write_pending);
const bool allowed = Impl::AllowsAccess(fault_vaddr, access);
const bool allowed = Impl::AllowsAccess(page, fault_vaddr, access);
pending = false;
if (waited && !allowed) {
FailFast("page remained inaccessible after waiting for its resolver");
@@ -762,12 +687,13 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex
page.write_watchers = 0;
}
const auto restored_protection = Impl::WatcherProtection(page);
Impl::Protect(PageStart(fault_vaddr), restored_protection, old_protection, true);
m_impl->Protect(page, PageStart(fault_vaddr), restored_protection, old_protection,
true);
if (page.write_watchers == 0) {
page.original_protection = 0;
}
Impl::PublishDelayedFaults(page, old_protection, restored_protection);
} else if (!Impl::AllowsAccess(fault_vaddr, access)) {
} else if (!Impl::AllowsAccess(page, fault_vaddr, access)) {
FailFast("fault completion left the page inaccessible");
}
page.resolving = false;
@@ -783,22 +709,4 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex
return true;
}
bool PageManager::HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept {
if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE ||
size > ADDRESS_SIZE - vaddr) {
return false;
}
const auto end = PageEnd(vaddr, size);
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
if (!IsMapped(page_vaddr, 1)) {
continue;
}
const auto fault_vaddr = std::max(page_vaddr, vaddr);
if (!HandleFault(PageFaultAccess::Write, fault_vaddr)) {
return false;
}
}
return true;
}
} // namespace Libs::Graphics
+2 -8
View File
@@ -13,11 +13,9 @@ namespace Libs::Graphics {
enum class PageFaultAccess { Read, Write, Execute, Unknown };
enum class PageFaultPhase { Invalidate, Complete, Release };
enum class PageWatchMode { Write, ReadWrite };
enum class GpuAccess { Read, Write, ReadWrite };
using PageFaultHandler = bool (*)(void* context, PageFaultAccess access, uint64_t vaddr,
uint64_t size, PageFaultPhase phase) noexcept;
class PageManager final {
public:
class BackingWrite final {
@@ -40,17 +38,13 @@ public:
[[nodiscard]] uint64_t GetPageSize() const;
[[nodiscard]] bool IsTracked(uint64_t vaddr) const noexcept;
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
[[nodiscard]] bool HasAnyMapping(uint64_t vaddr, uint64_t size) const noexcept;
[[nodiscard]] bool HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept;
void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
PageWatchMode mode = PageWatchMode::Write);
void OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
void OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
void OnGpuMap(uint64_t vaddr, uint64_t size);
void OnGpuUnmap(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
[[nodiscard]] bool HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept;
[[nodiscard]] std::vector<std::unique_ptr<BackingWrite>>
ReserveBackingWrites(std::span<const RangeSet::Range> ranges);
+10
View File
@@ -59,6 +59,16 @@ public:
return result;
}
[[nodiscard]] bool Contains(uint64_t address, uint64_t size) const {
const auto end = End(address, size);
auto it = m_ranges.upper_bound(address);
if (it == m_ranges.begin()) {
return false;
}
--it;
return it->first <= address && it->second >= end;
}
template <typename Func>
void ForEachIntersection(uint64_t address, uint64_t size, Func&& func) const {
const auto end = End(address, size);
+6
View File
@@ -18,6 +18,9 @@
#undef max
#elif defined(__APPLE__)
#include <pthread.h>
#elif defined(__linux__)
#include <sys/syscall.h>
#include <unistd.h>
#endif
namespace Libs::Graphics {
@@ -54,6 +57,9 @@ private:
#elif defined(__APPLE__)
// mach thread port is a nonzero per-thread id (0 is the "no owner" sentinel).
return static_cast<uint32_t>(pthread_mach_thread_np(pthread_self()));
#elif defined(__linux__)
static thread_local const uint32_t tid = static_cast<uint32_t>(::syscall(SYS_gettid));
return tid;
#else
EXIT("region tracking thread identity is unsupported on this platform\n");
#endif
@@ -1,13 +1,13 @@
#include "graphics/host_gpu/renderer/bufferCache.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include "common/assert.h"
#include "common/logging/log.h"
#include "common/profiler.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/resourceMutex.h"
#include "graphics/host_gpu/renderer/textureCache.h"
#include "kernel/memory.h"
#include <algorithm>
@@ -184,8 +184,7 @@ BufferCache::RecordDownloads(std::span<const DownloadCopy> copies) {
}
auto& download = m_download_buffer;
const auto [mapped, base_offset] =
download.Map(reservation_size, DOWNLOAD_ALIGNMENT);
const auto [mapped, base_offset] = download.Map(reservation_size, DOWNLOAD_ALIGNMENT);
if (mapped == nullptr) {
EXIT("BufferCache: download batch could not reserve the shared stream\n");
}
@@ -215,16 +214,15 @@ void BufferCache::PublishDownloads(std::span<const DownloadRange> downloads) {
}
}
void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies,
RetiredBuffer retire) {
void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire) {
if (copies.empty()) {
return;
}
auto downloads = RecordDownloads(copies);
const auto tick = m_scheduler.CurrentTick();
BeginBackingPublication(retire.address, retire.size, tick);
m_scheduler.DeferOperation(
[this, downloads = std::move(downloads), retire = std::move(retire), tick]() mutable {
m_scheduler.DeferOperation([this, downloads = std::move(downloads), retire = std::move(retire),
tick]() mutable {
PublishDownloads(downloads);
{
FaultSafeCacheLock lock(this, m_mutex);
@@ -232,8 +230,7 @@ void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies,
m_memory_tracker.ForEachDownloadRange<true>(
retire.address, retire.size,
[&](uint64_t address, uint64_t size) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(
m_gpu_modified_ranges, address, size,
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
"asynchronous garbage retirement");
},
[](uint64_t, uint64_t) noexcept {});
@@ -303,8 +300,7 @@ bool BufferCache::SynchronizeBacking(uint64_t vaddr, uint64_t size) {
for (;;) {
uint64_t tick = 0;
const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
const auto page_end =
(vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
const auto page_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
CacheRange affected {.address = page_begin, .size = page_end - page_begin};
{
FaultSafeCacheLock lock(this, m_mutex);
@@ -384,6 +380,73 @@ BufferBinding BufferCache::UploadTransient(const void* data, uint64_t size, uint
return {owner, owner->Handle(), 0};
}
void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("BufferCache: invalid memory-invalidation range\n");
}
(void)SynchronizeBacking(vaddr, size);
if (!HasPageOverlap(vaddr, size)) {
return;
}
m_memory_tracker.InvalidateRegion(vaddr, size,
[this, vaddr, size] { ReadMemory(vaddr, size); });
}
void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
(void)SynchronizeBacking(vaddr, size);
std::vector<DownloadCopy> copies;
{
FaultSafeCacheLock lock(this, m_mutex);
m_memory_tracker.ForEachDownloadRange<false>(
vaddr, size,
[&](uint64_t address, uint64_t bytes) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
"memory invalidation");
},
[&](uint64_t address, uint64_t bytes) noexcept {
for (const auto range: m_gpu_modified_ranges.Intersections(address, bytes)) {
for (uint64_t copied = 0; copied < range.size;) {
const auto copy_address = range.address + copied;
auto owner = m_buffers.upper_bound(copy_address);
if (owner == m_buffers.begin()) {
EXIT("BufferCache: invalidation readback has no buffer owner\n");
}
auto& cached = *std::prev(owner)->second;
if (!cached.buffer->IsInBounds(copy_address, 1)) {
EXIT(
"BufferCache: invalidation readback is outside its buffer owner\n");
}
const auto copy_size = std::min(range.size - copied,
cached.vaddr + cached.size - copy_address);
copies.push_back({cached.buffer, cached.buffer->Offset(copy_address),
copy_address, copy_size});
copied += copy_size;
}
}
});
}
if (copies.empty()) {
return;
}
auto downloads = RecordDownloads(copies);
m_scheduler.FinishCurrent();
PublishDownloads(downloads);
{
FaultSafeCacheLock lock(this, m_mutex);
m_memory_tracker.ForEachDownloadRange<true>(
vaddr, size,
[&](uint64_t address, uint64_t bytes) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
"memory invalidation completion");
},
[](uint64_t, uint64_t) noexcept {});
for (const auto& range: downloads) {
m_gpu_modified_ranges.Subtract(range.address, range.size);
}
}
}
bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept {
const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1);
@@ -544,8 +607,8 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
m_memory_tracker.ForEachDownloadRange<true>(
begin, bytes,
[&](uint64_t address, uint64_t download_size) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(
m_gpu_modified_ranges, address, download_size, "unmap retirement");
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address,
download_size, "unmap retirement");
},
[](uint64_t, uint64_t) noexcept {});
}
@@ -651,7 +714,6 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
if (command.IsInvalid() || command.IsExecute()) {
EXIT("BufferCache: buffer request requires a recording command buffer\n");
}
ValidateGpuAccess(vaddr, size, is_read, is_written);
std::lock_guard transaction(m_resource_mutex);
(void)SynchronizeBacking(vaddr, size);
@@ -725,8 +787,7 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size);
const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size);
const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size);
const bool invalidated =
!m_image_invalidated_ranges.Intersections(vaddr, size).empty();
const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
const bool requested_gpu_owned = !dirty.empty();
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
"image source");
@@ -831,9 +892,8 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
[&]() noexcept {
for (const auto& [address, upload_size]: uploads) {
cached.buffer->CopyFrom(
m_scheduler.Current(), m_staging_buffer,
stage_offset + address - stage_address, cached.buffer->Offset(address),
upload_size, vk::AccessFlagBits::eHostWrite);
m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address,
cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
}
});
DiscardGpuDirtyBytesLocked(vaddr, size, "staged image source transfer");
@@ -917,9 +977,8 @@ std::pair<std::shared_ptr<Buffer>, uint64_t> BufferCache::ObtainBufferForImageWr
[&]() noexcept {
for (const auto& [address, upload_size]: uploads) {
cached.buffer->CopyFrom(
m_scheduler.Current(), m_staging_buffer,
stage_offset + address - stage_address, cached.buffer->Offset(address),
upload_size, vk::AccessFlagBits::eHostWrite);
m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address,
cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
}
});
return {cached.buffer, cached.buffer->Offset(vaddr)};
@@ -939,19 +998,18 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
if (vaddr == 0) {
EXIT("BufferCache: invalid fill memory address\n");
}
ValidateGpuAccess(vaddr, size, false, true);
(void)m_texture_cache.ClearMeta(vaddr);
{
std::lock_guard transaction(m_resource_mutex);
const auto region = m_texture_cache.QueryRegion(vaddr, size);
if (!HasGpuDirtyBytes(vaddr, size) && !region.gpu_image_bytes) {
if (region.image_bytes) {
m_texture_cache.PrepareHostWrite(vaddr, size);
m_texture_cache.InvalidateMemory(vaddr, size);
}
std::array<uint32_t, 4096> values;
values.fill(value);
const std::span<const uint8_t> bytes {
reinterpret_cast<const uint8_t*>(values.data()), sizeof(values)};
const std::span<const uint8_t> bytes {reinterpret_cast<const uint8_t*>(values.data()),
sizeof(values)};
for (uint64_t offset = 0; offset < size;) {
const auto chunk = std::min<uint64_t>(size - offset, bytes.size());
WriteHostMemory(vaddr + offset, bytes.first(chunk));
@@ -981,12 +1039,6 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
(src_gds && (src_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - src_vaddr))) {
EXIT("BufferCache: invalid or overlapping copy range\n");
}
if (src_memory) {
ValidateGpuAccess(src_vaddr, size, true, false);
}
if (dst_memory) {
ValidateGpuAccess(dst_vaddr, size, false, true);
}
if (src_memory || dst_memory) {
std::lock_guard transaction(m_resource_mutex);
if (src_memory) {
@@ -1004,7 +1056,7 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
!HasGpuDirtyBytes(dst_vaddr, size) && !src_region.gpu_image_bytes &&
!dst_region.gpu_image_bytes) {
if (dst_region.image_bytes) {
m_texture_cache.PrepareHostWrite(dst_vaddr, size);
m_texture_cache.InvalidateMemory(dst_vaddr, size);
}
std::array<uint8_t, 64 * 1024> bytes;
for (uint64_t offset = 0; offset < size;) {
@@ -1039,8 +1091,8 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
auto& source = src.owner != nullptr ? *std::static_pointer_cast<Buffer>(src.owner)
: src_gds ? m_gds_buffer
: m_stream_buffer;
auto& destination = dst.owner != nullptr ? *std::static_pointer_cast<Buffer>(dst.owner)
: m_gds_buffer;
auto& destination =
dst.owner != nullptr ? *std::static_pointer_cast<Buffer>(dst.owner) : m_gds_buffer;
if (source.Handle() != src.buffer || destination.Handle() != dst.buffer) {
EXIT("BufferCache: resolved copy owner does not match its Vulkan handle\n");
}
@@ -1143,19 +1195,6 @@ void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) {
owner->second->tick_accessed_last = m_gc_tick;
}
void BufferCache::ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read,
bool is_written) const {
if ((!is_read && !is_written) || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
EXIT("BufferCache: invalid GPU access request\n");
}
if (is_read && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Read)) {
EXIT("BufferCache: GPU-read access denied\n");
}
if (is_written && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Write)) {
EXIT("BufferCache: GPU-write access denied\n");
}
}
void BufferCache::RunGarbageCollector() {
std::lock_guard transaction(m_resource_mutex);
const auto tick = m_gc_tick++;
@@ -1190,8 +1229,8 @@ void BufferCache::RunGarbageCollector() {
}
for (const auto address: candidates) {
auto& cached = *m_buffers.at(address);
m_memory_tracker.ValidateGpuDirtyOwnership(
m_gpu_modified_ranges, cached.vaddr, cached.size, "garbage collection");
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, cached.vaddr,
cached.size, "garbage collection");
retires.push_back({address, cached.size, cached.buffer});
// GC runs immediately before submission. Preserve every source referenced by commands
// already recorded in the active batch.
@@ -1205,13 +1244,13 @@ void BufferCache::RunGarbageCollector() {
m_memory_tracker.ForEachDownloadRange<false>(
retire.address, retire.size,
[&](uint64_t address, uint64_t size) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(
m_gpu_modified_ranges, address, size, "garbage collection");
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
"garbage collection");
},
[&](uint64_t address, uint64_t size) noexcept {
for (const auto range: m_gpu_modified_ranges.Intersections(address, size)) {
copies.push_back({retire.owner, range.address - retire.address, range.address,
range.size});
copies.push_back({retire.owner, range.address - retire.address,
range.address, range.size});
}
});
}
@@ -6,7 +6,7 @@
#include "common/threads.h"
#include "graphics/host_gpu/memoryTracker.h"
#include "graphics/host_gpu/rangeSet.h"
#include "graphics/host_gpu/renderer/streamBuffer.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
#include <map>
#include <memory>
@@ -49,6 +49,8 @@ public:
[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept;
void InvalidateMemory(uint64_t vaddr, uint64_t size);
void ReadMemory(uint64_t vaddr, uint64_t size);
void UnmapMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
bool is_written = false, bool is_read = true,
@@ -75,7 +77,6 @@ public:
void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
[[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size);
void PublishImageBuffer(uint64_t vaddr, uint64_t size);
void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const;
void RunGarbageCollector();
private:
@@ -99,15 +100,13 @@ private:
}
[[nodiscard]] static bool PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right,
uint64_t right_size) noexcept;
[[nodiscard]] static std::pair<uint64_t, uint64_t>
DownloadEnvelope(const DownloadCopy& copy);
[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
[[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset,
const void* source, uint64_t size);
[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
uint64_t size);
[[nodiscard]] std::vector<DownloadRange>
RecordDownloads(std::span<const DownloadCopy> copies);
[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
void PublishDownloads(std::span<const DownloadRange> downloads);
void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
void RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, uint64_t vaddr,
@@ -1,10 +1,9 @@
#include "graphics/host_gpu/renderer/gpuResourceManager.h"
#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
#include "common/assert.h"
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
namespace Libs::Graphics {
GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
@@ -36,7 +35,8 @@ bool GpuResourceManager::InvalidateMemory(PageFaultAccess access, uint64_t vaddr
}
bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
if (!m_page_manager.IsMapped(fault_vaddr, 1)) {
constexpr uint64_t fault_size = 8;
if (!IsMapped(fault_vaddr, fault_size)) {
return false;
}
if (CommandScheduler::InDeferredOperation()) {
@@ -47,10 +47,15 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
bool handled = false;
const auto resolve = [this, access, fault_vaddr, &handled](CommandProcessor& cp) {
cp.BeginReadbackTransaction();
(void)m_buffer_cache.SynchronizeBacking(fault_vaddr, 1);
{
ResourceMutex::FaultScope fault(m_resource_mutex);
handled = m_page_manager.HandleFault(access, fault_vaddr);
if (access == PageFaultAccess::Write) {
m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
} else {
m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
}
handled = true;
}
cp.EndReadbackTransaction();
};
@@ -68,58 +73,62 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
return handled;
}
void GpuResourceManager::PrepareHostWrite(uint64_t vaddr, uint64_t size) {
if (!m_page_manager.HasAnyMapping(vaddr, size)) {
return;
bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) {
if (!IsMapped(vaddr, size)) {
return false;
}
if (CommandScheduler::InDeferredOperation()) {
EXIT("unsupported host write from an asynchronous GPU completion, addr=0x%016" PRIx64
" size=0x%016" PRIx64 "\n",
EXIT("unsupported memory invalidation from an asynchronous GPU completion, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
const auto handle_range = [this, vaddr, size] {
if (!m_page_manager.HandleWriteRange(vaddr, size)) {
EXIT("failed to prepare host write, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
};
const auto resolve = [this, &handle_range](CommandProcessor& cp) {
const auto resolve = [this, vaddr, size](CommandProcessor& cp) {
cp.BeginReadbackTransaction();
{
ResourceMutex::FaultScope fault(m_resource_mutex);
handle_range();
m_buffer_cache.InvalidateMemory(vaddr, size);
m_texture_cache.InvalidateMemory(vaddr, size);
}
cp.EndReadbackTransaction();
};
if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
resolve(*cp);
return;
return true;
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported host write from a pre-owned resource transaction, addr=0x%016" PRIx64
" size=0x%016" PRIx64 "\n",
EXIT("unsupported memory invalidation from a pre-owned resource transaction, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
EXIT_IF(m_gpu == nullptr);
m_gpu->SendCommandSyncWithProcessor(resolve);
return true;
}
bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
return m_page_manager.IsMapped(vaddr, size);
}
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
m_page_manager.OnGpuMap(vaddr, size, access);
}
void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
if (!IsMapped(vaddr, size)) {
EXIT("cannot unmap an unmapped GPU resource range\n");
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
return false;
}
const auto unmap = [this, vaddr, size, access] {
m_texture_cache.UnmapMemory(vaddr, size);
std::shared_lock lock(m_mapped_ranges_mutex);
return m_mapped_ranges.Contains(vaddr, size);
}
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
{
std::lock_guard lock(m_mapped_ranges_mutex);
m_mapped_ranges.Add(vaddr, size);
}
m_page_manager.OnGpuMap(vaddr, size);
}
void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
const auto unmap = [this, vaddr, size] {
m_buffer_cache.UnmapMemory(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size, access);
m_texture_cache.UnmapMemory(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size);
std::lock_guard lock(m_mapped_ranges_mutex);
m_mapped_ranges.Subtract(vaddr, size);
};
if (m_gpu == nullptr) {
if (m_resource_mutex.IsOwnedByCurrentThread()) {
@@ -4,11 +4,12 @@
#include "common/abi.h"
#include "common/common.h"
#include "graphics/host_gpu/pageManager.h"
#include "graphics/host_gpu/renderer/bufferCache.h"
#include "graphics/host_gpu/renderer/resourceMutex.h"
#include "graphics/host_gpu/renderer/textureCache.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include <cstdint>
#include <shared_mutex>
namespace Libs::Graphics {
@@ -26,10 +27,10 @@ public:
void SetGpu(Gpu* gpu) noexcept { m_gpu = gpu; }
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
void PrepareHostWrite(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void MapMemory(uint64_t vaddr, uint64_t size);
void UnmapMemory(uint64_t vaddr, uint64_t size);
void RunGarbageCollector();
private:
@@ -42,6 +43,8 @@ private:
ResourceMutex m_resource_mutex;
BufferCache m_buffer_cache;
TextureCache m_texture_cache;
mutable std::shared_mutex m_mapped_ranges_mutex;
RangeSet m_mapped_ranges;
Gpu* m_gpu = nullptr;
};
@@ -1,4 +1,4 @@
#include "graphics/host_gpu/renderer/resourceMutex.h"
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
#include "common/assert.h"
@@ -1,4 +1,4 @@
#include "graphics/host_gpu/renderer/samplerCache.h"
#include "graphics/host_gpu/renderer/cache/samplerCache.h"
#include "common/assert.h"
#include "common/logging/log.h"
@@ -1,4 +1,4 @@
#include "graphics/host_gpu/renderer/streamBuffer.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
#include "common/assert.h"
#include "common/profiler.h"
@@ -1,4 +1,4 @@
#include "graphics/host_gpu/renderer/textureCache.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "common/assert.h"
#include "common/emulatorConfig.h"
@@ -7,13 +7,13 @@
#include "graphics/guest_gpu/gpu_format.h"
#include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/objects/textureCommon.h"
#include "graphics/host_gpu/renderer/bufferCache.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/renderer/imageView.h"
#include "graphics/host_gpu/renderer/image/imageView.h"
#include "graphics/host_gpu/renderer/image/textureCommon.h"
#include "graphics/host_gpu/renderer/image/tiler.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/resourceMutex.h"
#include "graphics/host_gpu/renderer/tiler.h"
#include "kernel/memory.h"
#include <algorithm>
@@ -58,8 +58,8 @@ private:
TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler,
PageManager& page_manager, BufferCache& buffer_cache,
ResourceMutex& resource_mutex)
: m_graphics(graphics), m_scheduler(scheduler),
m_memory_tracker(page_manager, PageWatchMode::Write), m_blit_helper(graphics, scheduler),
: m_graphics(graphics), m_scheduler(scheduler), m_page_manager(page_manager),
m_blit_helper(graphics, scheduler),
m_tiler(std::make_unique<TileManager>(graphics, scheduler,
buffer_cache.GetUtilityBuffer(MemoryUsage::Stream))),
m_buffer_cache(buffer_cache), m_resource_mutex(resource_mutex),
@@ -80,7 +80,7 @@ TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler
TextureCache::~TextureCache() {
for (uint32_t index = 0; index < m_slots.size(); index++) {
if (m_slots[index].image != nullptr && m_slots[index].image->registered) {
UnregisterImage({index, m_slots[index].generation}, false);
UnregisterImage({index, m_slots[index].generation});
}
m_slots[index].image.reset();
}
@@ -88,15 +88,34 @@ TextureCache::~TextureCache() {
bool TextureCache::SameBacking(const ImageInfo& cached, const ImageInfo& requested,
bool exact_format) {
const bool unit_extent =
requested.extent.width == 1 && requested.extent.height == 1 && requested.extent.depth == 1;
return cached.data == requested.data && cached.extent == requested.extent &&
cached.samples == requested.samples &&
cached.bytes_per_block == requested.bytes_per_block &&
(cached.type == requested.type || unit_extent) &&
(exact_format
? cached.pixel_format == requested.pixel_format
: ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format));
if (cached.data.address != requested.data.address) {
return false;
}
if (cached.data.size != requested.data.size) {
return false;
}
if (cached.extent != requested.extent) {
return false;
}
if (cached.samples != requested.samples) {
return false;
}
if (cached.bytes_per_block != requested.bytes_per_block) {
return false;
}
if (cached.tile_mode != requested.tile_mode) {
return false;
}
if (!ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format)) {
return false;
}
if (cached.type != requested.type && requested.extent != vk::Extent3D {1, 1, 1}) {
return false;
}
if (exact_format && cached.pixel_format != requested.pixel_format) {
return false;
}
return true;
}
TextureCache::BindingType TextureCache::UploadBinding(const Image& image) {
@@ -117,8 +136,7 @@ bool TextureCache::SafeToDownload(const Image& image) {
return false;
}
const auto range = image.info.data;
return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size) &&
!m_memory_tracker.IsRegionCpuModified(range.address, range.size);
return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size);
}
Image& TextureCache::ResolveImage(ImageId id) {
@@ -187,21 +205,17 @@ void TextureCache::RegisterImage(ImageId id) {
m_total_used_memory += image.AccountedSize();
}
void TextureCache::UnregisterImage(ImageId id, bool release_tracking) {
void TextureCache::UnregisterImage(ImageId id) {
auto& image = ResolveImage(id);
if (!image.registered) {
return;
}
UntrackImage(id);
std::vector<ImageOwnerIndex::ByteRange> releases;
if (!m_image_owner_index.Unregister(id, releases)) {
EXIT("TextureCache: image missing from owner index\n");
}
m_lru_cache.Free(image.lru_id);
if (release_tracking) {
for (const auto& range: releases) {
m_memory_tracker.UntrackMemory(range.address, range.size);
}
}
const auto accounted = image.AccountedSize();
if (accounted > m_total_used_memory) {
EXIT("TextureCache: image accounting underflow\n");
@@ -210,7 +224,7 @@ void TextureCache::UnregisterImage(ImageId id, bool release_tracking) {
image.registered = false;
}
void TextureCache::DeleteImage(ImageId id, bool release_tracking) {
void TextureCache::DeleteImage(ImageId id) {
auto owner = ResolveOwner(id);
if (owner == nullptr || !owner->registered) {
return;
@@ -224,7 +238,7 @@ void TextureCache::DeleteImage(ImageId id, bool release_tracking) {
}
}
for (const auto association: associations) {
ReleaseGpuTracking(association);
ClearGpuModified(association);
DeleteImage(association);
}
}
@@ -235,7 +249,7 @@ void TextureCache::DeleteImage(ImageId id, bool release_tracking) {
if (owner->info.metadata.kind == ImageMetadataKind::Htile) {
m_surface_metas.erase(owner->info.metadata.range.address);
}
UnregisterImage(id, release_tracking);
UnregisterImage(id);
const auto erase_slot = [this, id, retained = owner] {
auto& slot = m_slots[id.index];
if (slot.generation != id.generation || slot.image != retained) {
@@ -266,10 +280,10 @@ void TextureCache::DeleteImages(std::span<const ImageId> ids,
continue;
}
if (native_source == id) {
ReleaseGpuTracking(id);
ClearGpuModified(id);
} else if (owner->IsGpuModified()) {
DownloadImage(id);
ReleaseGpuTracking(id);
ClearGpuModified(id);
}
DeleteImage(id);
}
@@ -296,6 +310,121 @@ void TextureCache::TouchImage(Image& image) {
}
}
void TextureCache::TrackImage(ImageId id) {
auto& image = ResolveImage(id);
if (!image.registered) {
return;
}
const auto image_begin = image.info.data.address;
const auto image_end = image.info.data.End();
if (image_begin == image.track_addr && image_end == image.track_addr_end) {
return;
}
if (!image.IsTracked()) {
image.track_addr = image_begin;
image.track_addr_end = image_end;
m_page_manager.UpdatePageWatchers(true, image_begin, image.info.data.size);
return;
}
if (image_begin < image.track_addr) {
TrackImageHead(id);
}
if (image.track_addr_end < image_end) {
TrackImageTail(id);
}
}
void TextureCache::TrackImageHead(ImageId id) {
auto& image = ResolveImage(id);
if (!image.registered) {
return;
}
const auto image_begin = image.info.data.address;
if (image_begin == image.track_addr) {
return;
}
if (!image.IsTracked() || image_begin > image.track_addr) {
EXIT("TextureCache: invalid image head tracking range\n");
}
const auto size = image.track_addr - image_begin;
image.track_addr = image_begin;
m_page_manager.UpdatePageWatchers(true, image_begin, size);
}
void TextureCache::TrackImageTail(ImageId id) {
auto& image = ResolveImage(id);
if (!image.registered) {
return;
}
const auto image_end = image.info.data.End();
if (image_end == image.track_addr_end) {
return;
}
if (!image.IsTracked() || image.track_addr_end > image_end) {
EXIT("TextureCache: invalid image tail tracking range\n");
}
const auto address = image.track_addr_end;
const auto size = image_end - address;
image.track_addr_end = image_end;
m_page_manager.UpdatePageWatchers(true, address, size);
}
void TextureCache::UntrackImage(ImageId id) {
auto& image = ResolveImage(id);
if (!image.IsTracked()) {
return;
}
const auto address = image.track_addr;
const auto size = image.track_addr_end - image.track_addr;
image.track_addr = 0;
image.track_addr_end = 0;
if (size != 0) {
m_page_manager.UpdatePageWatchers(false, address, size);
}
}
void TextureCache::UntrackImageHead(ImageId id) {
auto& image = ResolveImage(id);
const auto begin = image.info.data.address;
if (!image.IsTracked() || begin < image.track_addr) {
return;
}
const auto address = (begin + TRACKER_PAGE_SIZE) & ~(TRACKER_PAGE_SIZE - 1);
const auto size = address - begin;
image.track_addr = address;
if (image.track_addr == image.track_addr_end) {
image.MarkMaybeCpuDirty();
if (image.NeedsMaybeCpuHash()) {
image.SetMaybeCpuHash(image.HashGuestEdges());
}
UntrackImage(id);
}
if (size != 0) {
m_page_manager.UpdatePageWatchers(false, begin, size);
}
}
void TextureCache::UntrackImageTail(ImageId id) {
auto& image = ResolveImage(id);
const auto end = image.info.data.End();
if (!image.IsTracked() || image.track_addr_end < end) {
return;
}
const auto address = end & ~(TRACKER_PAGE_SIZE - 1);
const auto size = end - address;
image.track_addr_end = address;
if (image.track_addr == image.track_addr_end) {
image.MarkMaybeCpuDirty();
if (image.NeedsMaybeCpuHash()) {
image.SetMaybeCpuHash(image.HashGuestEdges());
}
UntrackImage(id);
}
if (size != 0) {
m_page_manager.UpdatePageWatchers(false, address, size);
}
}
void TextureCache::TrackImageDownload(ImageId id) {
std::lock_guard transaction(m_resource_mutex);
CacheLock lock(*this, m_lock);
@@ -376,9 +505,6 @@ void TextureCache::ValidateImageDesc(const ImageDesc& desc) const {
}
void TextureCache::PrepareImageCopy(Image& image) {
const auto range = image.info.data;
m_memory_tracker.ForEachUploadRange(
range.address, range.size, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
if (image.IsCpuDirty()) {
image.RefreshComplete();
}
@@ -471,6 +597,7 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
RefreshCopySource(source_id);
auto& destination = ResolveImage(destination_id);
auto& source = ResolveImage(source_id);
TrackImage(destination_id);
if (source.backing.samples != destination.backing.samples) {
EXIT("TextureCache: cannot issue an unequal-sample image copy\n");
}
@@ -479,7 +606,6 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
if (source.info.data == destination.info.data) {
destination.MarkBufferModified();
}
RestoreGpuTracking(destination);
return;
}
const bool source_depth = source.info.IsDepth();
@@ -503,7 +629,6 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
destination.MarkGpuModified();
}
destination.ClearBufferModified();
RestoreGpuTracking(destination);
}
void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint32_t mip,
@@ -511,6 +636,7 @@ void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint3
RefreshCopySource(source_id);
auto& destination = ResolveImage(destination_id);
auto& source = ResolveImage(source_id);
TrackImage(destination_id);
if (source.IsBufferModified() || source.backing.samples != destination.backing.samples) {
EXIT("TextureCache: invalid mip-copy ownership or sample count\n");
}
@@ -520,7 +646,6 @@ void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint3
if (source.IsGpuModified()) {
destination.MarkGpuModified();
}
RestoreGpuTracking(destination);
}
ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingType binding,
@@ -590,7 +715,7 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
if (copied) {
DeleteImages(std::array {cached_id}, cached_id);
} else {
ReleaseGpuTracking(cached_id);
ClearGpuModified(cached_id);
DeleteImage(cached_id);
}
return replacement_id;
@@ -629,6 +754,12 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
(requested.IsVolume() || cached.info.IsVolume())) {
return {ExpandImage(requested, cached_id)};
}
if (requested.tile_mode != cached.info.tile_mode) {
if (safe_to_delete) {
DeleteImages(std::array {cached_id}, cached_id);
}
return {merged_id};
}
if (requested.pixel_format != cached.info.pixel_format ||
requested.data.size <= cached.info.data.size) {
const auto result_id = merged_id ? merged_id : cached_id;
@@ -641,12 +772,6 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
if (requested.type == cached.info.type && requested.resources > cached.info.resources) {
return {ExpandImage(requested, cached_id)};
}
if (requested.tile_mode != cached.info.tile_mode) {
if (safe_to_delete) {
DeleteImages(std::array {cached_id}, cached_id);
}
return {merged_id};
}
EXIT("TextureCache: unresolvable equal-address image overlap, address=0x%016" PRIx64
" requested=%ux%u "
"cached=%ux%u requested_size=0x%016" PRIx64 " cached_size=0x%016" PRIx64
@@ -903,30 +1028,20 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) {
if (image.info.data.Empty()) {
return;
}
TrackImage(id);
if (image.info.metadata.compression != VideoOutCompression::Uncompressed) {
m_memory_tracker.ForEachUploadRange(
image.info.data.address, image.info.data.size, false,
[](uint64_t, uint64_t) noexcept {}, []() noexcept {});
if (image.IsCpuDirty()) {
image.RefreshComplete();
}
return;
}
if (image.info.samples > 1) {
RestoreGpuTracking(image);
return;
}
bool data_gpu_owned = false;
bool data_imported = false;
bool uploaded = false;
m_memory_tracker.ForEachUploadRange(
image.info.data.address, image.info.data.size, false,
[&](uint64_t, uint64_t) noexcept { uploaded = true; },
[&]() noexcept {
uploaded |= image.IsBufferModified() || image.IsDefinitelyCpuDirty();
if (!uploaded) {
return;
}
const bool upload = image.IsBufferModified() || image.IsCpuDirty();
if (upload) {
const auto source =
m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size);
if (source.buffer == nullptr) {
@@ -935,7 +1050,7 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) {
data_gpu_owned |= source.gpu_owned;
data_imported = true;
UploadImage(image, desc, *source.buffer, source.offset);
});
}
if (data_imported) {
image.ClearBufferModified();
}
@@ -945,39 +1060,27 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) {
if (image.IsCpuDirty()) {
image.RefreshComplete();
}
RestoreGpuTracking(image);
}
void TextureCache::RefreshImage(ImageId id, const ImageDesc& desc) {
TrackImage(id);
auto& image = ResolveImage(id);
bool unchanged_maybe = false;
if (image.IsMaybeCpuDirty()) {
const auto hash = image.HashGuestEdges();
if (image.NeedsMaybeCpuHash()) {
image.SetMaybeCpuHash(hash);
return;
}
unchanged_maybe = !image.ResolveMaybeCpuHash(hash);
if (unchanged_maybe) {
m_memory_tracker.ForEachUploadRange(
image.info.data.address, image.info.data.size, false,
[](uint64_t, uint64_t) noexcept {}, []() noexcept {});
}
(void)image.ResolveMaybeCpuHash(hash);
}
bool cpu_dirty = image.IsBufferModified() || image.IsDefinitelyCpuDirty();
if (!unchanged_maybe) {
cpu_dirty |=
m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size);
}
if (image.info.metadata.compression != VideoOutCompression::Uncompressed) {
if (cpu_dirty) {
EXIT("TextureCache: compressed guest image refresh is unsupported\n");
}
RestoreGpuTracking(image);
return;
}
if (!cpu_dirty) {
RestoreGpuTracking(image);
return;
}
InitializeImage(id, desc);
@@ -1029,7 +1132,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
ImageId result {};
bool replacement_buffer = false;
bool replacing_image = false;
bool inserted_new = false;
{
std::lock_guard transaction(m_resource_mutex);
CacheLock lock(*this, m_lock);
@@ -1038,7 +1141,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
for (const auto id: candidates) {
const auto owner = ResolveOwner(id);
if (owner == nullptr || owner->info.data != desc.info.data) {
if (owner == nullptr) {
continue;
}
if (SameBacking(owner->info, desc.info, exact_format)) {
@@ -1080,19 +1183,18 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
replacement_buffer = resolved.IsBufferModified();
DeleteImage(result);
result = {};
replacing_image = true;
}
}
if (!result) {
result = InsertImage(desc.info);
inserted_new = true;
auto& inserted = ResolveImage(result);
if (replacement_buffer || m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address,
inserted.info.data.size)) {
inserted.MarkBufferModified();
} else if (replacing_image) {
m_memory_tracker.MarkRegionAsCpuModified(inserted.info.data.address,
inserted.info.data.size);
}
}
if (inserted_new) {
InitializeImage(result, desc);
} else {
RefreshImage(result, desc);
@@ -1101,9 +1203,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
auto& image = ResolveImage(result);
if (desc.type == BindingType::VideoOut &&
desc.info.metadata.compression != VideoOutCompression::Uncompressed) {
const bool guest_dirty =
image.IsBufferModified() || image.IsCpuDirty() ||
m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size);
const bool guest_dirty = image.IsBufferModified() || image.IsCpuDirty();
const bool native_current =
(image.usage.render_target || image.IsGpuModified()) && !guest_dirty;
if (!native_current) {
@@ -1252,6 +1352,7 @@ void TextureCache::MarkGpuWritten(ImageId id) {
if (!image.registered || image.depth_id) {
EXIT("TextureCache: cannot mark an unavailable image GPU-written\n");
}
TrackImage(id);
CommitGpuWrite(image);
}
@@ -1265,13 +1366,10 @@ void TextureCache::CommitGpuWrite(Image& image) {
}
m_buffer_cache.InvalidateImageAliases(range.address, range.size);
image.ClearBufferModified();
m_memory_tracker.ForEachUploadRange(
range.address, range.size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
if (image.IsCpuDirty()) {
image.RefreshComplete();
}
image.MarkGpuModified();
RestoreGpuTracking(image);
}
bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
@@ -1279,7 +1377,6 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
if (command.IsInvalid() || !GuestRange {address, size}.Valid()) {
EXIT("TextureCache: invalid image clear\n");
}
m_buffer_cache.ValidateGpuAccess(address, size, false, true);
std::lock_guard transaction(m_resource_mutex);
CacheLock lock(*this, m_lock);
ImageId selected {};
@@ -1335,8 +1432,7 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
if (m_buffer_cache.HasGpuDirtyBytes(address, size)) {
m_buffer_cache.DiscardGpuDirtyBytes(address, size);
}
if (image.IsBufferModified() || image.IsCpuDirty() ||
m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size)) {
if (image.IsBufferModified() || image.IsCpuDirty()) {
ImageDesc refresh {.info = image.info, .view_info = {}, .type = UploadBinding(image)};
InitializeImage(selected, refresh);
if (image.info.samples == 1 && (image.IsBufferModified() || image.IsCpuDirty())) {
@@ -1361,14 +1457,12 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
return true;
}
void TextureCache::PrepareHostWrite(uint64_t address, uint64_t size) {
void TextureCache::InvalidateMemory(uint64_t address, uint64_t size) {
if (!GuestRange {address, size}.Valid()) {
EXIT("TextureCache: invalid host-write range\n");
EXIT("TextureCache: invalid memory-invalidation range\n");
}
CacheLock lock(*this, m_lock);
InvalidateCpuAliases(address, size);
m_memory_tracker.ForEachDownloadRange<true>(address, size, [](uint64_t, uint64_t) noexcept {});
m_memory_tracker.MarkRegionAsCpuModified(address, size);
}
void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset) {
@@ -1568,17 +1662,14 @@ bool TextureCache::SynchronizeImageToBuffer(ImageId id) {
return false;
}
const auto range = image.info.data;
const bool refresh = image.IsDefinitelyCpuDirty() ||
m_memory_tracker.IsRegionCpuModified(range.address, range.size);
if (refresh) {
if (image.IsCpuDirty()) {
RefreshImage(id,
ImageDesc {.info = image.info, .view_info = {}, .type = UploadBinding(image)});
}
if (!image.IsGpuModified()) {
return true;
}
if (image.IsDefinitelyCpuDirty() || image.IsBufferModified() ||
m_memory_tracker.IsRegionCpuModified(range.address, range.size)) {
if (image.IsDefinitelyCpuDirty() || image.IsBufferModified()) {
EXIT("TextureCache: image mirror source is not native-current\n");
}
auto [destination, offset] =
@@ -1591,7 +1682,7 @@ bool TextureCache::SynchronizeImageToBuffer(ImageId id) {
m_buffer_cache.PublishImageBuffer(range.address, range.size);
image.MarkBufferModified();
RetainImage(m_scheduler.Current(), id);
ReleaseGpuTracking(id);
ClearGpuModified(id);
return true;
}
@@ -1633,7 +1724,7 @@ bool TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
if (!formatted_buffer_write) {
EXIT("TextureCache: buffer write aliases GPU-modified image\n");
}
ReleaseGpuTracking(id);
ClearGpuModified(id);
}
owner->MarkBufferModified();
found = true;
@@ -1660,50 +1751,40 @@ TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t address, uint64_t si
}
void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
const auto page_begin = address & ~(TRACKER_PAGE_SIZE - 1);
const auto page_end = (address + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
for (const auto id: FindImagesInRegion(address, size, true)) {
auto owner = ResolveOwner(id);
if (owner == nullptr || owner->depth_id) {
continue;
}
if (owner->Overlaps(address, size)) {
owner->InvalidateCpuWrite(address, size);
UntrackImage(id);
continue;
}
const auto image_begin = owner->info.data.address;
const auto image_end = owner->info.data.End();
if (page_end < image_end) {
UntrackImageHead(id);
} else if (image_begin < page_begin) {
UntrackImageTail(id);
} else {
owner->MarkMaybeCpuDirty();
if (owner->NeedsMaybeCpuHash()) {
owner->SetMaybeCpuHash(owner->HashGuestEdges());
}
}
}
void TextureCache::RestoreGpuTracking(const Image& image) {
if (!image.IsGpuModified()) {
return;
}
constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1;
const auto range = image.info.data;
const auto begin = range.address & ~page_mask;
const auto end = (range.End() + page_mask) & ~page_mask;
for (auto page = begin; page < end; page += TRACKER_PAGE_SIZE) {
if (!m_memory_tracker.IsRegionGpuModified(page, TRACKER_PAGE_SIZE) &&
!m_memory_tracker.IsRegionCpuModified(page, TRACKER_PAGE_SIZE)) {
m_memory_tracker.MarkRegionAsGpuModified(page, TRACKER_PAGE_SIZE);
UntrackImage(id);
}
}
}
void TextureCache::ReleaseGpuTracking(ImageId id) {
void TextureCache::ClearGpuModified(ImageId id) {
auto owner = ResolveOwner(id);
if (owner == nullptr || !owner->IsGpuModified()) {
return;
}
const auto released = owner->info.data;
owner->ClearGpuModified();
m_memory_tracker.ForEachDownloadRange<true>(released.address, released.size,
[](uint64_t, uint64_t) noexcept {});
for (const auto candidate: FindImagesInRegion(released.address, released.size, true)) {
const auto survivor = ResolveOwner(candidate);
if (survivor != nullptr && survivor.get() != owner.get() && !survivor->depth_id) {
RestoreGpuTracking(*survivor);
}
}
RestoreGpuTracking(*owner);
}
bool TextureCache::IsMeta(uint64_t address) {
@@ -1755,27 +1836,25 @@ bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t address, ui
return false;
}
if (phase == PageFaultPhase::Invalidate) {
const bool gpu_image =
m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase);
CpuFaultAction action = gpu_image ? CpuFaultAction::Download
: m_memory_tracker.BeginCpuFault(address, size, access);
{
CacheLock lock(*this, m_lock);
const bool tracked =
std::ranges::any_of(FindImagesInRegion(address, size, true), [&](ImageId id) {
const auto owner = ResolveOwner(id);
return owner != nullptr && !owner->depth_id && owner->IsTracked();
});
if (tracked) {
InvalidateCpuAliases(address, size);
}
return action != CpuFaultAction::Untracked;
return tracked;
}
if (phase == PageFaultPhase::Complete) {
const bool gpu_image = m_memory_tracker.IsRegionGpuModified(address, size);
return gpu_image ? m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase)
: m_memory_tracker.CompleteCpuFault(address, size, access, false);
}
if (phase != PageFaultPhase::Release) {
if (phase != PageFaultPhase::Complete && phase != PageFaultPhase::Release) {
return false;
}
(void)m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase);
return true;
CacheLock lock(*this, m_lock);
return std::ranges::any_of(FindImagesInRegion(address, size, true), [&](ImageId id) {
const auto owner = ResolveOwner(id);
return owner != nullptr && !owner->depth_id;
});
}
void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
@@ -1794,16 +1873,10 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
continue;
}
if (owner->IsGpuModified()) {
ReleaseGpuTracking(id);
ClearGpuModified(id);
}
DeleteImage(id);
}
m_memory_tracker.UntrackMemory(address, size);
for (const auto id: FindImagesInRegion(address, size, true)) {
if (const auto survivor = ResolveOwner(id); survivor != nullptr) {
RestoreGpuTracking(*survivor);
}
}
}
void TextureCache::RunGarbageCollector() {
@@ -1849,7 +1922,7 @@ void TextureCache::RunGarbageCollector() {
if (safe && !TryDownloadImage(id)) {
continue;
}
ReleaseGpuTracking(id);
ClearGpuModified(id);
}
DeleteImage(id);
if (m_total_used_memory < m_critical_gc_memory && aggressive) {
@@ -4,10 +4,11 @@
#include "common/abi.h"
#include "common/common.h"
#include "common/lruCache.h"
#include "graphics/host_gpu/memoryTracker.h"
#include "graphics/host_gpu/renderer/blitHelper.h"
#include "graphics/host_gpu/renderer/image.h"
#include "graphics/host_gpu/renderer/multiLevelPageTable.h"
#include "graphics/host_gpu/pageManager.h"
#include "graphics/host_gpu/regionManager.h"
#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
#include "graphics/host_gpu/renderer/image/blitHelper.h"
#include "graphics/host_gpu/renderer/image/image.h"
#include <compare>
#include <map>
@@ -64,7 +65,7 @@ public:
[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
uint32_t packed_clear);
void PrepareHostWrite(uint64_t address, uint64_t size);
void InvalidateMemory(uint64_t address, uint64_t size);
[[nodiscard]] bool SynchronizeImageToBuffer(uint64_t address, uint64_t size);
[[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
bool formatted_buffer_write = false);
@@ -109,11 +110,17 @@ private:
[[nodiscard]] ImageId InsertImage(const ImageInfo& info);
[[nodiscard]] ImageId GetNullImage(const ImageDesc& desc);
void RegisterImage(ImageId id);
void UnregisterImage(ImageId id, bool release_tracking);
void DeleteImage(ImageId id, bool release_tracking = true);
void UnregisterImage(ImageId id);
void DeleteImage(ImageId id);
void DeleteImages(std::span<const ImageId> ids, std::optional<ImageId> native_source = {});
void RetainImage(CommandBuffer& command, ImageId id);
void TouchImage(Image& image);
void TrackImage(ImageId id);
void TrackImageHead(ImageId id);
void TrackImageTail(ImageId id);
void UntrackImage(ImageId id);
void UntrackImageHead(ImageId id);
void UntrackImageTail(ImageId id);
void TrackImageDownload(ImageId id);
void TrackImageDownloadLocked(ImageId id, Image& image);
[[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested,
@@ -148,8 +155,7 @@ private:
void ValidateImageDesc(const ImageDesc& desc) const;
void InvalidateCpuAliases(uint64_t address, uint64_t size);
void RestoreGpuTracking(const Image& image);
void ReleaseGpuTracking(ImageId id);
void ClearGpuModified(ImageId id);
[[nodiscard]] bool SynchronizeImageToBuffer(ImageId id);
void DownloadImage(ImageId id);
@@ -160,7 +166,7 @@ private:
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
TrackingSpinLock m_lock;
MemoryTracker m_memory_tracker;
PageManager& m_page_manager;
BlitHelper m_blit_helper;
std::unique_ptr<TileManager> m_tiler;
BufferCache& m_buffer_cache;
@@ -7,9 +7,9 @@
#include "graphics/guest_gpu/hardwareContext.h"
#include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/objects/textureCommon.h"
#include "graphics/host_gpu/renderer/image/textureCommon.h"
#include "graphics/host_gpu/renderer/debug.h"
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/vulkanCommon.h"
@@ -3,7 +3,7 @@
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/host_gpu/renderer/renderTarget.h"
#include "graphics/host_gpu/renderer/textureCache.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include <cstdint>
+2 -2
View File
@@ -8,8 +8,8 @@
#include "graphics/host_gpu/renderer/colorRenderTarget.h"
#include "graphics/host_gpu/renderer/debug.h"
#include "graphics/host_gpu/renderer/depthRenderTarget.h"
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/imageView.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/image/imageView.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/vma.h"
@@ -10,10 +10,10 @@
#include "graphics/guest_gpu/hardwareContext.h"
#include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/objects/textureCommon.h"
#include "graphics/host_gpu/renderer/image/textureCommon.h"
#include "graphics/host_gpu/renderer/debug.h"
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/imageView.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/image/imageView.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/vulkanCommon.h"
@@ -2,9 +2,9 @@
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DEPTHRENDERTARGET_H_
#include "common/assert.h"
#include "graphics/host_gpu/renderer/imageView.h"
#include "graphics/host_gpu/renderer/image/imageView.h"
#include "graphics/host_gpu/renderer/renderTarget.h"
#include "graphics/host_gpu/renderer/textureCache.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include <cstdint>
@@ -1,11 +1,11 @@
#include "graphics/host_gpu/renderer/blitHelper.h"
#include "graphics/host_gpu/renderer/image/blitHelper.h"
#include "common/assert.h"
#include "gpu_blit_shaders/gpu_blit_color_to_ms_depth_spv.h"
#include "gpu_blit_shaders/gpu_blit_fs_triangle_spv.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/renderer/image.h"
#include "graphics/host_gpu/renderer/image/image.h"
#include "graphics/host_gpu/renderer/renderTarget.h"
#include <algorithm>
@@ -1,11 +1,11 @@
#include "graphics/host_gpu/renderer/image.h"
#include "graphics/host_gpu/renderer/image/image.h"
#include "common/assert.h"
#include "common/profiler.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/renderer/imageView.h"
#include "graphics/host_gpu/renderer/image/imageView.h"
#include "graphics/host_gpu/renderer/renderTarget.h"
#include "graphics/host_gpu/renderer/streamBuffer.h"
#include "kernel/memory.h"
#include <algorithm>
@@ -191,8 +191,7 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
}
void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
std::optional<ImageSubresourceRange> range,
vk::CommandBuffer command_buffer) {
std::optional<ImageSubresourceRange> range, vk::CommandBuffer command_buffer) {
const auto transfer_access =
vk::AccessFlagBits2::eTransferRead | vk::AccessFlagBits2::eTransferWrite;
vk::PipelineStageFlags2 destination_stage {};
@@ -201,8 +200,8 @@ void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destina
}
if (!destination_access ||
static_cast<bool>(destination_access & ~vk::AccessFlags2 {transfer_access})) {
destination_stage |= vk::PipelineStageFlagBits2::eAllGraphics |
vk::PipelineStageFlagBits2::eComputeShader;
destination_stage |=
vk::PipelineStageFlagBits2::eAllGraphics | vk::PipelineStageFlagBits2::eComputeShader;
}
const auto barriers =
GetBarriers(destination_layout, destination_access, destination_stage, range);
@@ -218,10 +217,9 @@ void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destina
command_buffer.pipelineBarrier2(dependency);
}
void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
uint64_t offset, uint64_t size) {
EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr ||
size == 0);
void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
uint64_t size) {
EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr || size == 0);
m_scheduler->EndRendering();
vk::BufferMemoryBarrier2 buffer_barrier {};
buffer_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands;
@@ -234,8 +232,7 @@ void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffe
buffer_barrier.offset = offset;
buffer_barrier.size = size;
const auto image_barriers =
GetBarriers(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite,
GetBarriers(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
vk::PipelineStageFlagBits2::eCopy, {});
vk::DependencyInfo dependency {};
dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion;
@@ -256,8 +253,7 @@ void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffe
dependency.pImageMemoryBarriers = nullptr;
command.pipelineBarrier2(dependency);
Transit(vk::ImageLayout::eGeneral,
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
command);
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
}
void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
@@ -276,8 +272,7 @@ void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buf
buffer_barrier.offset = offset;
buffer_barrier.size = size;
const auto image_barriers =
GetBarriers(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead,
GetBarriers(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead,
vk::PipelineStageFlagBits2::eCopy, {});
vk::DependencyInfo dependency {};
dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion;
@@ -299,8 +294,8 @@ void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buf
command.pipelineBarrier2(dependency);
}
std::pair<uint32_t, uint32_t>
Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_t depth) {
std::pair<uint32_t, uint32_t> Image::SanitizeCopyLayers(const Image& source,
const Image& destination, uint32_t depth) {
const auto source_type = source.backing.image_type;
const auto destination_type = destination.backing.image_type;
uint32_t source_layers = source.backing.layers;
@@ -312,13 +307,10 @@ Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_
destination_layers = 1;
}
if (source_type == destination_type) {
source_layers = destination_layers =
std::min(source_layers, destination_layers);
} else if (source_type == vk::ImageType::e2D &&
destination_type == vk::ImageType::e3D) {
source_layers = destination_layers = std::min(source_layers, destination_layers);
} else if (source_type == vk::ImageType::e2D && destination_type == vk::ImageType::e3D) {
source_layers = depth;
} else if (source_type == vk::ImageType::e3D &&
destination_type == vk::ImageType::e2D) {
} else if (source_type == vk::ImageType::e3D && destination_type == vk::ImageType::e2D) {
destination_layers = depth;
}
return {source_layers, destination_layers};
@@ -327,8 +319,7 @@ Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_
void Image::CopyImage(Image& source) {
EXIT_IF(m_scheduler == nullptr || source.backing.samples != backing.samples);
m_scheduler->EndRendering();
const uint32_t levels =
std::min(source.backing.mip_levels, backing.mip_levels);
const uint32_t levels = std::min(source.backing.mip_levels, backing.mip_levels);
const uint32_t base_depth = backing.image_type == vk::ImageType::e3D
? backing.extent.depth
: source.backing.extent.depth;
@@ -342,8 +333,7 @@ void Image::CopyImage(Image& source) {
const auto width = std::max(source.backing.extent.width >> level, 1u);
const auto height = std::max(source.backing.extent.height >> level, 1u);
const auto depth = std::max(base_depth >> level, 1u);
const auto [source_layers, destination_layers] =
SanitizeCopyLayers(source, *this, depth);
const auto [source_layers, destination_layers] = SanitizeCopyLayers(source, *this, depth);
vk::ImageCopy copy {};
copy.srcSubresource = {source_aspect, level, 0, 1};
copy.dstSubresource = {destination_aspect, level, 0, 1};
@@ -351,8 +341,7 @@ void Image::CopyImage(Image& source) {
if (source.backing.image_type == vk::ImageType::e3D) {
copy.extent = {width, height, depth};
} else {
copy.srcSubresource.layerCount =
std::min(source_layers, destination_layers);
copy.srcSubresource.layerCount = std::min(source_layers, destination_layers);
copy.dstSubresource.layerCount = copy.srcSubresource.layerCount;
copy.extent = {width, height, 1};
}
@@ -369,34 +358,30 @@ void Image::CopyImage(Image& source) {
return;
}
auto command = m_scheduler->Current().Handle();
source.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {}, command);
Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {}, command);
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
backing.image, vk::ImageLayout::eTransferDstOptimal,
static_cast<uint32_t>(copies.size()), copies.data());
Transit(vk::ImageLayout::eGeneral,
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
command);
Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
vk::ImageLayout::eTransferDstOptimal, static_cast<uint32_t>(copies.size()),
copies.data());
Transit(vk::ImageLayout::eGeneral,
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
}
void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
const ImageSubresourceRange& destination_range) {
EXIT_IF(m_scheduler == nullptr || backing.samples != 1 ||
source.backing.image_type != vk::ImageType::e2D ||
backing.image_type != vk::ImageType::e2D ||
source_range.level_count != 1 || destination_range.level_count != 1 ||
backing.image_type != vk::ImageType::e2D || source_range.level_count != 1 ||
destination_range.level_count != 1 ||
source_range.base_level >= source.backing.mip_levels ||
destination_range.base_level >= backing.mip_levels ||
source_range.base_layer >= source.backing.layers ||
destination_range.base_layer >= backing.layers);
const auto layers = std::min(
{source_range.layer_count, destination_range.layer_count,
const auto layers = std::min({source_range.layer_count, destination_range.layer_count,
source.backing.layers - source_range.base_layer,
backing.layers - destination_range.base_layer});
const auto source_width =
std::max(source.backing.extent.width >> source_range.base_level, 1u);
const auto source_width = std::max(source.backing.extent.width >> source_range.base_level, 1u);
const auto source_height =
std::max(source.backing.extent.height >> source_range.base_level, 1u);
const auto destination_width =
@@ -404,9 +389,8 @@ void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
const auto destination_height =
std::max(backing.extent.height >> destination_range.base_level, 1u);
const bool copy = source.backing.samples == 1;
EXIT_IF(layers == 0 || info.extent.width > source_width ||
info.extent.height > source_height || info.extent.width > destination_width ||
info.extent.height > destination_height ||
EXIT_IF(layers == 0 || info.extent.width > source_width || info.extent.height > source_height ||
info.extent.width > destination_width || info.extent.height > destination_height ||
(copy ? !ImageViewOps::FormatsCompatible(source.backing.format, backing.format)
: source.backing.format != backing.format));
auto resolved_source_range = source_range;
@@ -417,25 +401,23 @@ void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
m_scheduler->EndRendering();
auto command = m_scheduler->Current().Handle();
source.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, resolved_source_range, command);
Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, resolved_destination_range, command);
source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead,
resolved_source_range, command);
Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
resolved_destination_range, command);
if (copy) {
vk::ImageCopy region {};
region.srcSubresource = {vk::ImageAspectFlagBits::eColor,
resolved_source_range.base_level,
region.srcSubresource = {vk::ImageAspectFlagBits::eColor, resolved_source_range.base_level,
resolved_source_range.base_layer, layers};
region.dstSubresource = {vk::ImageAspectFlagBits::eColor,
resolved_destination_range.base_level,
resolved_destination_range.base_layer, layers};
region.extent = resolve_extent;
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
backing.image, vk::ImageLayout::eTransferDstOptimal, region);
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
vk::ImageLayout::eTransferDstOptimal, region);
} else {
vk::ImageResolve region {};
region.srcSubresource = {vk::ImageAspectFlagBits::eColor,
resolved_source_range.base_level,
region.srcSubresource = {vk::ImageAspectFlagBits::eColor, resolved_source_range.base_level,
resolved_source_range.base_layer, layers};
region.dstSubresource = {vk::ImageAspectFlagBits::eColor,
resolved_destination_range.base_level,
@@ -454,11 +436,10 @@ uint32_t Image::CopyRows(uint64_t row_size, uint32_t rows, uint64_t capacity) no
}
void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr ||
source.backing.samples != 1 || backing.samples != 1);
EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr || source.backing.samples != 1 ||
backing.samples != 1);
m_scheduler->EndRendering();
const uint32_t levels =
std::min(source.backing.mip_levels, backing.mip_levels);
const uint32_t levels = std::min(source.backing.mip_levels, backing.mip_levels);
const auto source_aspect =
FullAspectMask(source.backing.format) & ~vk::ImageAspectFlagBits::eStencil;
const auto destination_aspect =
@@ -488,10 +469,9 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
dependency.bufferMemoryBarrierCount = 1;
dependency.pBufferMemoryBarriers = &barrier;
auto command = m_scheduler->Current().Handle();
source.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {}, command);
Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {}, command);
source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
command);
Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
for (uint32_t level = 0; level < levels; level++) {
const auto width = std::max(source.backing.extent.width >> level, 1u);
const auto height = std::max(source.backing.extent.height >> level, 1u);
@@ -508,19 +488,16 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
const auto rows_per_copy = CopyRows(row_size, block_rows, buffer.Size());
EXIT_IF(slices == 0 || rows_per_copy == 0);
for (uint32_t slice = 0; slice < slices; slice++) {
for (uint32_t block_row = 0; block_row < block_rows;
block_row += rows_per_copy) {
for (uint32_t block_row = 0; block_row < block_rows; block_row += rows_per_copy) {
const auto copy_rows = std::min(rows_per_copy, block_rows - block_row);
const auto y = block_row * source_block;
const auto copy_height =
std::min(copy_rows * source_block, height - y);
const auto copy_height = std::min(copy_rows * source_block, height - y);
const auto copy_size = row_size * copy_rows;
vk::BufferImageCopy source_copy {};
source_copy.imageSubresource = {
source_aspect, level,
source.backing.image_type == vk::ImageType::e3D ? 0u : slice, 1};
source_copy.imageOffset = {
0, static_cast<int32_t>(y),
source_copy.imageOffset = {0, static_cast<int32_t>(y),
source.backing.image_type == vk::ImageType::e3D
? static_cast<int32_t>(slice)
: 0};
@@ -530,28 +507,24 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
destination_aspect, level,
backing.image_type == vk::ImageType::e3D ? 0u : slice, 1};
destination_copy.imageOffset.z =
backing.image_type == vk::ImageType::e3D
? static_cast<int32_t>(slice)
: 0;
backing.image_type == vk::ImageType::e3D ? static_cast<int32_t>(slice) : 0;
barrier.size = copy_size;
barrier.srcAccessMask = vk::AccessFlagBits2::eTransferRead;
barrier.dstAccessMask = vk::AccessFlagBits2::eTransferWrite;
command.pipelineBarrier2(dependency);
command.copyImageToBuffer(source.backing.image,
vk::ImageLayout::eTransferSrcOptimal,
buffer.Handle(), source_copy);
vk::ImageLayout::eTransferSrcOptimal, buffer.Handle(),
source_copy);
barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits2::eTransferRead;
command.pipelineBarrier2(dependency);
command.copyBufferToImage(buffer.Handle(), backing.image,
vk::ImageLayout::eTransferDstOptimal,
destination_copy);
vk::ImageLayout::eTransferDstOptimal, destination_copy);
}
}
}
Transit(vk::ImageLayout::eGeneral,
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
command);
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
}
void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
@@ -561,10 +534,8 @@ void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
const auto width = std::max(backing.extent.width >> mip, 1u);
const auto height = std::max(backing.extent.height >> mip, 1u);
const auto depth = std::max(backing.extent.depth >> mip, 1u);
EXIT_IF(width != source.backing.extent.width ||
height != source.backing.extent.height);
const auto [source_layers, destination_layers] =
SanitizeCopyLayers(source, *this, depth);
EXIT_IF(width != source.backing.extent.width || height != source.backing.extent.height);
const auto [source_layers, destination_layers] = SanitizeCopyLayers(source, *this, depth);
const auto aspects = FullAspectMask(source.backing.format);
EXIT_IF(aspects != FullAspectMask(backing.format));
std::array<vk::ImageCopy, 2> copies {};
@@ -580,16 +551,13 @@ void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
copy.extent = {width, height, depth};
}
auto command = m_scheduler->Current().Handle();
Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {}, command);
source.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {}, command);
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
backing.image, vk::ImageLayout::eTransferDstOptimal, copy_count,
copies.data());
Transit(vk::ImageLayout::eGeneral,
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
command);
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
vk::ImageLayout::eTransferDstOptimal, copy_count, copies.data());
Transit(vk::ImageLayout::eGeneral,
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
}
namespace ImageOps {
@@ -601,8 +569,7 @@ void Validate(const ImageInfo& info) {
if (info.pixel_format == vk::Format::eUndefined) {
const bool metadata_empty =
info.metadata.range.address == 0 && info.metadata.range.size == 0 &&
info.metadata.kind == ImageMetadataKind::None &&
info.metadata.control == 0 &&
info.metadata.kind == ImageMetadataKind::None && info.metadata.control == 0 &&
info.metadata.compression == VideoOutCompression::Uncompressed &&
!info.metadata.stencil_compressed;
if (info.data.Empty() || info.HasStencil() || !metadata_empty || info.extent.width == 0 ||
@@ -615,9 +582,9 @@ void Validate(const ImageInfo& info) {
}
if (info.extent.width == 0 || info.extent.height == 0 || info.extent.depth == 0 ||
info.resources.levels == 0 ||
info.resources.levels > info.mip_layout.size() || info.resources.layers == 0 ||
info.samples == 0 || vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} ||
info.resources.levels == 0 || info.resources.levels > info.mip_layout.size() ||
info.resources.layers == 0 || info.samples == 0 ||
vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} ||
info.bytes_per_block == 0 || (info.data.address != 0 && info.pitch == 0)) {
EXIT("invalid image geometry or format\n");
}
@@ -688,11 +655,11 @@ uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element) {
} // namespace ImageOps
Image::Image(GraphicContext& graphics, CommandScheduler& scheduler,
const ImageInfo& image_info)
Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageInfo& image_info)
: info(image_info), m_graphics(&graphics), m_scheduler(&scheduler) {
KYTY_PROFILER_FUNCTION();
ImageOps::Validate(info);
m_cpu_dirty = !info.data.Empty();
if (info.pixel_format == vk::Format::eUndefined) {
return;
}
@@ -3,7 +3,7 @@
#include "common/assert.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/imageInfo.h"
#include "graphics/host_gpu/renderer/image/imageInfo.h"
#include <compare>
#include <limits>
@@ -66,16 +66,16 @@ public:
[[nodiscard]] vk::ImageView FindView(const ImageViewInfo& view_info);
void AssociateDepth(ImageId image_id) { depth_id = image_id; }
using Barriers = std::vector<vk::ImageMemoryBarrier2>;
[[nodiscard]] Barriers
GetBarriers(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
[[nodiscard]] Barriers GetBarriers(vk::ImageLayout destination_layout,
vk::AccessFlags2 destination_access,
vk::PipelineStageFlags2 destination_stage,
std::optional<ImageSubresourceRange> range);
void Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
std::optional<ImageSubresourceRange> range, vk::CommandBuffer command_buffer);
void Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
uint64_t offset, uint64_t size);
void Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
uint64_t offset, uint64_t size);
void Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
uint64_t size);
void Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
uint64_t size);
void CopyImage(Image& source);
void Resolve(Image& source, const ImageSubresourceRange& source_range,
const ImageSubresourceRange& destination_range);
@@ -95,6 +95,11 @@ public:
[[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty || m_maybe_cpu_dirty; }
[[nodiscard]] bool IsDefinitelyCpuDirty() const { return m_cpu_dirty; }
[[nodiscard]] bool IsMaybeCpuDirty() const { return m_maybe_cpu_dirty; }
void MarkMaybeCpuDirty() {
if (!m_cpu_dirty) {
m_maybe_cpu_dirty = true;
}
}
[[nodiscard]] bool NeedsMaybeCpuHash() const {
return m_maybe_cpu_dirty && !m_maybe_hash_valid;
}
@@ -132,7 +137,8 @@ public:
void MarkBufferModified() noexcept { m_buffer_modified = true; }
void ClearBufferModified() noexcept { m_buffer_modified = false; }
[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size, bool pages = false) const noexcept {
[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size,
bool pages = false) const noexcept {
return pages ? ImagePageRangesOverlap(info.data.address, info.data.size, address, size)
: ImageRangeOverlaps(info.data.address, info.data.size, address, size);
}
@@ -142,6 +148,7 @@ public:
[[nodiscard]] bool SafeToDownload() const noexcept {
return IsGpuModified() && !IsBufferModified() && !IsCpuDirty();
}
[[nodiscard]] bool IsTracked() const noexcept { return track_addr != 0 && track_addr_end != 0; }
[[nodiscard]] uint64_t AccountedSize() const noexcept {
return backing.image == nullptr ? 0 : (info.data.size + 1023) & ~uint64_t {1023};
}
@@ -153,6 +160,8 @@ public:
ImageUsage usage;
ImageBinding binding;
bool registered = false;
uint64_t track_addr = 0;
uint64_t track_addr_end = 0;
ImageId depth_id {};
uint64_t tick_accessed_last = 0;
size_t lru_id = 0;
@@ -1,8 +1,8 @@
#include "graphics/host_gpu/renderer/imageView.h"
#include "graphics/host_gpu/renderer/image/imageView.h"
#include "common/assert.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/image.h"
#include "graphics/host_gpu/renderer/image/image.h"
#include <mutex>
@@ -2,8 +2,8 @@
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEVIEW_H_
#include "common/assert.h"
#include "graphics/host_gpu/renderer/imageInfo.h"
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/host_gpu/renderer/image/imageInfo.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/shader.h"
namespace Libs::Graphics {
@@ -103,10 +103,7 @@ IsSupportedSampledDepthUintResource(const ShaderRecompiler::IR::ImageResource& r
inline void ValidateStorageColorView(vk::Format image_format, vk::Format view_format,
uint32_t swizzle) noexcept {
const auto srgb_view = SrgbStorageViewFormat(image_format);
const bool srgb_storage_view =
srgb_view != vk::Format::eUndefined && view_format == srgb_view;
if ((image_format != view_format && !srgb_storage_view) ||
if (!ImageViewOps::FormatsCompatible(image_format, view_format) ||
!IsValidImageSwizzle(swizzle)) {
UnsupportedColorView("storage", image_format, view_format, swizzle);
}
@@ -122,7 +119,10 @@ IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resou
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D ||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.written &&
!resource.atomic && !resource.depth_compare;
(!resource.atomic ||
(resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint &&
resource.read)) &&
!resource.depth_compare;
}
inline void
@@ -1,9 +1,9 @@
#include "graphics/host_gpu/objects/textureCommon.h"
#include "graphics/host_gpu/renderer/image/textureCommon.h"
#include "common/assert.h"
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/guest_gpu/gpu_format.h"
#include "graphics/host_gpu/renderer/tiler.h"
#include "graphics/host_gpu/renderer/image/tiler.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include <algorithm>
@@ -102,6 +102,10 @@ constexpr RenderTargetFormatMapping kRenderTargetFormats[] = {
Prospero::ChannelType::kUNorm,
Prospero::ChannelOrder::kStandard,
{vk::Format::eR16G16B16A16Unorm, 8}},
{Prospero::ChannelLayout::k16_16_16_16,
Prospero::ChannelType::kUInt,
Prospero::ChannelOrder::kStandard,
{vk::Format::eR16G16B16A16Uint, 8}},
{Prospero::ChannelLayout::k16_16_16_16,
Prospero::ChannelType::kFloat,
Prospero::ChannelOrder::kStandard,
@@ -1,5 +1,5 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_TEXTURECOMMON_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_TEXTURECOMMON_H_
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_
#include "common/abi.h"
#include "common/common.h"
@@ -51,4 +51,4 @@ bool TextureBuildGpuTileInfos(uint64_t size,
} // namespace Libs::Graphics
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_TEXTURECOMMON_H_ */
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_ */
@@ -1,4 +1,4 @@
#include "graphics/host_gpu/renderer/tiler.h"
#include "graphics/host_gpu/renderer/image/tiler.h"
#include "common/assert.h"
#include "gpu_tiler_shaders/gpu_tiler_demote_d16_spv.h"
@@ -15,8 +15,8 @@
#include "gpu_tiler_shaders/gpu_tiler_swap_bgra16_spv.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/renderer/image.h"
#include "graphics/host_gpu/renderer/streamBuffer.h"
#include "graphics/host_gpu/renderer/image/image.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
#include <algorithm>
#include <array>
@@ -1,8 +1,8 @@
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "common/assert.h"
#include "common/profiler.h"
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include <array>
@@ -6,7 +6,7 @@
#include "common/common.h"
#include "common/threads.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/textureCache.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include "graphics/shader/shaderBindings.h"
@@ -1,4 +1,4 @@
#include "graphics/host_gpu/renderer/descriptors.h"
#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
#include "common/assert.h"
#include "common/common.h"
@@ -14,18 +14,18 @@
#include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/host_gpu/objects/textureCommon.h"
#include "graphics/host_gpu/renderer/debug.h"
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/imageView.h"
#include "graphics/host_gpu/renderer/image/imageView.h"
#include "graphics/host_gpu/renderer/image/textureCommon.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
#include "graphics/host_gpu/vma.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include "graphics/shader/recompiler/BindingLayout.h"
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/ir/BindingLayout.h"
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/shader.h"
#include <algorithm>
@@ -318,8 +318,8 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
const bool valid_2d_slice =
(is_color_2d && descriptor.Depth() == 0 && descriptor.BaseArray5() == 0) ||
(is_color_2d_array && descriptor.BaseArray5() <= descriptor.Depth());
const bool is_2d = resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D &&
valid_2d_slice;
const bool is_2d =
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && valid_2d_slice;
const bool is_2d_array =
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray &&
is_color_2d_array && descriptor.BaseArray5() <= descriptor.Depth();
@@ -333,18 +333,20 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
!Prospero::IsFmaskTextureFormat(descriptor.Format()) && (is_2d || is_2d_array) &&
TileGetBlockLayout(TileBlockFamily::Depth64KB, depth_bpe, depth_block);
const bool supported_standard_tile =
tile == Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB) &&
TileIsStandard4KBTextureSupported(descriptor.Format());
(tile == Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB) &&
TileIsStandard4KBTextureSupported(descriptor.Format())) ||
(tile == Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB) &&
TileIsStandard64KBTextureSupported(descriptor.Format()));
const bool supported_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kLinear) ||
tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) ||
supported_depth_tile || supported_standard_tile;
const auto swizzle = descriptor.DstSelXYZW();
const bool supported_swizzle =
IsValidImageSwizzle(descriptor.DstSelXYZW()) &&
(descriptor.DstSelXYZW() == DstSel(4, 5, 6, 7) || !resource.read);
IsValidImageSwizzle(swizzle) &&
(swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
const bool supported_mip_view = descriptor.BaseLevel() == 0 || is_1d || is_2d;
return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
supported_mip_view &&
descriptor.BaseLevel() == descriptor.LastLevel() &&
supported_mip_view && descriptor.BaseLevel() == descriptor.LastLevel() &&
descriptor.LastLevel() <= descriptor.MaxMip() && descriptor.MinLod() == 0 &&
supported_swizzle && descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth();
}
@@ -375,8 +377,10 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
const bool encoding_ok = IsSupportedStorageTextureEncoding(descriptor);
const bool uint_resource =
resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint;
const bool format_ok = Prospero::IsSupportedTextureFormat(format) &&
uint_resource == Prospero::IsUintTextureFormat(format);
const bool format_ok =
Prospero::IsSupportedTextureFormat(format) &&
uint_resource == Prospero::IsUintTextureFormat(format) &&
(!resource.atomic || format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt));
if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
return;
}
@@ -612,14 +616,12 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& reso
(address & (static_cast<uint64_t>(size.align) - 1u)) != 0);
if (storage) {
ValidateStorageTexture(resource, descriptor, size.size);
m_context.GetBufferCache().ValidateGpuAccess(address, size.size, resource.read,
resource.written);
}
const auto pixel_format = TextureGetFormat(format);
const auto storage_view_format = SrgbStorageViewFormat(pixel_format);
const auto view_format =
storage && storage_view_format != vk::Format::eUndefined ? storage_view_format
const auto view_format = storage && storage_view_format != vk::Format::eUndefined
? storage_view_format
: pixel_format;
const auto block_bytes = Prospero::BlockCompressedBytesPerBlock(format);
TextureCache::ImageDesc desc {};
@@ -2,9 +2,9 @@
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORS_H_
#include "common/assert.h"
#include "graphics/host_gpu/renderer/image.h"
#include "graphics/host_gpu/renderer/image/image.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/shaderBindings.h"
#include <cstdint>
@@ -1,4 +1,4 @@
#include "graphics/host_gpu/renderer/pipelineCache.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "common/assert.h"
#include "common/logging/log.h"
@@ -7,7 +7,7 @@
#include "graphics/host_gpu/renderer/colorRenderTarget.h"
#include "graphics/host_gpu/renderer/debug.h"
#include "graphics/host_gpu/renderer/depthRenderTarget.h"
#include "graphics/host_gpu/renderer/imageView.h"
#include "graphics/host_gpu/renderer/image/imageView.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
@@ -1,4 +1,4 @@
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
#include "common/assert.h"
#include "graphics/shader/shader.h"
@@ -2,7 +2,7 @@
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERRESOURCEBARRIER_H_
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include <vector>
@@ -1,6 +1,6 @@
#include "graphics/host_gpu/renderer/shaderSubgroup.h"
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/shader/recompiler/SpirvEmitter.h"
#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
namespace Libs::Graphics {
@@ -2,7 +2,7 @@
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
namespace Libs::Graphics {
@@ -6,14 +6,14 @@
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/debug.h"
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipelineCache.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/renderTarget.h"
#include "graphics/host_gpu/renderer/shaderSubgroup.h"
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/shader.h"
#include <algorithm>
+1 -1
View File
@@ -4,7 +4,7 @@
#include "common/abi.h"
#include "common/assert.h"
#include "common/common.h"
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/renderTarget.h"
#include "graphics/host_gpu/vulkanCommon.h"
@@ -10,17 +10,17 @@
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/guest_gpu/hardwareContext.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/descriptors.h"
#include "graphics/host_gpu/renderer/imageInfo.h"
#include "graphics/host_gpu/renderer/pipelineCache.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
#include "graphics/host_gpu/renderer/image/imageInfo.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/shaderSubgroup.h"
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/shader.h"
#include "kernel/eventQueue.h"
#include "kernel/pthread.h"
@@ -5,13 +5,13 @@
#include "common/assert.h"
#include "common/common.h"
#include "common/threads.h"
#include "graphics/host_gpu/renderer/bufferCache.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/gpuResourceManager.h"
#include "graphics/host_gpu/renderer/pipelineCache.h"
#include "graphics/host_gpu/renderer/samplerCache.h"
#include "graphics/host_gpu/renderer/textureCache.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/cache/samplerCache.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "kernel/eventQueue.h"
#include <memory>
+100 -51
View File
@@ -16,17 +16,18 @@
#include "graphics/host_gpu/renderer/colorRenderTarget.h"
#include "graphics/host_gpu/renderer/debug.h"
#include "graphics/host_gpu/renderer/depthRenderTarget.h"
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipelineCache.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/shaderSubgroup.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/shader.h"
#include "kernel/eventQueue.h"
#include "kernel/memory.h"
#include "kernel/pthread.h"
#include "libs/errno.h"
@@ -221,8 +222,7 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col
LogMrtState(draw_name, buffer, ps_input_info);
}
static void LogDrawInputState(const RenderCommandBuffer& buffer,
const RenderColorInfo& color,
static void LogDrawInputState(const RenderCommandBuffer& buffer, const RenderColorInfo& color,
const ShaderVertexInputInfo& vs_input_info,
uint32_t index_type_and_size, uint32_t index_count,
const void* index_addr) {
@@ -512,8 +512,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
auto& target = colors[i];
EXIT_IF(!target.image_id);
const auto old_image = cache.ResolveOwner(target.image_id);
if (old_image == nullptr ||
(!old_image->registered && !old_image->info.data.Empty()) ||
if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
old_image->binding.needs_rebind) {
if (old_image != nullptr) {
old_image->binding = {};
@@ -531,8 +530,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
attachment_samples, target.samples);
}
const auto& view = target.desc.view_info;
const auto layout =
image.binding.is_bound ? vk::ImageLayout::eGeneral
const auto layout = image.binding.is_bound ? vk::ImageLayout::eGeneral
: vk::ImageLayout::eColorAttachmentOptimal;
image.Transit(layout,
vk::AccessFlagBits2::eColorAttachmentRead |
@@ -561,8 +559,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
depth.depth_meta_clear_enable =
depth.htile &&
cache.IsMetaCleared(depth.htile_buffer_vaddr, depth.desc.view_info.base_layer);
depth.depth_load_clear_enable =
depth.depth_clear_enable || depth.depth_meta_clear_enable;
depth.depth_load_clear_enable = depth.depth_clear_enable || depth.depth_meta_clear_enable;
if (depth.depth_meta_clear_enable &&
!cache.TouchMeta(depth.htile_buffer_vaddr, depth.desc.view_info.base_layer, false)) {
EXIT("failed to consume HTile clear state\n");
@@ -572,8 +569,8 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
if (attachment_samples == 0) {
attachment_samples = depth.samples;
} else if (attachment_samples != depth.samples) {
EXIT("mixed color/depth sample counts are unsupported: %u and %u\n",
attachment_samples, depth.samples);
EXIT("mixed color/depth sample counts are unsupported: %u and %u\n", attachment_samples,
depth.samples);
}
const auto layout = depth_attachment_layout(depth);
const auto writes = depth.AttachmentWriteAspects();
@@ -595,11 +592,9 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
attachment.image_layout = layout;
attachment.clear_value[0] = std::bit_cast<uint32_t>(depth.depth_clear_value);
attachment.clear_value[1] = depth.stencil_clear_value;
attachment.has_depth =
static_cast<bool>(aspects & vk::ImageAspectFlagBits::eDepth);
attachment.has_depth = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eDepth);
attachment.depth_clear = depth.depth_load_clear_enable;
attachment.has_stencil =
static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
attachment.has_stencil = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
attachment.stencil_clear = depth.stencil_clear_enable;
}
if (attachment_samples == 0 ||
@@ -685,6 +680,85 @@ static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer
: buffer.num_records);
}
struct VertexBufferRange {
uint64_t base_address = 0;
uint64_t requested_end = 0;
uint64_t acquired_end = 0;
BufferBinding binding;
[[nodiscard]] uint64_t RequestedSize() const { return requested_end - base_address; }
};
static std::vector<BufferBinding> AcquireVertexBuffers(RenderCommandBuffer& buffer,
const ShaderVertexInputInfo& vs_input_info) {
// Collect the non-empty guest vertex ranges.
std::vector<VertexBufferRange> ranges;
ranges.reserve(vs_input_info.buffers_num);
for (int i = 0; i < vs_input_info.buffers_num; i++) {
const auto& vertex = vs_input_info.buffers[i];
const auto size = VertexBufferDescriptorSize(vertex);
if (size == 0) {
continue;
}
if (vertex.addr == 0 || size > UINT64_MAX - vertex.addr) {
EXIT("invalid vertex buffer range: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vertex.addr, size);
}
ranges.push_back({vertex.addr, vertex.addr + size});
}
std::ranges::sort(ranges, [](const VertexBufferRange& left, const VertexBufferRange& right) {
return left.base_address < right.base_address;
});
// Merge overlapping or touching ranges before acquiring host buffers.
std::vector<VertexBufferRange> merged_ranges;
merged_ranges.reserve(ranges.size());
for (const auto& range: ranges) {
if (!merged_ranges.empty() && merged_ranges.back().requested_end >= range.base_address) {
merged_ranges.back().requested_end =
std::max(merged_ranges.back().requested_end, range.requested_end);
continue;
}
merged_ranges.push_back(range);
}
auto& cache = buffer.GetContext().GetBufferCache();
for (auto& range: merged_ranges) {
// PPSA20298
const auto size =
Libs::LibKernel::Memory::ClampRangeSize(range.base_address, range.RequestedSize());
range.acquired_end = range.base_address + size;
range.binding = cache.ObtainBuffer(buffer, range.base_address, size);
}
// Rebuild slot bindings, offsetting non-empty slots into their acquired merged range.
std::vector<BufferBinding> bindings;
bindings.reserve(vs_input_info.buffers_num);
for (int i = 0; i < vs_input_info.buffers_num; i++) {
const auto& vertex = vs_input_info.buffers[i];
const auto size = VertexBufferDescriptorSize(vertex);
if (size == 0) {
auto owner = cache.ObtainNullBuffer();
bindings.push_back({owner, owner->Handle(), 0});
continue;
}
const auto range = std::ranges::find_if(merged_ranges, [&](const VertexBufferRange& value) {
return vertex.addr >= value.base_address && vertex.addr < value.acquired_end;
});
if (range == merged_ranges.end()) {
EXIT("vertex buffer address is outside the acquired range: addr=0x%016" PRIx64 "\n",
vertex.addr);
}
auto binding = range->binding;
binding.offset += vertex.addr - range->base_address;
bindings.push_back(std::move(binding));
}
return bindings;
}
static void SetDrawDebugPhase(RenderCommandBuffer& buffer, uint64_t submit_id,
const DrawCallInfo& draw, uint32_t phase) {
EXIT_IF(draw.name == nullptr);
@@ -823,37 +897,13 @@ static std::vector<BufferBinding> PrepareVertexBuffers(uint64_t
(void)submit_id;
LogDrawPhase(draw.name, "PrepareVertexBuffers");
std::vector<BufferBinding> bindings;
bindings.reserve(vs_input_info.buffers_num);
for (int i = 0; i < vs_input_info.buffers_num; i++) {
const auto& b = vs_input_info.buffers[i];
const auto size = VertexBufferDescriptorSize(b);
if (size == 0) {
auto owner = buffer.GetContext().GetBufferCache().ObtainNullBuffer();
bindings.push_back({owner, owner->Handle(), 0});
} else {
bindings.push_back(
buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, b.addr, size));
}
}
return bindings;
return AcquireVertexBuffers(buffer, vs_input_info);
}
static void RebindVertexBuffers(RenderCommandBuffer& buffer,
const ShaderVertexInputInfo& vs_input_info,
std::vector<BufferBinding>& bindings) {
EXIT_IF(bindings.size() != static_cast<size_t>(vs_input_info.buffers_num));
for (int i = 0; i < vs_input_info.buffers_num; i++) {
const auto& vertex = vs_input_info.buffers[i];
const auto size = VertexBufferDescriptorSize(vertex);
if (size == 0) {
auto owner = buffer.GetContext().GetBufferCache().ObtainNullBuffer();
bindings[i] = {owner, owner->Handle(), 0};
} else {
bindings[i] =
buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, vertex.addr, size);
}
}
bindings = AcquireVertexBuffers(buffer, vs_input_info);
}
static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer& buffer,
@@ -1228,8 +1278,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
}
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
uint32_t index_count,
void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_count,
uint32_t flags, uint32_t render_target_slice_offset,
uint32_t instance_count, uint32_t first_vertex,
uint32_t first_instance) {
@@ -1290,7 +1339,8 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
instance_count, first_instance};
DrawRenderState state {};
if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, false, state)) {
if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, false,
state)) {
ResetBindings();
return;
}
@@ -1369,8 +1419,7 @@ bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer
cache.MarkGpuWritten(dst.image_id);
auto& source = cache.GetImage(src.image_id);
auto& destination = cache.GetImage(dst.image_id);
destination.Resolve(source,
{src.base_mip_level, 1, src.base_array_layer, 1},
destination.Resolve(source, {src.base_mip_level, 1, src.base_array_layer, 1},
{dst.base_mip_level, 1, dst.base_array_layer, 1});
return true;
}
+1 -1
View File
@@ -5,7 +5,7 @@
#include "common/logging/log.h"
#include "common/threads.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/bufferCache.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/presentation/videoOut.h"
+109 -11
View File
@@ -19,12 +19,16 @@
#include <windows.h>
#undef min
#undef max
#else
#include <dlfcn.h>
// RenderDoc uses Windows-style names in its cross-platform API.
#define __cdecl
using HMODULE = void*;
#endif
namespace Libs::Graphics {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
using RenderDocDevicePointer = void*;
using RenderDocWindowHandle = void*;
@@ -107,6 +111,8 @@ static RenderDocDevicePointer GetRenderDocDevicePointer(vk::Instance instance) {
return VulkanHandleToPointer(instance);
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
static bool BindRenderDocApi(HMODULE module) {
auto* get_api = reinterpret_cast<pRENDERDOC_GetAPI>(GetProcAddress(module, "RENDERDOC_GetAPI"));
if (get_api == nullptr) {
@@ -147,10 +153,84 @@ static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) {
return info.info.win.window;
}
#else
static bool BindRenderDocApi(HMODULE module) {
auto* get_api = reinterpret_cast<pRENDERDOC_GetAPI>(::dlsym(module, "RENDERDOC_GetAPI"));
if (get_api == nullptr) {
return false;
}
void* api = nullptr;
if (get_api(eRENDERDOC_API_Version_1_4_2, &api) == 0 || api == nullptr) {
return false;
}
g_module = module;
g_api = static_cast<RenderDocApi*>(api);
g_api->SetCaptureFilePathTemplate("_RenderDoc/kyty");
static RenderDocInputButton capture_keys[] = {eRENDERDOC_Key_F1};
g_api->SetCaptureKeys(capture_keys, 1);
g_api->UnloadCrashHandler();
Dl_info info {};
if (::dladdr(reinterpret_cast<void*>(get_api), &info) != 0 && info.dli_fname != nullptr) {
LOGF("RenderDoc: bound API from %s\n", info.dli_fname);
} else {
LOGF("RenderDoc: bound API\n");
}
return true;
}
static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) {
if (window == nullptr) {
return nullptr;
}
SDL_SysWMinfo info {};
SDL_VERSION(&info.version);
if (SDL_GetWindowWMInfo(window, &info) != SDL_TRUE) {
return nullptr;
}
#if defined(SDL_VIDEO_DRIVER_X11)
if (info.subsystem == SDL_SYSWM_X11) {
// RenderDoc takes the raw xlib Window id in the pointer slot, not a Display*.
return reinterpret_cast<RenderDocWindowHandle>(
static_cast<uintptr_t>(info.info.x11.window));
}
#endif
// Wayland capture works without an active-window handle.
static std::atomic_bool logged = false;
if (!logged.exchange(true)) {
LOGF("RenderDoc: no native window handle for SDL subsystem %d (Wayland?); the in-app "
"overlay is unavailable, but --rd captures still work\n",
static_cast<int>(info.subsystem));
}
return nullptr;
}
#endif
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
static bool IsAvailable() {
return g_api != nullptr && g_device != nullptr && g_window != nullptr;
}
#else
static bool IsAvailable() {
return g_api != nullptr && g_device != nullptr;
}
#endif
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void RenderDocInit() {
bool expected = false;
if (!g_init_done.compare_exchange_strong(expected, true)) {
@@ -187,6 +267,33 @@ void RenderDocInit() {
}
}
#else
void RenderDocInit() {
bool expected = false;
if (!g_init_done.compare_exchange_strong(expected, true)) {
return;
}
// Prefer an injected RenderDoc instance.
auto* module = ::dlopen("librenderdoc.so", RTLD_NOW | RTLD_NOLOAD);
if (module == nullptr) {
module = ::dlopen("librenderdoc.so", RTLD_NOW);
}
if (module == nullptr) {
LOGF("RenderDoc: librenderdoc.so was not found; in-app capture disabled\n");
return;
}
if (!BindRenderDocApi(module)) {
LOGF("RenderDoc: API 1.4.2 is not available; in-app capture disabled\n");
::dlclose(module);
return;
}
}
#endif
void RenderDocSetActiveWindow(vk::Instance instance, SDL_Window* window) {
if (g_api == nullptr) {
return;
@@ -274,13 +381,4 @@ void RenderDocOnPresent() {
}
}
#else
void RenderDocInit() {}
void RenderDocSetActiveWindow(vk::Instance /*instance*/, SDL_Window* /*window*/) {}
void RenderDocRequestCapture() {}
void RenderDocOnPresent() {}
#endif
} // namespace Libs::Graphics
+1 -1
View File
@@ -12,7 +12,7 @@
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/renderer/imageInfo.h"
#include "graphics/host_gpu/renderer/image/imageInfo.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/presentation/presenter.h"
@@ -2,17 +2,17 @@
#include "common/assert.h"
#include "common/logging/log.h"
#include "graphics/shader/recompiler/BindingLayout.h"
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ResourceTracking.h"
#include "graphics/shader/recompiler/ScalarProvenance.h"
#include "graphics/shader/recompiler/ShaderCFG.h"
#include "graphics/shader/recompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/ShaderInfoCollection.h"
#include "graphics/shader/recompiler/SpirvEmitter.h"
#include "graphics/shader/recompiler/SrtPatcher.h"
#include "graphics/shader/recompiler/SrtWalker.h"
#include "graphics/shader/recompiler/ir/BindingLayout.h"
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/ResourceTracking.h"
#include "graphics/shader/recompiler/ir/ScalarProvenance.h"
#include "graphics/shader/recompiler/cfg/ShaderCFG.h"
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/recompiler/ir/ShaderInfoCollection.h"
#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
#include "graphics/shader/recompiler/ir/SrtPatcher.h"
#include "graphics/shader/recompiler/ir/SrtWalker.h"
#include <algorithm>
#include <array>
@@ -3,7 +3,7 @@
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include "graphics/shader/shader.h"
#include <optional>
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/ShaderCFG.h"
#include "graphics/shader/recompiler/cfg/ShaderCFG.h"
#include <algorithm>
#include <fmt/format.h>
@@ -3,7 +3,7 @@
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/shader/recompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include <vector>
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/ExportOps.h"
#include "graphics/shader/recompiler/decompiler/ExportOps.h"
#include <fmt/format.h>
@@ -1,7 +1,7 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_
#include "graphics/shader/recompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/ImageOps.h"
#include "graphics/shader/recompiler/decompiler/ImageOps.h"
#include <algorithm>
#include <fmt/format.h>
@@ -1,7 +1,7 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_
#include "graphics/shader/recompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/MemoryOps.h"
#include "graphics/shader/recompiler/decompiler/MemoryOps.h"
#include <fmt/format.h>
#include <iterator>
@@ -442,11 +442,6 @@ bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
inst.opcode == Opcode::DsReadAddtidB32)) {
SetUnsupported(inst, Family::DS, opcode, "DS swizzle/addtid is available only for LDS");
}
if (inst.gds && (inst.opcode == Opcode::DsAppend || inst.opcode == Opcode::DsConsume) &&
inst.offset != 0u) {
SetUnsupported(inst, Family::DS, opcode,
"GDS append/consume requires a zero instruction offset");
}
if (inst.opcode == Opcode::DsWriteAddtidB32 && data1 != 0u) {
SetUnsupported(inst, Family::DS, opcode,
"DS write addtid data1 operand is not implemented");
@@ -1,7 +1,7 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_
#include "graphics/shader/recompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/ScalarAluOps.h"
#include "graphics/shader/recompiler/decompiler/ScalarAluOps.h"
#include <iterator>
@@ -1,7 +1,7 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_
#include "graphics/shader/recompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
@@ -1,10 +1,10 @@
#include "graphics/shader/recompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/ExportOps.h"
#include "graphics/shader/recompiler/ImageOps.h"
#include "graphics/shader/recompiler/MemoryOps.h"
#include "graphics/shader/recompiler/ScalarAluOps.h"
#include "graphics/shader/recompiler/VectorAluOps.h"
#include "graphics/shader/recompiler/decompiler/ExportOps.h"
#include "graphics/shader/recompiler/decompiler/ImageOps.h"
#include "graphics/shader/recompiler/decompiler/MemoryOps.h"
#include "graphics/shader/recompiler/decompiler/ScalarAluOps.h"
#include "graphics/shader/recompiler/decompiler/VectorAluOps.h"
#include <algorithm>
#include <bit>
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/VectorAluOps.h"
#include "graphics/shader/recompiler/decompiler/VectorAluOps.h"
#include <fmt/format.h>
#include <iterator>
@@ -1,7 +1,7 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_
#include "graphics/shader/recompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/SpirvBuilder.h"
#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
#include <cstdio>
#include <cstring>
@@ -1,7 +1,7 @@
#include "graphics/shader/recompiler/SpirvEmitter.h"
#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
#include "graphics/shader/recompiler/SrtWalker.h"
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/ir/SrtWalker.h"
#include <algorithm>
#include <array>
@@ -160,8 +160,7 @@ bool ValidateInstructionContract(const IR::Instruction& inst, std::string* error
inst.dst.kind != IR::OperandKind::Null)) ||
((inst.op == IR::Opcode::DsAppend || inst.op == IR::Opcode::DsConsume) &&
(!ds_kind || !ds_resource || inst.src_count != 1 ||
inst.dst.kind != IR::OperandKind::Register ||
(kind == IR::ResourceKind::Gds && inst.memory.offset != 0))) ||
inst.dst.kind != IR::OperandKind::Register)) ||
((inst.op == IR::Opcode::DsMinF32 || inst.op == IR::Opcode::DsMaxF32) &&
(!ds_kind || !ds_resource || inst.src_count != 3 ||
inst.dst.kind != IR::OperandKind::Null)) ||
@@ -3,7 +3,7 @@
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include <vector>
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -1,7 +1,7 @@
#include "common/assert.h"
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/shader/recompiler/SpirvEmitter.h"
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -1025,15 +1025,47 @@ void EmitDispatcherSwitch(EmitterState& state, const IR::Program& program) {
EmitDispatcherExit(state);
}
size_t BufferLoadGroupSize(const IR::BasicBlock& block, size_t first_index) {
const auto& first = block.instructions[first_index];
if (first.op != IR::Opcode::BufferLoadDword || first.memory.component_index != 0u ||
first.memory.component_count <= 1u) {
return 1u;
}
size_t count = 1u;
while (first_index + count < block.instructions.size() &&
count < first.memory.component_count) {
const auto& next = block.instructions[first_index + count];
if (next.op != IR::Opcode::BufferLoadDword || next.pc != first.pc ||
next.memory.component_index != count ||
next.memory.component_count != first.memory.component_count) {
break;
}
count++;
}
return count;
}
void EmitBlockInstructions(EmitterState& state, const IR::BasicBlock& block) {
for (size_t i = 0; i < block.instructions.size();) {
const auto count = BufferLoadGroupSize(block, i);
if (count > 1u) {
EmitBufferLoadDwordGroup(state, block.instructions.data() + i,
static_cast<uint32_t>(count));
} else {
EmitInstruction(state, block.instructions[i]);
}
i += count;
}
}
void EmitDispatcherBlocks(EmitterState& state, const IR::Program& program) {
for (const auto& block: program.blocks) {
if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) {
continue;
}
state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)});
for (const auto& inst: block.instructions) {
EmitInstruction(state, inst);
}
EmitBlockInstructions(state, block);
EmitDispatcherTerminator(state, block.terminator);
}
}
@@ -1083,9 +1115,7 @@ void EmitFunction(EmitterState& state, const IR::Program& program) {
continue;
}
state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)});
for (const auto& inst: block.instructions) {
EmitInstruction(state, inst);
}
EmitBlockInstructions(state, block);
EmitTerminator(state, block.terminator);
}
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -1,5 +1,5 @@
#include "common/assert.h"
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -3,11 +3,11 @@
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/shader/recompiler/BindingLayout.h"
#include "graphics/shader/recompiler/BufferFormat.h"
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/SpirvBuilder.h"
#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
#include "graphics/shader/recompiler/ir/BindingLayout.h"
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include <algorithm>
#include <array>
@@ -990,11 +990,6 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
uint32_t UnpackTBufferFormat(EmitterState& state, const IR::Instruction& inst,
const Format::BufferFormatInfo& info);
bool EmitTypedTBufferLoad(EmitterState& state, const IR::Instruction& inst,
const Format::BufferFormatInfo& info);
bool EmitFormattedBufferLoad(EmitterState& state, const IR::Instruction& inst);
uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format,
uint32_t opcode_components);
@@ -1020,6 +1015,9 @@ void EmitBufferLoadSshort(EmitterState& state, const IR::Instruction& inst);
void EmitBufferLoadDword(EmitterState& state, const IR::Instruction& inst);
void EmitBufferLoadDwordGroup(EmitterState& state, const IR::Instruction* instructions,
uint32_t count);
void EmitBufferStoreDword(EmitterState& state, const IR::Instruction& inst);
void EmitFlatLoadUbyte(EmitterState& state, const IR::Instruction& inst);

Some files were not shown because too many files have changed in this diff Show More