mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-08-03 11:23:49 +00:00
Compare commits
66
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
302b579779 | ||
|
|
0b6bf01b36 | ||
|
|
66f640527d | ||
|
|
4631b96178 | ||
|
|
43f64e4ab4 | ||
|
|
e63f5b7d5c | ||
|
|
fa7c3c01bf | ||
|
|
89651f6f59 | ||
|
|
44d7f2a3e8 | ||
|
|
2dcb90066c | ||
|
|
51a33cc363 | ||
|
|
ed84370786 | ||
|
|
43f30d3ab2 | ||
|
|
0838142abd | ||
|
|
0f550d1fd0 | ||
|
|
c1a5927036 | ||
|
|
bc436548a9 | ||
|
|
a65d17a5d6 | ||
|
|
c690aeea62 | ||
|
|
f830d6b2e4 | ||
|
|
d68a477276 | ||
|
|
8977d4d2f0 | ||
|
|
4b4e3bf3cf | ||
|
|
a0bb129f02 | ||
|
|
846002c5eb | ||
|
|
d8a4c83cc7 | ||
|
|
68be13345a | ||
|
|
167da0abe0 | ||
|
|
48c31d61ee | ||
|
|
212282d693 | ||
|
|
6bca35d1f5 | ||
|
|
e4ad5fc988 | ||
|
|
c0d3d261ea | ||
|
|
3b75a5659a | ||
|
|
d475387171 | ||
|
|
2f5396c6a5 | ||
|
|
ecb48f90bb | ||
|
|
77aa28b27c | ||
|
|
d04938c88c | ||
|
|
85622befb8 | ||
|
|
3965d41d36 | ||
|
|
c508c4a9c0 | ||
|
|
e91dd39cb0 | ||
|
|
cc76827e63 | ||
|
|
832bc84100 | ||
|
|
65a0f0baa7 | ||
|
|
b9ae2537ef | ||
|
|
0b9edaa721 | ||
|
|
8a244677d7 | ||
|
|
f6e01e5403 | ||
|
|
861729fc6c | ||
|
|
687ce025c6 | ||
|
|
a21d1aaa47 | ||
|
|
ec11f31aa6 | ||
|
|
aeceaff028 | ||
|
|
e76f2d1af8 | ||
|
|
64845e2294 | ||
|
|
4b9030bd3d | ||
|
|
a6b61d7aa0 | ||
|
|
2b9cba457e | ||
|
|
3dc793d859 | ||
|
|
01d78b71fe | ||
|
|
4d79f19089 | ||
|
|
8fe4765f9e | ||
|
|
c71bb9fc9e | ||
|
|
f60f80b631 |
@@ -1,82 +0,0 @@
|
||||
name: Build KytyPS5 (Linux)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [ main, master ]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends --yes \
|
||||
clang \
|
||||
glslang-tools \
|
||||
libgl1-mesa-dev \
|
||||
libx11-dev \
|
||||
libxcursor-dev \
|
||||
libxext-dev \
|
||||
libxfixes-dev \
|
||||
libxi-dev \
|
||||
libxrandr-dev \
|
||||
libxss-dev \
|
||||
lld \
|
||||
ninja-build
|
||||
|
||||
- name: Install Qt 6.10.3
|
||||
uses: jurplel/install-qt-action@v4
|
||||
with:
|
||||
version: "6.10.3"
|
||||
host: linux
|
||||
target: desktop
|
||||
arch: linux_gcc_64
|
||||
cache: true
|
||||
|
||||
- name: Verify toolchain
|
||||
shell: bash
|
||||
run: |
|
||||
git --version
|
||||
cmake --version
|
||||
ninja --version
|
||||
clang++ --version
|
||||
ld.lld --version
|
||||
glslangValidator --version
|
||||
echo "Qt6_DIR=$Qt6_DIR"
|
||||
|
||||
- name: Configure
|
||||
shell: bash
|
||||
run: |
|
||||
cmake -S src -B _Build/linux \
|
||||
-G Ninja \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_C_COMPILER=clang \
|
||||
-DCMAKE_CXX_COMPILER=clang++ \
|
||||
-DCMAKE_PREFIX_PATH="$Qt6_DIR"
|
||||
|
||||
- name: Build
|
||||
shell: bash
|
||||
run: |
|
||||
cmake --build _Build/linux --target launcher --parallel
|
||||
|
||||
- name: Install
|
||||
shell: bash
|
||||
run: |
|
||||
cmake --install _Build/linux --prefix _Build/linux/install
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: KytyPS5-Linux
|
||||
path: _Build/linux/install/**
|
||||
if-no-files-found: error
|
||||
+308
-16
@@ -1,4 +1,4 @@
|
||||
name: Build KytyPS5
|
||||
name: Build and Release KytyPS5
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -7,7 +7,8 @@ on:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
windows:
|
||||
name: Build KytyPS5 (Windows)
|
||||
runs-on: windows-2022
|
||||
|
||||
steps:
|
||||
@@ -22,11 +23,31 @@ jobs:
|
||||
- name: Setup Ninja
|
||||
uses: seanmiddleditch/gha-setup-ninja@v5
|
||||
|
||||
- name: Install glslang
|
||||
- name: Locate vcpkg
|
||||
id: vcpkg
|
||||
shell: pwsh
|
||||
run: |
|
||||
$vcpkgRoot = Split-Path (Get-Command vcpkg).Source
|
||||
vcpkg install glslang[tools,opt]:x64-windows
|
||||
$portHash = (Get-FileHash "$vcpkgRoot\ports\glslang\vcpkg.json" -Algorithm SHA256).Hash
|
||||
"root=$vcpkgRoot" |
|
||||
Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
|
||||
"port_hash=$($portHash.ToLowerInvariant())" |
|
||||
Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
|
||||
|
||||
- name: Cache glslang
|
||||
id: glslang_cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ${{ steps.vcpkg.outputs.root }}\installed\x64-windows
|
||||
key: glslang-${{ runner.os }}-${{ runner.arch }}-${{ steps.vcpkg.outputs.port_hash }}-tools-opt
|
||||
|
||||
- name: Install glslang
|
||||
shell: pwsh
|
||||
run: |
|
||||
$vcpkgRoot = "${{ steps.vcpkg.outputs.root }}"
|
||||
if ("${{ steps.glslang_cache.outputs.cache-hit }}" -ne "true") {
|
||||
vcpkg install glslang[tools,opt]:x64-windows
|
||||
}
|
||||
"$vcpkgRoot\installed\x64-windows\tools\glslang" |
|
||||
Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
|
||||
@@ -62,52 +83,323 @@ jobs:
|
||||
- name: Build
|
||||
shell: cmd
|
||||
run: |
|
||||
cmake --build _Build/windows --target launcher --parallel
|
||||
cmake --build _Build/windows --target launcher audio_out2_port_tests virtual_memory_allocation_tests --parallel
|
||||
|
||||
- name: Test
|
||||
shell: cmd
|
||||
run: |
|
||||
ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|virtual_memory_allocation)$"
|
||||
|
||||
- name: Install
|
||||
shell: cmd
|
||||
run: |
|
||||
cmake --install _Build/windows --prefix _Build/windows/install
|
||||
|
||||
- name: Upload Artifacts
|
||||
- name: Upload Windows artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: KytyPS5
|
||||
name: KytyPS5-Windows-x64
|
||||
path: _Build/windows/install/**
|
||||
if-no-files-found: error
|
||||
|
||||
macos:
|
||||
name: Build KytyPS5 (macOS)
|
||||
runs-on: macos-15
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Select Xcode 26
|
||||
shell: bash
|
||||
run: |
|
||||
xcode_path="$(ls -d /Applications/Xcode_26*.app | tail -n 1)"
|
||||
test -n "$xcode_path"
|
||||
sudo xcode-select --switch "$xcode_path/Contents/Developer"
|
||||
xcodebuild -version
|
||||
|
||||
- name: Install build dependencies
|
||||
shell: bash
|
||||
run: brew install glslang ninja
|
||||
|
||||
- name: Install Qt 6.10.3
|
||||
uses: jurplel/install-qt-action@v4
|
||||
with:
|
||||
version: "6.10.3"
|
||||
host: mac
|
||||
target: desktop
|
||||
arch: clang_64
|
||||
cache: true
|
||||
|
||||
- name: Verify toolchain
|
||||
shell: bash
|
||||
run: |
|
||||
git --version
|
||||
cmake --version
|
||||
ninja --version
|
||||
clang++ --version
|
||||
glslangValidator --version
|
||||
echo "Host architecture: $(uname -m)"
|
||||
echo "QT_ROOT_DIR=$QT_ROOT_DIR"
|
||||
|
||||
- name: Configure
|
||||
shell: bash
|
||||
run: |
|
||||
cmake -S src -B _Build/macos \
|
||||
-G Ninja \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_OSX_ARCHITECTURES=x86_64 \
|
||||
-DCMAKE_C_COMPILER=clang \
|
||||
-DCMAKE_CXX_COMPILER=clang++ \
|
||||
-DCMAKE_PREFIX_PATH="$QT_ROOT_DIR"
|
||||
|
||||
- name: Build
|
||||
shell: bash
|
||||
run: |
|
||||
cmake --build _Build/macos \
|
||||
--target launcher audio_out2_port_tests virtual_memory_allocation_tests \
|
||||
--parallel
|
||||
|
||||
- name: Test
|
||||
shell: bash
|
||||
run: |
|
||||
ctest --test-dir _Build/macos --output-on-failure \
|
||||
-R '^(audio_out2_port|virtual_memory_allocation)$'
|
||||
|
||||
- name: Install
|
||||
shell: bash
|
||||
run: |
|
||||
cmake --install _Build/macos --prefix _Build/macos/install
|
||||
|
||||
- name: Bundle MoltenVK
|
||||
shell: bash
|
||||
env:
|
||||
MOLTENVK_VERSION: v1.4.2
|
||||
MOLTENVK_SHA256: f95765a6229cb7b915990a2890ce12ebe36a730b021545d3d52ae69ce4c4024e
|
||||
run: |
|
||||
archive="$RUNNER_TEMP/MoltenVK-macos.tar"
|
||||
package="$RUNNER_TEMP/MoltenVK"
|
||||
curl --fail --location --retry 3 \
|
||||
--output "$archive" \
|
||||
"https://github.com/KhronosGroup/MoltenVK/releases/download/$MOLTENVK_VERSION/MoltenVK-macos.tar"
|
||||
echo "$MOLTENVK_SHA256 $archive" | shasum -a 256 --check
|
||||
tar -xf "$archive" -C "$RUNNER_TEMP"
|
||||
install -m 755 \
|
||||
"$package/MoltenVK/dynamic/dylib/macOS/libMoltenVK.dylib" \
|
||||
_Build/macos/install/libMoltenVK.dylib
|
||||
install -m 644 "$package/LICENSE" _Build/macos/install/LICENSE.MoltenVK
|
||||
codesign --force --sign - --timestamp=none _Build/macos/install/libMoltenVK.dylib
|
||||
|
||||
- name: Verify artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
file _Build/macos/install/launcher
|
||||
file _Build/macos/install/kyty_emulator
|
||||
file _Build/macos/install/libMoltenVK.dylib
|
||||
lipo _Build/macos/install/launcher -verify_arch x86_64
|
||||
lipo _Build/macos/install/kyty_emulator -verify_arch x86_64
|
||||
lipo _Build/macos/install/libMoltenVK.dylib -verify_arch x86_64
|
||||
codesign --verify --strict _Build/macos/install/kyty_emulator
|
||||
codesign --verify --strict _Build/macos/install/libMoltenVK.dylib
|
||||
|
||||
- name: Upload macOS artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: KytyPS5-macOS-x86_64
|
||||
path: _Build/macos/install/**
|
||||
if-no-files-found: error
|
||||
|
||||
linux:
|
||||
name: Build KytyPS5 (Linux)
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends --yes \
|
||||
clang \
|
||||
glslang-tools \
|
||||
libasound2-dev \
|
||||
libdbus-1-dev \
|
||||
libgl1-mesa-dev \
|
||||
libpulse-dev \
|
||||
libudev-dev \
|
||||
libwayland-dev \
|
||||
libx11-dev \
|
||||
libxcursor-dev \
|
||||
libxext-dev \
|
||||
libxfixes-dev \
|
||||
libxi-dev \
|
||||
libxkbcommon-dev \
|
||||
libxrandr-dev \
|
||||
libxss-dev \
|
||||
lld \
|
||||
ninja-build \
|
||||
wayland-protocols
|
||||
|
||||
- name: Install Qt 6.10.3
|
||||
uses: jurplel/install-qt-action@v4
|
||||
with:
|
||||
version: "6.10.3"
|
||||
host: linux
|
||||
target: desktop
|
||||
arch: linux_gcc_64
|
||||
cache: true
|
||||
|
||||
- name: Verify toolchain
|
||||
shell: bash
|
||||
run: |
|
||||
git --version
|
||||
cmake --version
|
||||
ninja --version
|
||||
clang++ --version
|
||||
ld.lld --version
|
||||
glslangValidator --version
|
||||
echo "Qt6_DIR=$Qt6_DIR"
|
||||
|
||||
- name: Configure
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p _Build
|
||||
cmake -S src -B _Build/linux \
|
||||
-G Ninja \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_C_COMPILER=clang \
|
||||
-DCMAKE_CXX_COMPILER=clang++ \
|
||||
-DCMAKE_PREFIX_PATH="$Qt6_DIR" 2>&1 | tee _Build/configure.log
|
||||
exit "${PIPESTATUS[0]}"
|
||||
|
||||
- name: Verify SDL2 backends
|
||||
shell: bash
|
||||
run: |
|
||||
status=0
|
||||
for feature in SDL_ALSA SDL_PULSEAUDIO SDL_WAYLAND SDL_X11 SDL_LIBUDEV SDL_DBUS; do
|
||||
if grep -qE "^-- ${feature} +\\(Wanted: ON\\): ON" _Build/configure.log; then
|
||||
echo "ok ${feature}"
|
||||
else
|
||||
echo "FAIL ${feature} is not enabled"
|
||||
status=1
|
||||
fi
|
||||
done
|
||||
exit "$status"
|
||||
|
||||
- name: Build
|
||||
shell: bash
|
||||
run: |
|
||||
cmake --build _Build/linux \
|
||||
--target launcher page_manager_tests memory_tracker_tests \
|
||||
audio_out2_port_tests virtual_memory_allocation_tests \
|
||||
--parallel
|
||||
|
||||
- name: Test
|
||||
shell: bash
|
||||
run: |
|
||||
ctest --test-dir _Build/linux --output-on-failure \
|
||||
-R '^(audio_out2_port|page_manager|memory_tracker|virtual_memory_allocation)$'
|
||||
|
||||
- name: Install
|
||||
shell: bash
|
||||
run: |
|
||||
cmake --install _Build/linux --prefix _Build/linux/install
|
||||
|
||||
- name: Verify artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
file _Build/linux/install/launcher
|
||||
file _Build/linux/install/kyty_emulator
|
||||
file _Build/linux/install/kyty_emulator | grep -q "ELF 64-bit LSB .*x86-64"
|
||||
ldd _Build/linux/install/kyty_emulator > /dev/null
|
||||
readelf -d _Build/linux/install/launcher | grep -q 'RPATH.*\$ORIGIN/lib'
|
||||
while IFS= read -r -d '' binary; do
|
||||
while read -r dependency; do
|
||||
test -e "_Build/linux/install/lib/$dependency"
|
||||
done < <(
|
||||
readelf -d "$binary" |
|
||||
sed -n 's/.*Shared library: \[\(libQt6[^]]*\|libicu[^]]*\)\].*/\1/p'
|
||||
)
|
||||
done < <(
|
||||
find _Build/linux/install/launcher _Build/linux/install/plugins \
|
||||
-type f \( -name launcher -o -name '*.so' \) -print0
|
||||
)
|
||||
_Build/linux/install/kyty_emulator --help > /dev/null
|
||||
|
||||
- name: Upload Linux artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: KytyPS5-Linux-x86_64
|
||||
path: _Build/linux/install/**
|
||||
if-no-files-found: error
|
||||
|
||||
release:
|
||||
if: github.event_name == 'push'
|
||||
needs: build
|
||||
name: Release KytyPS5
|
||||
if: github.event_name == 'push' && github.repository == 'KytyPS5/KytyPS5'
|
||||
needs: [windows, macos, linux]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Download build
|
||||
- name: Download builds
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: KytyPS5
|
||||
path: KytyPS5
|
||||
path: artifacts
|
||||
|
||||
- name: Set release name
|
||||
shell: bash
|
||||
run: |
|
||||
echo "RELEASE_NAME=KytyPS5-$(date -u +'%Y-%m-%d')-${GITHUB_SHA::7}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Package build
|
||||
- name: Package builds
|
||||
shell: bash
|
||||
run: zip -r "$RELEASE_NAME.zip" KytyPS5
|
||||
run: |
|
||||
windows_dir="artifacts/KytyPS5-Windows-x64"
|
||||
macos_dir="artifacts/KytyPS5-macOS-x86_64"
|
||||
linux_dir="artifacts/KytyPS5-Linux-x86_64"
|
||||
test -d "$windows_dir"
|
||||
test -d "$macos_dir"
|
||||
test -d "$linux_dir"
|
||||
chmod a+x \
|
||||
"$macos_dir/launcher" \
|
||||
"$macos_dir/kyty_emulator" \
|
||||
"$macos_dir/libMoltenVK.dylib" \
|
||||
"$linux_dir/launcher" \
|
||||
"$linux_dir/kyty_emulator"
|
||||
(
|
||||
cd "$windows_dir"
|
||||
zip -r "$GITHUB_WORKSPACE/$RELEASE_NAME-Windows-x64.zip" .
|
||||
)
|
||||
(
|
||||
cd "$macos_dir"
|
||||
zip -r "$GITHUB_WORKSPACE/$RELEASE_NAME-macOS-x86_64.zip" .
|
||||
)
|
||||
tar -C "$linux_dir" -czf \
|
||||
"$GITHUB_WORKSPACE/$RELEASE_NAME-Linux-x86_64.tar.gz" .
|
||||
|
||||
- name: Create release
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
assets=(
|
||||
"$RELEASE_NAME-Windows-x64.zip"
|
||||
"$RELEASE_NAME-macOS-x86_64.zip"
|
||||
"$RELEASE_NAME-Linux-x86_64.tar.gz"
|
||||
)
|
||||
if gh release view "$RELEASE_NAME" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then
|
||||
gh release upload "$RELEASE_NAME" "$RELEASE_NAME.zip" --clobber --repo "$GITHUB_REPOSITORY"
|
||||
gh release upload "$RELEASE_NAME" "${assets[@]}" \
|
||||
--clobber \
|
||||
--repo "$GITHUB_REPOSITORY"
|
||||
else
|
||||
gh release create "$RELEASE_NAME" "$RELEASE_NAME.zip" \
|
||||
gh release create "$RELEASE_NAME" "${assets[@]}" \
|
||||
--generate-notes \
|
||||
--repo "$GITHUB_REPOSITORY" \
|
||||
--target "$GITHUB_SHA" \
|
||||
|
||||
+2
-1
@@ -2,4 +2,5 @@
|
||||
.vs/
|
||||
.idea/
|
||||
build/
|
||||
_Build/vscode-clang/
|
||||
_Build/vscode-clang/
|
||||
_Build/
|
||||
@@ -1,15 +1,16 @@
|
||||
# KytyPS5
|
||||
|
||||
[](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
|
||||
[](https://github.com/KytyPS5/KytyPS5/actions/workflows/build-linux.yml)
|
||||
[](#system-requirements)
|
||||
[](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
|
||||
[](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
|
||||
[](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
|
||||
[](#system-requirements)
|
||||
[](#current-status)
|
||||
[](LICENSE)
|
||||
|
||||
KytyPS5 is a free and open-source PlayStation 5 emulator written in C++ for Windows. It is based on
|
||||
a heavily modified version of [Kyty](https://github.com/InoriRus/Kyty). The project is in an early
|
||||
stage of development, so compatibility is limited and behavior may change significantly between
|
||||
builds.
|
||||
KytyPS5 is a free and open-source PlayStation 5 emulator written in C++ for Windows and Linux,
|
||||
with experimental macOS support. It is based on a heavily modified version of
|
||||
[Kyty](https://github.com/InoriRus/Kyty). The project is in an early stage of development, so
|
||||
compatibility is limited and behavior may change significantly between builds.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> KytyPS5 is not affiliated with Sony Interactive Entertainment or PlayStation. The project does
|
||||
@@ -23,7 +24,12 @@ KytyPS5 can boot 2D games and a selection of 3D games, including titles built wi
|
||||
|
||||
Development is focused on compatibility and boot reliability.
|
||||
|
||||
Linux support is planned, but Windows is the only supported platform at this time.
|
||||
Windows is the primary platform and receives the most testing. Linux builds and runs; see
|
||||
[Building on Linux](#building-on-linux).
|
||||
|
||||
macOS support is experimental. The emulator is built for x86-64 and runs on Apple Silicon under
|
||||
Rosetta 2, with Vulkan provided by MoltenVK. A small number of titles have been verified in-game
|
||||
on Apple Silicon hardware; see [Building on macOS](#building-on-macos).
|
||||
|
||||
## Bugs and Issues
|
||||
|
||||
@@ -45,7 +51,7 @@ graphical glitches, low compatibility, and poor performance.
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">
|
||||
<strong>Minecraft Legends</strong><br>
|
||||
<strong>Neptunia ReVerse</strong><br>
|
||||
<img src="docs/screenshots/ps5-04.png" width="300" alt="Minecraft Legends running in KytyPS5">
|
||||
</td>
|
||||
<td align="center">
|
||||
@@ -53,15 +59,28 @@ graphical glitches, low compatibility, and poor performance.
|
||||
<img src="docs/screenshots/ps5-05.png" width="300" alt="SILENT HILL: The Short Message running in KytyPS5">
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">
|
||||
<strong>Hellboy</strong><br>
|
||||
<img src="docs/screenshots/ps5-02.png" width="300" alt="Disgaea 6 running in KytyPS5">
|
||||
</td>
|
||||
<td align="center">
|
||||
<strong>Paleo Pines</strong><br>
|
||||
<img src="docs/screenshots/ps5-06.png" width="300" alt="Dreaming Sarah running in KytyPS5">
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<p align="center"><em>And many more...</em></p>
|
||||
|
||||
## Contributing
|
||||
|
||||
Testing games and submitting detailed bug reports are useful ways to contribute. Search existing
|
||||
issues first, then use the **Game Emulation Bug Report** template and attach the complete log file.
|
||||
|
||||
Code contributions should be focused, build successfully on Windows, and include relevant tests
|
||||
where practical. Because KytyPS5 is still evolving quickly, consider opening an issue before
|
||||
Code contributions should be focused, build successfully on the platforms they touch, and include
|
||||
relevant tests where practical. Windows is the primary target, so a change that alters shared code
|
||||
should not regress it; changes confined to a platform's own code paths only need to build there. Because KytyPS5 is still evolving quickly, consider opening an issue before
|
||||
starting a large change.
|
||||
|
||||
### Formatting
|
||||
@@ -96,11 +115,12 @@ the Vulkan/SPIR-V validation rules.
|
||||
|
||||
### System requirements
|
||||
|
||||
- Windows 10 version 1803
|
||||
- A 64-bit x86 processor
|
||||
- A Vulkan 1.3-capable GPU with current drivers
|
||||
- Windows 10 version 1803, a current Linux distribution, or macOS on Apple Silicon
|
||||
- A 64-bit x86 processor (on macOS, an Apple Silicon processor with Rosetta 2)
|
||||
- A Vulkan 1.3-capable GPU with current drivers (on macOS, Vulkan is provided by the bundled
|
||||
MoltenVK)
|
||||
|
||||
### Build requirements
|
||||
### Build requirements (Windows)
|
||||
|
||||
- Git
|
||||
- CMake 3.12 or newer
|
||||
@@ -134,11 +154,98 @@ cmake --install _Build/windows --prefix _Build/windows/install
|
||||
The finished application and its runtime dependencies will be placed in
|
||||
`_Build/windows/install`.
|
||||
|
||||
### Building on Linux
|
||||
|
||||
Install the toolchain and the libraries the bundled SDL2 needs. Without the audio, Wayland and
|
||||
udev development packages SDL2 quietly configures itself without those backends, and the resulting
|
||||
build has no working sound and no gamepad hotplug:
|
||||
|
||||
```bash
|
||||
sudo apt-get install --no-install-recommends \
|
||||
clang lld ninja-build cmake git glslang-tools \
|
||||
libgl1-mesa-dev libx11-dev libxcursor-dev libxext-dev libxfixes-dev \
|
||||
libxi-dev libxrandr-dev libxss-dev libxkbcommon-dev \
|
||||
libasound2-dev libpulse-dev libudev-dev libdbus-1-dev libwayland-dev wayland-protocols
|
||||
```
|
||||
|
||||
Qt 6 (Concurrent, Network, Widgets) is also required — either the distribution packages
|
||||
(`qt6-base-dev`) or an official Qt installation.
|
||||
|
||||
```bash
|
||||
git submodule update --init --recursive
|
||||
|
||||
cmake -S src -B _Build/linux -G Ninja -DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
|
||||
-DCMAKE_PREFIX_PATH="$Qt6_DIR"
|
||||
|
||||
cmake --build _Build/linux --target launcher --parallel
|
||||
cmake --install _Build/linux --prefix _Build/linux/install
|
||||
```
|
||||
|
||||
The install step copies the Qt libraries and plugins next to the binaries, so
|
||||
`_Build/linux/install` runs without a matching system Qt.
|
||||
|
||||
As on Windows, the MSVC compiler is not used; Clang is required. `cl.exe` is rejected at configure
|
||||
time.
|
||||
|
||||
Note that the CMake source root is `src`, not the repository root.
|
||||
|
||||
### Building on macOS
|
||||
|
||||
macOS builds target x86-64 and run under Rosetta 2 on Apple Silicon, so the PS5's x86-64 game
|
||||
code executes through the same translation layer as the emulator itself. Prebuilt archives are
|
||||
attached to releases; the steps below are for building from source.
|
||||
|
||||
Requirements:
|
||||
|
||||
- An Apple Silicon Mac with Rosetta 2 installed (`softwareupdate --install-rosetta`)
|
||||
- Xcode (or the Command Line Tools)
|
||||
- Homebrew packages: `brew install cmake ninja glslang`
|
||||
- Qt 6 (Concurrent, Network, Widgets) with x86-64 support. The official Qt installation is
|
||||
universal and works; Homebrew's Qt is arm64-only and will not link
|
||||
|
||||
```bash
|
||||
git submodule update --init --recursive
|
||||
|
||||
cmake -S src -B _Build/macos -G Ninja -DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_OSX_ARCHITECTURES=x86_64 \
|
||||
-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
|
||||
-DCMAKE_PREFIX_PATH="$Qt6_DIR"
|
||||
|
||||
cmake --build _Build/macos --target launcher --parallel
|
||||
cmake --install _Build/macos --prefix _Build/macos/install
|
||||
```
|
||||
|
||||
The build re-signs `kyty_emulator` with the JIT entitlements it needs to execute translated
|
||||
guest code; no manual signing step is required.
|
||||
|
||||
Vulkan comes from MoltenVK. Download `MoltenVK-macos.tar` from the
|
||||
[MoltenVK releases](https://github.com/KhronosGroup/MoltenVK/releases), then copy
|
||||
`MoltenVK/dynamic/dylib/macOS/libMoltenVK.dylib` next to `kyty_emulator` and ad-hoc sign it:
|
||||
|
||||
```bash
|
||||
codesign --force --sign - _Build/macos/install/libMoltenVK.dylib
|
||||
```
|
||||
|
||||
Release archives already include a signed `libMoltenVK.dylib`.
|
||||
|
||||
### Regression tests
|
||||
|
||||
Build every regression executable and run the registered tests with:
|
||||
|
||||
```powershell
|
||||
cmake --build _Build/windows --target kyty_tests
|
||||
ctest --test-dir _Build/windows --output-on-failure
|
||||
```
|
||||
|
||||
Use `_Build/linux` instead of `_Build/windows` for a Linux build.
|
||||
|
||||
### Visual Studio Code
|
||||
|
||||
A ready-made Visual Studio Code setup is included in [`.vscode`](.vscode). It configures CMake
|
||||
Tools to build the project with Ninja and `clang-cl` and provides launch profiles for both
|
||||
`launcher.exe` and `kyty_emulator.exe`.
|
||||
`launcher.exe` and `kyty_emulator.exe`. It is Windows-only: VS Code settings cannot select a
|
||||
compiler per platform, so on Linux configure from the command line as shown above.
|
||||
|
||||
Before using it:
|
||||
|
||||
@@ -160,6 +267,10 @@ To use the graphical launcher:
|
||||
.\_Build\windows\install\launcher.exe
|
||||
```
|
||||
|
||||
```bash
|
||||
./_Build/linux/install/launcher
|
||||
```
|
||||
|
||||
On first launch, add one or more game folders in the global settings. The launcher searches those
|
||||
folders recursively for game directories containing `eboot.bin`. Select a detected game and run it
|
||||
from the game list.
|
||||
@@ -170,7 +281,19 @@ The emulator can also be started directly with a legally obtained game directory
|
||||
.\_Build\windows\install\kyty_emulator.exe --game "D:\Games\ExampleGame"
|
||||
```
|
||||
|
||||
Run `kyty_emulator.exe --help` to see the available graphics, logging, validation, profiling, and
|
||||
```bash
|
||||
./_Build/linux/install/kyty_emulator --game "/games/ExampleGame"
|
||||
```
|
||||
|
||||
On macOS, point SDL at the MoltenVK library explicitly; the hardened runtime prevents it from
|
||||
being picked up from the executable's directory:
|
||||
|
||||
```bash
|
||||
cd _Build/macos/install
|
||||
SDL_VULKAN_LIBRARY="$PWD/libMoltenVK.dylib" ./kyty_emulator --game "/games/ExampleGame"
|
||||
```
|
||||
|
||||
Run `kyty_emulator --help` to see the available graphics, logging, validation, profiling, and
|
||||
debugging options.
|
||||
|
||||
### AI Use
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.6 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.7 MiB After Width: | Height: | Size: 2.6 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.4 MiB |
+124
-32
@@ -22,6 +22,8 @@ set(CMAKE_CXX_SCAN_FOR_MODULES OFF)
|
||||
include(utils.cmake)
|
||||
include(CTest)
|
||||
|
||||
option(KYTY_ENABLE_CLANG_TIDY "Run clang-tidy checks during builds" OFF)
|
||||
|
||||
set(KYTY_THIRD_PARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../3rdparty")
|
||||
|
||||
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
|
||||
@@ -129,7 +131,7 @@ if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0.0)
|
||||
common
|
||||
#launcher
|
||||
)
|
||||
list(APPEND KYTY_CLANG_TYDY
|
||||
list(APPEND KYTY_CLANG_TIDY
|
||||
kyty_emulator
|
||||
#common
|
||||
launcher
|
||||
@@ -156,16 +158,24 @@ file(GLOB kyty_emulator_src CONFIGURE_DEPENDS
|
||||
graphics/host_gpu/*.h
|
||||
graphics/host_gpu/renderer/*.cpp
|
||||
graphics/host_gpu/renderer/*.h
|
||||
graphics/host_gpu/renderer/cache/*.cpp
|
||||
graphics/host_gpu/renderer/cache/*.h
|
||||
graphics/host_gpu/renderer/image/*.cpp
|
||||
graphics/host_gpu/renderer/image/*.h
|
||||
graphics/host_gpu/renderer/pipeline/*.cpp
|
||||
graphics/host_gpu/renderer/pipeline/*.h
|
||||
graphics/shader/*.cpp
|
||||
graphics/shader/*.h
|
||||
graphics/shader/recompiler/*.cpp
|
||||
graphics/shader/recompiler/*.h
|
||||
graphics/shader/recompiler/shaderIR/*.cpp
|
||||
graphics/shader/recompiler/shaderIR/*.h
|
||||
graphics/shader/recompiler/spirvEmitter/*.cpp
|
||||
graphics/shader/recompiler/spirvEmitter/*.h
|
||||
graphics/host_gpu/objects/*.cpp
|
||||
graphics/host_gpu/objects/*.h
|
||||
graphics/shader/recompiler/cfg/*.cpp
|
||||
graphics/shader/recompiler/cfg/*.h
|
||||
graphics/shader/recompiler/decompiler/*.cpp
|
||||
graphics/shader/recompiler/decompiler/*.h
|
||||
graphics/shader/recompiler/emitter/*.cpp
|
||||
graphics/shader/recompiler/emitter/*.h
|
||||
graphics/shader/recompiler/ir/*.cpp
|
||||
graphics/shader/recompiler/ir/*.h
|
||||
graphics/presentation/*.cpp
|
||||
graphics/presentation/*.h
|
||||
graphics/presentation/window/*.cpp
|
||||
@@ -249,6 +259,26 @@ endif()
|
||||
|
||||
set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL2-static xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9)
|
||||
|
||||
# Linux system libraries required by the static FFmpeg archive.
|
||||
if(LINUX)
|
||||
find_package(Threads REQUIRED)
|
||||
list(APPEND kyty_emulator_link_libraries m ${CMAKE_DL_LIBS} Threads::Threads)
|
||||
|
||||
# Optional FFmpeg dependencies.
|
||||
find_package(ZLIB)
|
||||
if(ZLIB_FOUND)
|
||||
list(APPEND kyty_emulator_link_libraries ZLIB::ZLIB)
|
||||
endif()
|
||||
find_library(KYTY_BZ2_LIBRARY bz2)
|
||||
if(KYTY_BZ2_LIBRARY)
|
||||
list(APPEND kyty_emulator_link_libraries ${KYTY_BZ2_LIBRARY})
|
||||
endif()
|
||||
find_library(KYTY_LZMA_LIBRARY lzma)
|
||||
if(KYTY_LZMA_LIBRARY)
|
||||
list(APPEND kyty_emulator_link_libraries ${KYTY_LZMA_LIBRARY})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set(inc_headers
|
||||
${CMAKE_CURRENT_SOURCE_DIR}
|
||||
${KYTY_THIRD_PARTY_DIR}/SDL2/include
|
||||
@@ -282,6 +312,19 @@ function(add_kyty_full_emulator_test target source)
|
||||
target_link_libraries(${target} onecore)
|
||||
add_custom_command(TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $<TARGET_FILE_DIR:${target}>/libwinpthread-1.dll)
|
||||
endif()
|
||||
# The macOS x86_64 guest address space needs its .zerofill segments anchored
|
||||
# by linker flags, or the kernel kills the binary on load (posix_spawn EIO).
|
||||
configure_macos_guest_address_space(${target})
|
||||
endfunction()
|
||||
|
||||
function(configure_macos_guest_address_space target)
|
||||
if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
|
||||
(NOT CMAKE_OSX_ARCHITECTURES AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$")))
|
||||
target_sources(${target} PRIVATE kernel/macosGuestAddressSpace.cpp)
|
||||
target_compile_definitions(${target} PRIVATE KYTY_LINKED_GUEST_ADDRESS_SPACE=1)
|
||||
target_link_options(${target} PRIVATE
|
||||
-Wl,-ld_classic,-no_pie,-no_fixup_chains,-no_huge,-pagezero_size,0x40000,-segaddr,SYSTEM_MANAGED,0x40000,-segaddr,SYSTEM_RESERVED,0x7ffffc000,-segaddr,USER_AREA,0x7000000000,-image_base,0x700000000000)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
add_kyty_full_emulator_test(shader_cfg_tests ../tests/shaderCfgTests.cpp)
|
||||
@@ -289,8 +332,9 @@ add_kyty_full_emulator_test(shader_cfg_tests ../tests/shaderCfgTests.cpp)
|
||||
add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL
|
||||
../tests/ScalarProvenanceTests.cpp
|
||||
graphics/host_gpu/hostMemory.cpp
|
||||
graphics/shader/recompiler/ScalarProvenance.cpp
|
||||
graphics/shader/recompiler/SrtWalker.cpp
|
||||
graphics/shader/recompiler/ir/ReadLaneElimination.cpp
|
||||
graphics/shader/recompiler/ir/ScalarProvenance.cpp
|
||||
graphics/shader/recompiler/ir/SrtWalker.cpp
|
||||
)
|
||||
target_link_libraries(scalar_provenance_tests fmt::fmt)
|
||||
target_include_directories(scalar_provenance_tests PRIVATE ${inc_headers})
|
||||
@@ -308,7 +352,6 @@ add_executable(memory_tracker_tests EXCLUDE_FROM_ALL
|
||||
)
|
||||
target_link_libraries(memory_tracker_tests fmt::fmt common)
|
||||
target_include_directories(memory_tracker_tests PRIVATE ${inc_headers})
|
||||
target_compile_definitions(memory_tracker_tests PRIVATE KYTY_MEMORY_TRACKER_TESTS=1)
|
||||
|
||||
add_executable(shader_vertex_metadata_tests EXCLUDE_FROM_ALL
|
||||
../tests/ShaderVertexMetadataTests.cpp
|
||||
@@ -322,9 +365,9 @@ add_executable(shader_stage_runtime_tests EXCLUDE_FROM_ALL
|
||||
graphics/guest_gpu/gpu_format.cpp
|
||||
graphics/host_gpu/hostMemory.cpp
|
||||
graphics/shader/shaderStageRuntime.cpp
|
||||
graphics/shader/recompiler/ResourceMaterialization.cpp
|
||||
graphics/shader/recompiler/ScalarProvenance.cpp
|
||||
graphics/shader/recompiler/SrtWalker.cpp
|
||||
graphics/shader/recompiler/ir/ResourceMaterialization.cpp
|
||||
graphics/shader/recompiler/ir/ScalarProvenance.cpp
|
||||
graphics/shader/recompiler/ir/SrtWalker.cpp
|
||||
)
|
||||
target_link_libraries(shader_stage_runtime_tests fmt::fmt)
|
||||
target_include_directories(shader_stage_runtime_tests PRIVATE ${inc_headers})
|
||||
@@ -333,24 +376,32 @@ add_executable(resource_tracking_tests EXCLUDE_FROM_ALL
|
||||
../tests/ResourceTrackingTests.cpp
|
||||
graphics/guest_gpu/gpu_format.cpp
|
||||
graphics/host_gpu/hostMemory.cpp
|
||||
graphics/shader/recompiler/ScalarProvenance.cpp
|
||||
graphics/shader/recompiler/SrtWalker.cpp
|
||||
graphics/shader/recompiler/SrtPatcher.cpp
|
||||
graphics/shader/recompiler/ResourceTracking.cpp
|
||||
graphics/shader/recompiler/ResourceMaterialization.cpp
|
||||
graphics/shader/recompiler/ShaderInfoCollection.cpp
|
||||
graphics/shader/recompiler/BindingLayout.cpp
|
||||
graphics/shader/recompiler/ir/ScalarProvenance.cpp
|
||||
graphics/shader/recompiler/ir/SrtWalker.cpp
|
||||
graphics/shader/recompiler/ir/SrtPatcher.cpp
|
||||
graphics/shader/recompiler/ir/ResourceTracking.cpp
|
||||
graphics/shader/recompiler/ir/ResourceMaterialization.cpp
|
||||
graphics/shader/recompiler/ir/ShaderInfoCollection.cpp
|
||||
graphics/shader/recompiler/ir/BindingLayout.cpp
|
||||
)
|
||||
target_link_libraries(resource_tracking_tests fmt::fmt)
|
||||
target_include_directories(resource_tracking_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(resource_mutex_tests EXCLUDE_FROM_ALL
|
||||
../tests/ResourceMutexTests.cpp
|
||||
graphics/host_gpu/renderer/resourceMutex.cpp
|
||||
graphics/host_gpu/renderer/cache/resourceMutex.cpp
|
||||
)
|
||||
target_link_libraries(resource_mutex_tests common)
|
||||
target_include_directories(resource_mutex_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(audio_out2_port_tests EXCLUDE_FROM_ALL
|
||||
../tests/AudioOut2PortTests.cpp
|
||||
libs/libAudio2.cpp
|
||||
loader/timer.cpp
|
||||
)
|
||||
target_link_libraries(audio_out2_port_tests common fmt::fmt)
|
||||
target_include_directories(audio_out2_port_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(event_queue_lifetime_tests EXCLUDE_FROM_ALL
|
||||
../tests/EventQueueLifetimeTests.cpp
|
||||
kernel/eventQueue.cpp
|
||||
@@ -392,13 +443,29 @@ add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemo
|
||||
target_compile_definitions(virtual_memory_allocation_tests PRIVATE
|
||||
KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1)
|
||||
|
||||
# These tests use exceptions.
|
||||
if(NOT KYTY_CLANG_CL)
|
||||
foreach(kyty_exception_test scalar_provenance_tests resource_tracking_tests
|
||||
virtual_memory_allocation_tests)
|
||||
target_compile_options(${kyty_exception_test} PRIVATE -fexceptions)
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
if(BUILD_TESTING)
|
||||
add_test(NAME shader_cfg COMMAND $<TARGET_FILE:shader_cfg_tests>)
|
||||
add_test(NAME scalar_provenance COMMAND $<TARGET_FILE:scalar_provenance_tests>)
|
||||
add_test(NAME image_page_table COMMAND $<TARGET_FILE:image_page_table_tests>)
|
||||
add_test(NAME memory_tracker COMMAND $<TARGET_FILE:memory_tracker_tests>)
|
||||
add_test(NAME page_manager COMMAND $<TARGET_FILE:page_manager_tests>)
|
||||
add_test(NAME shader_vertex_metadata COMMAND $<TARGET_FILE:shader_vertex_metadata_tests>)
|
||||
add_test(NAME shader_stage_runtime COMMAND $<TARGET_FILE:shader_stage_runtime_tests>)
|
||||
add_test(NAME resource_tracking COMMAND $<TARGET_FILE:resource_tracking_tests>)
|
||||
add_test(NAME resource_mutex COMMAND $<TARGET_FILE:resource_mutex_tests>)
|
||||
add_test(NAME event_queue_lifetime COMMAND $<TARGET_FILE:event_queue_lifetime_tests>)
|
||||
add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
|
||||
add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
|
||||
add_test(NAME virtual_memory_allocation
|
||||
COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
|
||||
add_test(NAME command_scheduler_timeline
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --scheduler-only)
|
||||
add_test(NAME stream_buffer_ring
|
||||
@@ -408,28 +475,46 @@ if(BUILD_TESTING)
|
||||
add_test(NAME gpu_tiler
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --gpu-tiler-only)
|
||||
|
||||
add_test(NAME texture_cache_layered_image
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --layered-image-only)
|
||||
add_test(NAME texture_cache_image_views
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --image-view-cache-only)
|
||||
add_test(NAME texture_cache_storage_sampled
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --storage-sampled-only)
|
||||
add_test(NAME texture_cache_depth_readback
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --depth-readback-only)
|
||||
add_test(NAME buffer_cache_dirty_gc
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-gc-only)
|
||||
|
||||
if(WIN32)
|
||||
# These tests still depend on the Windows multisample-depth path.
|
||||
add_test(NAME texture_cache_image_overlap
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --image-overlap-only)
|
||||
add_test(NAME texture_cache_htile_clear
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --htile-clear-only)
|
||||
add_test(NAME texture_cache_layered_image
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --layered-image-only)
|
||||
add_test(NAME texture_cache_image_views
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --image-view-cache-only)
|
||||
add_test(NAME texture_cache_storage_sampled
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --storage-sampled-only)
|
||||
add_test(NAME texture_cache_depth_readback
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --depth-readback-only)
|
||||
add_test(NAME buffer_cache_ranges
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-range-only)
|
||||
add_test(NAME buffer_cache_dirty_gc
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-gc-only)
|
||||
endif()
|
||||
|
||||
add_custom_target(kyty_tests DEPENDS
|
||||
shader_cfg_tests
|
||||
scalar_provenance_tests
|
||||
image_page_table_tests
|
||||
memory_tracker_tests
|
||||
page_manager_tests
|
||||
shader_vertex_metadata_tests
|
||||
shader_stage_runtime_tests
|
||||
resource_tracking_tests
|
||||
resource_mutex_tests
|
||||
event_queue_lifetime_tests
|
||||
shader_recompiler_compute_tests
|
||||
virtual_memory_allocation_tests
|
||||
)
|
||||
endif()
|
||||
|
||||
|
||||
add_executable(kyty_emulator main.cpp ${kyty_emulator_src})
|
||||
configure_macos_guest_address_space(kyty_emulator)
|
||||
|
||||
target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries})
|
||||
if (WIN32)
|
||||
@@ -444,6 +529,8 @@ endif()
|
||||
if (CLANG AND NOT KYTY_CLANG_CL)
|
||||
target_link_libraries(kyty_emulator pthread)
|
||||
endif()
|
||||
# dlopen/dlsym/dladdr for RenderDoc.
|
||||
target_link_libraries(kyty_emulator ${CMAKE_DL_LIBS})
|
||||
target_include_directories(kyty_emulator PRIVATE ${inc_headers})
|
||||
|
||||
clang_tidy_check(kyty_emulator "" "${check_headers}" "${inc_headers}")
|
||||
@@ -456,7 +543,12 @@ set(KYTY_EMULATOR_MAP_LINK_PATH "${CMAKE_CURRENT_BINARY_DIR}/${KYTY_EMULATOR_MAP
|
||||
set(KYTY_EMULATOR_PDB_LINK_PATH "${CMAKE_CURRENT_BINARY_DIR}/kyty_emulator.pdb")
|
||||
|
||||
if(KYTY_CLANG_CL)
|
||||
set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "/DYNAMICBASE:NO /DEBUG:FULL /PDB:${KYTY_EMULATOR_PDB_LINK_PATH} /lldmap:${KYTY_EMULATOR_MAP_LINK_PATH}")
|
||||
target_link_options(kyty_emulator PRIVATE
|
||||
"/DYNAMICBASE:NO"
|
||||
"/DEBUG:FULL"
|
||||
"/PDB:${KYTY_EMULATOR_PDB_LINK_PATH}"
|
||||
"/lldmap:${KYTY_EMULATOR_MAP_LINK_PATH}"
|
||||
)
|
||||
add_custom_command(TARGET kyty_emulator POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $<TARGET_FILE_DIR:kyty_emulator>/libwinpthread-1.dll)
|
||||
elseif(WIN32 OR LINUX)
|
||||
set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "${KYTY_LD_OPTIONS} -Wl,-Map=${KYTY_EMULATOR_MAP_LINK_PATH}")
|
||||
|
||||
@@ -6,6 +6,14 @@
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
#include <windows.h> // IWYU pragma: keep
|
||||
#elif defined(__APPLE__)
|
||||
#include <csignal>
|
||||
#include <sys/ucontext.h>
|
||||
#else
|
||||
#include <csignal>
|
||||
#include <initializer_list>
|
||||
#include <ucontext.h> // IWYU pragma: keep
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
// IWYU pragma: no_include <errhandlingapi.h>
|
||||
@@ -16,7 +24,7 @@
|
||||
|
||||
namespace Common::HostException {
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
#if !defined(__APPLE__)
|
||||
|
||||
static std::atomic<Handler> g_handler {nullptr};
|
||||
static std::atomic_uint32_t g_install_state {0};
|
||||
@@ -30,7 +38,9 @@ static_assert(decltype(g_install_state)::is_always_lock_free);
|
||||
std::fputs(reason != nullptr ? reason : "unspecified", stderr);
|
||||
std::fputc('\n', stderr);
|
||||
std::fflush(stderr);
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
TerminateProcess(GetCurrentProcess(), static_cast<UINT>(EXCEPTION_NONCONTINUABLE_EXCEPTION));
|
||||
#endif
|
||||
std::_Exit(321);
|
||||
}
|
||||
|
||||
@@ -48,6 +58,21 @@ public:
|
||||
KYTY_CLASS_NO_COPY(FilterScope);
|
||||
};
|
||||
|
||||
static Handler LoadInstalledHandler() noexcept {
|
||||
if (g_install_state.load(std::memory_order_acquire) == 0) {
|
||||
FailFast("host exception handler is not installed");
|
||||
}
|
||||
|
||||
const auto handler = g_handler.load(std::memory_order_acquire);
|
||||
if (handler == nullptr) {
|
||||
FailFast("host exception callback is null");
|
||||
}
|
||||
return handler;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
|
||||
static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
|
||||
FilterScope filter_scope;
|
||||
|
||||
@@ -106,22 +131,176 @@ static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
|
||||
info.r14 = exception->ContextRecord->R14;
|
||||
info.r15 = exception->ContextRecord->R15;
|
||||
|
||||
if (g_install_state.load(std::memory_order_acquire) == 0) {
|
||||
FailFast("host exception handler is not installed");
|
||||
const auto handler = LoadInstalledHandler();
|
||||
|
||||
return handler(info) ? EXCEPTION_CONTINUE_EXECUTION : EXCEPTION_CONTINUE_SEARCH;
|
||||
}
|
||||
|
||||
#elif defined(__APPLE__)
|
||||
|
||||
static std::atomic<Handler> g_handler {nullptr};
|
||||
static std::atomic_uint32_t g_install_state {0};
|
||||
static thread_local bool g_in_exception_filter = false;
|
||||
|
||||
static_assert(decltype(g_handler)::is_always_lock_free);
|
||||
static_assert(decltype(g_install_state)::is_always_lock_free);
|
||||
|
||||
[[noreturn]] static void FailFast(const char* reason) noexcept {
|
||||
std::fputs("HostException fail-fast: ", stderr);
|
||||
std::fputs(reason != nullptr ? reason : "unspecified", stderr);
|
||||
std::fputc('\n', stderr);
|
||||
std::fflush(stderr);
|
||||
std::_Exit(321);
|
||||
}
|
||||
|
||||
// Translate the x86-64 page-fault error code (mcontext __es.__err) into an access type.
|
||||
// bit 1 (0x2) = write, bit 4 (0x10) = instruction fetch, otherwise a read.
|
||||
static AccessViolationType DecodeAccess(uint64_t err) {
|
||||
if ((err & 0x10u) != 0) {
|
||||
return AccessViolationType::Execute;
|
||||
}
|
||||
if ((err & 0x2u) != 0) {
|
||||
return AccessViolationType::Write;
|
||||
}
|
||||
return AccessViolationType::Read;
|
||||
}
|
||||
|
||||
// POSIX signal handler that mirrors the Windows vectored handler: build an ExceptionInfo
|
||||
// from the mcontext and dispatch. A resolved fault (handler returns true) simply returns,
|
||||
// re-executing the faulting instruction against the now-fixed protection. An unresolved
|
||||
// fault restores the default disposition so the retry terminates the process.
|
||||
static void SignalHandler(int sig, siginfo_t* si, void* uctx) {
|
||||
if (g_in_exception_filter) {
|
||||
FailFast("nested exception while resolving a host fault");
|
||||
}
|
||||
g_in_exception_filter = true;
|
||||
|
||||
auto* uc = static_cast<ucontext_t*>(uctx);
|
||||
const auto* mc = uc->uc_mcontext;
|
||||
const auto& ss = mc->__ss;
|
||||
|
||||
ExceptionInfo info {};
|
||||
info.exception_address = ss.__rip;
|
||||
info.native_code = static_cast<uint32_t>(si->si_code);
|
||||
info.native_context = uctx;
|
||||
|
||||
if (sig == SIGILL) {
|
||||
info.type = ExceptionType::IllegalInstruction;
|
||||
} else {
|
||||
info.type = ExceptionType::AccessViolation;
|
||||
info.access_violation_type = DecodeAccess(mc->__es.__err);
|
||||
info.access_violation_vaddr = reinterpret_cast<uint64_t>(si->si_addr);
|
||||
}
|
||||
|
||||
info.rax = ss.__rax;
|
||||
info.rbx = ss.__rbx;
|
||||
info.rcx = ss.__rcx;
|
||||
info.rdx = ss.__rdx;
|
||||
info.rsi = ss.__rsi;
|
||||
info.rdi = ss.__rdi;
|
||||
info.rbp = ss.__rbp;
|
||||
info.rsp = ss.__rsp;
|
||||
info.r8 = ss.__r8;
|
||||
info.r9 = ss.__r9;
|
||||
info.r10 = ss.__r10;
|
||||
info.r11 = ss.__r11;
|
||||
info.r12 = ss.__r12;
|
||||
info.r13 = ss.__r13;
|
||||
info.r14 = ss.__r14;
|
||||
info.r15 = ss.__r15;
|
||||
|
||||
const auto handler = g_handler.load(std::memory_order_acquire);
|
||||
if (handler == nullptr) {
|
||||
FailFast("host exception callback is null");
|
||||
}
|
||||
|
||||
return handler(info) ? EXCEPTION_CONTINUE_EXECUTION : EXCEPTION_CONTINUE_SEARCH;
|
||||
const bool resolved = handler(info);
|
||||
g_in_exception_filter = false;
|
||||
|
||||
if (resolved) {
|
||||
return; // retry the faulting instruction against the fixed mapping
|
||||
}
|
||||
|
||||
// Unresolved: restore the default action so the re-executed instruction terminates.
|
||||
struct sigaction dfl {};
|
||||
dfl.sa_handler = SIG_DFL;
|
||||
sigemptyset(&dfl.sa_mask);
|
||||
sigaction(sig, &dfl, nullptr);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
// x86-64 page-fault error bits.
|
||||
constexpr uint64_t PAGE_FAULT_ERROR_WRITE = 0x02;
|
||||
constexpr uint64_t PAGE_FAULT_ERROR_INSTRUCTION = 0x10;
|
||||
|
||||
// Let the kernel handle an unresolved fault on retry.
|
||||
static void ChainToDefault(int signal_number) noexcept {
|
||||
struct sigaction restore {};
|
||||
restore.sa_handler = SIG_DFL;
|
||||
sigemptyset(&restore.sa_mask);
|
||||
restore.sa_flags = 0;
|
||||
::sigaction(signal_number, &restore, nullptr);
|
||||
}
|
||||
|
||||
static void SignalHandler(int signal_number, siginfo_t* signal_info, void* native_context) {
|
||||
FilterScope filter_scope;
|
||||
|
||||
auto* context = static_cast<ucontext_t*>(native_context);
|
||||
auto* gregs = context->uc_mcontext.gregs;
|
||||
|
||||
ExceptionInfo info {};
|
||||
info.exception_address = static_cast<uint64_t>(gregs[REG_RIP]);
|
||||
info.native_code = static_cast<uint32_t>(signal_number);
|
||||
info.native_context = context;
|
||||
|
||||
if (signal_number == SIGSEGV || signal_number == SIGBUS) {
|
||||
info.type = ExceptionType::AccessViolation;
|
||||
const auto error_code = static_cast<uint64_t>(gregs[REG_ERR]);
|
||||
if ((error_code & PAGE_FAULT_ERROR_INSTRUCTION) != 0) {
|
||||
info.access_violation_type = AccessViolationType::Execute;
|
||||
} else if ((error_code & PAGE_FAULT_ERROR_WRITE) != 0) {
|
||||
info.access_violation_type = AccessViolationType::Write;
|
||||
} else {
|
||||
info.access_violation_type = AccessViolationType::Read;
|
||||
}
|
||||
info.access_violation_vaddr = reinterpret_cast<uint64_t>(signal_info->si_addr);
|
||||
} else if (signal_number == SIGILL) {
|
||||
info.type = ExceptionType::IllegalInstruction;
|
||||
} else {
|
||||
ChainToDefault(signal_number);
|
||||
return;
|
||||
}
|
||||
|
||||
info.rax = static_cast<uint64_t>(gregs[REG_RAX]);
|
||||
info.rbx = static_cast<uint64_t>(gregs[REG_RBX]);
|
||||
info.rcx = static_cast<uint64_t>(gregs[REG_RCX]);
|
||||
info.rdx = static_cast<uint64_t>(gregs[REG_RDX]);
|
||||
info.rsi = static_cast<uint64_t>(gregs[REG_RSI]);
|
||||
info.rdi = static_cast<uint64_t>(gregs[REG_RDI]);
|
||||
info.rbp = static_cast<uint64_t>(gregs[REG_RBP]);
|
||||
info.rsp = static_cast<uint64_t>(gregs[REG_RSP]);
|
||||
info.r8 = static_cast<uint64_t>(gregs[REG_R8]);
|
||||
info.r9 = static_cast<uint64_t>(gregs[REG_R9]);
|
||||
info.r10 = static_cast<uint64_t>(gregs[REG_R10]);
|
||||
info.r11 = static_cast<uint64_t>(gregs[REG_R11]);
|
||||
info.r12 = static_cast<uint64_t>(gregs[REG_R12]);
|
||||
info.r13 = static_cast<uint64_t>(gregs[REG_R13]);
|
||||
info.r14 = static_cast<uint64_t>(gregs[REG_R14]);
|
||||
info.r15 = static_cast<uint64_t>(gregs[REG_R15]);
|
||||
|
||||
const auto handler = LoadInstalledHandler();
|
||||
|
||||
if (handler(info)) {
|
||||
return;
|
||||
}
|
||||
|
||||
ChainToDefault(signal_number);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
bool InstallHandler(Handler handler) {
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
if (handler == nullptr) {
|
||||
return false;
|
||||
}
|
||||
@@ -133,19 +312,52 @@ bool InstallHandler(Handler handler) {
|
||||
|
||||
g_handler.store(handler, std::memory_order_release);
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
if (AddVectoredExceptionHandler(1, ExceptionFilter) == nullptr) {
|
||||
g_handler.store(nullptr, std::memory_order_release);
|
||||
g_install_state.store(0, std::memory_order_release);
|
||||
printf("AddVectoredExceptionHandler() failed\n");
|
||||
return false;
|
||||
}
|
||||
#elif defined(__APPLE__)
|
||||
struct sigaction sa {};
|
||||
sa.sa_sigaction = SignalHandler;
|
||||
sa.sa_flags = SA_SIGINFO;
|
||||
sigemptyset(&sa.sa_mask);
|
||||
// The guest signal-dispatch path (KernelRaiseException) interrupts threads with
|
||||
// SIGUSR1; block it while a fault is being resolved so a stop-the-world request
|
||||
// cannot preempt the handler between the protection fix and the retry.
|
||||
sigaddset(&sa.sa_mask, SIGUSR1);
|
||||
|
||||
// macOS raises SIGBUS for protection faults on some paths and SIGSEGV on others;
|
||||
// SIGILL covers instructions the host cannot execute (routed to the x64 emulator).
|
||||
bool ok = sigaction(SIGSEGV, &sa, nullptr) == 0 && sigaction(SIGBUS, &sa, nullptr) == 0 &&
|
||||
sigaction(SIGILL, &sa, nullptr) == 0;
|
||||
if (!ok) {
|
||||
g_handler.store(nullptr, std::memory_order_release);
|
||||
g_install_state.store(0, std::memory_order_release);
|
||||
printf("sigaction() failed to install the host fault handler\n");
|
||||
return false;
|
||||
}
|
||||
#else
|
||||
struct sigaction action {};
|
||||
action.sa_sigaction = SignalHandler;
|
||||
sigemptyset(&action.sa_mask);
|
||||
// Fault resolution needs the normal thread stack.
|
||||
action.sa_flags = SA_SIGINFO | SA_RESTART;
|
||||
|
||||
for (const int signal_number: {SIGSEGV, SIGBUS, SIGILL}) {
|
||||
if (::sigaction(signal_number, &action, nullptr) != 0) {
|
||||
g_handler.store(nullptr, std::memory_order_release);
|
||||
g_install_state.store(0, std::memory_order_release);
|
||||
printf("sigaction(%d) failed\n", signal_number);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
g_install_state.store(2, std::memory_order_release);
|
||||
return true;
|
||||
#else
|
||||
(void)handler;
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace Common::HostException
|
||||
|
||||
@@ -19,10 +19,10 @@ class LeastRecentlyUsedCache {
|
||||
|
||||
public:
|
||||
[[nodiscard]] size_t Insert(Object object, Tick tick) {
|
||||
const auto id = Build();
|
||||
const auto id = Build();
|
||||
auto& item = m_items[id];
|
||||
item.object = std::move(object);
|
||||
item.tick = tick;
|
||||
item.object = std::move(object);
|
||||
item.tick = tick;
|
||||
Attach(item);
|
||||
return id;
|
||||
}
|
||||
@@ -49,8 +49,7 @@ public:
|
||||
|
||||
template <typename Function>
|
||||
void ForEachItemBelow(Tick tick, Function&& function) {
|
||||
constexpr bool ReturnsBool =
|
||||
std::is_same_v<std::invoke_result_t<Function, Object>, bool>;
|
||||
constexpr bool ReturnsBool = std::is_same_v<std::invoke_result_t<Function, Object>, bool>;
|
||||
for (auto* item = m_first; item != nullptr;) {
|
||||
if (item->tick > tick) {
|
||||
return;
|
||||
@@ -87,10 +86,10 @@ private:
|
||||
m_last = &item;
|
||||
return;
|
||||
}
|
||||
item.prev = m_last;
|
||||
item.prev = m_last;
|
||||
m_last->next = &item;
|
||||
item.next = nullptr;
|
||||
m_last = &item;
|
||||
item.next = nullptr;
|
||||
m_last = &item;
|
||||
}
|
||||
|
||||
void Detach(Item& item) {
|
||||
|
||||
@@ -23,6 +23,10 @@ struct sys_dbg_stack_info_t {
|
||||
size_t commited_size;
|
||||
size_t total_size;
|
||||
size_t code_size;
|
||||
|
||||
// Full stack reservation reported by pthread.
|
||||
uintptr_t reserved_addr;
|
||||
size_t reserved_size;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
@@ -8,12 +8,56 @@
|
||||
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <execinfo.h>
|
||||
#include <pthread.h>
|
||||
#include <sys/param.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#if defined(__APPLE__)
|
||||
#include <libgen.h> // POSIX basename() lives here on macOS, not in <cstring>
|
||||
#endif
|
||||
|
||||
void SysStackWalk(void** /*stack*/, int* depth) {
|
||||
*depth = 0;
|
||||
// Avoid unwinding a guest-owned stack.
|
||||
static bool OnOwnStack() {
|
||||
const char* probe = reinterpret_cast<const char*>(&probe);
|
||||
|
||||
pthread_attr_t attr {};
|
||||
#if defined(__APPLE__)
|
||||
const auto* top = static_cast<const char*>(pthread_get_stackaddr_np(pthread_self()));
|
||||
const auto size = pthread_get_stacksize_np(pthread_self());
|
||||
(void)attr;
|
||||
return top != nullptr && size != 0 && probe < top && probe >= top - size;
|
||||
#else
|
||||
if (pthread_getattr_np(pthread_self(), &attr) != 0) {
|
||||
return false;
|
||||
}
|
||||
void* base = nullptr;
|
||||
size_t size = 0;
|
||||
const bool ok = pthread_attr_getstack(&attr, &base, &size) == 0 && base != nullptr && size != 0;
|
||||
pthread_attr_destroy(&attr);
|
||||
if (!ok) {
|
||||
return false;
|
||||
}
|
||||
const auto* low = static_cast<const char*>(base);
|
||||
return probe >= low && probe < low + size;
|
||||
#endif
|
||||
}
|
||||
|
||||
void SysStackWalk(void** stack, int* depth) {
|
||||
if (stack == nullptr || depth == nullptr || *depth <= 0) {
|
||||
if (depth != nullptr) {
|
||||
*depth = 0;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (!OnOwnStack()) {
|
||||
*depth = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
const int n = ::backtrace(stack, *depth);
|
||||
*depth = (n < 0 ? 0 : n);
|
||||
}
|
||||
|
||||
void SysStackUsagePrint(sys_dbg_stack_info_t& stack) {
|
||||
@@ -30,6 +74,33 @@ void SysStackUsage(sys_dbg_stack_info_t& s) {
|
||||
|
||||
[[maybe_unused]] int result = 0;
|
||||
|
||||
memset(&s, 0, sizeof(sys_dbg_stack_info_t));
|
||||
|
||||
// Record the reservation before the Linux /proc walk.
|
||||
{
|
||||
pthread_attr_t self_attr {};
|
||||
#if defined(__APPLE__)
|
||||
void* stack_top = pthread_get_stackaddr_np(pthread_self());
|
||||
const size_t stack_size = pthread_get_stacksize_np(pthread_self());
|
||||
if (stack_top != nullptr && stack_size != 0) {
|
||||
s.reserved_addr = reinterpret_cast<uintptr_t>(stack_top) - stack_size;
|
||||
s.reserved_size = stack_size;
|
||||
}
|
||||
(void)self_attr;
|
||||
#else
|
||||
if (pthread_getattr_np(pthread_self(), &self_attr) == 0) {
|
||||
void* stack_base = nullptr;
|
||||
size_t stack_size = 0;
|
||||
if (pthread_attr_getstack(&self_attr, &stack_base, &stack_size) == 0 &&
|
||||
stack_base != nullptr && stack_size != 0) {
|
||||
s.reserved_addr = reinterpret_cast<uintptr_t>(stack_base);
|
||||
s.reserved_size = stack_size;
|
||||
}
|
||||
pthread_attr_destroy(&self_attr);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
char str[1024];
|
||||
char str2[1024];
|
||||
result = sprintf(str, "/proc/%d/exe", static_cast<int>(pid));
|
||||
@@ -43,8 +114,6 @@ void SysStackUsage(sys_dbg_stack_info_t& s) {
|
||||
|
||||
result = sprintf(str, "/proc/%d/maps", static_cast<int>(pid));
|
||||
|
||||
memset(&s, 0, sizeof(sys_dbg_stack_info_t));
|
||||
|
||||
FILE* f = fopen(str, "r");
|
||||
|
||||
if (f == nullptr) {
|
||||
@@ -118,6 +187,11 @@ void SysStackUsage(sys_dbg_stack_info_t& s) {
|
||||
}
|
||||
|
||||
result = fclose(f);
|
||||
|
||||
if (s.reserved_addr == 0) {
|
||||
s.reserved_addr = s.addr;
|
||||
s.reserved_size = s.total_size;
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -11,7 +11,11 @@
|
||||
|
||||
#include <cerrno>
|
||||
#include <cstdlib>
|
||||
#include <dirent.h>
|
||||
#include <fcntl.h>
|
||||
#include <filesystem>
|
||||
#include <sys/stat.h>
|
||||
#include <system_error>
|
||||
#include <unistd.h>
|
||||
#include <utime.h>
|
||||
|
||||
@@ -39,10 +43,43 @@ struct sys_file_t {
|
||||
};
|
||||
};
|
||||
|
||||
// Darwin uses BSD timestamp member names.
|
||||
#if defined(__APPLE__)
|
||||
#define KYTY_STAT_ATIME_NS(st) ((st).st_atimespec.tv_nsec)
|
||||
#define KYTY_STAT_MTIME_NS(st) ((st).st_mtimespec.tv_nsec)
|
||||
#else
|
||||
#define KYTY_STAT_ATIME_NS(st) ((st).st_atim.tv_nsec)
|
||||
#define KYTY_STAT_MTIME_NS(st) ((st).st_mtim.tv_nsec)
|
||||
#endif
|
||||
|
||||
static std::filesystem::path get_internal_name(const std::filesystem::path& name) {
|
||||
return name.is_absolute() ? name : (std::filesystem::path(".") / name);
|
||||
}
|
||||
|
||||
// Pass access-pattern hints to the host.
|
||||
static void apply_cache_hint(FILE* f, sys_file_cache_type_t cache_type) {
|
||||
if (f == nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
#if !defined(__APPLE__)
|
||||
int advice = POSIX_FADV_NORMAL;
|
||||
switch (cache_type) {
|
||||
case SYS_FILE_CACHE_RANDOM_ACCESS: advice = POSIX_FADV_RANDOM; break;
|
||||
case SYS_FILE_CACHE_SEQUENTIAL_SCAN: advice = POSIX_FADV_SEQUENTIAL; break;
|
||||
case SYS_FILE_CACHE_AUTO:
|
||||
default: return;
|
||||
}
|
||||
::posix_fadvise(fileno(f), 0, 0, advice);
|
||||
#else
|
||||
if (cache_type == SYS_FILE_CACHE_SEQUENTIAL_SCAN) {
|
||||
::fcntl(fileno(f), F_RDAHEAD, 1);
|
||||
} else if (cache_type == SYS_FILE_CACHE_RANDOM_ACCESS) {
|
||||
::fcntl(fileno(f), F_RDAHEAD, 0);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void SysFileRead(void* data, uint32_t size, sys_file_t& f, uint32_t* bytes_read) {
|
||||
if (f.type == SYS_FILE_FILE) {
|
||||
size_t w = fread(data, 1, size, f.f);
|
||||
@@ -135,8 +172,7 @@ sys_file_t* SysFileCreate(const std::filesystem::path& file_name) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
sys_file_t* SysFileOpenR(const std::filesystem::path& file_name,
|
||||
sys_file_cache_type_t /*cache_type*/) {
|
||||
sys_file_t* SysFileOpenR(const std::filesystem::path& file_name, sys_file_cache_type_t cache_type) {
|
||||
auto* ret = new sys_file_t;
|
||||
|
||||
ret->type = SYS_FILE_FILE;
|
||||
@@ -150,6 +186,8 @@ sys_file_t* SysFileOpenR(const std::filesystem::path& file_name,
|
||||
ret->type = SYS_FILE_ERROR;
|
||||
}
|
||||
|
||||
apply_cache_hint(f, cache_type);
|
||||
|
||||
ret->f = f;
|
||||
|
||||
return ret;
|
||||
@@ -179,8 +217,7 @@ sys_file_t* SysFileCreate() {
|
||||
return ret;
|
||||
}
|
||||
|
||||
sys_file_t* SysFileOpenW(const std::filesystem::path& file_name,
|
||||
sys_file_cache_type_t /*cache_type*/) {
|
||||
sys_file_t* SysFileOpenW(const std::filesystem::path& file_name, sys_file_cache_type_t cache_type) {
|
||||
auto* ret = new sys_file_t;
|
||||
|
||||
auto real_name = get_internal_name(file_name);
|
||||
@@ -194,13 +231,15 @@ sys_file_t* SysFileOpenW(const std::filesystem::path& file_name,
|
||||
ret->type = SYS_FILE_FILE;
|
||||
}
|
||||
|
||||
apply_cache_hint(f, cache_type);
|
||||
|
||||
ret->f = f;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
sys_file_t* SysFileOpenRw(const std::filesystem::path& file_name,
|
||||
sys_file_cache_type_t /*cache_type*/) {
|
||||
sys_file_cache_type_t cache_type) {
|
||||
auto* ret = new sys_file_t;
|
||||
|
||||
auto real_name = get_internal_name(file_name);
|
||||
@@ -214,6 +253,8 @@ sys_file_t* SysFileOpenRw(const std::filesystem::path& file_name,
|
||||
ret->type = SYS_FILE_FILE;
|
||||
}
|
||||
|
||||
apply_cache_hint(f, cache_type);
|
||||
|
||||
ret->f = f;
|
||||
|
||||
return ret;
|
||||
@@ -239,11 +280,17 @@ uint64_t SysFileSize(sys_file_t& f) {
|
||||
[[maybe_unused]] int result = 0;
|
||||
|
||||
if (f.type == SYS_FILE_FILE) {
|
||||
uint32_t pos = ftell(f.f);
|
||||
result = fseek(f.f, 0, SEEK_END);
|
||||
uint32_t size = ftell(f.f);
|
||||
result = fseek(f.f, pos, SEEK_SET);
|
||||
return size;
|
||||
// Preserve sizes above 4 GiB.
|
||||
const off_t pos = ftello(f.f);
|
||||
if (pos < 0) {
|
||||
return 0;
|
||||
}
|
||||
if (fseeko(f.f, 0, SEEK_END) != 0) {
|
||||
return 0;
|
||||
}
|
||||
const off_t size = ftello(f.f);
|
||||
result = fseeko(f.f, pos, SEEK_SET);
|
||||
return (size < 0 ? 0 : static_cast<uint64_t>(size));
|
||||
}
|
||||
|
||||
if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) {
|
||||
@@ -260,8 +307,18 @@ uint64_t SysFileSize(const std::filesystem::path& file_name) {
|
||||
return size;
|
||||
}
|
||||
|
||||
bool SysFileTruncate(sys_file_t& /*f*/, uint64_t /*size*/) {
|
||||
return false;
|
||||
bool SysFileTruncate(sys_file_t& f, uint64_t size) {
|
||||
bool ok = false;
|
||||
if (f.type == SYS_FILE_FILE) {
|
||||
// Flush before resizing and restore the caller's position.
|
||||
const auto position = ftell(f.f);
|
||||
fflush(f.f);
|
||||
ok = (ftruncate(fileno(f.f), static_cast<off_t>(size)) == 0);
|
||||
if (position >= 0) {
|
||||
fseek(f.f, position, SEEK_SET);
|
||||
}
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool SysFileUnlink(sys_file_t& /*f*/, const std::filesystem::path& name) {
|
||||
@@ -382,6 +439,7 @@ SysFileTimeStruct SysFileGetLastAccessTimeUtc(const std::filesystem::path& name)
|
||||
} else {
|
||||
r.is_invalid = false;
|
||||
r.time = s.st_atime;
|
||||
r.nanos = KYTY_STAT_ATIME_NS(s);
|
||||
}
|
||||
|
||||
return r;
|
||||
@@ -400,6 +458,7 @@ SysFileTimeStruct SysFileGetLastWriteTimeUtc(const std::filesystem::path& name)
|
||||
} else {
|
||||
r.is_invalid = false;
|
||||
r.time = s.st_mtime;
|
||||
r.nanos = KYTY_STAT_MTIME_NS(s);
|
||||
}
|
||||
|
||||
return r;
|
||||
@@ -419,13 +478,36 @@ void SysFileGetLastAccessAndWriteTimeUtc(const std::filesystem::path& name, SysF
|
||||
a.is_invalid = false;
|
||||
w.is_invalid = false;
|
||||
a.time = s.st_atime;
|
||||
a.nanos = KYTY_STAT_ATIME_NS(s);
|
||||
w.time = s.st_mtime;
|
||||
w.nanos = KYTY_STAT_MTIME_NS(s);
|
||||
}
|
||||
}
|
||||
|
||||
void SysFileGetLastAccessAndWriteTimeUtc(sys_file_t& /*f*/, SysFileTimeStruct& /*a*/,
|
||||
SysFileTimeStruct& /*w*/) {
|
||||
EXIT("not implemented\n");
|
||||
void SysFileGetLastAccessAndWriteTimeUtc(sys_file_t& f, SysFileTimeStruct& a,
|
||||
SysFileTimeStruct& w) {
|
||||
if (f.type == SYS_FILE_FILE) {
|
||||
struct stat s {};
|
||||
|
||||
const bool ok = (0 == fstat(fileno(f.f), &s));
|
||||
|
||||
a.is_invalid = w.is_invalid = !ok;
|
||||
|
||||
if (ok) {
|
||||
a.time = s.st_atime;
|
||||
a.nanos = KYTY_STAT_ATIME_NS(s);
|
||||
w.time = s.st_mtime;
|
||||
w.nanos = KYTY_STAT_MTIME_NS(s);
|
||||
}
|
||||
} else if (f.type == SYS_FILE_MEMORY_STAT || f.type == SYS_FILE_MEMORY_DYN) {
|
||||
// Memory-backed files use the current time.
|
||||
SysTimeStruct t {};
|
||||
SysGetSystemTimeUtc(t);
|
||||
SysSystemToFileTimeUtc(t, a);
|
||||
SysSystemToFileTimeUtc(t, w);
|
||||
} else {
|
||||
a.is_invalid = w.is_invalid = true;
|
||||
}
|
||||
}
|
||||
|
||||
bool SysFileSetLastAccessTimeUtc(const std::filesystem::path& name, SysFileTimeStruct& access) {
|
||||
@@ -531,27 +613,110 @@ bool SysFileSetLastAccessAndWriteTimeUtc(const std::filesystem::path& name,
|
||||
// }
|
||||
}
|
||||
|
||||
void SysFileFindFiles(const std::filesystem::path& /*path*/,
|
||||
std::vector<sys_file_find_t>& /*out*/) {
|
||||
EXIT("not implemented\n");
|
||||
// Recursively collect regular files.
|
||||
void SysFileFindFiles(const std::filesystem::path& path, std::vector<sys_file_find_t>& out) {
|
||||
auto real_path = get_internal_name(path);
|
||||
|
||||
DIR* dir = opendir(real_path.string().c_str());
|
||||
if (dir == nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const dirent* entry = readdir(dir); entry != nullptr; entry = readdir(dir)) {
|
||||
const std::string file_name(entry->d_name);
|
||||
|
||||
if (file_name == "." || file_name == "..") {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto child = real_path / file_name;
|
||||
struct stat s {};
|
||||
|
||||
// lstat, so a symlink is never followed into a cycle during the recursive walk.
|
||||
if (0 != lstat(child.string().c_str(), &s)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (S_ISDIR(s.st_mode)) {
|
||||
SysFileFindFiles(child, out);
|
||||
} else if (S_ISREG(s.st_mode)) {
|
||||
sys_file_find_t r {};
|
||||
|
||||
r.path_with_name = child;
|
||||
r.size = static_cast<uint64_t>(s.st_size);
|
||||
r.last_access_time.is_invalid = false;
|
||||
r.last_access_time.time = s.st_atime;
|
||||
r.last_access_time.nanos = KYTY_STAT_ATIME_NS(s);
|
||||
r.last_write_time.is_invalid = false;
|
||||
r.last_write_time.time = s.st_mtime;
|
||||
r.last_write_time.nanos = KYTY_STAT_MTIME_NS(s);
|
||||
|
||||
out.push_back(r);
|
||||
}
|
||||
}
|
||||
|
||||
closedir(dir);
|
||||
}
|
||||
|
||||
void SysFileGetDents(const std::filesystem::path& /*path*/, std::vector<sys_dir_entry_t>& /*out*/) {
|
||||
EXIT("not implemented\n");
|
||||
// Keep "." and ".." to match FindFirstFileW.
|
||||
void SysFileGetDents(const std::filesystem::path& path, std::vector<sys_dir_entry_t>& out) {
|
||||
auto real_path = get_internal_name(path);
|
||||
|
||||
DIR* dir = opendir(real_path.string().c_str());
|
||||
if (dir == nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const dirent* entry = readdir(dir); entry != nullptr; entry = readdir(dir)) {
|
||||
sys_dir_entry_t r {};
|
||||
|
||||
r.name = entry->d_name;
|
||||
|
||||
if (entry->d_type == DT_UNKNOWN) {
|
||||
// Some filesystems do not populate d_type.
|
||||
struct stat s {};
|
||||
r.is_file = 0 == lstat((real_path / r.name).string().c_str(), &s) && S_ISREG(s.st_mode);
|
||||
} else {
|
||||
r.is_file = entry->d_type != DT_DIR;
|
||||
}
|
||||
|
||||
out.push_back(r);
|
||||
}
|
||||
|
||||
closedir(dir);
|
||||
}
|
||||
|
||||
bool SysFileCopyFile(const std::filesystem::path& /*src*/, const std::filesystem::path& /*dst*/) {
|
||||
EXIT("not implemented\n");
|
||||
return false;
|
||||
bool SysFileCopyFile(const std::filesystem::path& src, const std::filesystem::path& dst) {
|
||||
std::error_code error;
|
||||
return std::filesystem::copy_file(get_internal_name(src), get_internal_name(dst),
|
||||
std::filesystem::copy_options::overwrite_existing, error) &&
|
||||
!error;
|
||||
}
|
||||
|
||||
bool SysFileMoveFile(const std::filesystem::path& /*src*/, const std::filesystem::path& /*dst*/) {
|
||||
EXIT("not implemented\n");
|
||||
return false;
|
||||
bool SysFileMoveFile(const std::filesystem::path& src, const std::filesystem::path& dst) {
|
||||
auto real_src = get_internal_name(src);
|
||||
auto real_dst = get_internal_name(dst);
|
||||
|
||||
// Match MoveFileW: fail when the destination exists.
|
||||
std::error_code error;
|
||||
if (std::filesystem::exists(real_dst, error)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return 0 == rename(real_src.string().c_str(), real_dst.string().c_str());
|
||||
}
|
||||
|
||||
void SysFileRemoveReadonly(const std::filesystem::path& /*name*/) {
|
||||
EXIT("not implemented\n");
|
||||
void SysFileRemoveReadonly(const std::filesystem::path& name) {
|
||||
auto real_name = get_internal_name(name);
|
||||
auto real_name_str = real_name.string();
|
||||
|
||||
struct stat s {};
|
||||
|
||||
if (0 != stat(real_name_str.c_str(), &s)) {
|
||||
return;
|
||||
}
|
||||
|
||||
chmod(real_name_str.c_str(), s.st_mode | S_IWUSR);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -8,9 +8,16 @@
|
||||
#include "common/platform/sysVirtual.h"
|
||||
#include "common/virtualMemory.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <map>
|
||||
#include <pthread.h>
|
||||
#include <sys/mman.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#if defined(__APPLE__)
|
||||
#include <mach/mach.h>
|
||||
#include <mach/mach_vm.h>
|
||||
#endif
|
||||
|
||||
// IWYU pragma: no_include <asm/mman-common.h>
|
||||
// IWYU pragma: no_include <asm/mman.h>
|
||||
@@ -31,8 +38,8 @@ void SysVirtualInit() {
|
||||
pthread_mutexattr_t attr {};
|
||||
|
||||
pthread_mutexattr_init(&attr);
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
|
||||
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_FAST_NP);
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
|
||||
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_FAST_NP); // glibc-only fast mutex
|
||||
#else
|
||||
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL);
|
||||
#endif
|
||||
@@ -76,6 +83,70 @@ static VirtualMemory::Mode get_protection_flag(int mode) {
|
||||
}
|
||||
}
|
||||
|
||||
// Keep automatic mappings inside the guest and GPU-addressable low window.
|
||||
#ifdef KYTY_FIXED_NOREPLACE
|
||||
static constexpr uintptr_t LOW_ARENA_LIMIT = 0x000000FC00000000ULL; // libc mspace window ceiling
|
||||
static constexpr uintptr_t LOW_ARENA_FLOOR = 0x000000A000000000ULL; // 640 GiB
|
||||
static constexpr uintptr_t LOW_ARENA_GRAIN = 0x0000000000010000ULL; // 64 KiB
|
||||
|
||||
static_assert(LOW_ARENA_LIMIT <= 0x0000010000000000ULL,
|
||||
"arena must stay inside the GPU page tracker's 1<<40 window");
|
||||
static_assert(LOW_ARENA_FLOOR < LOW_ARENA_LIMIT, "arena floor must sit below its ceiling");
|
||||
|
||||
static std::atomic<uintptr_t> g_low_arena_next {LOW_ARENA_LIMIT};
|
||||
#endif
|
||||
|
||||
// Caller holds g_virtual_mutex.
|
||||
static void record_alloc(uintptr_t addr, size_t size) {
|
||||
auto next = g_allocs->upper_bound(addr);
|
||||
if (next != g_allocs->begin()) {
|
||||
auto it = std::prev(next);
|
||||
const auto alloc_addr = it->first;
|
||||
const auto alloc_end = alloc_addr + it->second;
|
||||
if (alloc_addr <= addr && addr + size <= alloc_end) {
|
||||
g_allocs->erase(it);
|
||||
if (alloc_addr < addr) {
|
||||
(*g_allocs)[alloc_addr] = addr - alloc_addr;
|
||||
}
|
||||
if (addr + size < alloc_end) {
|
||||
(*g_allocs)[addr + size] = alloc_end - (addr + size);
|
||||
}
|
||||
}
|
||||
}
|
||||
(*g_allocs)[addr] = size;
|
||||
}
|
||||
|
||||
#ifdef KYTY_FIXED_NOREPLACE
|
||||
static uintptr_t align_up_to(uintptr_t addr, uint64_t alignment) {
|
||||
return (addr + alignment - 1) & ~(alignment - 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
// Freed arena addresses are not reused while GPU caches remain keyed by address.
|
||||
static void* map_anonymous(uintptr_t addr, size_t size, int protect, int flags) {
|
||||
if (addr != 0) {
|
||||
return mmap(reinterpret_cast<void*>(addr), size, protect, flags, -1, 0); // NOLINT
|
||||
}
|
||||
|
||||
#ifdef KYTY_FIXED_NOREPLACE
|
||||
const auto step = align_up_to(size, LOW_ARENA_GRAIN);
|
||||
for (int attempt = 0; attempt < 256; attempt++) {
|
||||
const auto top = g_low_arena_next.fetch_sub(step, std::memory_order_relaxed);
|
||||
if (top < step || top - step < LOW_ARENA_FLOOR) {
|
||||
break;
|
||||
}
|
||||
const auto hint = (top - step) & ~(LOW_ARENA_GRAIN - 1);
|
||||
void* ptr = mmap(reinterpret_cast<void*>(hint), size, protect, flags | MAP_FIXED_NOREPLACE,
|
||||
-1, 0); // NOLINT
|
||||
if (ptr != MAP_FAILED) {
|
||||
return ptr;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
return mmap(nullptr, size, protect, flags, -1, 0); // NOLINT
|
||||
}
|
||||
|
||||
uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
|
||||
EXIT_IF(g_allocs == nullptr);
|
||||
|
||||
@@ -83,16 +154,15 @@ uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mo
|
||||
|
||||
int protect = get_protection_flag(mode);
|
||||
|
||||
void* ptr =
|
||||
mmap(reinterpret_cast<void*>(addr), size, protect, MAP_PRIVATE | MAP_ANON, -1, 0); // NOLINT
|
||||
void* ptr = map_anonymous(addr, size, protect, MAP_PRIVATE | MAP_ANON);
|
||||
|
||||
auto ret_addr = reinterpret_cast<uintptr_t>(ptr);
|
||||
|
||||
if (ptr != MAP_FAILED) {
|
||||
pthread_mutex_lock(&g_virtual_mutex);
|
||||
(*g_allocs)[ret_addr] = size;
|
||||
uintptr_t page_start = ret_addr >> 12u;
|
||||
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
|
||||
record_alloc(ret_addr, size);
|
||||
uintptr_t page_start = ret_addr >> 12u;
|
||||
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
|
||||
for (uintptr_t page = page_start; page <= page_end; page++) {
|
||||
(*g_protects)[page] = protect;
|
||||
}
|
||||
@@ -117,18 +187,43 @@ uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::
|
||||
auto addr = static_cast<uintptr_t>(address);
|
||||
int protect = get_protection_flag(mode);
|
||||
|
||||
void* ptr =
|
||||
mmap(reinterpret_cast<void*>(addr), size, protect, MAP_PRIVATE | MAP_ANON, -1, 0); // NOLINT
|
||||
void* ptr = map_anonymous(addr, size, protect, MAP_PRIVATE | MAP_ANON);
|
||||
|
||||
auto ret_addr = reinterpret_cast<uintptr_t>(ptr);
|
||||
|
||||
if (ptr != MAP_FAILED && ((ret_addr & (alignment - 1)) != 0)) {
|
||||
munmap(ptr, size);
|
||||
|
||||
ptr = mmap(reinterpret_cast<void*>(addr), size + alignment, protect,
|
||||
MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); // NOLINT
|
||||
ptr =
|
||||
map_anonymous(addr, size + alignment, protect, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE);
|
||||
ret_addr = reinterpret_cast<uintptr_t>(ptr);
|
||||
if (ptr != MAP_FAILED) {
|
||||
#if defined(__APPLE__)
|
||||
// Carve the aligned subrange out of the live mapping with MAP_FIXED (in-place
|
||||
// replacement) and trim the slack; never munmap the whole range first, or a
|
||||
// concurrent host mapping (dyld, Rosetta, Metal) could claim the hole and be
|
||||
// destroyed by the MAP_FIXED. Other platforms keep the original path below.
|
||||
auto aligned_addr = align_up(ret_addr, alignment);
|
||||
// NOLINTNEXTLINE
|
||||
void* fixed = mmap(reinterpret_cast<void*>(aligned_addr), size, protect,
|
||||
MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0);
|
||||
if (fixed == MAP_FAILED) {
|
||||
munmap(ptr, size + alignment);
|
||||
ret_addr = 0;
|
||||
ptr = MAP_FAILED;
|
||||
} else {
|
||||
if (aligned_addr > ret_addr) {
|
||||
munmap(reinterpret_cast<void*>(ret_addr), aligned_addr - ret_addr);
|
||||
}
|
||||
const uintptr_t tail_start = aligned_addr + size;
|
||||
const uintptr_t resv_end = ret_addr + size + alignment;
|
||||
if (resv_end > tail_start) {
|
||||
munmap(reinterpret_cast<void*>(tail_start), resv_end - tail_start);
|
||||
}
|
||||
ptr = fixed;
|
||||
ret_addr = aligned_addr;
|
||||
}
|
||||
#else
|
||||
munmap(ptr, size + alignment);
|
||||
auto aligned_addr = align_up(ret_addr, alignment);
|
||||
#ifdef KYTY_FIXED_NOREPLACE
|
||||
@@ -146,6 +241,7 @@ uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::
|
||||
ret_addr = 0;
|
||||
ptr = MAP_FAILED;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,9 +250,9 @@ uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&g_virtual_mutex);
|
||||
(*g_allocs)[ret_addr] = size;
|
||||
uintptr_t page_start = ret_addr >> 12u;
|
||||
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
|
||||
record_alloc(ret_addr, size);
|
||||
uintptr_t page_start = ret_addr >> 12u;
|
||||
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
|
||||
for (uintptr_t page = page_start; page <= page_end; page++) {
|
||||
(*g_protects)[page] = protect;
|
||||
}
|
||||
@@ -165,6 +261,27 @@ uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::
|
||||
return ret_addr;
|
||||
}
|
||||
|
||||
#if defined(__APPLE__)
|
||||
// macOS has no /proc/self/maps; query the Mach VM map directly. mach_vm_region returns
|
||||
// the first mapped region at or above `region_addr`; if it begins before the end of the
|
||||
// requested range, the range overlaps an existing mapping.
|
||||
static bool is_mapped(void* ptr, size_t length) {
|
||||
auto query_addr = reinterpret_cast<mach_vm_address_t>(ptr);
|
||||
mach_vm_address_t region_addr = query_addr;
|
||||
mach_vm_size_t region_size = 0;
|
||||
vm_region_basic_info_data_64_t info {};
|
||||
mach_msg_type_number_t count = VM_REGION_BASIC_INFO_COUNT_64;
|
||||
mach_port_t object_name = MACH_PORT_NULL;
|
||||
|
||||
kern_return_t kr =
|
||||
mach_vm_region(mach_task_self(), ®ion_addr, ®ion_size, VM_REGION_BASIC_INFO_64,
|
||||
reinterpret_cast<vm_region_info_t>(&info), &count, &object_name);
|
||||
if (kr != KERN_SUCCESS) {
|
||||
return false; // no region at or above the address → unmapped
|
||||
}
|
||||
return region_addr < (query_addr + length);
|
||||
}
|
||||
#else
|
||||
static bool is_mapped(void* ptr, size_t length) {
|
||||
FILE* file = fopen("/proc/self/maps", "r");
|
||||
char line[1024];
|
||||
@@ -189,6 +306,7 @@ static bool is_mapped(void* ptr, size_t length) {
|
||||
fclose(file);
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
bool SysVirtualAllocFixed(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
|
||||
EXIT_IF(g_allocs == nullptr);
|
||||
@@ -218,9 +336,9 @@ bool SysVirtualAllocFixed(uint64_t address, uint64_t size, VirtualMemory::Mode m
|
||||
|
||||
if (ptr != MAP_FAILED) {
|
||||
pthread_mutex_lock(&g_virtual_mutex);
|
||||
(*g_allocs)[ret_addr] = size;
|
||||
uintptr_t page_start = ret_addr >> 12u;
|
||||
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
|
||||
record_alloc(ret_addr, size);
|
||||
uintptr_t page_start = ret_addr >> 12u;
|
||||
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
|
||||
for (uintptr_t page = page_start; page <= page_end; page++) {
|
||||
(*g_protects)[page] = protect;
|
||||
}
|
||||
@@ -249,18 +367,44 @@ uint64_t SysVirtualReserveAligned(uint64_t address, uint64_t size, uint64_t alig
|
||||
|
||||
auto addr = static_cast<uintptr_t>(address);
|
||||
|
||||
void* ptr = mmap(reinterpret_cast<void*>(addr), size, PROT_NONE,
|
||||
MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); // NOLINT
|
||||
void* ptr = map_anonymous(addr, size, PROT_NONE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE);
|
||||
|
||||
auto ret_addr = reinterpret_cast<uintptr_t>(ptr);
|
||||
|
||||
if (ptr != MAP_FAILED && ((ret_addr & (alignment - 1)) != 0)) {
|
||||
munmap(ptr, size);
|
||||
|
||||
ptr = mmap(reinterpret_cast<void*>(addr), size + alignment, PROT_NONE,
|
||||
MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); // NOLINT
|
||||
ptr = map_anonymous(addr, size + alignment, PROT_NONE,
|
||||
MAP_PRIVATE | MAP_ANON | MAP_NORESERVE);
|
||||
ret_addr = reinterpret_cast<uintptr_t>(ptr);
|
||||
if (ptr != MAP_FAILED) {
|
||||
#if defined(__APPLE__)
|
||||
// Carve the aligned subrange out of the live reservation with MAP_FIXED (an
|
||||
// in-place replacement), then trim the slack. The range must never be
|
||||
// returned to the OS in between: another thread (dyld, Rosetta, Metal,
|
||||
// malloc) could claim the hole, and the subsequent MAP_FIXED would silently
|
||||
// destroy its mapping. Other platforms keep the original path below.
|
||||
auto aligned_addr = align_up(ret_addr, alignment);
|
||||
// NOLINTNEXTLINE
|
||||
void* fixed = mmap(reinterpret_cast<void*>(aligned_addr), size, PROT_NONE,
|
||||
MAP_FIXED | MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0);
|
||||
if (fixed == MAP_FAILED) {
|
||||
munmap(ptr, size + alignment);
|
||||
ret_addr = 0;
|
||||
ptr = MAP_FAILED;
|
||||
} else {
|
||||
if (aligned_addr > ret_addr) {
|
||||
munmap(reinterpret_cast<void*>(ret_addr), aligned_addr - ret_addr);
|
||||
}
|
||||
const uintptr_t tail_start = aligned_addr + size;
|
||||
const uintptr_t resv_end = ret_addr + size + alignment;
|
||||
if (resv_end > tail_start) {
|
||||
munmap(reinterpret_cast<void*>(tail_start), resv_end - tail_start);
|
||||
}
|
||||
ptr = fixed;
|
||||
ret_addr = aligned_addr;
|
||||
}
|
||||
#else
|
||||
munmap(ptr, size + alignment);
|
||||
auto aligned_addr = align_up(ret_addr, alignment);
|
||||
#ifdef KYTY_FIXED_NOREPLACE
|
||||
@@ -278,6 +422,7 @@ uint64_t SysVirtualReserveAligned(uint64_t address, uint64_t size, uint64_t alig
|
||||
ret_addr = 0;
|
||||
ptr = MAP_FAILED;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@@ -286,12 +431,7 @@ uint64_t SysVirtualReserveAligned(uint64_t address, uint64_t size, uint64_t alig
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&g_virtual_mutex);
|
||||
(*g_allocs)[ret_addr] = size;
|
||||
uintptr_t page_start = ret_addr >> 12u;
|
||||
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
|
||||
for (uintptr_t page = page_start; page <= page_end; page++) {
|
||||
(*g_protects)[page] = PROT_NONE;
|
||||
}
|
||||
record_alloc(ret_addr, size);
|
||||
pthread_mutex_unlock(&g_virtual_mutex);
|
||||
|
||||
return ret_addr;
|
||||
@@ -324,12 +464,7 @@ bool SysVirtualReserveFixed(uint64_t address, uint64_t size) {
|
||||
|
||||
if (ptr != MAP_FAILED) {
|
||||
pthread_mutex_lock(&g_virtual_mutex);
|
||||
(*g_allocs)[ret_addr] = size;
|
||||
uintptr_t page_start = ret_addr >> 12u;
|
||||
uintptr_t page_end = (ret_addr + size - 1) >> 12u;
|
||||
for (uintptr_t page = page_start; page <= page_end; page++) {
|
||||
(*g_protects)[page] = PROT_NONE;
|
||||
}
|
||||
record_alloc(ret_addr, size);
|
||||
pthread_mutex_unlock(&g_virtual_mutex);
|
||||
|
||||
return true;
|
||||
@@ -339,7 +474,29 @@ bool SysVirtualReserveFixed(uint64_t address, uint64_t size) {
|
||||
}
|
||||
|
||||
bool SysVirtualDecommit(uint64_t address, uint64_t size) {
|
||||
return SysVirtualProtect(address, size, VirtualMemory::Mode::NoAccess);
|
||||
// Drop physical pages while preserving the reservation.
|
||||
if (!SysVirtualProtect(address, size, VirtualMemory::Mode::NoAccess)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (size != 0) {
|
||||
#if defined(__APPLE__)
|
||||
constexpr int RECLAIM_ADVICE = MADV_FREE;
|
||||
#else
|
||||
constexpr int RECLAIM_ADVICE = MADV_DONTNEED;
|
||||
#endif
|
||||
const auto page_size = static_cast<uintptr_t>(sysconf(_SC_PAGESIZE));
|
||||
if (page_size != 0) {
|
||||
// Do not discard pages outside the requested range.
|
||||
const auto begin = (static_cast<uintptr_t>(address) + page_size - 1) & ~(page_size - 1);
|
||||
const auto end = (static_cast<uintptr_t>(address) + size) & ~(page_size - 1);
|
||||
if (end > begin) {
|
||||
::madvise(reinterpret_cast<void*>(begin), end - begin, RECLAIM_ADVICE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SysVirtualFree(uint64_t address) {
|
||||
@@ -391,15 +548,34 @@ bool SysVirtualFreeRange(uint64_t address, uint64_t size) {
|
||||
pthread_mutex_unlock(&g_virtual_mutex);
|
||||
return false;
|
||||
}
|
||||
auto it = std::prev(next);
|
||||
const auto alloc_addr = it->first;
|
||||
const auto alloc_end = alloc_addr + it->second;
|
||||
if (addr < alloc_addr || end > alloc_end || munmap(reinterpret_cast<void*>(addr), size) != 0) {
|
||||
|
||||
// A reservation may have been split into several adjacent records.
|
||||
auto first = std::prev(next);
|
||||
const auto alloc_addr = first->first;
|
||||
if (addr < alloc_addr || alloc_addr + first->second <= addr) {
|
||||
pthread_mutex_unlock(&g_virtual_mutex);
|
||||
return false;
|
||||
}
|
||||
|
||||
g_allocs->erase(it);
|
||||
auto last = first;
|
||||
uintptr_t cursor = alloc_addr + first->second;
|
||||
while (cursor < end) {
|
||||
auto following = std::next(last);
|
||||
if (following == g_allocs->end() || following->first != cursor) {
|
||||
pthread_mutex_unlock(&g_virtual_mutex);
|
||||
return false;
|
||||
}
|
||||
last = following;
|
||||
cursor = following->first + following->second;
|
||||
}
|
||||
const auto alloc_end = cursor;
|
||||
|
||||
if (munmap(reinterpret_cast<void*>(addr), size) != 0) {
|
||||
pthread_mutex_unlock(&g_virtual_mutex);
|
||||
return false;
|
||||
}
|
||||
|
||||
g_allocs->erase(first, std::next(last));
|
||||
if (alloc_addr < addr) {
|
||||
(*g_allocs)[alloc_addr] = addr - alloc_addr;
|
||||
}
|
||||
|
||||
@@ -27,7 +27,9 @@ struct SysFileTimeStruct {
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
FILETIME time;
|
||||
#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
|
||||
// Nanoseconds preserve sub-second file timestamps.
|
||||
time_t time;
|
||||
long nanos;
|
||||
#endif
|
||||
bool is_invalid;
|
||||
};
|
||||
@@ -139,7 +141,7 @@ inline void SysFileToSystemTimeUtc(const SysFileTimeStruct& f, SysTimeStruct& t)
|
||||
t.Hour = i.tm_hour;
|
||||
t.Minute = i.tm_min;
|
||||
t.Second = (i.tm_sec == 60 ? 59 : i.tm_sec);
|
||||
t.Milliseconds = 0;
|
||||
t.Milliseconds = static_cast<uint16_t>((f.nanos / 1000000) % 1000);
|
||||
}
|
||||
|
||||
inline void SysTimeTToSystem(time_t t, SysTimeStruct& s) {
|
||||
@@ -168,10 +170,11 @@ inline void SysSystemToFileTimeUtc(const SysTimeStruct& f, SysFileTimeStruct& t)
|
||||
|
||||
// Retrieves the current local date and time.
|
||||
inline void SysGetSystemTime(SysTimeStruct& t) {
|
||||
time_t st {};
|
||||
// Preserve millisecond precision.
|
||||
timespec now {};
|
||||
struct tm i {};
|
||||
|
||||
if (time(&st) == static_cast<time_t>(-1) || localtime_r(&st, &i) == nullptr) {
|
||||
if (clock_gettime(CLOCK_REALTIME, &now) != 0 || localtime_r(&now.tv_sec, &i) == nullptr) {
|
||||
t.is_invalid = true;
|
||||
return;
|
||||
}
|
||||
@@ -183,15 +186,16 @@ inline void SysGetSystemTime(SysTimeStruct& t) {
|
||||
t.Hour = i.tm_hour;
|
||||
t.Minute = i.tm_min;
|
||||
t.Second = (i.tm_sec == 60 ? 59 : i.tm_sec);
|
||||
t.Milliseconds = 0;
|
||||
t.Milliseconds = static_cast<uint16_t>((now.tv_nsec / 1000000) % 1000);
|
||||
}
|
||||
|
||||
// Retrieves the current system date and time in Coordinated Universal Time (UTC).
|
||||
inline void SysGetSystemTimeUtc(SysTimeStruct& t) {
|
||||
time_t st {};
|
||||
// Preserve millisecond precision.
|
||||
timespec now {};
|
||||
struct tm i {};
|
||||
|
||||
if (time(&st) == static_cast<time_t>(-1) || gmtime_r(&st, &i) == nullptr) {
|
||||
if (clock_gettime(CLOCK_REALTIME, &now) != 0 || gmtime_r(&now.tv_sec, &i) == nullptr) {
|
||||
t.is_invalid = true;
|
||||
return;
|
||||
}
|
||||
@@ -203,7 +207,7 @@ inline void SysGetSystemTimeUtc(SysTimeStruct& t) {
|
||||
t.Hour = i.tm_hour;
|
||||
t.Minute = i.tm_min;
|
||||
t.Second = (i.tm_sec == 60 ? 59 : i.tm_sec);
|
||||
t.Milliseconds = 0;
|
||||
t.Milliseconds = static_cast<uint16_t>((now.tv_nsec / 1000000) % 1000);
|
||||
}
|
||||
|
||||
inline void SysQueryPerformanceFrequency(uint64_t* freq) {
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cerrno>
|
||||
#include <chrono> // IWYU pragma: keep
|
||||
#include <condition_variable> // IWYU pragma: keep
|
||||
#include <mutex>
|
||||
@@ -14,6 +15,12 @@
|
||||
#define KYTY_WIN_CS
|
||||
#endif
|
||||
|
||||
// macOS has no clock_nanosleep.
|
||||
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS && !defined(__APPLE__)
|
||||
#define KYTY_POSIX_HIGH_RES_SLEEP
|
||||
#include <ctime>
|
||||
#endif
|
||||
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
@@ -121,6 +128,42 @@ static SleepConditionVariableCS_func_t ResolveSleepConditionVariableCS() {
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef KYTY_POSIX_HIGH_RES_SLEEP
|
||||
// Spin for very short waits; use an absolute deadline for longer waits.
|
||||
static void SleepHighResolutionNanos(uint64_t nanos) {
|
||||
if (nanos == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
constexpr uint64_t NANOS_PER_SEC = 1000000000;
|
||||
constexpr uint64_t SPIN_LIMIT_NS = 50000; // below this a context switch dominates
|
||||
|
||||
timespec deadline {};
|
||||
if (clock_gettime(CLOCK_MONOTONIC, &deadline) != 0) {
|
||||
std::this_thread::sleep_for(std::chrono::nanoseconds(nanos));
|
||||
return;
|
||||
}
|
||||
|
||||
auto target_nsec = static_cast<uint64_t>(deadline.tv_nsec) + nanos;
|
||||
deadline.tv_sec += static_cast<time_t>(target_nsec / NANOS_PER_SEC);
|
||||
deadline.tv_nsec = static_cast<long>(target_nsec % NANOS_PER_SEC);
|
||||
|
||||
if (nanos <= SPIN_LIMIT_NS) {
|
||||
timespec now {};
|
||||
do {
|
||||
if (clock_gettime(CLOCK_MONOTONIC, &now) != 0) {
|
||||
return;
|
||||
}
|
||||
} while (now.tv_sec < deadline.tv_sec ||
|
||||
(now.tv_sec == deadline.tv_sec && now.tv_nsec < deadline.tv_nsec));
|
||||
return;
|
||||
}
|
||||
|
||||
while (clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &deadline, nullptr) == EINTR) {
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
namespace Common {
|
||||
|
||||
using thread_id_t = std::thread::id;
|
||||
@@ -249,6 +292,8 @@ void Thread::Sleep(uint32_t millis) {
|
||||
void Thread::SleepMicro(uint32_t micros) {
|
||||
#ifdef KYTY_WIN_CS
|
||||
SleepHighResolution100ns(static_cast<uint64_t>(micros) * 10);
|
||||
#elif defined(KYTY_POSIX_HIGH_RES_SLEEP)
|
||||
SleepHighResolutionNanos(static_cast<uint64_t>(micros) * 1000);
|
||||
#else
|
||||
std::this_thread::sleep_for(std::chrono::microseconds(micros));
|
||||
#endif
|
||||
@@ -257,6 +302,8 @@ void Thread::SleepMicro(uint32_t micros) {
|
||||
void Thread::SleepNano(uint64_t nanos) {
|
||||
#ifdef KYTY_WIN_CS
|
||||
SleepHighResolution100ns((nanos + 99) / 100);
|
||||
#elif defined(KYTY_POSIX_HIGH_RES_SLEEP)
|
||||
SleepHighResolutionNanos(nanos);
|
||||
#else
|
||||
std::this_thread::sleep_for(std::chrono::nanoseconds(nanos));
|
||||
#endif
|
||||
|
||||
@@ -11,7 +11,7 @@ template <typename Result, typename... Args>
|
||||
class UniqueFunction {
|
||||
class CallableBase {
|
||||
public:
|
||||
virtual ~CallableBase() = default;
|
||||
virtual ~CallableBase() = default;
|
||||
virtual Result Invoke(Args&&... args) = 0;
|
||||
};
|
||||
|
||||
@@ -20,9 +20,7 @@ class UniqueFunction {
|
||||
public:
|
||||
explicit Callable(Function function): m_function(std::move(function)) {}
|
||||
|
||||
Result Invoke(Args&&... args) override {
|
||||
return m_function(std::forward<Args>(args)...);
|
||||
}
|
||||
Result Invoke(Args&&... args) override { return m_function(std::forward<Args>(args)...); }
|
||||
|
||||
private:
|
||||
Function m_function;
|
||||
|
||||
@@ -58,25 +58,6 @@ bool FlushInstructionCache(uint64_t address, uint64_t size) {
|
||||
return SysVirtualFlushInstructionCache(address, size);
|
||||
}
|
||||
|
||||
bool PatchReplace(uint64_t vaddr, uint64_t value) {
|
||||
Mode old_mode {};
|
||||
Protect(vaddr, 8, Mode::ReadWrite, &old_mode);
|
||||
|
||||
auto* ptr = reinterpret_cast<uint64_t*>(vaddr);
|
||||
|
||||
bool ret = (*ptr != value);
|
||||
|
||||
*ptr = value;
|
||||
|
||||
Protect(vaddr, 8, old_mode);
|
||||
|
||||
if (IsExecute(old_mode)) {
|
||||
FlushInstructionCache(vaddr, 8);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
} // namespace VirtualMemory
|
||||
|
||||
} // namespace Common
|
||||
|
||||
@@ -37,7 +37,6 @@ bool Free(uint64_t address);
|
||||
bool FreeRange(uint64_t address, uint64_t size);
|
||||
bool Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode = nullptr);
|
||||
bool FlushInstructionCache(uint64_t address, uint64_t size);
|
||||
bool PatchReplace(uint64_t vaddr, uint64_t value);
|
||||
|
||||
} // namespace VirtualMemory
|
||||
|
||||
|
||||
+19
-17
@@ -105,7 +105,7 @@ static void ClearDebugTextureFolder() {
|
||||
}
|
||||
}
|
||||
|
||||
static void Init(const Config::ConfigOptions& cfg) {
|
||||
static void Init(const Config::ConfigOptions& cfg, const std::filesystem::path& param_json) {
|
||||
EXIT_IF(!Common::Thread::IsMainThread());
|
||||
|
||||
auto* slist = Common::SubsystemsList::Instance();
|
||||
@@ -127,12 +127,21 @@ static void Init(const Config::ConfigOptions& cfg) {
|
||||
slist->InitAll(true);
|
||||
|
||||
Config::Load(cfg);
|
||||
slist->Add(log, {core, config});
|
||||
slist->InitAll(true);
|
||||
|
||||
if (Common::File::IsFileExisting(param_json)) {
|
||||
Loader::SystemContentLoadParamSfo(param_json);
|
||||
if (const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
|
||||
flexible_memory_size != 0) {
|
||||
Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
|
||||
}
|
||||
}
|
||||
|
||||
slist->Add(audio, {core, log, pthread, memory});
|
||||
slist->Add(controller, {core, log, config});
|
||||
slist->Add(file_system, {core, log, pthread});
|
||||
slist->Add(graphics, {core, log, pthread, memory, config, profiler, controller});
|
||||
slist->Add(log, {core, config});
|
||||
slist->Add(memory, {core, log});
|
||||
slist->Add(network, {core, log, pthread});
|
||||
slist->Add(profiler, {core, config});
|
||||
@@ -159,13 +168,14 @@ static void LoadElf(const std::filesystem::path& elf, bool dbg_print_reloc = fal
|
||||
}
|
||||
}
|
||||
|
||||
static void Execute() {
|
||||
static void Execute(const std::filesystem::path& game_patch) {
|
||||
auto patch_path = game_patch;
|
||||
Common::Thread guest_thread(
|
||||
[](void* /*unused*/) {
|
||||
[](void* param) {
|
||||
auto* rt = Common::Singleton<Loader::RuntimeLinker>::Instance();
|
||||
rt->Execute();
|
||||
rt->Execute(*static_cast<const std::filesystem::path*>(param));
|
||||
},
|
||||
nullptr);
|
||||
&patch_path);
|
||||
Libs::Graphics::WindowRun();
|
||||
std::quick_exit(0);
|
||||
}
|
||||
@@ -179,7 +189,8 @@ void Run(const RunOptions& options) {
|
||||
EXIT("ELF is required\n");
|
||||
}
|
||||
|
||||
Init(options.config);
|
||||
const auto param_json = options.app0_dir / "sce_sys" / "param.json";
|
||||
Init(options.config, param_json);
|
||||
|
||||
ClearDebugTextureFolder();
|
||||
|
||||
@@ -191,15 +202,6 @@ void Run(const RunOptions& options) {
|
||||
Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/app0");
|
||||
Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/hostapp");
|
||||
|
||||
auto param_json = options.app0_dir / "sce_sys" / "param.json";
|
||||
if (Common::File::IsFileExisting(param_json)) {
|
||||
Loader::SystemContentLoadParamSfo(param_json);
|
||||
if (auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
|
||||
flexible_memory_size != 0) {
|
||||
Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
|
||||
}
|
||||
}
|
||||
|
||||
MountSandboxDirs();
|
||||
|
||||
auto* rt = Common::Singleton<Loader::RuntimeLinker>::Instance();
|
||||
@@ -207,7 +209,7 @@ void Run(const RunOptions& options) {
|
||||
|
||||
LoadElf(options.elf);
|
||||
|
||||
Execute();
|
||||
Execute(options.game_patch);
|
||||
}
|
||||
|
||||
} // namespace Emulator
|
||||
|
||||
@@ -12,6 +12,7 @@ struct RunOptions {
|
||||
Config::ConfigOptions config;
|
||||
std::filesystem::path app0_dir;
|
||||
std::filesystem::path elf;
|
||||
std::filesystem::path game_patch;
|
||||
};
|
||||
|
||||
void Run(const RunOptions& options);
|
||||
|
||||
@@ -158,7 +158,7 @@ private:
|
||||
void CheckBuffer() const { GetScheduler().CheckActive(); }
|
||||
GpuResourceManager& GetGpuResources() const { return m_renderer.GetGpuResources(); }
|
||||
|
||||
RenderContext& m_renderer;
|
||||
RenderContext& m_renderer;
|
||||
HW::Context m_ctx;
|
||||
HW::UserConfig m_ucfg;
|
||||
HW::Shader m_sh_ctx;
|
||||
@@ -170,9 +170,9 @@ private:
|
||||
uint64_t m_dispatch_indirect_args_base_addr = 0;
|
||||
uint32_t m_num_instances = 1;
|
||||
|
||||
uint32_t m_de_count = 0;
|
||||
uint32_t m_ce_count = 0;
|
||||
bool m_ce_complete = false;
|
||||
uint32_t m_de_count = 0;
|
||||
uint32_t m_ce_count = 0;
|
||||
bool m_ce_complete = false;
|
||||
bool m_readback_active = false;
|
||||
|
||||
uint32_t m_const_ram[0x3000] = {0};
|
||||
|
||||
@@ -65,10 +65,23 @@ constexpr uint32_t GcrKnownMask = GcrGl2MetadataInvalidate | GcrGl0V
|
||||
GcrGl2Writeback | GcrOrder012 | GcrOrder210;
|
||||
constexpr uint32_t RegisterSelectorMask = 0x70000000u;
|
||||
|
||||
uint32_t NormalizeRegisterOffset(uint32_t raw_offset) {
|
||||
return (raw_offset & ~RegisterSelectorMask);
|
||||
constexpr uint32_t NormalizeRegisterOffset(uint32_t raw_offset) {
|
||||
return raw_offset & ~RegisterSelectorMask;
|
||||
}
|
||||
|
||||
// Indirect Cx descriptors retain their selector. Selector 1 offsets 0..31 address the
|
||||
// SPI_PS_INPUT_CNTL register bank; ordinary context-register offsets remain unchanged.
|
||||
constexpr uint32_t DecodeIndirectCxRegisterOffset(uint32_t raw_offset) {
|
||||
const auto offset = NormalizeRegisterOffset(raw_offset);
|
||||
return (raw_offset & RegisterSelectorMask) == Pm4::CX_PS_SHADER_USAGE_BASE && offset < 32u
|
||||
? Pm4::SPI_PS_INPUT_CNTL_0 + offset
|
||||
: offset;
|
||||
}
|
||||
|
||||
static_assert(DecodeIndirectCxRegisterOffset(Pm4::CX_PS_SHADER_USAGE_BASE + 2u) ==
|
||||
Pm4::SPI_PS_INPUT_CNTL_0 + 2u);
|
||||
static_assert(DecodeIndirectCxRegisterOffset(Pm4::DB_Z_INFO) == Pm4::DB_Z_INFO);
|
||||
|
||||
bool ReleaseMemGcrNeedsBarrier(uint32_t eop_event_type, uint32_t gcr_cntl) {
|
||||
return eop_event_type != 0x28u ||
|
||||
(gcr_cntl & (GcrGl2MetadataInvalidate | GcrGl0VectorInvalidate | GcrGl1Invalidate |
|
||||
@@ -1904,17 +1917,23 @@ KYTY_CP_OP_PARSER(CpOpCopyData) {
|
||||
|
||||
EXIT_NOT_IMPLEMENTED(cmd_id != KYTY_PM4(6, Pm4::IT_COPY_DATA, 0u));
|
||||
|
||||
const uint32_t control = buffer[0];
|
||||
const uint32_t src_sel = ((control & 0xfu) << 1u) | ((control >> 30u) & 0x1u);
|
||||
const uint32_t dst_sel = ((control >> 8u) & 0xfu) << 1u;
|
||||
const uint8_t src_cache = static_cast<uint8_t>((control >> 13u) & 0x3u);
|
||||
const uint8_t dst_cache = static_cast<uint8_t>((control >> 25u) & 0x3u);
|
||||
const uint8_t write_confirm = static_cast<uint8_t>((control >> 20u) & 0x1u);
|
||||
const uint32_t num_bytes = ((control >> 16u) & 0x1u) != 0 ? 8u : 4u;
|
||||
const uint64_t src = buffer[1] | (static_cast<uint64_t>(buffer[2]) << 32u);
|
||||
const uint64_t dst = buffer[3] | (static_cast<uint64_t>(buffer[4]) << 32u);
|
||||
if (src_sel == (9u << 1u)) {
|
||||
if (dst_sel != (2u << 1u) || dst == 0 || (dst & (num_bytes - 1u)) != 0) {
|
||||
const uint32_t control = buffer[0];
|
||||
const uint32_t src_sel = ((control & 0xfu) << 1u) | ((control >> 30u) & 0x1u);
|
||||
const uint32_t dst_sel = ((control >> 8u) & 0xfu) << 1u;
|
||||
const uint8_t src_cache = static_cast<uint8_t>((control >> 13u) & 0x3u);
|
||||
const uint8_t dst_cache = static_cast<uint8_t>((control >> 25u) & 0x3u);
|
||||
const uint8_t write_confirm = static_cast<uint8_t>((control >> 20u) & 0x1u);
|
||||
const uint32_t num_bytes = ((control >> 16u) & 0x1u) != 0 ? 8u : 4u;
|
||||
const uint64_t src = buffer[1] | (static_cast<uint64_t>(buffer[2]) << 32u);
|
||||
const uint64_t dst = buffer[3] | (static_cast<uint64_t>(buffer[4]) << 32u);
|
||||
uint32_t reference_clock_dst = 0;
|
||||
switch (src_sel) {
|
||||
case 9u: reference_clock_dst = 2u; break;
|
||||
case 18u: reference_clock_dst = 4u; break;
|
||||
default: break;
|
||||
}
|
||||
if (reference_clock_dst != 0) {
|
||||
if (dst_sel != reference_clock_dst || dst == 0 || (dst & (num_bytes - 1u)) != 0) {
|
||||
EXIT("unsupported reference-clock copyData, src_sel=0x%02" PRIx32
|
||||
" dst_sel=0x%02" PRIx32 " dst=0x%016" PRIx64 " size=%u\n",
|
||||
src_sel, dst_sel, dst, num_bytes);
|
||||
@@ -2321,14 +2340,18 @@ KYTY_CP_OP_PARSER(CpOpIndirectCxRegs) {
|
||||
EXIT("indirect CX registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw);
|
||||
}
|
||||
for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) {
|
||||
auto cmd_offset = indirect_buffer[0];
|
||||
auto value = indirect_buffer[1];
|
||||
// Keep the encoded offset for packet control values, and use the decoded offset only
|
||||
// for register dispatch.
|
||||
auto raw_cmd_offset = indirect_buffer[0];
|
||||
auto cmd_offset = DecodeIndirectCxRegisterOffset(raw_cmd_offset);
|
||||
auto value = indirect_buffer[1];
|
||||
|
||||
if (HwCtxTrySetFakeRegister(cmd_offset, value)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cmd_offset == 0xffffffffu) {
|
||||
// The sentinel is an encoded descriptor value and must be checked before normalization.
|
||||
if (raw_cmd_offset == 0xffffffffu) {
|
||||
static bool logged = false;
|
||||
if (!logged) {
|
||||
LOGF("\t temporary: skipping indirect CX sentinel pair offset = 0xffffffff, value "
|
||||
@@ -3373,6 +3396,12 @@ void GraphicsInitJmpTablesCxIndirect() {
|
||||
g_hw_ctx_indirect_func[Pm4::DB_COUNT_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
|
||||
HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
|
||||
};
|
||||
for (auto cmd_offset = Pm4::DB_SRESULTS_COMPARE_STATE0;
|
||||
cmd_offset <= Pm4::DB_SRESULTS_COMPARE_STATE1; cmd_offset++) {
|
||||
g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
|
||||
HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
|
||||
};
|
||||
}
|
||||
g_hw_ctx_indirect_func[Pm4::DB_RENDER_OVERRIDE] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
|
||||
HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
|
||||
};
|
||||
|
||||
@@ -72,6 +72,7 @@ enum class ChannelLayout : uint32_t {
|
||||
k32_32 = 11,
|
||||
k16_16_16_16 = 12,
|
||||
k32_32_32_32 = 14,
|
||||
k5_6_5 = 16,
|
||||
k5_5_5_1 = 17,
|
||||
k4_4_4_4 = 19,
|
||||
kBc1 = 35,
|
||||
@@ -374,6 +375,8 @@ enum class BufferFormat : uint32_t {
|
||||
k32_32_32_32UInt = 75,
|
||||
k32_32_32_32SInt = 76,
|
||||
k32_32_32_32Float = 77,
|
||||
k8Srgb = 128,
|
||||
k8_8Srgb = 129,
|
||||
k8_8_8_8Srgb = 130,
|
||||
k9_9_9_5Float = 132,
|
||||
k5_6_5UNorm = 133,
|
||||
|
||||
@@ -39,6 +39,7 @@ constexpr FormatInfo kFormatInfo[] = {
|
||||
{GpuEnumValue(BufferFormat::k16_16Float), 4, 0, 4, true, false},
|
||||
{GpuEnumValue(BufferFormat::k11_11_10Float), 4, 0, 4, true, false},
|
||||
{GpuEnumValue(BufferFormat::k10_10_10_2UNorm), 4, 0, 4, true, false},
|
||||
{GpuEnumValue(BufferFormat::k10_10_10_2UInt), 4, 0, 4, true, true},
|
||||
{GpuEnumValue(BufferFormat::k8_8_8_8UNorm), 4, 0, 4, true, false},
|
||||
{GpuEnumValue(BufferFormat::k8_8_8_8SNorm), 4, 0, 4, true, false},
|
||||
{GpuEnumValue(BufferFormat::k8_8_8_8UInt), 4, 0, 4, true, true},
|
||||
@@ -57,6 +58,8 @@ constexpr FormatInfo kFormatInfo[] = {
|
||||
{GpuEnumValue(BufferFormat::k32_32_32_32UInt), 16, 0, 16, true, true},
|
||||
{GpuEnumValue(BufferFormat::k32_32_32_32SInt), 16, 0, 16, false, false},
|
||||
{GpuEnumValue(BufferFormat::k32_32_32_32Float), 16, 0, 16, true, false},
|
||||
{GpuEnumValue(BufferFormat::k8Srgb), 1, 0, 0, true, false},
|
||||
{GpuEnumValue(BufferFormat::k8_8Srgb), 2, 0, 0, true, false},
|
||||
{GpuEnumValue(BufferFormat::k8_8_8_8Srgb), 4, 0, 4, true, false},
|
||||
{GpuEnumValue(BufferFormat::k9_9_9_5Float), 4, 0, 0, true, false},
|
||||
{GpuEnumValue(BufferFormat::k5_6_5UNorm), 2, 0, 2, true, false},
|
||||
|
||||
@@ -407,11 +407,16 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw
|
||||
uint32_t write_control) {
|
||||
const uint32_t dst_sel = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu);
|
||||
const uint32_t cache_policy = (write_control >> 25u) & 0x3u;
|
||||
const uint32_t increment = (write_control >> 16u) & 0x1u;
|
||||
const uint32_t increment = (write_control >> 16u) & 0x1u;
|
||||
const uint32_t write_confirm = (write_control >> 20u) & 0x1u;
|
||||
|
||||
if (dst_sel != 0 && dst_sel != 2 && dst_sel != 4 && dst_sel != 5) {
|
||||
EXIT("unsupported writeData destination selector 0x%02" PRIx32 "\n", dst_sel);
|
||||
switch (dst_sel) {
|
||||
case 0:
|
||||
case 2:
|
||||
case 4:
|
||||
case 5:
|
||||
case 6: break;
|
||||
default: EXIT("unsupported writeData destination selector 0x%02" PRIx32 "\n", dst_sel);
|
||||
}
|
||||
EXIT_NOT_IMPLEMENTED(increment != 0);
|
||||
|
||||
@@ -962,9 +967,8 @@ void CommandProcessor::DrawIndexOffset(uint32_t index_offset, uint32_t index_cou
|
||||
auto* index_addr = reinterpret_cast<const void*>(
|
||||
m_index_base_addr + static_cast<uint64_t>(index_offset) * index_size);
|
||||
|
||||
m_renderer.GetRenderExecutor().DrawIndex(m_submit_id, CurrentBuffer(),
|
||||
m_index_type_and_size, index_count, index_addr,
|
||||
flags, 1, m_num_instances);
|
||||
m_renderer.GetRenderExecutor().DrawIndex(m_submit_id, CurrentBuffer(), m_index_type_and_size,
|
||||
index_count, index_addr, flags, 1, m_num_instances);
|
||||
}
|
||||
|
||||
void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiator, bool indexed) {
|
||||
@@ -1190,8 +1194,8 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g
|
||||
}
|
||||
}
|
||||
|
||||
m_renderer.GetRenderExecutor().DispatchDirect(
|
||||
m_submit_id, CurrentBuffer(), thread_group_x, thread_group_y, thread_group_z, mode);
|
||||
m_renderer.GetRenderExecutor().DispatchDirect(m_submit_id, CurrentBuffer(), thread_group_x,
|
||||
thread_group_y, thread_group_z, mode);
|
||||
}
|
||||
|
||||
constexpr uint32_t DispatchInitiatorUseThreadDimensions = 1u << 5u;
|
||||
@@ -1237,16 +1241,16 @@ void CommandProcessor::DrawIndexAuto(uint32_t index_count, uint32_t flags,
|
||||
uint32_t first_vertex, uint32_t first_instance) {
|
||||
CheckBuffer();
|
||||
|
||||
m_renderer.GetRenderExecutor().DrawAuto(
|
||||
m_submit_id, CurrentBuffer(), index_count, flags, render_target_slice_offset,
|
||||
instance_count, first_vertex, first_instance);
|
||||
m_renderer.GetRenderExecutor().DrawAuto(m_submit_id, CurrentBuffer(), index_count, flags,
|
||||
render_target_slice_offset, instance_count,
|
||||
first_vertex, first_instance);
|
||||
}
|
||||
|
||||
void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) {
|
||||
BufferFlush();
|
||||
|
||||
m_renderer.GetVideoOut().WaitFlipDone(static_cast<int>(video_out_handle),
|
||||
static_cast<int>(display_buffer_index));
|
||||
static_cast<int>(display_buffer_index));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@@ -1317,8 +1321,8 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
|
||||
if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) {
|
||||
auto* dst = static_cast<uint32_t*>(dst_gpu_addr);
|
||||
SynchronizeGpu();
|
||||
Sync::ReadGds(m_renderer.GetBufferCache().GetGdsBuffer(), dst,
|
||||
value & 0xffffu, value >> 16u);
|
||||
Sync::ReadGds(m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu,
|
||||
value >> 16u);
|
||||
Sync::WriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst, value & 0xffffu,
|
||||
value >> 16u);
|
||||
return;
|
||||
@@ -1486,8 +1490,7 @@ void CommandProcessor::EmitGlobalBarrier() {
|
||||
barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands;
|
||||
barrier.srcAccessMask = vk::AccessFlagBits2::eMemoryWrite;
|
||||
barrier.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands;
|
||||
barrier.dstAccessMask =
|
||||
vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite;
|
||||
barrier.dstAccessMask = vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite;
|
||||
|
||||
vk::DependencyInfo dependency {};
|
||||
dependency.memoryBarrierCount = 1;
|
||||
|
||||
@@ -110,7 +110,6 @@ void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t star
|
||||
auto* cmd = cmd_buffer + start_dw;
|
||||
auto dw = num_dw;
|
||||
while (dw != 0) {
|
||||
EXIT_NOT_IMPLEMENTED(dw < 2);
|
||||
EXIT_NOT_IMPLEMENTED(dw > num_dw);
|
||||
|
||||
auto cmd_id = *cmd++;
|
||||
@@ -120,6 +119,9 @@ void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t star
|
||||
uint32_t len = 0;
|
||||
|
||||
const auto packet_type = static_cast<PacketType>(cmd_id >> 30u);
|
||||
// Type-2 packets are header-only padding; every other packet type requires a body.
|
||||
EXIT_NOT_IMPLEMENTED(dw < 2 && packet_type != PacketType::Type2);
|
||||
|
||||
switch (packet_type) {
|
||||
case PacketType::Type3: {
|
||||
const bool sh_gx = (cmd_id & 0x2u) == 0;
|
||||
|
||||
@@ -385,6 +385,9 @@ constexpr uint32_t SPI_SHADER_POS_FORMAT = 0x1C3;
|
||||
constexpr uint32_t SPI_SHADER_Z_FORMAT = 0x1C4;
|
||||
constexpr uint32_t SPI_SHADER_COL_FORMAT = 0x1C5;
|
||||
|
||||
// Indirect Cx descriptor selector for the 32-entry PS input-control register bank.
|
||||
constexpr uint32_t CX_PS_SHADER_USAGE_BASE = 0x10000000u;
|
||||
|
||||
constexpr uint32_t CB_BLEND0_CONTROL = 0x1E0;
|
||||
constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_SHIFT = 0;
|
||||
constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_MASK = 0x1F;
|
||||
|
||||
@@ -65,41 +65,41 @@ struct TileVolumeLayout {
|
||||
};
|
||||
|
||||
bool TileGetBlockLayout(TileBlockFamily family, uint32_t bytes_per_element,
|
||||
TileBlockLayout& layout);
|
||||
TileBlockLayout& layout);
|
||||
bool TileGetBlockOffset(const TileBlockLayout& layout, uint32_t x, uint32_t y, uint32_t z,
|
||||
uint32_t& byte_offset);
|
||||
uint32_t& byte_offset);
|
||||
bool TileGetBlockXor(const TileBlockLayout& layout, uint32_t block_x, uint32_t block_y,
|
||||
uint32_t& byte_offset);
|
||||
uint32_t& byte_offset);
|
||||
bool TileGetBlockXor(const TileBlockLayout& layout, uint32_t block_x, uint32_t block_y,
|
||||
uint32_t block_z, uint32_t& byte_offset);
|
||||
uint32_t block_z, uint32_t& byte_offset);
|
||||
bool TileIsStandard256BTextureSupported(uint32_t format);
|
||||
bool TileIsStandard4KBTextureSupported(uint32_t format);
|
||||
bool TileIsStandard64KBTextureSupported(uint32_t format);
|
||||
bool TileGetTextureVolumeLayout(uint32_t format, uint32_t width, uint32_t height, uint32_t depth,
|
||||
uint32_t levels, uint32_t tile, TileVolumeLayout& layout);
|
||||
uint32_t levels, uint32_t tile, TileVolumeLayout& layout);
|
||||
|
||||
bool TileGetHtileSize(uint32_t width, uint32_t height, TileSizeAlign& htile_size);
|
||||
bool TileGetDepthSize(uint32_t width, uint32_t height, uint32_t pitch, uint32_t z_format,
|
||||
uint32_t stencil_format, bool htile, TileSizeAlign& stencil_size,
|
||||
TileSizeAlign& htile_size, TileSizeAlign& depth_size,
|
||||
uint32_t stencil_format, bool htile, TileSizeAlign& stencil_size,
|
||||
TileSizeAlign& htile_size, TileSizeAlign& depth_size,
|
||||
uint32_t num_fragments_log2 = 0);
|
||||
uint32_t TileGetRenderTargetPitch(uint32_t width, uint32_t bytes_per_element,
|
||||
uint32_t num_fragments_log2 = 0);
|
||||
uint32_t TileGetDepthPitch(uint32_t width, uint32_t bytes_per_element,
|
||||
uint32_t num_fragments_log2 = 0);
|
||||
bool TileGetRenderTargetSize(uint32_t width, uint32_t height, uint32_t pitch,
|
||||
uint32_t bytes_per_element, TileSizeAlign& total_size,
|
||||
uint32_t bytes_per_element, TileSizeAlign& total_size,
|
||||
uint32_t num_fragments_log2 = 0);
|
||||
bool TileGetRenderTargetMipLayout(uint32_t width, uint32_t height, uint32_t pitch,
|
||||
uint32_t bytes_per_element, uint32_t levels,
|
||||
TileSizeAlign& total_size, TileSizeOffset* level_sizes,
|
||||
uint32_t bytes_per_element, uint32_t levels,
|
||||
TileSizeAlign& total_size, TileSizeOffset* level_sizes,
|
||||
TilePaddedSize* padded_size);
|
||||
void TileGetTextureSize(uint32_t format, uint32_t width, uint32_t height, uint32_t pitch,
|
||||
uint32_t levels, uint32_t tile, TileSizeAlign* total_size,
|
||||
TileSizeOffset* level_sizes, TilePaddedSize* padded_size);
|
||||
void TileGetTextureTotalSize(uint32_t format, uint32_t width, uint32_t height, uint32_t depth,
|
||||
uint32_t pitch, uint32_t levels, uint32_t tile, bool volume_texture,
|
||||
TileSizeAlign& total_size);
|
||||
TileSizeAlign& total_size);
|
||||
uint32_t TileGetTexturePitch(uint32_t format, uint32_t width, uint32_t levels, uint32_t tile);
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
|
||||
@@ -60,19 +60,19 @@ struct VulkanImage {
|
||||
VulkanImage() = default;
|
||||
KYTY_CLASS_NO_COPY(VulkanImage);
|
||||
|
||||
vk::Format format = vk::Format::eUndefined;
|
||||
vk::ImageType image_type = vk::ImageType::e2D;
|
||||
vk::Extent3D extent = {1, 1, 1};
|
||||
uint32_t guest_pitch = 0;
|
||||
uint32_t layers = 1;
|
||||
uint32_t mip_levels = 1;
|
||||
uint32_t samples = 1;
|
||||
vk::ImageUsageFlags usage = {};
|
||||
vk::ImageCreateFlags flags = {};
|
||||
vk::Image image = nullptr;
|
||||
VulkanImageState state;
|
||||
vk::Format format = vk::Format::eUndefined;
|
||||
vk::ImageType image_type = vk::ImageType::e2D;
|
||||
vk::Extent3D extent = {1, 1, 1};
|
||||
uint32_t guest_pitch = 0;
|
||||
uint32_t layers = 1;
|
||||
uint32_t mip_levels = 1;
|
||||
uint32_t samples = 1;
|
||||
vk::ImageUsageFlags usage = {};
|
||||
vk::ImageCreateFlags flags = {};
|
||||
vk::Image image = nullptr;
|
||||
VulkanImageState state;
|
||||
std::vector<VulkanImageState> subresource_states;
|
||||
Graphics::VulkanMemory memory;
|
||||
Graphics::VulkanMemory memory;
|
||||
};
|
||||
|
||||
struct VulkanBuffer {
|
||||
|
||||
@@ -30,7 +30,7 @@ bool IsAccessible(DWORD protect, HostMemoryAccess access) {
|
||||
} // namespace
|
||||
|
||||
bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAccess access,
|
||||
uint64_t& accessible_size) {
|
||||
uint64_t& accessible_size) {
|
||||
accessible_size = 0;
|
||||
if (addr == 0 || requested_size == 0) {
|
||||
return false;
|
||||
|
||||
@@ -8,7 +8,7 @@ namespace Libs::Graphics {
|
||||
enum class HostMemoryAccess { Read, Mapped };
|
||||
|
||||
bool HostMemoryQueryRange(uint64_t addr, uint64_t requested_size, HostMemoryAccess access,
|
||||
uint64_t& accessible_size);
|
||||
uint64_t& accessible_size);
|
||||
bool HostMemoryQueryReadable(uint64_t addr, uint64_t requested_size, uint64_t& readable_size);
|
||||
bool HostMemoryIsReadable(uint64_t addr);
|
||||
bool HostMemoryRangeIsReadable(uint64_t addr, uint64_t size);
|
||||
|
||||
@@ -4,16 +4,6 @@
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
#if defined(KYTY_MEMORY_TRACKER_TESTS)
|
||||
namespace {
|
||||
std::atomic<MemoryTracker::UnmapContentionHook> g_unmap_contention_hook {nullptr};
|
||||
}
|
||||
|
||||
void MemoryTracker::SetUnmapContentionHook(UnmapContentionHook hook) noexcept {
|
||||
g_unmap_contention_hook.store(hook, std::memory_order_release);
|
||||
}
|
||||
#endif
|
||||
|
||||
static_assert(std::atomic<void*>::is_always_lock_free);
|
||||
|
||||
MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode)
|
||||
@@ -31,6 +21,7 @@ MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_
|
||||
|
||||
MemoryTracker::~MemoryTracker() = default;
|
||||
|
||||
#if KYTY_BUILD == KYTY_BUILD_DEBUG
|
||||
void MemoryTracker::ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
|
||||
const char* operation) const noexcept {
|
||||
if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr ||
|
||||
@@ -68,6 +59,7 @@ void MemoryTracker::ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t va
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
void MemoryTracker::ValidateRange(uint64_t vaddr, uint64_t size) {
|
||||
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
|
||||
@@ -94,7 +86,6 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
|
||||
bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
|
||||
CheckNotInUploadCallback();
|
||||
std::lock_guard access(m_access_mutex);
|
||||
RequireMapped(vaddr, size);
|
||||
return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
std::scoped_lock lock(manager->lock);
|
||||
return manager->IsModified<DirtySource::Cpu>(offset, bytes);
|
||||
@@ -104,7 +95,6 @@ bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
|
||||
bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
|
||||
CheckNotInUploadCallback();
|
||||
std::lock_guard access(m_access_mutex);
|
||||
RequireMapped(vaddr, size);
|
||||
return Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
std::scoped_lock lock(manager->lock);
|
||||
return manager->IsModified<DirtySource::Gpu>(offset, bytes);
|
||||
@@ -114,7 +104,6 @@ bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
|
||||
void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
|
||||
CheckNotInUploadCallback();
|
||||
std::lock_guard access(m_access_mutex);
|
||||
RequireMapped(vaddr, size);
|
||||
Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
std::scoped_lock lock(manager->lock);
|
||||
const auto changed =
|
||||
@@ -126,7 +115,6 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
|
||||
void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
|
||||
CheckNotInUploadCallback();
|
||||
std::lock_guard access(m_access_mutex);
|
||||
RequireMapped(vaddr, size);
|
||||
Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
std::scoped_lock lock(manager->lock);
|
||||
const auto changed =
|
||||
@@ -138,7 +126,6 @@ void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
|
||||
void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
|
||||
CheckNotInUploadCallback();
|
||||
std::lock_guard access(m_access_mutex);
|
||||
RequireMapped(vaddr, size);
|
||||
Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
std::scoped_lock lock(manager->lock);
|
||||
if (!manager->IsFullyModified<DirtySource::Gpu>(offset, bytes)) {
|
||||
@@ -151,8 +138,6 @@ void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
|
||||
}
|
||||
|
||||
void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
|
||||
RequireMapped(vaddr, size);
|
||||
|
||||
std::vector<RegionManager*> managers;
|
||||
managers.reserve((vaddr % TRACKER_REGION_SIZE + size + TRACKER_REGION_SIZE - 1) /
|
||||
TRACKER_REGION_SIZE);
|
||||
@@ -174,7 +159,6 @@ void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
|
||||
const auto changed =
|
||||
manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
|
||||
manager->ApplyProtection(changed, false);
|
||||
manager->Untrack(manager->GetCpuAddr() + offset, bytes);
|
||||
});
|
||||
locks.clear();
|
||||
}
|
||||
@@ -185,109 +169,4 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
|
||||
UntrackMemoryLocked(vaddr, size);
|
||||
}
|
||||
|
||||
void MemoryTracker::UnmapMemory(uint64_t vaddr, uint64_t size) {
|
||||
CheckNotInUploadCallback();
|
||||
std::unique_lock access(m_access_mutex, std::try_to_lock);
|
||||
if (!access.owns_lock()) {
|
||||
#if defined(KYTY_MEMORY_TRACKER_TESTS)
|
||||
if (const auto hook = g_unmap_contention_hook.load(std::memory_order_acquire);
|
||||
hook != nullptr) {
|
||||
hook();
|
||||
}
|
||||
#endif
|
||||
access.lock();
|
||||
}
|
||||
UntrackMemoryLocked(vaddr, size);
|
||||
m_page_manager.OnGpuUnmap(vaddr, size);
|
||||
}
|
||||
|
||||
bool MemoryTracker::InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept {
|
||||
switch (phase) {
|
||||
case PageFaultPhase::Release: return true;
|
||||
case PageFaultPhase::Invalidate: {
|
||||
const auto action = BeginCpuFault(vaddr, size);
|
||||
switch (action) {
|
||||
case CpuFaultAction::Untracked: return false;
|
||||
case CpuFaultAction::Continue: return true;
|
||||
case CpuFaultAction::Download:
|
||||
EXIT("generic region invalidation cannot download GPU-dirty memory\n");
|
||||
}
|
||||
}
|
||||
case PageFaultPhase::Complete:
|
||||
return CompleteCpuFault(vaddr, size, PageFaultAccess::Write, false);
|
||||
}
|
||||
EXIT("unsupported region invalidation phase\n");
|
||||
}
|
||||
|
||||
bool MemoryTracker::InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr, uint64_t size,
|
||||
PageFaultPhase phase) noexcept {
|
||||
switch (phase) {
|
||||
case PageFaultPhase::Release: return true;
|
||||
case PageFaultPhase::Invalidate: {
|
||||
const bool gpu_modified = Iterate<false>(
|
||||
vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
std::scoped_lock lock(manager->lock);
|
||||
return manager->IsModified<DirtySource::Gpu>(offset, bytes);
|
||||
});
|
||||
if (!gpu_modified) {
|
||||
return false;
|
||||
}
|
||||
const auto action = BeginCpuFault(vaddr, size);
|
||||
if (access != PageFaultAccess::Write || action != CpuFaultAction::Download) {
|
||||
EXIT("virtual GPU write fault requires write access to GPU-dirty memory\n");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
case PageFaultPhase::Complete: {
|
||||
if (access != PageFaultAccess::Write) {
|
||||
EXIT("virtual GPU write completion requires write access\n");
|
||||
}
|
||||
bool completed = false;
|
||||
Iterate<false>(
|
||||
vaddr, size, [&completed](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
std::scoped_lock lock(manager->lock);
|
||||
if (completed) {
|
||||
EXIT("virtual GPU write fault spans multiple tracked regions\n");
|
||||
}
|
||||
completed =
|
||||
manager->CompleteVirtualGpuWrite(manager->GetCpuAddr() + offset, bytes);
|
||||
});
|
||||
return completed;
|
||||
}
|
||||
}
|
||||
EXIT("unsupported virtual GPU write invalidation phase\n");
|
||||
}
|
||||
|
||||
CpuFaultAction MemoryTracker::BeginCpuFault(uint64_t vaddr, uint64_t size,
|
||||
PageFaultAccess access) noexcept {
|
||||
CheckNotInUploadCallback();
|
||||
CpuFaultAction action = CpuFaultAction::Untracked;
|
||||
Iterate<false>(
|
||||
vaddr, size, [&action, access](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
std::scoped_lock lock(manager->lock);
|
||||
if (action != CpuFaultAction::Untracked) {
|
||||
EXIT("CPU fault spans multiple tracked regions\n");
|
||||
}
|
||||
action = manager->BeginCpuFault(manager->GetCpuAddr() + offset, bytes, access);
|
||||
});
|
||||
return action;
|
||||
}
|
||||
|
||||
bool MemoryTracker::CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access,
|
||||
bool downloaded) noexcept {
|
||||
CheckNotInUploadCallback();
|
||||
bool found = false;
|
||||
Iterate<false>(
|
||||
vaddr, size,
|
||||
[&found, access, downloaded](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
std::scoped_lock lock(manager->lock);
|
||||
if (found) {
|
||||
EXIT("CPU fault completion spans multiple tracked regions\n");
|
||||
}
|
||||
found = manager->CompleteCpuFault(manager->GetCpuAddr() + offset, bytes, access,
|
||||
downloaded);
|
||||
});
|
||||
return found;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
|
||||
@@ -30,28 +30,63 @@ public:
|
||||
void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
|
||||
void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
|
||||
void UntrackMemory(uint64_t vaddr, uint64_t size);
|
||||
void UnmapMemory(uint64_t vaddr, uint64_t size);
|
||||
[[nodiscard]] CpuFaultAction
|
||||
BeginCpuFault(uint64_t vaddr, uint64_t size,
|
||||
PageFaultAccess access = PageFaultAccess::Write) noexcept;
|
||||
[[nodiscard]] bool CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access,
|
||||
bool downloaded) noexcept;
|
||||
[[nodiscard]] bool InvalidateRegion(uint64_t vaddr, uint64_t size,
|
||||
PageFaultPhase phase) noexcept;
|
||||
[[nodiscard]] bool InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr,
|
||||
uint64_t size, PageFaultPhase phase) noexcept;
|
||||
template <typename Flush>
|
||||
void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) {
|
||||
static_assert(std::is_invocable_v<Flush&>);
|
||||
CheckNotInUploadCallback();
|
||||
ValidateRange(vaddr, size);
|
||||
|
||||
const auto update_cpu_state = [this, vaddr, size] {
|
||||
std::lock_guard access(m_access_mutex);
|
||||
std::vector<RegionManager*> managers;
|
||||
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
|
||||
managers.push_back(manager);
|
||||
});
|
||||
std::vector<std::unique_lock<TrackingSpinLock>> locks;
|
||||
locks.reserve(managers.size());
|
||||
for (auto* manager: managers) {
|
||||
locks.emplace_back(manager->lock);
|
||||
}
|
||||
const bool gpu_modified = Iterate<false>(
|
||||
vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
return manager->IsModified<DirtySource::Gpu>(offset, bytes);
|
||||
});
|
||||
if (gpu_modified) {
|
||||
return true;
|
||||
}
|
||||
Iterate<false>(vaddr, size,
|
||||
[](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
const auto changed = manager->ChangeState<DirtySource::Cpu, true>(
|
||||
manager->GetCpuAddr() + offset, bytes);
|
||||
manager->ApplyProtection(changed, false);
|
||||
});
|
||||
return false;
|
||||
};
|
||||
|
||||
if (!update_cpu_state()) {
|
||||
return;
|
||||
}
|
||||
std::forward<Flush>(on_flush)();
|
||||
if (update_cpu_state()) {
|
||||
EXIT("memory invalidation retained GPU-owned pages\n");
|
||||
}
|
||||
}
|
||||
#if KYTY_BUILD == KYTY_BUILD_DEBUG
|
||||
void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
|
||||
const char* operation) const noexcept;
|
||||
void ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
|
||||
const char* operation);
|
||||
#else
|
||||
void ValidateGpuDirtyPages(const RangeSet&, uint64_t, uint64_t, const char*) const noexcept {}
|
||||
void ValidateGpuDirtyOwnership(const RangeSet&, uint64_t, uint64_t, const char*) {}
|
||||
#endif
|
||||
|
||||
template <bool clear, typename Preflight, typename Func>
|
||||
void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Preflight&& preflight, Func&& func) {
|
||||
static_assert(std::is_nothrow_invocable_v<Preflight&, uint64_t, uint64_t>);
|
||||
static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
|
||||
CheckNotInUploadCallback();
|
||||
std::lock_guard access(m_access_mutex);
|
||||
RequireMapped(vaddr, size);
|
||||
std::lock_guard access(m_access_mutex);
|
||||
std::vector<RegionManager*> managers;
|
||||
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
|
||||
managers.push_back(manager);
|
||||
@@ -63,9 +98,6 @@ public:
|
||||
}
|
||||
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
const auto address = manager->GetCpuAddr() + offset;
|
||||
if (manager->HasPendingFault(address, bytes)) {
|
||||
EXIT("GPU download synchronization raced a pending CPU fault\n");
|
||||
}
|
||||
manager->template ForEachModifiedRange<DirtySource::Gpu, false>(address, bytes,
|
||||
preflight);
|
||||
});
|
||||
@@ -91,11 +123,6 @@ public:
|
||||
vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward<Func>(func));
|
||||
}
|
||||
|
||||
#if defined(KYTY_MEMORY_TRACKER_TESTS)
|
||||
using UnmapContentionHook = void (*)() noexcept;
|
||||
static void SetUnmapContentionHook(UnmapContentionHook hook) noexcept;
|
||||
#endif
|
||||
|
||||
template <typename RangeFunc, typename UploadFunc>
|
||||
void ForEachUploadRange(uint64_t vaddr, uint64_t size, bool is_written, RangeFunc&& range_func,
|
||||
UploadFunc&& upload_func) {
|
||||
@@ -103,12 +130,10 @@ public:
|
||||
static_assert(std::is_nothrow_invocable_v<UploadFunc&>);
|
||||
CheckNotInUploadCallback();
|
||||
std::unique_lock access(m_access_mutex);
|
||||
RequireMapped(vaddr, size);
|
||||
Iterate<true>(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {});
|
||||
const auto* previous_upload_owner = std::exchange(s_upload_owner, this);
|
||||
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
manager->lock.lock();
|
||||
manager->Track(manager->GetCpuAddr() + offset, bytes);
|
||||
manager->ForEachModifiedRange<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset,
|
||||
bytes, range_func);
|
||||
if (!is_written) {
|
||||
@@ -168,16 +193,8 @@ private:
|
||||
return false;
|
||||
}
|
||||
|
||||
static void ValidateRange(uint64_t vaddr, uint64_t size);
|
||||
void UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
|
||||
void RequireMapped(uint64_t vaddr, uint64_t size) const {
|
||||
ValidateRange(vaddr, size);
|
||||
if (!m_page_manager.IsMapped(vaddr, size)) {
|
||||
EXIT("memory tracker range [0x%llx, 0x%llx) is not mapped\n",
|
||||
static_cast<unsigned long long>(vaddr),
|
||||
static_cast<unsigned long long>(vaddr + size));
|
||||
}
|
||||
}
|
||||
static void ValidateRange(uint64_t vaddr, uint64_t size);
|
||||
void UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
|
||||
RegionManager* GetOrCreateRegion(uint64_t index);
|
||||
|
||||
std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
#include "graphics/host_gpu/pageManager.h"
|
||||
|
||||
#include "graphics/host_gpu/regionDefinitions.h"
|
||||
#include "kernel/memory.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <cstdarg>
|
||||
@@ -19,6 +21,15 @@
|
||||
#include <windows.h>
|
||||
#undef min
|
||||
#undef max
|
||||
#elif defined(__APPLE__)
|
||||
#include <pthread.h>
|
||||
#include <sys/mman.h>
|
||||
#include <unistd.h>
|
||||
#else
|
||||
#include <execinfo.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace Libs::Graphics {
|
||||
@@ -28,19 +39,22 @@ constexpr uint64_t PAGE_SIZE = TRACKER_PAGE_SIZE;
|
||||
constexpr uint64_t REGION_SIZE = TRACKER_REGION_SIZE;
|
||||
constexpr uint64_t ADDRESS_SIZE = TRACKER_ADDRESS_SIZE;
|
||||
constexpr uint64_t REGION_COUNT = ADDRESS_SIZE / REGION_SIZE;
|
||||
|
||||
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
|
||||
// The tracker reuses Win32 memory-protection tags as internal page-state values (on
|
||||
// Windows they come from <windows.h> and are what VirtualQuery returns). Mirror the
|
||||
// canonical Win32 numeric values so the shared state-machine logic is identical.
|
||||
constexpr uint32_t PAGE_NOACCESS = 0x01;
|
||||
constexpr uint32_t PAGE_READONLY = 0x02;
|
||||
constexpr uint32_t PAGE_READWRITE = 0x04;
|
||||
#endif
|
||||
constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE;
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
constexpr uint32_t NO_ACCESS_PROTECTION = PAGE_NOACCESS;
|
||||
constexpr uint32_t READ_ONLY_PROTECTION = PAGE_READONLY;
|
||||
constexpr uint32_t NO_ACCESS_PROTECTION = PAGE_NOACCESS;
|
||||
constexpr uint32_t READ_ONLY_PROTECTION = PAGE_READONLY;
|
||||
constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
|
||||
#else
|
||||
constexpr uint32_t NO_ACCESS_PROTECTION = 0;
|
||||
constexpr uint32_t READ_ONLY_PROTECTION = 1;
|
||||
constexpr uint32_t READ_WRITE_PROTECTION = 2;
|
||||
#endif
|
||||
|
||||
thread_local bool g_in_fault_resolution = false;
|
||||
// Zero is the unknown protection sentinel.
|
||||
constexpr uint32_t UNKNOWN_PROTECTION = 0;
|
||||
|
||||
[[noreturn]] void FailFast(const char* reason = nullptr) noexcept {
|
||||
std::fputs("PageManager fail-fast: ", stderr);
|
||||
@@ -56,6 +70,10 @@ thread_local bool g_in_fault_resolution = false;
|
||||
std::fprintf(stderr, " frame[%u]=0x%016" PRIxPTR " image_rva=0x%016" PRIxPTR "\n", i,
|
||||
address, address >= image_base ? address - image_base : 0);
|
||||
}
|
||||
#elif !defined(__APPLE__)
|
||||
void* frames[16] {};
|
||||
const int frame_count = ::backtrace(frames, static_cast<int>(std::size(frames)));
|
||||
::backtrace_symbols_fd(frames, frame_count, STDERR_FILENO);
|
||||
#endif
|
||||
std::fflush(stderr);
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
@@ -75,11 +93,31 @@ thread_local bool g_in_fault_resolution = false;
|
||||
std::_Exit(322);
|
||||
}
|
||||
|
||||
Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
|
||||
switch (protection) {
|
||||
case NO_ACCESS_PROTECTION: return Common::VirtualMemory::Mode::NoAccess;
|
||||
case READ_ONLY_PROTECTION: return Common::VirtualMemory::Mode::Read;
|
||||
case READ_WRITE_PROTECTION: return Common::VirtualMemory::Mode::ReadWrite;
|
||||
default: Fatal("unmappable protection 0x%08" PRIx32, protection);
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t CurrentThread() noexcept {
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
return GetCurrentThreadId();
|
||||
#elif defined(__APPLE__)
|
||||
return static_cast<uint32_t>(pthread_mach_thread_np(pthread_self()));
|
||||
#elif defined(__linux__)
|
||||
static thread_local const uint32_t tid = [] {
|
||||
const auto raw = static_cast<uint32_t>(::syscall(SYS_gettid));
|
||||
if (raw == 0) {
|
||||
FailFast("gettid returned the reserved zero owner token");
|
||||
}
|
||||
return raw;
|
||||
}();
|
||||
return tid;
|
||||
#else
|
||||
FailFast();
|
||||
FailFast("page tracking thread identity is unsupported on this platform");
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -116,28 +154,41 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
|
||||
|
||||
struct PageManager::Impl {
|
||||
struct PageState {
|
||||
std::atomic_flag lock = ATOMIC_FLAG_INIT;
|
||||
uint32_t mappings = 0;
|
||||
uint32_t gpu_read_mappings = 0;
|
||||
uint32_t gpu_write_mappings = 0;
|
||||
uint32_t write_watchers = 0;
|
||||
uint32_t access_watchers = 0;
|
||||
uint32_t original_protection = 0;
|
||||
uint32_t backing_writer = 0;
|
||||
bool resolving = false;
|
||||
bool resolving_read_write = false;
|
||||
bool late_read_pending = false;
|
||||
bool late_write_pending = false;
|
||||
std::atomic_flag lock = ATOMIC_FLAG_INIT;
|
||||
uint32_t write_watchers = 0;
|
||||
uint32_t access_watchers = 0;
|
||||
uint32_t original_protection = 0;
|
||||
uint32_t backing_writer = 0;
|
||||
// Shadow the protection applied through Protect().
|
||||
uint32_t current_protection = UNKNOWN_PROTECTION;
|
||||
bool resolving = false;
|
||||
};
|
||||
|
||||
struct Region {
|
||||
std::array<PageState, REGION_PAGES> pages;
|
||||
};
|
||||
|
||||
Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) {
|
||||
if (fault_handler == nullptr) {
|
||||
Fatal("null fault handler");
|
||||
class PageRangeGuard final {
|
||||
public:
|
||||
explicit PageRangeGuard(std::span<PageState*> pages): m_pages(pages) {
|
||||
for (auto* page: m_pages) {
|
||||
while (page->lock.test_and_set(std::memory_order_acquire)) {
|
||||
std::atomic_signal_fence(std::memory_order_seq_cst);
|
||||
}
|
||||
}
|
||||
}
|
||||
~PageRangeGuard() {
|
||||
for (auto it = m_pages.rbegin(); it != m_pages.rend(); ++it) {
|
||||
(*it)->lock.clear(std::memory_order_release);
|
||||
}
|
||||
}
|
||||
KYTY_CLASS_NO_COPY(PageRangeGuard);
|
||||
|
||||
private:
|
||||
std::span<PageState*> m_pages;
|
||||
};
|
||||
|
||||
Impl() {
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
SYSTEM_INFO info {};
|
||||
GetSystemInfo(&info);
|
||||
@@ -145,8 +196,16 @@ struct PageManager::Impl {
|
||||
Fatal("unsupported host page size 0x%08" PRIx32,
|
||||
static_cast<uint32_t>(info.dwPageSize));
|
||||
}
|
||||
#elif defined(__APPLE__)
|
||||
// Under Rosetta the host page size is 4 KB, matching TRACKER_PAGE_SIZE.
|
||||
if (static_cast<uint64_t>(getpagesize()) != PAGE_SIZE) {
|
||||
Fatal("unsupported host page size 0x%08" PRIx32, static_cast<uint32_t>(getpagesize()));
|
||||
}
|
||||
#else
|
||||
Fatal("page-fault invalidation is not implemented on this platform");
|
||||
const auto host_page_size = ::sysconf(_SC_PAGESIZE);
|
||||
if (host_page_size < 0 || static_cast<uint64_t>(host_page_size) != PAGE_SIZE) {
|
||||
Fatal("unsupported host page size %ld", static_cast<long>(host_page_size));
|
||||
}
|
||||
#endif
|
||||
regions = std::make_unique<std::atomic<Region*>[]>(REGION_COUNT);
|
||||
for (uint64_t i = 0; i < REGION_COUNT; i++) {
|
||||
@@ -158,9 +217,8 @@ struct PageManager::Impl {
|
||||
for (const auto& region: region_storage) {
|
||||
for (auto& page: region->pages) {
|
||||
SpinGuard lock(page.lock);
|
||||
if (page.mappings != 0 || page.gpu_read_mappings != 0 ||
|
||||
page.gpu_write_mappings != 0 || page.write_watchers != 0 ||
|
||||
page.access_watchers != 0 || page.backing_writer != 0 || page.resolving) {
|
||||
if (page.write_watchers != 0 || page.access_watchers != 0 ||
|
||||
page.backing_writer != 0 || page.resolving) {
|
||||
FailFast("PageManager destroyed with live page state");
|
||||
}
|
||||
}
|
||||
@@ -202,300 +260,181 @@ struct PageManager::Impl {
|
||||
return page.original_protection;
|
||||
}
|
||||
|
||||
static void PublishDelayedFaults(PageState& page, uint32_t old_protection,
|
||||
uint32_t new_protection) {
|
||||
if (old_protection == NO_ACCESS_PROTECTION && new_protection != NO_ACCESS_PROTECTION) {
|
||||
page.late_read_pending = true;
|
||||
}
|
||||
if ((old_protection == NO_ACCESS_PROTECTION ||
|
||||
old_protection == READ_ONLY_PROTECTION) &&
|
||||
new_protection == READ_WRITE_PROTECTION) {
|
||||
page.late_write_pending = true;
|
||||
static void InitializeProtection(std::span<PageState*> pages) {
|
||||
for (auto* page: pages) {
|
||||
page->original_protection = READ_WRITE_PROTECTION;
|
||||
page->current_protection = READ_WRITE_PROTECTION;
|
||||
}
|
||||
}
|
||||
|
||||
static uint32_t QueryProtection(uint64_t vaddr) {
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
MEMORY_BASIC_INFORMATION info {};
|
||||
if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
|
||||
sizeof(info)) == 0 ||
|
||||
info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) {
|
||||
Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32
|
||||
", protection=0x%08" PRIx32 ")",
|
||||
vaddr, static_cast<uint32_t>(info.State), static_cast<uint32_t>(info.Protect));
|
||||
void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
|
||||
std::span<const uint32_t> expected_old) noexcept {
|
||||
const auto size = pages.size() * PAGE_SIZE;
|
||||
if (pages.size() != expected_old.size()) {
|
||||
FailFast("protection range state size mismatch");
|
||||
}
|
||||
return info.Protect;
|
||||
#else
|
||||
(void)vaddr;
|
||||
Fatal("page query is unsupported on this platform");
|
||||
#endif
|
||||
}
|
||||
|
||||
static bool AllowsAccess(uint64_t vaddr, PageFaultAccess access) noexcept {
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
MEMORY_BASIC_INFORMATION info {};
|
||||
if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
|
||||
sizeof(info)) == 0 ||
|
||||
info.State != MEM_COMMIT) {
|
||||
return false;
|
||||
}
|
||||
switch (access) {
|
||||
case PageFaultAccess::Read:
|
||||
return info.Protect == PAGE_READONLY || info.Protect == PAGE_READWRITE;
|
||||
case PageFaultAccess::Write: return info.Protect == PAGE_READWRITE;
|
||||
default: return false;
|
||||
}
|
||||
#else
|
||||
(void)vaddr;
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
static void Protect(uint64_t vaddr, uint32_t protection, uint32_t expected_old,
|
||||
bool fault_path) noexcept {
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
DWORD old_protection = 0;
|
||||
if (VirtualProtect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), PAGE_SIZE,
|
||||
protection, &old_protection) == 0 ||
|
||||
old_protection != expected_old) {
|
||||
if (fault_path) {
|
||||
FailFast("VirtualProtect fault transition did not match expected protection");
|
||||
for (size_t i = 0; i < pages.size(); i++) {
|
||||
const auto actual = pages[i]->current_protection;
|
||||
if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) {
|
||||
Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
|
||||
", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
|
||||
vaddr + i * PAGE_SIZE, actual, expected_old[i], protection);
|
||||
}
|
||||
Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
|
||||
", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
|
||||
vaddr, static_cast<uint32_t>(old_protection), expected_old, protection);
|
||||
}
|
||||
#else
|
||||
(void)vaddr;
|
||||
(void)protection;
|
||||
(void)fault_path;
|
||||
FailFast("page protection is unsupported on this platform");
|
||||
#endif
|
||||
if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
|
||||
ToMemoryMode(protection))) {
|
||||
Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
|
||||
protection);
|
||||
}
|
||||
for (auto* page: pages) {
|
||||
page->current_protection = protection;
|
||||
}
|
||||
}
|
||||
|
||||
void Protect(PageState& page, uint64_t vaddr, uint32_t protection,
|
||||
uint32_t expected_old) noexcept {
|
||||
PageState* pages[] = {&page};
|
||||
uint32_t expected[] = {expected_old};
|
||||
ProtectRange(pages, vaddr, protection, expected);
|
||||
}
|
||||
|
||||
std::unique_ptr<std::atomic<Region*>[]> regions;
|
||||
std::vector<std::unique_ptr<Region>> region_storage;
|
||||
std::mutex region_mutex;
|
||||
PageFaultHandler fault_handler = nullptr;
|
||||
void* fault_context = nullptr;
|
||||
};
|
||||
|
||||
static_assert(std::atomic<void*>::is_always_lock_free);
|
||||
|
||||
PageManager::PageManager(PageFaultHandler fault_handler, void* fault_context)
|
||||
: m_impl(std::make_unique<Impl>(fault_handler, fault_context)) {}
|
||||
PageManager::PageManager(): m_impl(std::make_unique<Impl>()) {}
|
||||
|
||||
PageManager::~PageManager() = default;
|
||||
|
||||
uint64_t PageManager::GetPageSize() const {
|
||||
if (g_in_fault_resolution) {
|
||||
FailFast("nested page fault while resolving a watched page");
|
||||
}
|
||||
return PAGE_SIZE;
|
||||
}
|
||||
|
||||
bool PageManager::IsTracked(uint64_t vaddr) const noexcept {
|
||||
if (g_in_fault_resolution) {
|
||||
FailFast("IsTracked called during fault resolution");
|
||||
}
|
||||
auto* region = m_impl->FindRegion(vaddr);
|
||||
if (region == nullptr) {
|
||||
return false;
|
||||
}
|
||||
auto& page = m_impl->GetPage(*region, vaddr);
|
||||
SpinGuard lock(page.lock);
|
||||
return page.write_watchers != 0 || page.access_watchers != 0;
|
||||
}
|
||||
|
||||
bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
|
||||
if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
|
||||
return false;
|
||||
}
|
||||
const auto end = PageStart(vaddr + size - 1) + PAGE_SIZE;
|
||||
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
|
||||
auto* region = m_impl->FindRegion(page_vaddr);
|
||||
if (region == nullptr) {
|
||||
return false;
|
||||
}
|
||||
auto& page = m_impl->GetPage(*region, page_vaddr);
|
||||
SpinGuard lock(page.lock);
|
||||
if (page.mappings == 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PageManager::HasAnyMapping(uint64_t vaddr, uint64_t size) const noexcept {
|
||||
if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE ||
|
||||
size > ADDRESS_SIZE - vaddr) {
|
||||
return false;
|
||||
}
|
||||
const auto end = PageEnd(vaddr, size);
|
||||
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
|
||||
auto* region = m_impl->FindRegion(page_vaddr);
|
||||
if (region == nullptr) {
|
||||
continue;
|
||||
}
|
||||
auto& page = m_impl->GetPage(*region, page_vaddr);
|
||||
SpinGuard lock(page.lock);
|
||||
if (page.mappings != 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept {
|
||||
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
|
||||
FailFast("HasGpuAccess received an invalid GPU access mode");
|
||||
}
|
||||
const bool need_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
|
||||
const bool need_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
|
||||
if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
|
||||
return false;
|
||||
}
|
||||
const auto end = PageEnd(vaddr, size);
|
||||
for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
|
||||
auto* region = m_impl->FindRegion(addr);
|
||||
if (region == nullptr) {
|
||||
return false;
|
||||
}
|
||||
auto& page = m_impl->GetPage(*region, addr);
|
||||
SpinGuard lock(page.lock);
|
||||
if ((need_read && page.gpu_read_mappings == 0) ||
|
||||
(need_write && page.gpu_write_mappings == 0)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
|
||||
PageWatchMode mode) {
|
||||
if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
|
||||
Fatal("invalid watcher mode");
|
||||
}
|
||||
const auto end = PageEnd(vaddr, size);
|
||||
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
|
||||
auto* region =
|
||||
track ? m_impl->GetOrCreateRegion(page_vaddr) : m_impl->FindRegion(page_vaddr);
|
||||
const auto begin = PageStart(vaddr);
|
||||
const auto end = PageEnd(vaddr, size);
|
||||
for (auto chunk_begin = begin; chunk_begin < end;) {
|
||||
const auto chunk_end = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE);
|
||||
auto* region =
|
||||
track ? m_impl->GetOrCreateRegion(chunk_begin) : m_impl->FindRegion(chunk_begin);
|
||||
if (region == nullptr) {
|
||||
Fatal("untracking unknown page 0x%016" PRIx64, page_vaddr);
|
||||
Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin);
|
||||
}
|
||||
auto& page = m_impl->GetPage(*region, page_vaddr);
|
||||
SpinGuard lock(page.lock);
|
||||
if (page.resolving && track) {
|
||||
FailFast("new page watcher raced active fault resolution");
|
||||
|
||||
const auto page_count = static_cast<size_t>((chunk_end - chunk_begin) / PAGE_SIZE);
|
||||
std::vector<Impl::PageState*> pages;
|
||||
pages.reserve(page_count);
|
||||
for (auto address = chunk_begin; address < chunk_end; address += PAGE_SIZE) {
|
||||
pages.push_back(&m_impl->GetPage(*region, address));
|
||||
}
|
||||
if (page.mappings == 0) {
|
||||
Fatal("watching unmapped page 0x%016" PRIx64, page_vaddr);
|
||||
Impl::PageRangeGuard lock(pages);
|
||||
|
||||
std::vector<uint8_t> first_watchers(page_count);
|
||||
for (size_t i = 0; i < page_count; i++) {
|
||||
auto& page = *pages[i];
|
||||
const auto address = chunk_begin + i * PAGE_SIZE;
|
||||
if (page.resolving && track) {
|
||||
FailFast("new page watcher raced active fault resolution");
|
||||
}
|
||||
auto& watchers =
|
||||
(mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
|
||||
if (track) {
|
||||
if (watchers == std::numeric_limits<uint32_t>::max()) {
|
||||
Fatal("watcher overflow at 0x%016" PRIx64, address);
|
||||
}
|
||||
first_watchers[i] = page.write_watchers == 0 && page.access_watchers == 0;
|
||||
} else {
|
||||
if (watchers == 0) {
|
||||
Fatal("watcher underflow at 0x%016" PRIx64, address);
|
||||
}
|
||||
if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) {
|
||||
Fatal("backing write ownership changed at 0x%016" PRIx64, address);
|
||||
}
|
||||
}
|
||||
}
|
||||
auto& watchers =
|
||||
(mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
|
||||
|
||||
if (track) {
|
||||
if (watchers == std::numeric_limits<uint32_t>::max()) {
|
||||
Fatal("watcher overflow at 0x%016" PRIx64, page_vaddr);
|
||||
}
|
||||
const bool first_watcher = page.write_watchers == 0 && page.access_watchers == 0;
|
||||
if (first_watcher) {
|
||||
page.original_protection = Impl::QueryProtection(page_vaddr);
|
||||
for (size_t first = 0; first < page_count;) {
|
||||
while (first < page_count && first_watchers[first] == 0) {
|
||||
first++;
|
||||
}
|
||||
auto last = first;
|
||||
while (last < page_count && first_watchers[last] != 0) {
|
||||
last++;
|
||||
}
|
||||
if (first != last) {
|
||||
Impl::InitializeProtection(std::span {pages}.subspan(first, last - first));
|
||||
}
|
||||
first = last;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<uint32_t> old_protections(page_count);
|
||||
std::vector<uint32_t> new_protections(page_count);
|
||||
std::vector<uint8_t> transitions(page_count);
|
||||
for (size_t i = 0; i < page_count; i++) {
|
||||
auto& page = *pages[i];
|
||||
auto& watchers =
|
||||
(mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
|
||||
const auto old_protection = Impl::WatcherProtection(page);
|
||||
watchers++;
|
||||
const auto new_protection = Impl::WatcherProtection(page);
|
||||
if (new_protection != old_protection) {
|
||||
Impl::Protect(page_vaddr, new_protection, old_protection, false);
|
||||
if (track) {
|
||||
watchers++;
|
||||
} else {
|
||||
watchers--;
|
||||
}
|
||||
switch (new_protection) {
|
||||
case NO_ACCESS_PROTECTION:
|
||||
page.late_read_pending = false;
|
||||
page.late_write_pending = false;
|
||||
const auto new_protection = Impl::WatcherProtection(page);
|
||||
old_protections[i] = old_protection;
|
||||
new_protections[i] = new_protection;
|
||||
if (new_protection != old_protection && (track || page.backing_writer == 0)) {
|
||||
transitions[i] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t first = 0; first < page_count;) {
|
||||
while (first < page_count && transitions[first] == 0) {
|
||||
first++;
|
||||
}
|
||||
if (first == page_count) {
|
||||
break;
|
||||
}
|
||||
const auto protection = new_protections[first];
|
||||
auto current = first + 1;
|
||||
auto last = current;
|
||||
for (; current < page_count && new_protections[current] == protection; current++) {
|
||||
if (old_protections[current] != new_protections[current] &&
|
||||
transitions[current] == 0) {
|
||||
break;
|
||||
case READ_ONLY_PROTECTION: page.late_write_pending = false; break;
|
||||
default: break;
|
||||
}
|
||||
if (transitions[current] != 0) {
|
||||
last = current + 1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (watchers == 0) {
|
||||
Fatal("watcher underflow at 0x%016" PRIx64, page_vaddr);
|
||||
}
|
||||
if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) {
|
||||
Fatal("backing write ownership changed at 0x%016" PRIx64, page_vaddr);
|
||||
}
|
||||
const auto old_protection = Impl::WatcherProtection(page);
|
||||
watchers--;
|
||||
const auto new_protection = Impl::WatcherProtection(page);
|
||||
if (page.backing_writer == 0 && new_protection != old_protection) {
|
||||
Impl::Protect(page_vaddr, new_protection, old_protection, false);
|
||||
}
|
||||
if (page.backing_writer == 0) {
|
||||
Impl::PublishDelayedFaults(page, old_protection, new_protection);
|
||||
}
|
||||
if (page.backing_writer == 0 && page.write_watchers == 0 && page.access_watchers == 0) {
|
||||
page.original_protection = 0;
|
||||
m_impl->ProtectRange(std::span {pages}.subspan(first, last - first),
|
||||
chunk_begin + first * PAGE_SIZE, protection,
|
||||
std::span {old_protections}.subspan(first, last - first));
|
||||
first = current;
|
||||
}
|
||||
|
||||
for (auto* page: pages) {
|
||||
if (!track && page->backing_writer == 0 && page->write_watchers == 0 &&
|
||||
page->access_watchers == 0) {
|
||||
page->original_protection = 0;
|
||||
}
|
||||
}
|
||||
chunk_begin = chunk_end;
|
||||
}
|
||||
}
|
||||
|
||||
void PageManager::OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access) {
|
||||
if (g_in_fault_resolution) {
|
||||
FailFast("GPU mapping changed during fault resolution");
|
||||
}
|
||||
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
|
||||
FailFast("GPU map received an invalid access mode");
|
||||
}
|
||||
const bool gpu_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
|
||||
const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
|
||||
const auto end = PageEnd(vaddr, size);
|
||||
for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
|
||||
auto& page = m_impl->GetPage(*m_impl->GetOrCreateRegion(addr), addr);
|
||||
SpinGuard lock(page.lock);
|
||||
if (page.resolving || page.mappings == std::numeric_limits<uint32_t>::max() ||
|
||||
(gpu_read && page.gpu_read_mappings == std::numeric_limits<uint32_t>::max()) ||
|
||||
(gpu_write && page.gpu_write_mappings == std::numeric_limits<uint32_t>::max())) {
|
||||
Fatal("invalid map state at 0x%016" PRIx64, addr);
|
||||
}
|
||||
page.mappings++;
|
||||
page.gpu_read_mappings += gpu_read ? 1u : 0u;
|
||||
page.gpu_write_mappings += gpu_write ? 1u : 0u;
|
||||
}
|
||||
}
|
||||
void PageManager::OnGpuMap(uint64_t, uint64_t) {}
|
||||
|
||||
void PageManager::OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access) {
|
||||
if (g_in_fault_resolution) {
|
||||
FailFast("GPU unmapping changed during fault resolution");
|
||||
}
|
||||
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
|
||||
FailFast("GPU unmap received an invalid access mode");
|
||||
}
|
||||
const bool gpu_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
|
||||
const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
|
||||
const auto end = PageEnd(vaddr, size);
|
||||
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
|
||||
auto* region = m_impl->FindRegion(page_vaddr);
|
||||
if (region == nullptr) {
|
||||
Fatal("unmapping unknown page 0x%016" PRIx64, page_vaddr);
|
||||
}
|
||||
auto& page = m_impl->GetPage(*region, page_vaddr);
|
||||
SpinGuard lock(page.lock);
|
||||
if (page.resolving || page.mappings == 0 || (gpu_read && page.gpu_read_mappings == 0) ||
|
||||
(gpu_write && page.gpu_write_mappings == 0) ||
|
||||
(page.mappings == 1 && (page.write_watchers != 0 || page.access_watchers != 0))) {
|
||||
Fatal("invalid unmap state at 0x%016" PRIx64, page_vaddr);
|
||||
}
|
||||
page.mappings--;
|
||||
page.gpu_read_mappings -= gpu_read ? 1u : 0u;
|
||||
page.gpu_write_mappings -= gpu_write ? 1u : 0u;
|
||||
if (page.mappings == 0) {
|
||||
if (page.gpu_read_mappings != 0 || page.gpu_write_mappings != 0) {
|
||||
FailFast("GPU unmap left nonzero GPU mapping counts");
|
||||
}
|
||||
page.late_read_pending = false;
|
||||
page.late_write_pending = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
void PageManager::OnGpuUnmap(uint64_t, uint64_t) {}
|
||||
|
||||
PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr,
|
||||
uint64_t size) noexcept
|
||||
@@ -539,9 +478,6 @@ PageManager::ReserveBackingWrites(std::span<const RangeSet::Range> ranges) {
|
||||
}
|
||||
|
||||
void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
|
||||
if (g_in_fault_resolution) {
|
||||
FailFast("backing write began during fault resolution");
|
||||
}
|
||||
const auto end = PageEnd(vaddr, size);
|
||||
const auto writer = CurrentThread();
|
||||
for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) {
|
||||
@@ -551,20 +487,15 @@ void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
|
||||
}
|
||||
auto& page = m_impl->GetPage(*region, address);
|
||||
SpinGuard lock(page.lock);
|
||||
if (page.mappings == 0 || page.resolving || page.backing_writer != 0 ||
|
||||
page.access_watchers == 0) {
|
||||
if (page.resolving || page.backing_writer != 0 || page.access_watchers == 0) {
|
||||
Fatal("backing write races page resolution at 0x%016" PRIx64, address);
|
||||
}
|
||||
page.resolving = true;
|
||||
page.resolving_read_write = true;
|
||||
page.backing_writer = writer;
|
||||
page.resolving = true;
|
||||
page.backing_writer = writer;
|
||||
}
|
||||
}
|
||||
|
||||
void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
|
||||
if (g_in_fault_resolution) {
|
||||
FailFast("backing write ended during fault resolution");
|
||||
}
|
||||
const auto end = PageEnd(vaddr, size);
|
||||
const auto writer = CurrentThread();
|
||||
for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) {
|
||||
@@ -580,144 +511,14 @@ void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
|
||||
const auto old_protection = NO_ACCESS_PROTECTION;
|
||||
const auto new_protection = Impl::WatcherProtection(page);
|
||||
if (new_protection != old_protection) {
|
||||
Impl::Protect(address, new_protection, old_protection, false);
|
||||
m_impl->Protect(page, address, new_protection, old_protection);
|
||||
}
|
||||
Impl::PublishDelayedFaults(page, old_protection, new_protection);
|
||||
if (page.write_watchers == 0 && page.access_watchers == 0) {
|
||||
page.original_protection = 0;
|
||||
}
|
||||
page.backing_writer = 0;
|
||||
page.resolving = false;
|
||||
page.resolving_read_write = false;
|
||||
page.backing_writer = 0;
|
||||
page.resolving = false;
|
||||
}
|
||||
}
|
||||
|
||||
bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
|
||||
if (g_in_fault_resolution) {
|
||||
FailFast("nested HandleFault call");
|
||||
}
|
||||
auto* region = m_impl->FindRegion(fault_vaddr);
|
||||
if (region == nullptr) {
|
||||
return false;
|
||||
}
|
||||
auto& page = m_impl->GetPage(*region, fault_vaddr);
|
||||
bool waited = false;
|
||||
while (true) {
|
||||
SpinGuard lock(page.lock);
|
||||
if (access == PageFaultAccess::Read && page.late_read_pending &&
|
||||
Impl::AllowsAccess(fault_vaddr, access)) {
|
||||
page.late_read_pending = false;
|
||||
return true;
|
||||
}
|
||||
if (access == PageFaultAccess::Write && page.late_write_pending &&
|
||||
Impl::AllowsAccess(fault_vaddr, access)) {
|
||||
page.late_write_pending = false;
|
||||
return true;
|
||||
}
|
||||
if (page.resolving) {
|
||||
if (page.backing_writer == CurrentThread()) {
|
||||
FailFast("backing writer faulted on its own reserved page");
|
||||
}
|
||||
if ((!page.resolving_read_write && access != PageFaultAccess::Write) ||
|
||||
(page.resolving_read_write && access != PageFaultAccess::Read &&
|
||||
access != PageFaultAccess::Write)) {
|
||||
FailFast("fault access is incompatible with the active resolver");
|
||||
}
|
||||
waited = true;
|
||||
continue;
|
||||
}
|
||||
if (page.write_watchers == 0 && page.access_watchers == 0) {
|
||||
if (access != PageFaultAccess::Read && access != PageFaultAccess::Write) {
|
||||
return false;
|
||||
}
|
||||
bool& pending = (access == PageFaultAccess::Read ? page.late_read_pending
|
||||
: page.late_write_pending);
|
||||
const bool allowed = Impl::AllowsAccess(fault_vaddr, access);
|
||||
pending = false;
|
||||
if (waited && !allowed) {
|
||||
FailFast("page remained inaccessible after waiting for its resolver");
|
||||
}
|
||||
// More than one CPU can fault before a protection transition becomes visible. The first
|
||||
// delayed fault consumes the hint bit; later faults must also resume once the mapped
|
||||
// page already permits the requested access. A genuinely read-only/no-access page still
|
||||
// falls through to the guest exception path.
|
||||
return allowed;
|
||||
}
|
||||
if ((access != PageFaultAccess::Read && access != PageFaultAccess::Write) ||
|
||||
(access == PageFaultAccess::Read && page.access_watchers == 0)) {
|
||||
FailFast("fault access is incompatible with active page watchers");
|
||||
}
|
||||
page.resolving = true;
|
||||
page.resolving_read_write = page.access_watchers != 0;
|
||||
break;
|
||||
}
|
||||
g_in_fault_resolution = true;
|
||||
const bool handled = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1,
|
||||
PageFaultPhase::Invalidate);
|
||||
g_in_fault_resolution = false;
|
||||
{
|
||||
SpinGuard lock(page.lock);
|
||||
if (!handled || !page.resolving) {
|
||||
FailFast("fault invalidation did not preserve the resolving state");
|
||||
}
|
||||
}
|
||||
g_in_fault_resolution = true;
|
||||
const bool completed = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1,
|
||||
PageFaultPhase::Complete);
|
||||
g_in_fault_resolution = false;
|
||||
{
|
||||
SpinGuard lock(page.lock);
|
||||
if (!completed || !page.resolving) {
|
||||
FailFast("fault completion did not preserve the resolving state");
|
||||
}
|
||||
if (page.write_watchers != 0 || page.access_watchers != 0) {
|
||||
const auto old_protection = Impl::WatcherProtection(page);
|
||||
const bool read_only_fault = access == PageFaultAccess::Read;
|
||||
if (read_only_fault && page.access_watchers == 0) {
|
||||
FailFast("read fault completed without a read/write watcher");
|
||||
}
|
||||
page.access_watchers = 0;
|
||||
if (!read_only_fault) {
|
||||
page.write_watchers = 0;
|
||||
}
|
||||
const auto restored_protection = Impl::WatcherProtection(page);
|
||||
Impl::Protect(PageStart(fault_vaddr), restored_protection, old_protection, true);
|
||||
if (page.write_watchers == 0) {
|
||||
page.original_protection = 0;
|
||||
}
|
||||
Impl::PublishDelayedFaults(page, old_protection, restored_protection);
|
||||
} else if (!Impl::AllowsAccess(fault_vaddr, access)) {
|
||||
FailFast("fault completion left the page inaccessible");
|
||||
}
|
||||
page.resolving = false;
|
||||
page.resolving_read_write = false;
|
||||
}
|
||||
g_in_fault_resolution = true;
|
||||
const bool released = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1,
|
||||
PageFaultPhase::Release);
|
||||
g_in_fault_resolution = false;
|
||||
if (!released) {
|
||||
FailFast("fault release callback failed");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PageManager::HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept {
|
||||
if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE ||
|
||||
size > ADDRESS_SIZE - vaddr) {
|
||||
return false;
|
||||
}
|
||||
const auto end = PageEnd(vaddr, size);
|
||||
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
|
||||
if (!IsMapped(page_vaddr, 1)) {
|
||||
continue;
|
||||
}
|
||||
const auto fault_vaddr = std::max(page_vaddr, vaddr);
|
||||
if (!HandleFault(PageFaultAccess::Write, fault_vaddr)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
|
||||
@@ -11,12 +11,7 @@
|
||||
namespace Libs::Graphics {
|
||||
|
||||
enum class PageFaultAccess { Read, Write, Execute, Unknown };
|
||||
enum class PageFaultPhase { Invalidate, Complete, Release };
|
||||
enum class PageWatchMode { Write, ReadWrite };
|
||||
enum class GpuAccess { Read, Write, ReadWrite };
|
||||
|
||||
using PageFaultHandler = bool (*)(void* context, PageFaultAccess access, uint64_t vaddr,
|
||||
uint64_t size, PageFaultPhase phase) noexcept;
|
||||
|
||||
class PageManager final {
|
||||
public:
|
||||
@@ -32,27 +27,21 @@ public:
|
||||
uint64_t m_size = 0;
|
||||
};
|
||||
|
||||
PageManager(PageFaultHandler fault_handler, void* fault_context);
|
||||
PageManager();
|
||||
// The owner must stop all PageManager callers before destruction.
|
||||
~PageManager();
|
||||
|
||||
KYTY_CLASS_NO_COPY(PageManager);
|
||||
|
||||
[[nodiscard]] uint64_t GetPageSize() const;
|
||||
[[nodiscard]] bool IsTracked(uint64_t vaddr) const noexcept;
|
||||
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
|
||||
[[nodiscard]] bool HasAnyMapping(uint64_t vaddr, uint64_t size) const noexcept;
|
||||
[[nodiscard]] bool HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept;
|
||||
|
||||
void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
|
||||
PageWatchMode mode = PageWatchMode::Write);
|
||||
void OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
|
||||
void OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
|
||||
void OnGpuMap(uint64_t vaddr, uint64_t size);
|
||||
void OnGpuUnmap(uint64_t vaddr, uint64_t size);
|
||||
|
||||
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
|
||||
[[nodiscard]] bool HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept;
|
||||
[[nodiscard]] std::vector<std::unique_ptr<BackingWrite>>
|
||||
ReserveBackingWrites(std::span<const RangeSet::Range> ranges);
|
||||
ReserveBackingWrites(std::span<const RangeSet::Range> ranges);
|
||||
|
||||
private:
|
||||
void BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept;
|
||||
|
||||
@@ -59,6 +59,16 @@ public:
|
||||
return result;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool Contains(uint64_t address, uint64_t size) const {
|
||||
const auto end = End(address, size);
|
||||
auto it = m_ranges.upper_bound(address);
|
||||
if (it == m_ranges.begin()) {
|
||||
return false;
|
||||
}
|
||||
--it;
|
||||
return it->first <= address && it->second >= end;
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void ForEachIntersection(uint64_t address, uint64_t size, Func&& func) const {
|
||||
const auto end = End(address, size);
|
||||
|
||||
@@ -16,12 +16,15 @@
|
||||
#include <windows.h>
|
||||
#undef min
|
||||
#undef max
|
||||
#elif defined(__APPLE__)
|
||||
#include <pthread.h>
|
||||
#elif defined(__linux__)
|
||||
#include <sys/syscall.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
enum class CpuFaultAction { Untracked, Continue, Download };
|
||||
|
||||
class TrackingSpinLock final {
|
||||
public:
|
||||
void lock() noexcept {
|
||||
@@ -49,6 +52,12 @@ private:
|
||||
static uint32_t CurrentThread() noexcept {
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
return GetCurrentThreadId();
|
||||
#elif defined(__APPLE__)
|
||||
// mach thread port is a nonzero per-thread id (0 is the "no owner" sentinel).
|
||||
return static_cast<uint32_t>(pthread_mach_thread_np(pthread_self()));
|
||||
#elif defined(__linux__)
|
||||
static thread_local const uint32_t tid = static_cast<uint32_t>(::syscall(SYS_gettid));
|
||||
return tid;
|
||||
#else
|
||||
EXIT("region tracking thread identity is unsupported on this platform\n");
|
||||
#endif
|
||||
@@ -74,18 +83,6 @@ public:
|
||||
KYTY_CLASS_NO_COPY(RegionManager);
|
||||
|
||||
[[nodiscard]] uint64_t GetCpuAddr() const { return m_cpu_addr; }
|
||||
void Track(uint64_t vaddr, uint64_t size) {
|
||||
const auto [start, end] = GetPageRange(vaddr, size);
|
||||
for (auto page = start; page < end; page++) {
|
||||
m_tracked.set(page);
|
||||
}
|
||||
}
|
||||
void Untrack(uint64_t vaddr, uint64_t size) {
|
||||
const auto [start, end] = GetPageRange(vaddr, size);
|
||||
for (auto page = start; page < end; page++) {
|
||||
m_tracked.reset(page);
|
||||
}
|
||||
}
|
||||
template <DirtySource source>
|
||||
[[nodiscard]] bool IsModified(uint64_t offset, uint64_t size) const {
|
||||
const auto [start, end] = GetPageRange(m_cpu_addr + offset, size);
|
||||
@@ -115,15 +112,15 @@ public:
|
||||
const auto [start, end] = GetPageRange(vaddr, size);
|
||||
if constexpr (source == DirtySource::Cpu && enable) {
|
||||
for (auto page = start; page < end; page++) {
|
||||
if (m_gpu_dirty.test(page) || m_fault_pending.test(page)) {
|
||||
EXIT("CPU dirty state conflicts with GPU dirty or pending fault state\n");
|
||||
if (m_gpu_dirty.test(page)) {
|
||||
EXIT("CPU dirty state conflicts with GPU dirty state\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
if constexpr (source == DirtySource::Gpu && enable) {
|
||||
for (auto page = start; page < end; page++) {
|
||||
if (m_cpu_dirty.test(page) || m_fault_pending.test(page)) {
|
||||
EXIT("GPU dirty state conflicts with CPU dirty or pending fault state\n");
|
||||
if (m_cpu_dirty.test(page)) {
|
||||
EXIT("GPU dirty state conflicts with CPU dirty state\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -140,110 +137,10 @@ public:
|
||||
return changed;
|
||||
}
|
||||
|
||||
[[nodiscard]] CpuFaultAction BeginCpuFault(uint64_t vaddr, uint64_t size,
|
||||
PageFaultAccess access = PageFaultAccess::Write) {
|
||||
if (access != PageFaultAccess::Read && access != PageFaultAccess::Write) {
|
||||
EXIT("unsupported CPU fault access while beginning ownership transfer\n");
|
||||
}
|
||||
const auto [start, end] = GetPageRange(vaddr, size);
|
||||
const bool tracked = m_tracked.test(start);
|
||||
for (auto page = start; page < end; page++) {
|
||||
if (m_tracked.test(page) != tracked) {
|
||||
EXIT("CPU fault spans mixed tracked and untracked pages\n");
|
||||
}
|
||||
if (m_fault_pending.test(page)) {
|
||||
return CpuFaultAction::Untracked;
|
||||
}
|
||||
if (m_cpu_dirty.test(page) != m_writable.test(page) ||
|
||||
(m_gpu_dirty.test(page) && (m_cpu_dirty.test(page) || m_writable.test(page)))) {
|
||||
EXIT("inconsistent CPU fault page state\n");
|
||||
}
|
||||
}
|
||||
if (!tracked) {
|
||||
return CpuFaultAction::Untracked;
|
||||
}
|
||||
bool gpu_dirty = m_gpu_dirty.test(start);
|
||||
bool writable = m_writable.test(start);
|
||||
for (auto page = start + 1; page < end; page++) {
|
||||
if (m_gpu_dirty.test(page) != gpu_dirty || m_writable.test(page) != writable) {
|
||||
EXIT("CPU fault spans pages with incompatible dirty or writable state\n");
|
||||
}
|
||||
}
|
||||
for (auto page = start; page < end; page++) {
|
||||
if (!gpu_dirty && access == PageFaultAccess::Write) {
|
||||
m_cpu_dirty.set(page);
|
||||
m_writable.set(page);
|
||||
}
|
||||
m_fault_pending.set(page);
|
||||
}
|
||||
return gpu_dirty ? CpuFaultAction::Download : CpuFaultAction::Continue;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access,
|
||||
bool downloaded) {
|
||||
const auto [start, end] = GetPageRange(vaddr, size);
|
||||
for (auto page = start; page < end; page++) {
|
||||
if (!m_fault_pending.test(page)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (auto page = start; page < end; page++) {
|
||||
const bool gpu_dirty = m_gpu_dirty.test(page);
|
||||
if (gpu_dirty != downloaded) {
|
||||
EXIT("CPU fault download result disagrees with GPU dirty state\n");
|
||||
}
|
||||
if (gpu_dirty) {
|
||||
m_gpu_dirty.reset(page);
|
||||
switch (access) {
|
||||
case PageFaultAccess::Read: break;
|
||||
case PageFaultAccess::Write:
|
||||
m_cpu_dirty.set(page);
|
||||
m_writable.set(page);
|
||||
break;
|
||||
default: EXIT("unsupported CPU fault access after GPU download\n");
|
||||
}
|
||||
}
|
||||
m_fault_pending.reset(page);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool HasPendingFault(uint64_t vaddr, uint64_t size) const {
|
||||
const auto [start, end] = GetPageRange(vaddr, size);
|
||||
for (auto page = start; page < end; page++) {
|
||||
if (m_fault_pending.test(page)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool CompleteVirtualGpuWrite(uint64_t vaddr, uint64_t size) {
|
||||
const auto [start, end] = GetPageRange(vaddr, size);
|
||||
for (auto page = start; page < end; page++) {
|
||||
if (!m_fault_pending.test(page)) {
|
||||
return false;
|
||||
}
|
||||
if (!m_gpu_dirty.test(page)) {
|
||||
EXIT("virtual GPU write completion found a non-GPU-dirty page\n");
|
||||
}
|
||||
}
|
||||
for (auto page = start; page < end; page++) {
|
||||
m_gpu_dirty.reset(page);
|
||||
m_cpu_dirty.set(page);
|
||||
m_writable.set(page);
|
||||
m_fault_pending.reset(page);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <DirtySource source, bool clear, typename Func>
|
||||
RegionBits ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) {
|
||||
const auto [start, end] = GetPageRange(vaddr, size);
|
||||
auto mask = GetBits<source>();
|
||||
if constexpr (source == DirtySource::Cpu) {
|
||||
mask &= ~m_fault_pending;
|
||||
}
|
||||
for (auto page = 0u; page < start; page++) {
|
||||
mask.reset(page);
|
||||
}
|
||||
@@ -340,8 +237,6 @@ private:
|
||||
RegionBits m_cpu_dirty;
|
||||
RegionBits m_gpu_dirty;
|
||||
RegionBits m_writable;
|
||||
RegionBits m_fault_pending;
|
||||
RegionBits m_tracked;
|
||||
};
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
|
||||
+123
-192
@@ -1,13 +1,13 @@
|
||||
#include "graphics/host_gpu/renderer/bufferCache.h"
|
||||
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/logging/log.h"
|
||||
#include "common/profiler.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
|
||||
#include "graphics/host_gpu/renderer/cache/textureCache.h"
|
||||
#include "graphics/host_gpu/renderer/commandScheduler.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/resourceMutex.h"
|
||||
#include "graphics/host_gpu/renderer/textureCache.h"
|
||||
#include "kernel/memory.h"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -123,24 +123,6 @@ struct BufferCache::RetiredBuffer {
|
||||
std::shared_ptr<Buffer> owner;
|
||||
};
|
||||
|
||||
struct BufferCache::FaultReadback {
|
||||
PageFaultAccess access = PageFaultAccess::Unknown;
|
||||
uint64_t vaddr = 0;
|
||||
uint64_t size = 0;
|
||||
std::vector<DownloadRange> ranges;
|
||||
bool installed = false;
|
||||
|
||||
[[nodiscard]] bool Active() const noexcept { return !ranges.empty(); }
|
||||
|
||||
void Reset() {
|
||||
access = PageFaultAccess::Unknown;
|
||||
vaddr = 0;
|
||||
size = 0;
|
||||
installed = false;
|
||||
ranges.clear();
|
||||
}
|
||||
};
|
||||
|
||||
struct BufferCache::PendingBackingPublication {
|
||||
uint64_t address = 0;
|
||||
uint64_t size = 0;
|
||||
@@ -183,9 +165,8 @@ BufferCache::RecordDownloads(std::span<const DownloadCopy> copies) {
|
||||
return {};
|
||||
}
|
||||
|
||||
auto& download = m_download_buffer;
|
||||
const auto [mapped, base_offset] =
|
||||
download.Map(reservation_size, DOWNLOAD_ALIGNMENT);
|
||||
auto& download = m_download_buffer;
|
||||
const auto [mapped, base_offset] = download.Map(reservation_size, DOWNLOAD_ALIGNMENT);
|
||||
if (mapped == nullptr) {
|
||||
EXIT("BufferCache: download batch could not reserve the shared stream\n");
|
||||
}
|
||||
@@ -215,40 +196,38 @@ void BufferCache::PublishDownloads(std::span<const DownloadRange> downloads) {
|
||||
}
|
||||
}
|
||||
|
||||
void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies,
|
||||
RetiredBuffer retire) {
|
||||
void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire) {
|
||||
if (copies.empty()) {
|
||||
return;
|
||||
}
|
||||
auto downloads = RecordDownloads(copies);
|
||||
const auto tick = m_scheduler.CurrentTick();
|
||||
auto downloads = RecordDownloads(copies);
|
||||
const auto tick = m_scheduler.CurrentTick();
|
||||
BeginBackingPublication(retire.address, retire.size, tick);
|
||||
m_scheduler.DeferOperation(
|
||||
[this, downloads = std::move(downloads), retire = std::move(retire), tick]() mutable {
|
||||
PublishDownloads(downloads);
|
||||
{
|
||||
FaultSafeCacheLock lock(this, m_mutex);
|
||||
if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
|
||||
m_memory_tracker.ForEachDownloadRange<true>(
|
||||
retire.address, retire.size,
|
||||
[&](uint64_t address, uint64_t size) noexcept {
|
||||
m_memory_tracker.ValidateGpuDirtyPages(
|
||||
m_gpu_modified_ranges, address, size,
|
||||
"asynchronous garbage retirement");
|
||||
},
|
||||
[](uint64_t, uint64_t) noexcept {});
|
||||
}
|
||||
for (const auto& range: downloads) {
|
||||
m_gpu_modified_ranges.Subtract(range.address, range.size);
|
||||
}
|
||||
if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
|
||||
!m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
|
||||
EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
|
||||
}
|
||||
m_memory_tracker.UntrackMemory(retire.address, retire.size);
|
||||
}
|
||||
CompleteBackingPublication(retire.address, retire.size, tick);
|
||||
});
|
||||
m_scheduler.DeferOperation([this, downloads = std::move(downloads), retire = std::move(retire),
|
||||
tick]() mutable {
|
||||
PublishDownloads(downloads);
|
||||
{
|
||||
FaultSafeCacheLock lock(this, m_mutex);
|
||||
if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
|
||||
m_memory_tracker.ForEachDownloadRange<true>(
|
||||
retire.address, retire.size,
|
||||
[&](uint64_t address, uint64_t size) noexcept {
|
||||
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
|
||||
"asynchronous garbage retirement");
|
||||
},
|
||||
[](uint64_t, uint64_t) noexcept {});
|
||||
}
|
||||
for (const auto& range: downloads) {
|
||||
m_gpu_modified_ranges.Subtract(range.address, range.size);
|
||||
}
|
||||
if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
|
||||
!m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
|
||||
EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
|
||||
}
|
||||
m_memory_tracker.UntrackMemory(retire.address, retire.size);
|
||||
}
|
||||
CompleteBackingPublication(retire.address, retire.size, tick);
|
||||
});
|
||||
}
|
||||
|
||||
BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
|
||||
@@ -256,7 +235,7 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
|
||||
ResourceMutex& resource_mutex)
|
||||
: m_graphics(graphics), m_scheduler(scheduler),
|
||||
m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize),
|
||||
m_fault_readback(std::make_unique<FaultReadback>()), m_memory_tracker(page_manager),
|
||||
m_memory_tracker(page_manager),
|
||||
m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB),
|
||||
m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
|
||||
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
|
||||
@@ -280,9 +259,6 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
|
||||
}
|
||||
|
||||
BufferCache::~BufferCache() {
|
||||
if (m_fault_readback->Active()) {
|
||||
EXIT("BufferCache: destroyed with an active fault readback\n");
|
||||
}
|
||||
if (!m_gpu_modified_ranges.Empty()) {
|
||||
EXIT("BufferCache: destroyed with pending GPU-modified ranges\n");
|
||||
}
|
||||
@@ -301,14 +277,13 @@ BufferCache::~BufferCache() {
|
||||
bool BufferCache::SynchronizeBacking(uint64_t vaddr, uint64_t size) {
|
||||
bool waited = false;
|
||||
for (;;) {
|
||||
uint64_t tick = 0;
|
||||
uint64_t tick = 0;
|
||||
const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
|
||||
const auto page_end =
|
||||
(vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
|
||||
const auto page_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
|
||||
CacheRange affected {.address = page_begin, .size = page_end - page_begin};
|
||||
{
|
||||
FaultSafeCacheLock lock(this, m_mutex);
|
||||
bool changed = true;
|
||||
bool changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const auto& [address, cached]: m_buffers) {
|
||||
@@ -384,91 +359,71 @@ BufferBinding BufferCache::UploadTransient(const void* data, uint64_t size, uint
|
||||
return {owner, owner->Handle(), 0};
|
||||
}
|
||||
|
||||
bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
|
||||
PageFaultPhase phase) noexcept {
|
||||
const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1);
|
||||
if (size == 0 || size > page + TRACKER_PAGE_SIZE - vaddr) {
|
||||
EXIT("BufferCache: invalid page-fault range\n");
|
||||
void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
|
||||
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
|
||||
size > TRACKER_ADDRESS_SIZE - vaddr) {
|
||||
EXIT("BufferCache: invalid memory-invalidation range\n");
|
||||
}
|
||||
|
||||
if (phase == PageFaultPhase::Complete) {
|
||||
FaultSafeCacheLock lock(this, m_mutex);
|
||||
auto& fault = *m_fault_readback;
|
||||
if (!fault.Active()) {
|
||||
return m_memory_tracker.CompleteCpuFault(vaddr, size, access, false);
|
||||
}
|
||||
if (fault.access != access || fault.vaddr != vaddr || fault.size != size ||
|
||||
fault.installed) {
|
||||
EXIT("BufferCache: mismatched fault readback completion\n");
|
||||
}
|
||||
PublishDownloads(fault.ranges);
|
||||
if (!m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) {
|
||||
EXIT("BufferCache: failed to complete downloaded CPU fault\n");
|
||||
}
|
||||
fault.installed = true;
|
||||
return true;
|
||||
(void)SynchronizeBacking(vaddr, size);
|
||||
if (!HasPageOverlap(vaddr, size)) {
|
||||
return;
|
||||
}
|
||||
m_memory_tracker.InvalidateRegion(vaddr, size,
|
||||
[this, vaddr, size] { ReadMemory(vaddr, size); });
|
||||
}
|
||||
|
||||
if (phase == PageFaultPhase::Release) {
|
||||
FaultSafeCacheLock lock(this, m_mutex);
|
||||
auto& fault = *m_fault_readback;
|
||||
if (fault.Active()) {
|
||||
if (fault.access != access || fault.vaddr != vaddr || fault.size != size ||
|
||||
!fault.installed) {
|
||||
EXIT("BufferCache: mismatched fault readback release\n");
|
||||
}
|
||||
for (const auto& range: fault.ranges) {
|
||||
m_gpu_modified_ranges.Subtract(range.address, range.size);
|
||||
}
|
||||
fault.Reset();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
if (phase != PageFaultPhase::Invalidate) {
|
||||
EXIT("BufferCache: unsupported page-fault phase\n");
|
||||
}
|
||||
|
||||
const auto action = m_memory_tracker.BeginCpuFault(vaddr, size, access);
|
||||
if (action != CpuFaultAction::Download) {
|
||||
return action == CpuFaultAction::Continue;
|
||||
}
|
||||
|
||||
auto& fault = *m_fault_readback;
|
||||
void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
|
||||
(void)SynchronizeBacking(vaddr, size);
|
||||
std::vector<DownloadCopy> copies;
|
||||
{
|
||||
FaultSafeCacheLock lock(this, m_mutex);
|
||||
if (fault.Active()) {
|
||||
EXIT("BufferCache: nested fault readback\n");
|
||||
}
|
||||
fault.access = access;
|
||||
fault.vaddr = vaddr;
|
||||
fault.size = size;
|
||||
|
||||
m_gpu_modified_ranges.ForEachIntersection(
|
||||
page, TRACKER_PAGE_SIZE, [&](RangeSet::Range range) {
|
||||
auto owner = m_buffers.upper_bound(range.address);
|
||||
if (owner == m_buffers.begin()) {
|
||||
EXIT("BufferCache: fault readback has no buffer owner\n");
|
||||
m_memory_tracker.ForEachDownloadRange<false>(
|
||||
vaddr, size,
|
||||
[&](uint64_t address, uint64_t bytes) noexcept {
|
||||
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
|
||||
"memory invalidation");
|
||||
},
|
||||
[&](uint64_t address, uint64_t bytes) noexcept {
|
||||
for (const auto range: m_gpu_modified_ranges.Intersections(address, bytes)) {
|
||||
for (uint64_t copied = 0; copied < range.size;) {
|
||||
const auto copy_address = range.address + copied;
|
||||
auto owner = m_buffers.upper_bound(copy_address);
|
||||
if (owner == m_buffers.begin()) {
|
||||
EXIT("BufferCache: invalidation readback has no buffer owner\n");
|
||||
}
|
||||
auto& cached = *std::prev(owner)->second;
|
||||
if (!cached.buffer->IsInBounds(copy_address, 1)) {
|
||||
EXIT(
|
||||
"BufferCache: invalidation readback is outside its buffer owner\n");
|
||||
}
|
||||
const auto copy_size = std::min(range.size - copied,
|
||||
cached.vaddr + cached.size - copy_address);
|
||||
copies.push_back({cached.buffer, cached.buffer->Offset(copy_address),
|
||||
copy_address, copy_size});
|
||||
copied += copy_size;
|
||||
}
|
||||
}
|
||||
--owner;
|
||||
auto& cached = *owner->second;
|
||||
if (!cached.buffer->IsInBounds(range.address, range.size)) {
|
||||
EXIT("BufferCache: fault readback is outside its buffer owner\n");
|
||||
}
|
||||
copies.push_back({cached.buffer, cached.buffer->Offset(range.address),
|
||||
range.address, range.size});
|
||||
});
|
||||
if (copies.empty()) {
|
||||
EXIT("BufferCache: GPU-dirty fault page has no dirty byte ranges\n");
|
||||
}
|
||||
if (copies.empty()) {
|
||||
return;
|
||||
}
|
||||
auto downloads = RecordDownloads(copies);
|
||||
m_scheduler.FinishCurrent();
|
||||
PublishDownloads(downloads);
|
||||
{
|
||||
FaultSafeCacheLock lock(this, m_mutex);
|
||||
m_memory_tracker.ForEachDownloadRange<true>(
|
||||
vaddr, size,
|
||||
[&](uint64_t address, uint64_t bytes) noexcept {
|
||||
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
|
||||
"memory invalidation completion");
|
||||
},
|
||||
[](uint64_t, uint64_t) noexcept {});
|
||||
for (const auto& range: downloads) {
|
||||
m_gpu_modified_ranges.Subtract(range.address, range.size);
|
||||
}
|
||||
}
|
||||
fault.ranges = RecordDownloads(copies);
|
||||
if (!fault.Active()) {
|
||||
EXIT("BufferCache: GPU-dirty fault page has no dirty byte ranges\n");
|
||||
}
|
||||
m_scheduler.FinishCurrent();
|
||||
return true;
|
||||
}
|
||||
|
||||
void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
|
||||
@@ -544,8 +499,8 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
|
||||
m_memory_tracker.ForEachDownloadRange<true>(
|
||||
begin, bytes,
|
||||
[&](uint64_t address, uint64_t download_size) noexcept {
|
||||
m_memory_tracker.ValidateGpuDirtyPages(
|
||||
m_gpu_modified_ranges, address, download_size, "unmap retirement");
|
||||
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address,
|
||||
download_size, "unmap retirement");
|
||||
},
|
||||
[](uint64_t, uint64_t) noexcept {});
|
||||
}
|
||||
@@ -651,7 +606,6 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
|
||||
if (command.IsInvalid() || command.IsExecute()) {
|
||||
EXIT("BufferCache: buffer request requires a recording command buffer\n");
|
||||
}
|
||||
ValidateGpuAccess(vaddr, size, is_read, is_written);
|
||||
std::lock_guard transaction(m_resource_mutex);
|
||||
(void)SynchronizeBacking(vaddr, size);
|
||||
|
||||
@@ -722,12 +676,11 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
|
||||
|
||||
{
|
||||
FaultSafeCacheLock lock(this, m_mutex);
|
||||
const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size);
|
||||
const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size);
|
||||
const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size);
|
||||
const bool invalidated =
|
||||
!m_image_invalidated_ranges.Intersections(vaddr, size).empty();
|
||||
const bool requested_gpu_owned = !dirty.empty();
|
||||
const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size);
|
||||
const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size);
|
||||
const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size);
|
||||
const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
|
||||
const bool requested_gpu_owned = !dirty.empty();
|
||||
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
|
||||
"image source");
|
||||
|
||||
@@ -807,8 +760,8 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
|
||||
}
|
||||
|
||||
FaultSafeCacheLock lock(this, m_mutex);
|
||||
const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size);
|
||||
const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
|
||||
const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size);
|
||||
const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
|
||||
const bool requested_gpu_owned = !dirty.empty();
|
||||
auto owner = find_owner();
|
||||
if (requested_gpu_owned && owner == m_buffers.end()) {
|
||||
@@ -831,9 +784,8 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
|
||||
[&]() noexcept {
|
||||
for (const auto& [address, upload_size]: uploads) {
|
||||
cached.buffer->CopyFrom(
|
||||
m_scheduler.Current(), m_staging_buffer,
|
||||
stage_offset + address - stage_address, cached.buffer->Offset(address),
|
||||
upload_size, vk::AccessFlagBits::eHostWrite);
|
||||
m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address,
|
||||
cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
|
||||
}
|
||||
});
|
||||
DiscardGpuDirtyBytesLocked(vaddr, size, "staged image source transfer");
|
||||
@@ -917,9 +869,8 @@ std::pair<std::shared_ptr<Buffer>, uint64_t> BufferCache::ObtainBufferForImageWr
|
||||
[&]() noexcept {
|
||||
for (const auto& [address, upload_size]: uploads) {
|
||||
cached.buffer->CopyFrom(
|
||||
m_scheduler.Current(), m_staging_buffer,
|
||||
stage_offset + address - stage_address, cached.buffer->Offset(address),
|
||||
upload_size, vk::AccessFlagBits::eHostWrite);
|
||||
m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address,
|
||||
cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
|
||||
}
|
||||
});
|
||||
return {cached.buffer, cached.buffer->Offset(vaddr)};
|
||||
@@ -939,19 +890,18 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
|
||||
if (vaddr == 0) {
|
||||
EXIT("BufferCache: invalid fill memory address\n");
|
||||
}
|
||||
ValidateGpuAccess(vaddr, size, false, true);
|
||||
(void)m_texture_cache.ClearMeta(vaddr);
|
||||
{
|
||||
std::lock_guard transaction(m_resource_mutex);
|
||||
const auto region = m_texture_cache.QueryRegion(vaddr, size);
|
||||
if (!HasGpuDirtyBytes(vaddr, size) && !region.gpu_image_bytes) {
|
||||
if (region.image_bytes) {
|
||||
m_texture_cache.PrepareHostWrite(vaddr, size);
|
||||
m_texture_cache.InvalidateMemory(vaddr, size);
|
||||
}
|
||||
std::array<uint32_t, 4096> values;
|
||||
values.fill(value);
|
||||
const std::span<const uint8_t> bytes {
|
||||
reinterpret_cast<const uint8_t*>(values.data()), sizeof(values)};
|
||||
const std::span<const uint8_t> bytes {reinterpret_cast<const uint8_t*>(values.data()),
|
||||
sizeof(values)};
|
||||
for (uint64_t offset = 0; offset < size;) {
|
||||
const auto chunk = std::min<uint64_t>(size - offset, bytes.size());
|
||||
WriteHostMemory(vaddr + offset, bytes.first(chunk));
|
||||
@@ -981,18 +931,12 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
|
||||
(src_gds && (src_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - src_vaddr))) {
|
||||
EXIT("BufferCache: invalid or overlapping copy range\n");
|
||||
}
|
||||
if (src_memory) {
|
||||
ValidateGpuAccess(src_vaddr, size, true, false);
|
||||
}
|
||||
if (dst_memory) {
|
||||
ValidateGpuAccess(dst_vaddr, size, false, true);
|
||||
}
|
||||
if (src_memory || dst_memory) {
|
||||
std::lock_guard transaction(m_resource_mutex);
|
||||
if (src_memory) {
|
||||
(void)SynchronizeBacking(src_vaddr, size);
|
||||
}
|
||||
const auto src_region =
|
||||
const auto src_region =
|
||||
src_memory ? m_texture_cache.QueryRegion(src_vaddr, size) : TextureCache::RegionInfo {};
|
||||
const auto dst_region =
|
||||
dst_memory ? m_texture_cache.QueryRegion(dst_vaddr, size) : TextureCache::RegionInfo {};
|
||||
@@ -1004,7 +948,7 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
|
||||
!HasGpuDirtyBytes(dst_vaddr, size) && !src_region.gpu_image_bytes &&
|
||||
!dst_region.gpu_image_bytes) {
|
||||
if (dst_region.image_bytes) {
|
||||
m_texture_cache.PrepareHostWrite(dst_vaddr, size);
|
||||
m_texture_cache.InvalidateMemory(dst_vaddr, size);
|
||||
}
|
||||
std::array<uint8_t, 64 * 1024> bytes;
|
||||
for (uint64_t offset = 0; offset < size;) {
|
||||
@@ -1037,10 +981,10 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
|
||||
EXIT("BufferCache: resolved Vulkan copy ranges overlap\n");
|
||||
}
|
||||
auto& source = src.owner != nullptr ? *std::static_pointer_cast<Buffer>(src.owner)
|
||||
: src_gds ? m_gds_buffer
|
||||
: m_stream_buffer;
|
||||
auto& destination = dst.owner != nullptr ? *std::static_pointer_cast<Buffer>(dst.owner)
|
||||
: m_gds_buffer;
|
||||
: src_gds ? m_gds_buffer
|
||||
: m_stream_buffer;
|
||||
auto& destination =
|
||||
dst.owner != nullptr ? *std::static_pointer_cast<Buffer>(dst.owner) : m_gds_buffer;
|
||||
if (source.Handle() != src.buffer || destination.Handle() != dst.buffer) {
|
||||
EXIT("BufferCache: resolved copy owner does not match its Vulkan handle\n");
|
||||
}
|
||||
@@ -1107,7 +1051,7 @@ void BufferCache::BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_
|
||||
|
||||
void BufferCache::CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) {
|
||||
std::lock_guard lock(m_publication_mutex);
|
||||
const auto publication =
|
||||
const auto publication =
|
||||
std::ranges::find_if(m_pending_backing_publications, [&](const auto& pending) {
|
||||
return pending.address == vaddr && pending.size == size && pending.tick == tick;
|
||||
});
|
||||
@@ -1143,26 +1087,13 @@ void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) {
|
||||
owner->second->tick_accessed_last = m_gc_tick;
|
||||
}
|
||||
|
||||
void BufferCache::ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read,
|
||||
bool is_written) const {
|
||||
if ((!is_read && !is_written) || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
|
||||
EXIT("BufferCache: invalid GPU access request\n");
|
||||
}
|
||||
if (is_read && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Read)) {
|
||||
EXIT("BufferCache: GPU-read access denied\n");
|
||||
}
|
||||
if (is_written && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Write)) {
|
||||
EXIT("BufferCache: GPU-write access denied\n");
|
||||
}
|
||||
}
|
||||
|
||||
void BufferCache::RunGarbageCollector() {
|
||||
std::lock_guard transaction(m_resource_mutex);
|
||||
const auto tick = m_gc_tick++;
|
||||
if (m_graphics.CanReportMemoryUsage()) {
|
||||
m_total_used_memory = m_graphics.GetDeviceMemoryUsage();
|
||||
}
|
||||
if (m_total_used_memory < m_trigger_gc_memory || m_fault_readback->Active()) {
|
||||
if (m_total_used_memory < m_trigger_gc_memory) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1170,7 +1101,7 @@ void BufferCache::RunGarbageCollector() {
|
||||
const uint64_t age = std::min<uint64_t>(aggressive ? 80 : 160, tick);
|
||||
const size_t limit = aggressive ? 64 : 32;
|
||||
|
||||
std::vector<RetiredBuffer> retires;
|
||||
std::vector<RetiredBuffer> retires;
|
||||
std::vector<std::pair<RetiredBuffer, std::vector<DownloadCopy>>> dirty_retires;
|
||||
{
|
||||
FaultSafeCacheLock lock(this, m_mutex);
|
||||
@@ -1190,8 +1121,8 @@ void BufferCache::RunGarbageCollector() {
|
||||
}
|
||||
for (const auto address: candidates) {
|
||||
auto& cached = *m_buffers.at(address);
|
||||
m_memory_tracker.ValidateGpuDirtyOwnership(
|
||||
m_gpu_modified_ranges, cached.vaddr, cached.size, "garbage collection");
|
||||
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, cached.vaddr,
|
||||
cached.size, "garbage collection");
|
||||
retires.push_back({address, cached.size, cached.buffer});
|
||||
// GC runs immediately before submission. Preserve every source referenced by commands
|
||||
// already recorded in the active batch.
|
||||
@@ -1205,13 +1136,13 @@ void BufferCache::RunGarbageCollector() {
|
||||
m_memory_tracker.ForEachDownloadRange<false>(
|
||||
retire.address, retire.size,
|
||||
[&](uint64_t address, uint64_t size) noexcept {
|
||||
m_memory_tracker.ValidateGpuDirtyPages(
|
||||
m_gpu_modified_ranges, address, size, "garbage collection");
|
||||
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
|
||||
"garbage collection");
|
||||
},
|
||||
[&](uint64_t address, uint64_t size) noexcept {
|
||||
for (const auto range: m_gpu_modified_ranges.Intersections(address, size)) {
|
||||
copies.push_back({retire.owner, range.address - retire.address, range.address,
|
||||
range.size});
|
||||
copies.push_back({retire.owner, range.address - retire.address,
|
||||
range.address, range.size});
|
||||
}
|
||||
});
|
||||
}
|
||||
Vendored
+14
-19
@@ -6,7 +6,7 @@
|
||||
#include "common/threads.h"
|
||||
#include "graphics/host_gpu/memoryTracker.h"
|
||||
#include "graphics/host_gpu/rangeSet.h"
|
||||
#include "graphics/host_gpu/renderer/streamBuffer.h"
|
||||
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
@@ -47,9 +47,9 @@ public:
|
||||
~BufferCache();
|
||||
KYTY_CLASS_NO_COPY(BufferCache);
|
||||
|
||||
[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
|
||||
PageFaultPhase phase) noexcept;
|
||||
void UnmapMemory(uint64_t vaddr, uint64_t size);
|
||||
void InvalidateMemory(uint64_t vaddr, uint64_t size);
|
||||
void ReadMemory(uint64_t vaddr, uint64_t size);
|
||||
void UnmapMemory(uint64_t vaddr, uint64_t size);
|
||||
[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
|
||||
bool is_written = false, bool is_read = true,
|
||||
bool is_formatted = false);
|
||||
@@ -71,12 +71,11 @@ public:
|
||||
[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
|
||||
[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
|
||||
void InvalidateImageAliases(uint64_t vaddr, uint64_t size);
|
||||
void BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
|
||||
void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
|
||||
void BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
|
||||
void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
|
||||
[[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size);
|
||||
void PublishImageBuffer(uint64_t vaddr, uint64_t size);
|
||||
void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const;
|
||||
void RunGarbageCollector();
|
||||
void RunGarbageCollector();
|
||||
|
||||
private:
|
||||
friend struct BufferCacheTestAccess;
|
||||
@@ -89,25 +88,22 @@ private:
|
||||
struct DownloadCopy;
|
||||
struct DownloadRange;
|
||||
struct RetiredBuffer;
|
||||
struct FaultReadback;
|
||||
struct PendingBackingPublication;
|
||||
static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64;
|
||||
[[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept;
|
||||
[[nodiscard]] static uint64_t AlignUp(uint64_t value);
|
||||
static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64;
|
||||
[[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept;
|
||||
[[nodiscard]] static uint64_t AlignUp(uint64_t value);
|
||||
[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
|
||||
return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
|
||||
}
|
||||
[[nodiscard]] static bool PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right,
|
||||
uint64_t right_size) noexcept;
|
||||
[[nodiscard]] static std::pair<uint64_t, uint64_t>
|
||||
DownloadEnvelope(const DownloadCopy& copy);
|
||||
[[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
|
||||
uint64_t right_size) noexcept;
|
||||
[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
|
||||
[[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
|
||||
void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset,
|
||||
const void* source, uint64_t size);
|
||||
[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
|
||||
uint64_t size);
|
||||
[[nodiscard]] std::vector<DownloadRange>
|
||||
RecordDownloads(std::span<const DownloadCopy> copies);
|
||||
[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
|
||||
void PublishDownloads(std::span<const DownloadRange> downloads);
|
||||
void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
|
||||
void RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, uint64_t vaddr,
|
||||
@@ -121,7 +117,6 @@ private:
|
||||
Common::Mutex m_mutex;
|
||||
std::shared_ptr<Buffer> m_null_buffer;
|
||||
std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
|
||||
std::unique_ptr<FaultReadback> m_fault_readback;
|
||||
RangeSet m_gpu_modified_ranges;
|
||||
RangeSet m_image_invalidated_ranges;
|
||||
std::mutex m_publication_mutex;
|
||||
+44
-57
@@ -1,42 +1,20 @@
|
||||
#include "graphics/host_gpu/renderer/gpuResourceManager.h"
|
||||
#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
|
||||
#include "graphics/guest_gpu/graphicsRun.h"
|
||||
#include "graphics/host_gpu/renderer/commandScheduler.h"
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
|
||||
: m_page_manager(FaultThunk, this),
|
||||
m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex),
|
||||
: m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex),
|
||||
m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache, m_resource_mutex) {}
|
||||
|
||||
GpuResourceManager::~GpuResourceManager() = default;
|
||||
|
||||
bool GpuResourceManager::FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr,
|
||||
uint64_t size, PageFaultPhase phase) noexcept {
|
||||
return static_cast<GpuResourceManager*>(context)->InvalidateMemory(access, vaddr, size, phase);
|
||||
}
|
||||
|
||||
bool GpuResourceManager::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
|
||||
PageFaultPhase phase) noexcept {
|
||||
// Let the authoritative image materialize first. A clean overlapping buffer marks a write
|
||||
// fault CPU-dirty when it begins ownership transfer; doing that before image preflight would
|
||||
// make the image appear to race a real CPU write. Completion and release retain buffer-first
|
||||
// ordering so its pending fault is gone before TextureCache publishes the downloaded backing.
|
||||
if (phase == PageFaultPhase::Invalidate) {
|
||||
const bool image_handled = m_texture_cache.InvalidateMemory(access, vaddr, size, phase);
|
||||
const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase);
|
||||
return buffer_handled || image_handled;
|
||||
}
|
||||
const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase);
|
||||
const bool image_handled = m_texture_cache.InvalidateMemory(access, vaddr, size, phase);
|
||||
return buffer_handled || image_handled;
|
||||
}
|
||||
|
||||
bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
|
||||
if (!m_page_manager.IsMapped(fault_vaddr, 1)) {
|
||||
constexpr uint64_t fault_size = 8;
|
||||
if (!IsMapped(fault_vaddr, fault_size)) {
|
||||
return false;
|
||||
}
|
||||
if (CommandScheduler::InDeferredOperation()) {
|
||||
@@ -47,10 +25,15 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
|
||||
bool handled = false;
|
||||
const auto resolve = [this, access, fault_vaddr, &handled](CommandProcessor& cp) {
|
||||
cp.BeginReadbackTransaction();
|
||||
(void)m_buffer_cache.SynchronizeBacking(fault_vaddr, 1);
|
||||
{
|
||||
ResourceMutex::FaultScope fault(m_resource_mutex);
|
||||
handled = m_page_manager.HandleFault(access, fault_vaddr);
|
||||
if (access == PageFaultAccess::Write) {
|
||||
m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
|
||||
m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
|
||||
} else {
|
||||
m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
|
||||
}
|
||||
handled = true;
|
||||
}
|
||||
cp.EndReadbackTransaction();
|
||||
};
|
||||
@@ -68,58 +51,62 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
|
||||
return handled;
|
||||
}
|
||||
|
||||
void GpuResourceManager::PrepareHostWrite(uint64_t vaddr, uint64_t size) {
|
||||
if (!m_page_manager.HasAnyMapping(vaddr, size)) {
|
||||
return;
|
||||
bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) {
|
||||
if (!IsMapped(vaddr, size)) {
|
||||
return false;
|
||||
}
|
||||
if (CommandScheduler::InDeferredOperation()) {
|
||||
EXIT("unsupported host write from an asynchronous GPU completion, addr=0x%016" PRIx64
|
||||
" size=0x%016" PRIx64 "\n",
|
||||
EXIT("unsupported memory invalidation from an asynchronous GPU completion, "
|
||||
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
|
||||
vaddr, size);
|
||||
}
|
||||
const auto handle_range = [this, vaddr, size] {
|
||||
if (!m_page_manager.HandleWriteRange(vaddr, size)) {
|
||||
EXIT("failed to prepare host write, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
|
||||
vaddr, size);
|
||||
}
|
||||
};
|
||||
const auto resolve = [this, &handle_range](CommandProcessor& cp) {
|
||||
const auto resolve = [this, vaddr, size](CommandProcessor& cp) {
|
||||
cp.BeginReadbackTransaction();
|
||||
{
|
||||
ResourceMutex::FaultScope fault(m_resource_mutex);
|
||||
handle_range();
|
||||
m_buffer_cache.InvalidateMemory(vaddr, size);
|
||||
m_texture_cache.InvalidateMemory(vaddr, size);
|
||||
}
|
||||
cp.EndReadbackTransaction();
|
||||
};
|
||||
if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
|
||||
resolve(*cp);
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
if (m_resource_mutex.IsOwnedByCurrentThread()) {
|
||||
EXIT("unsupported host write from a pre-owned resource transaction, addr=0x%016" PRIx64
|
||||
" size=0x%016" PRIx64 "\n",
|
||||
EXIT("unsupported memory invalidation from a pre-owned resource transaction, "
|
||||
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
|
||||
vaddr, size);
|
||||
}
|
||||
EXIT_IF(m_gpu == nullptr);
|
||||
m_gpu->SendCommandSyncWithProcessor(resolve);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
|
||||
return m_page_manager.IsMapped(vaddr, size);
|
||||
}
|
||||
|
||||
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
|
||||
m_page_manager.OnGpuMap(vaddr, size, access);
|
||||
}
|
||||
|
||||
void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
|
||||
if (!IsMapped(vaddr, size)) {
|
||||
EXIT("cannot unmap an unmapped GPU resource range\n");
|
||||
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
|
||||
size > TRACKER_ADDRESS_SIZE - vaddr) {
|
||||
return false;
|
||||
}
|
||||
const auto unmap = [this, vaddr, size, access] {
|
||||
m_texture_cache.UnmapMemory(vaddr, size);
|
||||
std::shared_lock lock(m_mapped_ranges_mutex);
|
||||
return m_mapped_ranges.Contains(vaddr, size);
|
||||
}
|
||||
|
||||
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
|
||||
{
|
||||
std::lock_guard lock(m_mapped_ranges_mutex);
|
||||
m_mapped_ranges.Add(vaddr, size);
|
||||
}
|
||||
m_page_manager.OnGpuMap(vaddr, size);
|
||||
}
|
||||
|
||||
void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
|
||||
const auto unmap = [this, vaddr, size] {
|
||||
m_buffer_cache.UnmapMemory(vaddr, size);
|
||||
m_page_manager.OnGpuUnmap(vaddr, size, access);
|
||||
m_texture_cache.UnmapMemory(vaddr, size);
|
||||
m_page_manager.OnGpuUnmap(vaddr, size);
|
||||
std::lock_guard lock(m_mapped_ranges_mutex);
|
||||
m_mapped_ranges.Subtract(vaddr, size);
|
||||
};
|
||||
if (m_gpu == nullptr) {
|
||||
if (m_resource_mutex.IsOwnedByCurrentThread()) {
|
||||
+14
-16
@@ -4,11 +4,12 @@
|
||||
#include "common/abi.h"
|
||||
#include "common/common.h"
|
||||
#include "graphics/host_gpu/pageManager.h"
|
||||
#include "graphics/host_gpu/renderer/bufferCache.h"
|
||||
#include "graphics/host_gpu/renderer/resourceMutex.h"
|
||||
#include "graphics/host_gpu/renderer/textureCache.h"
|
||||
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
|
||||
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
|
||||
#include "graphics/host_gpu/renderer/cache/textureCache.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <shared_mutex>
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
@@ -26,23 +27,20 @@ public:
|
||||
void SetGpu(Gpu* gpu) noexcept { m_gpu = gpu; }
|
||||
|
||||
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
|
||||
void PrepareHostWrite(uint64_t vaddr, uint64_t size);
|
||||
[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
|
||||
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
|
||||
void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
|
||||
void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
|
||||
void MapMemory(uint64_t vaddr, uint64_t size);
|
||||
void UnmapMemory(uint64_t vaddr, uint64_t size);
|
||||
void RunGarbageCollector();
|
||||
|
||||
private:
|
||||
static bool FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr, uint64_t size,
|
||||
PageFaultPhase phase) noexcept;
|
||||
[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
|
||||
PageFaultPhase phase) noexcept;
|
||||
|
||||
PageManager m_page_manager;
|
||||
ResourceMutex m_resource_mutex;
|
||||
BufferCache m_buffer_cache;
|
||||
TextureCache m_texture_cache;
|
||||
Gpu* m_gpu = nullptr;
|
||||
PageManager m_page_manager;
|
||||
ResourceMutex m_resource_mutex;
|
||||
BufferCache m_buffer_cache;
|
||||
TextureCache m_texture_cache;
|
||||
mutable std::shared_mutex m_mapped_ranges_mutex;
|
||||
RangeSet m_mapped_ranges;
|
||||
Gpu* m_gpu = nullptr;
|
||||
};
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
#include "graphics/host_gpu/renderer/resourceMutex.h"
|
||||
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
|
||||
+5
-2
@@ -1,4 +1,4 @@
|
||||
#include "graphics/host_gpu/renderer/samplerCache.h"
|
||||
#include "graphics/host_gpu/renderer/cache/samplerCache.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/logging/log.h"
|
||||
@@ -56,7 +56,10 @@ vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) {
|
||||
case Prospero::SamplerAnisoRatio::kFour: aniso_ratio = 4.0f; break;
|
||||
case Prospero::SamplerAnisoRatio::kEight: aniso_ratio = 8.0f; break;
|
||||
case Prospero::SamplerAnisoRatio::kSixteen: aniso_ratio = 16.0f; break;
|
||||
default: EXIT("unknown ratio: %d\n", static_cast<int>(r.MaxAnisoRatio()));
|
||||
default:
|
||||
EXIT("unknown ratio: %d dwords=%08x,%08x,%08x,%08x\n",
|
||||
static_cast<int>(r.MaxAnisoRatio()), r.fields[0], r.fields[1], r.fields[2],
|
||||
r.fields[3]);
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+11
-13
@@ -1,4 +1,4 @@
|
||||
#include "graphics/host_gpu/renderer/streamBuffer.h"
|
||||
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/profiler.h"
|
||||
@@ -135,8 +135,8 @@ void Buffer::Write(uint64_t offset, const void* source, uint64_t size) {
|
||||
void Buffer::Flush(uint64_t offset, uint64_t size) {
|
||||
EXIT_IF(m_mapped.empty() || offset > m_size || size > m_size - offset);
|
||||
if (!m_is_coherent && size != 0) {
|
||||
const auto result = vmaFlushAllocation(m_graphics->allocator, m_buffer->memory.allocation,
|
||||
offset, size);
|
||||
const auto result =
|
||||
vmaFlushAllocation(m_graphics->allocator, m_buffer->memory.allocation, offset, size);
|
||||
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
|
||||
}
|
||||
}
|
||||
@@ -144,8 +144,8 @@ void Buffer::Flush(uint64_t offset, uint64_t size) {
|
||||
vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::AccessFlags source,
|
||||
vk::AccessFlags destination) const {
|
||||
if (Handle() == nullptr || size == 0 || offset > m_size || size > m_size - offset) {
|
||||
EXIT("Buffer: invalid DMA barrier, handle=%p offset=0x%016" PRIx64
|
||||
" size=0x%016" PRIx64 " capacity=0x%016" PRIx64 "\n",
|
||||
EXIT("Buffer: invalid DMA barrier, handle=%p offset=0x%016" PRIx64 " size=0x%016" PRIx64
|
||||
" capacity=0x%016" PRIx64 "\n",
|
||||
static_cast<const void*>(Handle()), offset, size, m_size);
|
||||
}
|
||||
vk::BufferMemoryBarrier barrier {};
|
||||
@@ -175,10 +175,9 @@ void Buffer::CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t sou
|
||||
command.EndRendering();
|
||||
const vk::BufferMemoryBarrier before[] = {
|
||||
source.Barrier(source_offset, size, source_before, vk::AccessFlagBits::eTransferRead),
|
||||
Barrier(destination_offset, size, destination_before,
|
||||
vk::AccessFlagBits::eTransferWrite),
|
||||
Barrier(destination_offset, size, destination_before, vk::AccessFlagBits::eTransferWrite),
|
||||
};
|
||||
const auto host_access = vk::AccessFlagBits::eHostRead | vk::AccessFlagBits::eHostWrite;
|
||||
const auto host_access = vk::AccessFlagBits::eHostRead | vk::AccessFlagBits::eHostWrite;
|
||||
auto before_stage = vk::PipelineStageFlags {vk::PipelineStageFlagBits::eAllCommands};
|
||||
if (static_cast<bool>((source_before | destination_before) & host_access)) {
|
||||
before_stage |= vk::PipelineStageFlagBits::eHost;
|
||||
@@ -214,9 +213,8 @@ void Buffer::Fill(uint64_t offset, uint64_t size, uint32_t value) {
|
||||
vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion,
|
||||
0, nullptr, 1, &before, 0, nullptr);
|
||||
native.fillBuffer(Handle(), offset, size, value);
|
||||
const auto after =
|
||||
Barrier(offset, size, vk::AccessFlagBits::eTransferWrite,
|
||||
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite);
|
||||
const auto after = Barrier(offset, size, vk::AccessFlagBits::eTransferWrite,
|
||||
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite);
|
||||
native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
|
||||
vk::PipelineStageFlagBits::eAllCommands,
|
||||
vk::DependencyFlagBits::eByRegion, 0, nullptr, 1, &after, 0, nullptr);
|
||||
@@ -250,8 +248,8 @@ std::pair<uint8_t*, uint64_t> StreamBuffer::Map(uint64_t size, uint64_t alignmen
|
||||
if (Mapped().empty()) {
|
||||
return {nullptr, 0};
|
||||
}
|
||||
uint64_t mapped_size = size;
|
||||
const auto atom = Graphics().physical_device_properties.limits.nonCoherentAtomSize;
|
||||
uint64_t mapped_size = size;
|
||||
const auto atom = Graphics().physical_device_properties.limits.nonCoherentAtomSize;
|
||||
if (!NormalizeReservation(IsCoherent(), atom, mapped_size, alignment)) {
|
||||
return {nullptr, 0};
|
||||
}
|
||||
Vendored
+14
-15
@@ -54,16 +54,15 @@ public:
|
||||
[[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept;
|
||||
void Write(uint64_t offset, const void* source, uint64_t size);
|
||||
void Flush(uint64_t offset, uint64_t size);
|
||||
void CopyFrom(
|
||||
CommandBuffer& command, const Buffer& source, uint64_t source_offset,
|
||||
uint64_t destination_offset, uint64_t size,
|
||||
vk::AccessFlags source_before = vk::AccessFlagBits::eMemoryWrite,
|
||||
vk::AccessFlags destination_before =
|
||||
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
|
||||
vk::AccessFlags source_after =
|
||||
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
|
||||
vk::AccessFlags destination_after =
|
||||
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite);
|
||||
void CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,
|
||||
uint64_t destination_offset, uint64_t size,
|
||||
vk::AccessFlags source_before = vk::AccessFlagBits::eMemoryWrite,
|
||||
vk::AccessFlags destination_before = vk::AccessFlagBits::eMemoryRead |
|
||||
vk::AccessFlagBits::eMemoryWrite,
|
||||
vk::AccessFlags source_after = vk::AccessFlagBits::eMemoryRead |
|
||||
vk::AccessFlagBits::eMemoryWrite,
|
||||
vk::AccessFlags destination_after = vk::AccessFlagBits::eMemoryRead |
|
||||
vk::AccessFlagBits::eMemoryWrite);
|
||||
void Fill(uint64_t offset, uint64_t size, uint32_t value);
|
||||
|
||||
protected:
|
||||
@@ -107,13 +106,13 @@ private:
|
||||
uint64_t upper_bound = 0;
|
||||
};
|
||||
|
||||
void ReserveWatches(std::vector<Watch>& watches, size_t grow_size);
|
||||
void ReserveWatches(std::vector<Watch>& watches, size_t grow_size);
|
||||
[[nodiscard]] static bool NormalizeReservation(bool coherent, uint64_t atom, uint64_t& size,
|
||||
uint64_t& alignment);
|
||||
[[nodiscard]] bool WaitPendingOperations(const std::vector<Watch>& watches,
|
||||
std::optional<size_t> invalidation_mark,
|
||||
uint64_t requested_upper_bound, bool allow_wait,
|
||||
size_t& wait_cursor, uint64_t& wait_bound);
|
||||
[[nodiscard]] bool WaitPendingOperations(const std::vector<Watch>& watches,
|
||||
std::optional<size_t> invalidation_mark,
|
||||
uint64_t requested_upper_bound, bool allow_wait,
|
||||
size_t& wait_cursor, uint64_t& wait_bound);
|
||||
|
||||
uint64_t m_offset = 0;
|
||||
uint64_t m_mapped_size = 0;
|
||||
+224
-182
@@ -1,4 +1,4 @@
|
||||
#include "graphics/host_gpu/renderer/textureCache.h"
|
||||
#include "graphics/host_gpu/renderer/cache/textureCache.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/emulatorConfig.h"
|
||||
@@ -7,13 +7,13 @@
|
||||
#include "graphics/guest_gpu/gpu_format.h"
|
||||
#include "graphics/guest_gpu/tile.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/objects/textureCommon.h"
|
||||
#include "graphics/host_gpu/renderer/bufferCache.h"
|
||||
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
|
||||
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
|
||||
#include "graphics/host_gpu/renderer/commandScheduler.h"
|
||||
#include "graphics/host_gpu/renderer/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/image/textureCommon.h"
|
||||
#include "graphics/host_gpu/renderer/image/tiler.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/resourceMutex.h"
|
||||
#include "graphics/host_gpu/renderer/tiler.h"
|
||||
#include "kernel/memory.h"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -58,8 +58,8 @@ private:
|
||||
TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler,
|
||||
PageManager& page_manager, BufferCache& buffer_cache,
|
||||
ResourceMutex& resource_mutex)
|
||||
: m_graphics(graphics), m_scheduler(scheduler),
|
||||
m_memory_tracker(page_manager, PageWatchMode::Write), m_blit_helper(graphics, scheduler),
|
||||
: m_graphics(graphics), m_scheduler(scheduler), m_page_manager(page_manager),
|
||||
m_blit_helper(graphics, scheduler),
|
||||
m_tiler(std::make_unique<TileManager>(graphics, scheduler,
|
||||
buffer_cache.GetUtilityBuffer(MemoryUsage::Stream))),
|
||||
m_buffer_cache(buffer_cache), m_resource_mutex(resource_mutex),
|
||||
@@ -80,7 +80,7 @@ TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler
|
||||
TextureCache::~TextureCache() {
|
||||
for (uint32_t index = 0; index < m_slots.size(); index++) {
|
||||
if (m_slots[index].image != nullptr && m_slots[index].image->registered) {
|
||||
UnregisterImage({index, m_slots[index].generation}, false);
|
||||
UnregisterImage({index, m_slots[index].generation});
|
||||
}
|
||||
m_slots[index].image.reset();
|
||||
}
|
||||
@@ -88,15 +88,34 @@ TextureCache::~TextureCache() {
|
||||
|
||||
bool TextureCache::SameBacking(const ImageInfo& cached, const ImageInfo& requested,
|
||||
bool exact_format) {
|
||||
const bool unit_extent =
|
||||
requested.extent.width == 1 && requested.extent.height == 1 && requested.extent.depth == 1;
|
||||
return cached.data == requested.data && cached.extent == requested.extent &&
|
||||
cached.samples == requested.samples &&
|
||||
cached.bytes_per_block == requested.bytes_per_block &&
|
||||
(cached.type == requested.type || unit_extent) &&
|
||||
(exact_format
|
||||
? cached.pixel_format == requested.pixel_format
|
||||
: ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format));
|
||||
if (cached.data.address != requested.data.address) {
|
||||
return false;
|
||||
}
|
||||
if (cached.data.size != requested.data.size) {
|
||||
return false;
|
||||
}
|
||||
if (cached.extent != requested.extent) {
|
||||
return false;
|
||||
}
|
||||
if (cached.samples != requested.samples) {
|
||||
return false;
|
||||
}
|
||||
if (cached.bytes_per_block != requested.bytes_per_block) {
|
||||
return false;
|
||||
}
|
||||
if (cached.tile_mode != requested.tile_mode) {
|
||||
return false;
|
||||
}
|
||||
if (!ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format)) {
|
||||
return false;
|
||||
}
|
||||
if (cached.type != requested.type && requested.extent != vk::Extent3D {1, 1, 1}) {
|
||||
return false;
|
||||
}
|
||||
if (exact_format && cached.pixel_format != requested.pixel_format) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
TextureCache::BindingType TextureCache::UploadBinding(const Image& image) {
|
||||
@@ -117,8 +136,7 @@ bool TextureCache::SafeToDownload(const Image& image) {
|
||||
return false;
|
||||
}
|
||||
const auto range = image.info.data;
|
||||
return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size) &&
|
||||
!m_memory_tracker.IsRegionCpuModified(range.address, range.size);
|
||||
return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size);
|
||||
}
|
||||
|
||||
Image& TextureCache::ResolveImage(ImageId id) {
|
||||
@@ -187,21 +205,17 @@ void TextureCache::RegisterImage(ImageId id) {
|
||||
m_total_used_memory += image.AccountedSize();
|
||||
}
|
||||
|
||||
void TextureCache::UnregisterImage(ImageId id, bool release_tracking) {
|
||||
void TextureCache::UnregisterImage(ImageId id) {
|
||||
auto& image = ResolveImage(id);
|
||||
if (!image.registered) {
|
||||
return;
|
||||
}
|
||||
UntrackImage(id);
|
||||
std::vector<ImageOwnerIndex::ByteRange> releases;
|
||||
if (!m_image_owner_index.Unregister(id, releases)) {
|
||||
EXIT("TextureCache: image missing from owner index\n");
|
||||
}
|
||||
m_lru_cache.Free(image.lru_id);
|
||||
if (release_tracking) {
|
||||
for (const auto& range: releases) {
|
||||
m_memory_tracker.UntrackMemory(range.address, range.size);
|
||||
}
|
||||
}
|
||||
const auto accounted = image.AccountedSize();
|
||||
if (accounted > m_total_used_memory) {
|
||||
EXIT("TextureCache: image accounting underflow\n");
|
||||
@@ -210,7 +224,7 @@ void TextureCache::UnregisterImage(ImageId id, bool release_tracking) {
|
||||
image.registered = false;
|
||||
}
|
||||
|
||||
void TextureCache::DeleteImage(ImageId id, bool release_tracking) {
|
||||
void TextureCache::DeleteImage(ImageId id) {
|
||||
auto owner = ResolveOwner(id);
|
||||
if (owner == nullptr || !owner->registered) {
|
||||
return;
|
||||
@@ -224,7 +238,7 @@ void TextureCache::DeleteImage(ImageId id, bool release_tracking) {
|
||||
}
|
||||
}
|
||||
for (const auto association: associations) {
|
||||
ReleaseGpuTracking(association);
|
||||
ClearGpuModified(association);
|
||||
DeleteImage(association);
|
||||
}
|
||||
}
|
||||
@@ -235,7 +249,7 @@ void TextureCache::DeleteImage(ImageId id, bool release_tracking) {
|
||||
if (owner->info.metadata.kind == ImageMetadataKind::Htile) {
|
||||
m_surface_metas.erase(owner->info.metadata.range.address);
|
||||
}
|
||||
UnregisterImage(id, release_tracking);
|
||||
UnregisterImage(id);
|
||||
const auto erase_slot = [this, id, retained = owner] {
|
||||
auto& slot = m_slots[id.index];
|
||||
if (slot.generation != id.generation || slot.image != retained) {
|
||||
@@ -266,10 +280,10 @@ void TextureCache::DeleteImages(std::span<const ImageId> ids,
|
||||
continue;
|
||||
}
|
||||
if (native_source == id) {
|
||||
ReleaseGpuTracking(id);
|
||||
ClearGpuModified(id);
|
||||
} else if (owner->IsGpuModified()) {
|
||||
DownloadImage(id);
|
||||
ReleaseGpuTracking(id);
|
||||
ClearGpuModified(id);
|
||||
}
|
||||
DeleteImage(id);
|
||||
}
|
||||
@@ -296,6 +310,121 @@ void TextureCache::TouchImage(Image& image) {
|
||||
}
|
||||
}
|
||||
|
||||
void TextureCache::TrackImage(ImageId id) {
|
||||
auto& image = ResolveImage(id);
|
||||
if (!image.registered) {
|
||||
return;
|
||||
}
|
||||
const auto image_begin = image.info.data.address;
|
||||
const auto image_end = image.info.data.End();
|
||||
if (image_begin == image.track_addr && image_end == image.track_addr_end) {
|
||||
return;
|
||||
}
|
||||
if (!image.IsTracked()) {
|
||||
image.track_addr = image_begin;
|
||||
image.track_addr_end = image_end;
|
||||
m_page_manager.UpdatePageWatchers(true, image_begin, image.info.data.size);
|
||||
return;
|
||||
}
|
||||
if (image_begin < image.track_addr) {
|
||||
TrackImageHead(id);
|
||||
}
|
||||
if (image.track_addr_end < image_end) {
|
||||
TrackImageTail(id);
|
||||
}
|
||||
}
|
||||
|
||||
void TextureCache::TrackImageHead(ImageId id) {
|
||||
auto& image = ResolveImage(id);
|
||||
if (!image.registered) {
|
||||
return;
|
||||
}
|
||||
const auto image_begin = image.info.data.address;
|
||||
if (image_begin == image.track_addr) {
|
||||
return;
|
||||
}
|
||||
if (!image.IsTracked() || image_begin > image.track_addr) {
|
||||
EXIT("TextureCache: invalid image head tracking range\n");
|
||||
}
|
||||
const auto size = image.track_addr - image_begin;
|
||||
image.track_addr = image_begin;
|
||||
m_page_manager.UpdatePageWatchers(true, image_begin, size);
|
||||
}
|
||||
|
||||
void TextureCache::TrackImageTail(ImageId id) {
|
||||
auto& image = ResolveImage(id);
|
||||
if (!image.registered) {
|
||||
return;
|
||||
}
|
||||
const auto image_end = image.info.data.End();
|
||||
if (image_end == image.track_addr_end) {
|
||||
return;
|
||||
}
|
||||
if (!image.IsTracked() || image.track_addr_end > image_end) {
|
||||
EXIT("TextureCache: invalid image tail tracking range\n");
|
||||
}
|
||||
const auto address = image.track_addr_end;
|
||||
const auto size = image_end - address;
|
||||
image.track_addr_end = image_end;
|
||||
m_page_manager.UpdatePageWatchers(true, address, size);
|
||||
}
|
||||
|
||||
void TextureCache::UntrackImage(ImageId id) {
|
||||
auto& image = ResolveImage(id);
|
||||
if (!image.IsTracked()) {
|
||||
return;
|
||||
}
|
||||
const auto address = image.track_addr;
|
||||
const auto size = image.track_addr_end - image.track_addr;
|
||||
image.track_addr = 0;
|
||||
image.track_addr_end = 0;
|
||||
if (size != 0) {
|
||||
m_page_manager.UpdatePageWatchers(false, address, size);
|
||||
}
|
||||
}
|
||||
|
||||
void TextureCache::UntrackImageHead(ImageId id) {
|
||||
auto& image = ResolveImage(id);
|
||||
const auto begin = image.info.data.address;
|
||||
if (!image.IsTracked() || begin < image.track_addr) {
|
||||
return;
|
||||
}
|
||||
const auto address = (begin + TRACKER_PAGE_SIZE) & ~(TRACKER_PAGE_SIZE - 1);
|
||||
const auto size = address - begin;
|
||||
image.track_addr = address;
|
||||
if (image.track_addr == image.track_addr_end) {
|
||||
image.MarkMaybeCpuDirty();
|
||||
if (image.NeedsMaybeCpuHash()) {
|
||||
image.SetMaybeCpuHash(image.HashGuestEdges());
|
||||
}
|
||||
UntrackImage(id);
|
||||
}
|
||||
if (size != 0) {
|
||||
m_page_manager.UpdatePageWatchers(false, begin, size);
|
||||
}
|
||||
}
|
||||
|
||||
void TextureCache::UntrackImageTail(ImageId id) {
|
||||
auto& image = ResolveImage(id);
|
||||
const auto end = image.info.data.End();
|
||||
if (!image.IsTracked() || image.track_addr_end < end) {
|
||||
return;
|
||||
}
|
||||
const auto address = end & ~(TRACKER_PAGE_SIZE - 1);
|
||||
const auto size = end - address;
|
||||
image.track_addr_end = address;
|
||||
if (image.track_addr == image.track_addr_end) {
|
||||
image.MarkMaybeCpuDirty();
|
||||
if (image.NeedsMaybeCpuHash()) {
|
||||
image.SetMaybeCpuHash(image.HashGuestEdges());
|
||||
}
|
||||
UntrackImage(id);
|
||||
}
|
||||
if (size != 0) {
|
||||
m_page_manager.UpdatePageWatchers(false, address, size);
|
||||
}
|
||||
}
|
||||
|
||||
void TextureCache::TrackImageDownload(ImageId id) {
|
||||
std::lock_guard transaction(m_resource_mutex);
|
||||
CacheLock lock(*this, m_lock);
|
||||
@@ -376,9 +505,6 @@ void TextureCache::ValidateImageDesc(const ImageDesc& desc) const {
|
||||
}
|
||||
|
||||
void TextureCache::PrepareImageCopy(Image& image) {
|
||||
const auto range = image.info.data;
|
||||
m_memory_tracker.ForEachUploadRange(
|
||||
range.address, range.size, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
|
||||
if (image.IsCpuDirty()) {
|
||||
image.RefreshComplete();
|
||||
}
|
||||
@@ -471,6 +597,7 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
|
||||
RefreshCopySource(source_id);
|
||||
auto& destination = ResolveImage(destination_id);
|
||||
auto& source = ResolveImage(source_id);
|
||||
TrackImage(destination_id);
|
||||
if (source.backing.samples != destination.backing.samples) {
|
||||
EXIT("TextureCache: cannot issue an unequal-sample image copy\n");
|
||||
}
|
||||
@@ -479,7 +606,6 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
|
||||
if (source.info.data == destination.info.data) {
|
||||
destination.MarkBufferModified();
|
||||
}
|
||||
RestoreGpuTracking(destination);
|
||||
return;
|
||||
}
|
||||
const bool source_depth = source.info.IsDepth();
|
||||
@@ -503,7 +629,6 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
|
||||
destination.MarkGpuModified();
|
||||
}
|
||||
destination.ClearBufferModified();
|
||||
RestoreGpuTracking(destination);
|
||||
}
|
||||
|
||||
void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint32_t mip,
|
||||
@@ -511,6 +636,7 @@ void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint3
|
||||
RefreshCopySource(source_id);
|
||||
auto& destination = ResolveImage(destination_id);
|
||||
auto& source = ResolveImage(source_id);
|
||||
TrackImage(destination_id);
|
||||
if (source.IsBufferModified() || source.backing.samples != destination.backing.samples) {
|
||||
EXIT("TextureCache: invalid mip-copy ownership or sample count\n");
|
||||
}
|
||||
@@ -520,7 +646,6 @@ void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint3
|
||||
if (source.IsGpuModified()) {
|
||||
destination.MarkGpuModified();
|
||||
}
|
||||
RestoreGpuTracking(destination);
|
||||
}
|
||||
|
||||
ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingType binding,
|
||||
@@ -590,7 +715,7 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
|
||||
if (copied) {
|
||||
DeleteImages(std::array {cached_id}, cached_id);
|
||||
} else {
|
||||
ReleaseGpuTracking(cached_id);
|
||||
ClearGpuModified(cached_id);
|
||||
DeleteImage(cached_id);
|
||||
}
|
||||
return replacement_id;
|
||||
@@ -629,6 +754,12 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
|
||||
(requested.IsVolume() || cached.info.IsVolume())) {
|
||||
return {ExpandImage(requested, cached_id)};
|
||||
}
|
||||
if (requested.tile_mode != cached.info.tile_mode) {
|
||||
if (safe_to_delete) {
|
||||
DeleteImages(std::array {cached_id}, cached_id);
|
||||
}
|
||||
return {merged_id};
|
||||
}
|
||||
if (requested.pixel_format != cached.info.pixel_format ||
|
||||
requested.data.size <= cached.info.data.size) {
|
||||
const auto result_id = merged_id ? merged_id : cached_id;
|
||||
@@ -641,12 +772,6 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
|
||||
if (requested.type == cached.info.type && requested.resources > cached.info.resources) {
|
||||
return {ExpandImage(requested, cached_id)};
|
||||
}
|
||||
if (requested.tile_mode != cached.info.tile_mode) {
|
||||
if (safe_to_delete) {
|
||||
DeleteImages(std::array {cached_id}, cached_id);
|
||||
}
|
||||
return {merged_id};
|
||||
}
|
||||
EXIT("TextureCache: unresolvable equal-address image overlap, address=0x%016" PRIx64
|
||||
" requested=%ux%u "
|
||||
"cached=%ux%u requested_size=0x%016" PRIx64 " cached_size=0x%016" PRIx64
|
||||
@@ -903,39 +1028,29 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) {
|
||||
if (image.info.data.Empty()) {
|
||||
return;
|
||||
}
|
||||
TrackImage(id);
|
||||
if (image.info.metadata.compression != VideoOutCompression::Uncompressed) {
|
||||
m_memory_tracker.ForEachUploadRange(
|
||||
image.info.data.address, image.info.data.size, false,
|
||||
[](uint64_t, uint64_t) noexcept {}, []() noexcept {});
|
||||
if (image.IsCpuDirty()) {
|
||||
image.RefreshComplete();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (image.info.samples > 1) {
|
||||
RestoreGpuTracking(image);
|
||||
return;
|
||||
}
|
||||
bool data_gpu_owned = false;
|
||||
bool data_imported = false;
|
||||
bool uploaded = false;
|
||||
m_memory_tracker.ForEachUploadRange(
|
||||
image.info.data.address, image.info.data.size, false,
|
||||
[&](uint64_t, uint64_t) noexcept { uploaded = true; },
|
||||
[&]() noexcept {
|
||||
uploaded |= image.IsBufferModified() || image.IsDefinitelyCpuDirty();
|
||||
if (!uploaded) {
|
||||
return;
|
||||
}
|
||||
const auto source =
|
||||
m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size);
|
||||
if (source.buffer == nullptr) {
|
||||
EXIT("TextureCache: failed to obtain image upload source\n");
|
||||
}
|
||||
data_gpu_owned |= source.gpu_owned;
|
||||
data_imported = true;
|
||||
UploadImage(image, desc, *source.buffer, source.offset);
|
||||
});
|
||||
bool data_gpu_owned = false;
|
||||
bool data_imported = false;
|
||||
const bool upload = image.IsBufferModified() || image.IsCpuDirty();
|
||||
if (upload) {
|
||||
const auto source =
|
||||
m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size);
|
||||
if (source.buffer == nullptr) {
|
||||
EXIT("TextureCache: failed to obtain image upload source\n");
|
||||
}
|
||||
data_gpu_owned |= source.gpu_owned;
|
||||
data_imported = true;
|
||||
UploadImage(image, desc, *source.buffer, source.offset);
|
||||
}
|
||||
if (data_imported) {
|
||||
image.ClearBufferModified();
|
||||
}
|
||||
@@ -945,39 +1060,27 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) {
|
||||
if (image.IsCpuDirty()) {
|
||||
image.RefreshComplete();
|
||||
}
|
||||
RestoreGpuTracking(image);
|
||||
}
|
||||
|
||||
void TextureCache::RefreshImage(ImageId id, const ImageDesc& desc) {
|
||||
auto& image = ResolveImage(id);
|
||||
bool unchanged_maybe = false;
|
||||
TrackImage(id);
|
||||
auto& image = ResolveImage(id);
|
||||
if (image.IsMaybeCpuDirty()) {
|
||||
const auto hash = image.HashGuestEdges();
|
||||
if (image.NeedsMaybeCpuHash()) {
|
||||
image.SetMaybeCpuHash(hash);
|
||||
return;
|
||||
}
|
||||
unchanged_maybe = !image.ResolveMaybeCpuHash(hash);
|
||||
if (unchanged_maybe) {
|
||||
m_memory_tracker.ForEachUploadRange(
|
||||
image.info.data.address, image.info.data.size, false,
|
||||
[](uint64_t, uint64_t) noexcept {}, []() noexcept {});
|
||||
}
|
||||
(void)image.ResolveMaybeCpuHash(hash);
|
||||
}
|
||||
bool cpu_dirty = image.IsBufferModified() || image.IsDefinitelyCpuDirty();
|
||||
if (!unchanged_maybe) {
|
||||
cpu_dirty |=
|
||||
m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size);
|
||||
}
|
||||
if (image.info.metadata.compression != VideoOutCompression::Uncompressed) {
|
||||
if (cpu_dirty) {
|
||||
EXIT("TextureCache: compressed guest image refresh is unsupported\n");
|
||||
}
|
||||
RestoreGpuTracking(image);
|
||||
return;
|
||||
}
|
||||
if (!cpu_dirty) {
|
||||
RestoreGpuTracking(image);
|
||||
return;
|
||||
}
|
||||
InitializeImage(id, desc);
|
||||
@@ -1029,7 +1132,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
|
||||
|
||||
ImageId result {};
|
||||
bool replacement_buffer = false;
|
||||
bool replacing_image = false;
|
||||
bool inserted_new = false;
|
||||
{
|
||||
std::lock_guard transaction(m_resource_mutex);
|
||||
CacheLock lock(*this, m_lock);
|
||||
@@ -1038,7 +1141,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
|
||||
|
||||
for (const auto id: candidates) {
|
||||
const auto owner = ResolveOwner(id);
|
||||
if (owner == nullptr || owner->info.data != desc.info.data) {
|
||||
if (owner == nullptr) {
|
||||
continue;
|
||||
}
|
||||
if (SameBacking(owner->info, desc.info, exact_format)) {
|
||||
@@ -1079,20 +1182,19 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
|
||||
}
|
||||
replacement_buffer = resolved.IsBufferModified();
|
||||
DeleteImage(result);
|
||||
result = {};
|
||||
replacing_image = true;
|
||||
result = {};
|
||||
}
|
||||
}
|
||||
if (!result) {
|
||||
result = InsertImage(desc.info);
|
||||
inserted_new = true;
|
||||
auto& inserted = ResolveImage(result);
|
||||
if (replacement_buffer || m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address,
|
||||
inserted.info.data.size)) {
|
||||
inserted.MarkBufferModified();
|
||||
} else if (replacing_image) {
|
||||
m_memory_tracker.MarkRegionAsCpuModified(inserted.info.data.address,
|
||||
inserted.info.data.size);
|
||||
}
|
||||
}
|
||||
if (inserted_new) {
|
||||
InitializeImage(result, desc);
|
||||
} else {
|
||||
RefreshImage(result, desc);
|
||||
@@ -1101,9 +1203,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
|
||||
auto& image = ResolveImage(result);
|
||||
if (desc.type == BindingType::VideoOut &&
|
||||
desc.info.metadata.compression != VideoOutCompression::Uncompressed) {
|
||||
const bool guest_dirty =
|
||||
image.IsBufferModified() || image.IsCpuDirty() ||
|
||||
m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size);
|
||||
const bool guest_dirty = image.IsBufferModified() || image.IsCpuDirty();
|
||||
const bool native_current =
|
||||
(image.usage.render_target || image.IsGpuModified()) && !guest_dirty;
|
||||
if (!native_current) {
|
||||
@@ -1252,6 +1352,7 @@ void TextureCache::MarkGpuWritten(ImageId id) {
|
||||
if (!image.registered || image.depth_id) {
|
||||
EXIT("TextureCache: cannot mark an unavailable image GPU-written\n");
|
||||
}
|
||||
TrackImage(id);
|
||||
CommitGpuWrite(image);
|
||||
}
|
||||
|
||||
@@ -1265,13 +1366,10 @@ void TextureCache::CommitGpuWrite(Image& image) {
|
||||
}
|
||||
m_buffer_cache.InvalidateImageAliases(range.address, range.size);
|
||||
image.ClearBufferModified();
|
||||
m_memory_tracker.ForEachUploadRange(
|
||||
range.address, range.size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
|
||||
if (image.IsCpuDirty()) {
|
||||
image.RefreshComplete();
|
||||
}
|
||||
image.MarkGpuModified();
|
||||
RestoreGpuTracking(image);
|
||||
}
|
||||
|
||||
bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
|
||||
@@ -1279,7 +1377,6 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
|
||||
if (command.IsInvalid() || !GuestRange {address, size}.Valid()) {
|
||||
EXIT("TextureCache: invalid image clear\n");
|
||||
}
|
||||
m_buffer_cache.ValidateGpuAccess(address, size, false, true);
|
||||
std::lock_guard transaction(m_resource_mutex);
|
||||
CacheLock lock(*this, m_lock);
|
||||
ImageId selected {};
|
||||
@@ -1335,8 +1432,7 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
|
||||
if (m_buffer_cache.HasGpuDirtyBytes(address, size)) {
|
||||
m_buffer_cache.DiscardGpuDirtyBytes(address, size);
|
||||
}
|
||||
if (image.IsBufferModified() || image.IsCpuDirty() ||
|
||||
m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size)) {
|
||||
if (image.IsBufferModified() || image.IsCpuDirty()) {
|
||||
ImageDesc refresh {.info = image.info, .view_info = {}, .type = UploadBinding(image)};
|
||||
InitializeImage(selected, refresh);
|
||||
if (image.info.samples == 1 && (image.IsBufferModified() || image.IsCpuDirty())) {
|
||||
@@ -1361,14 +1457,12 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
|
||||
return true;
|
||||
}
|
||||
|
||||
void TextureCache::PrepareHostWrite(uint64_t address, uint64_t size) {
|
||||
void TextureCache::InvalidateMemory(uint64_t address, uint64_t size) {
|
||||
if (!GuestRange {address, size}.Valid()) {
|
||||
EXIT("TextureCache: invalid host-write range\n");
|
||||
EXIT("TextureCache: invalid memory-invalidation range\n");
|
||||
}
|
||||
CacheLock lock(*this, m_lock);
|
||||
InvalidateCpuAliases(address, size);
|
||||
m_memory_tracker.ForEachDownloadRange<true>(address, size, [](uint64_t, uint64_t) noexcept {});
|
||||
m_memory_tracker.MarkRegionAsCpuModified(address, size);
|
||||
}
|
||||
|
||||
void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset) {
|
||||
@@ -1567,18 +1661,15 @@ bool TextureCache::SynchronizeImageToBuffer(ImageId id) {
|
||||
if (!plan.valid) {
|
||||
return false;
|
||||
}
|
||||
const auto range = image.info.data;
|
||||
const bool refresh = image.IsDefinitelyCpuDirty() ||
|
||||
m_memory_tracker.IsRegionCpuModified(range.address, range.size);
|
||||
if (refresh) {
|
||||
const auto range = image.info.data;
|
||||
if (image.IsCpuDirty()) {
|
||||
RefreshImage(id,
|
||||
ImageDesc {.info = image.info, .view_info = {}, .type = UploadBinding(image)});
|
||||
}
|
||||
if (!image.IsGpuModified()) {
|
||||
return true;
|
||||
}
|
||||
if (image.IsDefinitelyCpuDirty() || image.IsBufferModified() ||
|
||||
m_memory_tracker.IsRegionCpuModified(range.address, range.size)) {
|
||||
if (image.IsDefinitelyCpuDirty() || image.IsBufferModified()) {
|
||||
EXIT("TextureCache: image mirror source is not native-current\n");
|
||||
}
|
||||
auto [destination, offset] =
|
||||
@@ -1591,7 +1682,7 @@ bool TextureCache::SynchronizeImageToBuffer(ImageId id) {
|
||||
m_buffer_cache.PublishImageBuffer(range.address, range.size);
|
||||
image.MarkBufferModified();
|
||||
RetainImage(m_scheduler.Current(), id);
|
||||
ReleaseGpuTracking(id);
|
||||
ClearGpuModified(id);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1633,7 +1724,7 @@ bool TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
|
||||
if (!formatted_buffer_write) {
|
||||
EXIT("TextureCache: buffer write aliases GPU-modified image\n");
|
||||
}
|
||||
ReleaseGpuTracking(id);
|
||||
ClearGpuModified(id);
|
||||
}
|
||||
owner->MarkBufferModified();
|
||||
found = true;
|
||||
@@ -1660,50 +1751,40 @@ TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t address, uint64_t si
|
||||
}
|
||||
|
||||
void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
|
||||
const auto page_begin = address & ~(TRACKER_PAGE_SIZE - 1);
|
||||
const auto page_end = (address + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
|
||||
for (const auto id: FindImagesInRegion(address, size, true)) {
|
||||
auto owner = ResolveOwner(id);
|
||||
if (owner == nullptr || owner->depth_id) {
|
||||
continue;
|
||||
}
|
||||
owner->InvalidateCpuWrite(address, size);
|
||||
if (owner->NeedsMaybeCpuHash()) {
|
||||
owner->SetMaybeCpuHash(owner->HashGuestEdges());
|
||||
if (owner->Overlaps(address, size)) {
|
||||
owner->InvalidateCpuWrite(address, size);
|
||||
UntrackImage(id);
|
||||
continue;
|
||||
}
|
||||
const auto image_begin = owner->info.data.address;
|
||||
const auto image_end = owner->info.data.End();
|
||||
if (page_end < image_end) {
|
||||
UntrackImageHead(id);
|
||||
} else if (image_begin < page_begin) {
|
||||
UntrackImageTail(id);
|
||||
} else {
|
||||
owner->MarkMaybeCpuDirty();
|
||||
if (owner->NeedsMaybeCpuHash()) {
|
||||
owner->SetMaybeCpuHash(owner->HashGuestEdges());
|
||||
}
|
||||
UntrackImage(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TextureCache::RestoreGpuTracking(const Image& image) {
|
||||
if (!image.IsGpuModified()) {
|
||||
return;
|
||||
}
|
||||
constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1;
|
||||
const auto range = image.info.data;
|
||||
const auto begin = range.address & ~page_mask;
|
||||
const auto end = (range.End() + page_mask) & ~page_mask;
|
||||
for (auto page = begin; page < end; page += TRACKER_PAGE_SIZE) {
|
||||
if (!m_memory_tracker.IsRegionGpuModified(page, TRACKER_PAGE_SIZE) &&
|
||||
!m_memory_tracker.IsRegionCpuModified(page, TRACKER_PAGE_SIZE)) {
|
||||
m_memory_tracker.MarkRegionAsGpuModified(page, TRACKER_PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TextureCache::ReleaseGpuTracking(ImageId id) {
|
||||
void TextureCache::ClearGpuModified(ImageId id) {
|
||||
auto owner = ResolveOwner(id);
|
||||
if (owner == nullptr || !owner->IsGpuModified()) {
|
||||
return;
|
||||
}
|
||||
const auto released = owner->info.data;
|
||||
owner->ClearGpuModified();
|
||||
m_memory_tracker.ForEachDownloadRange<true>(released.address, released.size,
|
||||
[](uint64_t, uint64_t) noexcept {});
|
||||
for (const auto candidate: FindImagesInRegion(released.address, released.size, true)) {
|
||||
const auto survivor = ResolveOwner(candidate);
|
||||
if (survivor != nullptr && survivor.get() != owner.get() && !survivor->depth_id) {
|
||||
RestoreGpuTracking(*survivor);
|
||||
}
|
||||
}
|
||||
RestoreGpuTracking(*owner);
|
||||
}
|
||||
|
||||
bool TextureCache::IsMeta(uint64_t address) {
|
||||
@@ -1745,39 +1826,6 @@ bool TextureCache::TouchMeta(uint64_t address, uint32_t slice, bool is_clear) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t address, uint64_t size,
|
||||
PageFaultPhase phase) noexcept {
|
||||
if ((access != PageFaultAccess::Read && access != PageFaultAccess::Write) ||
|
||||
!GuestRange {address, size}.Valid()) {
|
||||
return false;
|
||||
}
|
||||
if (access == PageFaultAccess::Read) {
|
||||
return false;
|
||||
}
|
||||
if (phase == PageFaultPhase::Invalidate) {
|
||||
const bool gpu_image =
|
||||
m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase);
|
||||
CpuFaultAction action = gpu_image ? CpuFaultAction::Download
|
||||
: m_memory_tracker.BeginCpuFault(address, size, access);
|
||||
{
|
||||
CacheLock lock(*this, m_lock);
|
||||
InvalidateCpuAliases(address, size);
|
||||
}
|
||||
return action != CpuFaultAction::Untracked;
|
||||
}
|
||||
|
||||
if (phase == PageFaultPhase::Complete) {
|
||||
const bool gpu_image = m_memory_tracker.IsRegionGpuModified(address, size);
|
||||
return gpu_image ? m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase)
|
||||
: m_memory_tracker.CompleteCpuFault(address, size, access, false);
|
||||
}
|
||||
if (phase != PageFaultPhase::Release) {
|
||||
return false;
|
||||
}
|
||||
(void)m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase);
|
||||
return true;
|
||||
}
|
||||
|
||||
void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
|
||||
if (!GuestRange {address, size}.Valid()) {
|
||||
EXIT("TextureCache: invalid unmap range\n");
|
||||
@@ -1794,16 +1842,10 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
|
||||
continue;
|
||||
}
|
||||
if (owner->IsGpuModified()) {
|
||||
ReleaseGpuTracking(id);
|
||||
ClearGpuModified(id);
|
||||
}
|
||||
DeleteImage(id);
|
||||
}
|
||||
m_memory_tracker.UntrackMemory(address, size);
|
||||
for (const auto id: FindImagesInRegion(address, size, true)) {
|
||||
if (const auto survivor = ResolveOwner(id); survivor != nullptr) {
|
||||
RestoreGpuTracking(*survivor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TextureCache::RunGarbageCollector() {
|
||||
@@ -1849,7 +1891,7 @@ void TextureCache::RunGarbageCollector() {
|
||||
if (safe && !TryDownloadImage(id)) {
|
||||
continue;
|
||||
}
|
||||
ReleaseGpuTracking(id);
|
||||
ClearGpuModified(id);
|
||||
}
|
||||
DeleteImage(id);
|
||||
if (m_total_used_memory < m_critical_gc_memory && aggressive) {
|
||||
Vendored
+19
-15
@@ -4,10 +4,11 @@
|
||||
#include "common/abi.h"
|
||||
#include "common/common.h"
|
||||
#include "common/lruCache.h"
|
||||
#include "graphics/host_gpu/memoryTracker.h"
|
||||
#include "graphics/host_gpu/renderer/blitHelper.h"
|
||||
#include "graphics/host_gpu/renderer/image.h"
|
||||
#include "graphics/host_gpu/renderer/multiLevelPageTable.h"
|
||||
#include "graphics/host_gpu/pageManager.h"
|
||||
#include "graphics/host_gpu/regionManager.h"
|
||||
#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
|
||||
#include "graphics/host_gpu/renderer/image/blitHelper.h"
|
||||
#include "graphics/host_gpu/renderer/image/image.h"
|
||||
|
||||
#include <compare>
|
||||
#include <map>
|
||||
@@ -64,7 +65,7 @@ public:
|
||||
|
||||
[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
|
||||
uint32_t packed_clear);
|
||||
void PrepareHostWrite(uint64_t address, uint64_t size);
|
||||
void InvalidateMemory(uint64_t address, uint64_t size);
|
||||
[[nodiscard]] bool SynchronizeImageToBuffer(uint64_t address, uint64_t size);
|
||||
[[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
|
||||
bool formatted_buffer_write = false);
|
||||
@@ -75,11 +76,9 @@ public:
|
||||
[[nodiscard]] bool ClearMeta(uint64_t address);
|
||||
[[nodiscard]] bool TouchMeta(uint64_t address, uint32_t slice, bool is_clear);
|
||||
|
||||
[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t address, uint64_t size,
|
||||
PageFaultPhase phase) noexcept;
|
||||
void UnmapMemory(uint64_t address, uint64_t size);
|
||||
void ProcessDownloadImages();
|
||||
void RunGarbageCollector();
|
||||
void UnmapMemory(uint64_t address, uint64_t size);
|
||||
void ProcessDownloadImages();
|
||||
void RunGarbageCollector();
|
||||
|
||||
private:
|
||||
enum class TransferDirection { Upload, Download };
|
||||
@@ -109,11 +108,17 @@ private:
|
||||
[[nodiscard]] ImageId InsertImage(const ImageInfo& info);
|
||||
[[nodiscard]] ImageId GetNullImage(const ImageDesc& desc);
|
||||
void RegisterImage(ImageId id);
|
||||
void UnregisterImage(ImageId id, bool release_tracking);
|
||||
void DeleteImage(ImageId id, bool release_tracking = true);
|
||||
void UnregisterImage(ImageId id);
|
||||
void DeleteImage(ImageId id);
|
||||
void DeleteImages(std::span<const ImageId> ids, std::optional<ImageId> native_source = {});
|
||||
void RetainImage(CommandBuffer& command, ImageId id);
|
||||
void TouchImage(Image& image);
|
||||
void TrackImage(ImageId id);
|
||||
void TrackImageHead(ImageId id);
|
||||
void TrackImageTail(ImageId id);
|
||||
void UntrackImage(ImageId id);
|
||||
void UntrackImageHead(ImageId id);
|
||||
void UntrackImageTail(ImageId id);
|
||||
void TrackImageDownload(ImageId id);
|
||||
void TrackImageDownloadLocked(ImageId id, Image& image);
|
||||
[[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested,
|
||||
@@ -148,8 +153,7 @@ private:
|
||||
void ValidateImageDesc(const ImageDesc& desc) const;
|
||||
|
||||
void InvalidateCpuAliases(uint64_t address, uint64_t size);
|
||||
void RestoreGpuTracking(const Image& image);
|
||||
void ReleaseGpuTracking(ImageId id);
|
||||
void ClearGpuModified(ImageId id);
|
||||
|
||||
[[nodiscard]] bool SynchronizeImageToBuffer(ImageId id);
|
||||
void DownloadImage(ImageId id);
|
||||
@@ -160,7 +164,7 @@ private:
|
||||
GraphicContext& m_graphics;
|
||||
CommandScheduler& m_scheduler;
|
||||
TrackingSpinLock m_lock;
|
||||
MemoryTracker m_memory_tracker;
|
||||
PageManager& m_page_manager;
|
||||
BlitHelper m_blit_helper;
|
||||
std::unique_ptr<TileManager> m_tiler;
|
||||
BufferCache& m_buffer_cache;
|
||||
@@ -7,9 +7,9 @@
|
||||
#include "graphics/guest_gpu/hardwareContext.h"
|
||||
#include "graphics/guest_gpu/tile.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/objects/textureCommon.h"
|
||||
#include "graphics/host_gpu/renderer/debug.h"
|
||||
#include "graphics/host_gpu/renderer/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/image/textureCommon.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/renderContext.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
@@ -23,10 +23,10 @@ static std::atomic<uint32_t> g_render_color_log_count = 0;
|
||||
|
||||
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
|
||||
void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer,
|
||||
RenderColorInfo& r,
|
||||
uint32_t render_target_slice_offset,
|
||||
uint32_t render_target_slot, bool ignore_target_mask,
|
||||
bool exact_format) {
|
||||
RenderColorInfo& r,
|
||||
uint32_t render_target_slice_offset,
|
||||
uint32_t render_target_slot, bool ignore_target_mask,
|
||||
bool exact_format) {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
const auto& hw = buffer.GetRegisters();
|
||||
|
||||
@@ -79,10 +79,8 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
|
||||
const auto view = ResolveTargetViewInfo(
|
||||
rt.view.base_array_slice_index, rt.view.last_array_slice_index, render_target_slice_offset);
|
||||
switch (view.type) {
|
||||
case TargetViewType::Image2D: break;
|
||||
case TargetViewType::Image2DArray:
|
||||
EXIT("layered render-target views are unsupported: base=%u count=%u\n", view.base_layer,
|
||||
view.layer_count);
|
||||
case TargetViewType::Image2D:
|
||||
case TargetViewType::Image2DArray: break;
|
||||
case TargetViewType::Unsupported:
|
||||
EXIT("invalid render-target view: base=%u last=%u draw_offset=%u\n",
|
||||
rt.view.base_array_slice_index, rt.view.last_array_slice_index,
|
||||
@@ -241,12 +239,12 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
|
||||
}
|
||||
|
||||
TextureCache::ImageDesc desc {};
|
||||
desc.type = TextureCache::BindingType::RenderTarget;
|
||||
desc.info.data = {rt.base.addr, backing_size};
|
||||
desc.info.pixel_format = target_format.format;
|
||||
desc.info.guest_format = ImageOps::RenderTargetTransferFormat(bytes_per_element);
|
||||
desc.info.type = Prospero::ImageType::kColor2D;
|
||||
desc.info.extent = {width, height, 1};
|
||||
desc.type = TextureCache::BindingType::RenderTarget;
|
||||
desc.info.data = {rt.base.addr, backing_size};
|
||||
desc.info.pixel_format = target_format.format;
|
||||
desc.info.guest_format = ImageOps::RenderTargetTransferFormat(bytes_per_element);
|
||||
desc.info.type = Prospero::ImageType::kColor2D;
|
||||
desc.info.extent = {width, height, 1};
|
||||
desc.info.resources = {levels, view.image_layers};
|
||||
desc.info.pitch = pitch;
|
||||
desc.info.bytes_per_block = bytes_per_element;
|
||||
@@ -275,20 +273,20 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
|
||||
desc.view_info.base_layer = view.base_layer;
|
||||
desc.view_info.layer_count = view.layer_count;
|
||||
desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
|
||||
auto& texture_cache = m_context.GetTextureCache();
|
||||
r.desc = std::move(desc);
|
||||
r.image_id = texture_cache.FindImage(r.desc, exact_format);
|
||||
r.type = RenderColorType::RenderTexture;
|
||||
r.base_addr = rt.base.addr;
|
||||
r.image_view = nullptr;
|
||||
r.format = r.desc.view_info.format;
|
||||
r.extent = view_extent;
|
||||
r.base_mip_level = rt.view.current_mip_level;
|
||||
r.buffer_size = backing_size;
|
||||
r.samples = samples;
|
||||
r.export_mapping = target_format.export_mapping;
|
||||
r.color_clear_enable = false;
|
||||
r.color_clear_value = {};
|
||||
auto& texture_cache = m_context.GetTextureCache();
|
||||
r.desc = std::move(desc);
|
||||
r.image_id = texture_cache.FindImage(r.desc, exact_format);
|
||||
r.type = RenderColorType::RenderTexture;
|
||||
r.base_addr = rt.base.addr;
|
||||
r.image_view = nullptr;
|
||||
r.format = r.desc.view_info.format;
|
||||
r.extent = view_extent;
|
||||
r.base_mip_level = rt.view.current_mip_level;
|
||||
r.buffer_size = backing_size;
|
||||
r.samples = samples;
|
||||
r.export_mapping = target_format.export_mapping;
|
||||
r.color_clear_enable = false;
|
||||
r.color_clear_value = {};
|
||||
BindRenderTarget(r.image_id);
|
||||
}
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_COLORRENDERTARGET_H_
|
||||
|
||||
#include "graphics/guest_gpu/gpu_defs.h"
|
||||
#include "graphics/host_gpu/renderer/cache/textureCache.h"
|
||||
#include "graphics/host_gpu/renderer/renderTarget.h"
|
||||
#include "graphics/host_gpu/renderer/textureCache.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
@@ -44,13 +44,13 @@ CommandSlot* CommandScheduler::CommandPool::CreateSlot() {
|
||||
allocate.commandPool = m_pool;
|
||||
allocate.level = vk::CommandBufferLevel::ePrimary;
|
||||
allocate.commandBufferCount = 1;
|
||||
vk::CommandBuffer buffer = nullptr;
|
||||
vk::CommandBuffer buffer = nullptr;
|
||||
EXIT_IF(graphics.device.allocateCommandBuffers(&allocate, &buffer) != vk::Result::eSuccess);
|
||||
|
||||
vk::FenceCreateInfo fence_create {};
|
||||
fence_create.sType = vk::StructureType::eFenceCreateInfo;
|
||||
fence_create.flags = vk::FenceCreateFlagBits::eSignaled;
|
||||
vk::Fence fence = nullptr;
|
||||
vk::Fence fence = nullptr;
|
||||
if (graphics.device.createFence(&fence_create, nullptr, &fence) != vk::Result::eSuccess) {
|
||||
graphics.device.freeCommandBuffers(m_pool, 1, &buffer);
|
||||
EXIT("failed to create command-buffer fence\n");
|
||||
@@ -70,9 +70,9 @@ CommandSlot* CommandScheduler::CommandPool::Allocate(GraphicContext& graphics) {
|
||||
Create(graphics);
|
||||
}
|
||||
EXIT_IF(m_graphics != &graphics);
|
||||
auto found = std::ranges::find_if(m_slots, [](const auto& slot) { return !slot.busy; });
|
||||
auto* slot = found != m_slots.end() ? &*found : CreateSlot();
|
||||
slot->busy = true;
|
||||
auto found = std::ranges::find_if(m_slots, [](const auto& slot) { return !slot.busy; });
|
||||
auto* slot = found != m_slots.end() ? &*found : CreateSlot();
|
||||
slot->busy = true;
|
||||
slot->Reset();
|
||||
return slot;
|
||||
}
|
||||
@@ -331,8 +331,7 @@ void CommandScheduler::WaitPriorityOperations(uint64_t tick) {
|
||||
EXIT_IF(g_deferred_callback_scheduler == this);
|
||||
std::unique_lock lock(m_operation_mutex);
|
||||
m_operation_available.wait(lock, [this, tick] {
|
||||
const bool active_before_or_at =
|
||||
m_priority_active && m_priority_active_tick <= tick;
|
||||
const bool active_before_or_at = m_priority_active && m_priority_active_tick <= tick;
|
||||
const bool queued_before_or_at =
|
||||
!m_priority_operations.empty() && m_priority_operations.front().tick <= tick;
|
||||
return !active_before_or_at && !queued_before_or_at;
|
||||
|
||||
@@ -47,21 +47,21 @@ public:
|
||||
void FinishCurrent();
|
||||
// Deferred callbacks can observe an externally owned drain, but cannot initiate shutdown:
|
||||
// the priority runner cannot join itself.
|
||||
void Shutdown();
|
||||
void Wait(uint64_t tick);
|
||||
void PopPendingOperations();
|
||||
void DrainPriorityOperations();
|
||||
void WaitPriorityOperations(uint64_t tick);
|
||||
void DeferOperation(Common::UniqueFunction<void>&& operation);
|
||||
void DeferPriorityOperation(Common::UniqueFunction<void>&& operation);
|
||||
void Shutdown();
|
||||
void Wait(uint64_t tick);
|
||||
void PopPendingOperations();
|
||||
void DrainPriorityOperations();
|
||||
void WaitPriorityOperations(uint64_t tick);
|
||||
void DeferOperation(Common::UniqueFunction<void>&& operation);
|
||||
void DeferPriorityOperation(Common::UniqueFunction<void>&& operation);
|
||||
[[nodiscard]] static bool InDeferredOperation() noexcept;
|
||||
|
||||
[[nodiscard]] bool Active() const noexcept { return m_current >= 0; }
|
||||
void CheckActive() const;
|
||||
RenderCommandBuffer& Current() const;
|
||||
[[nodiscard]] uint64_t CurrentTick() const noexcept { return m_master.CurrentTick(); }
|
||||
[[nodiscard]] bool IsFree(uint64_t tick);
|
||||
[[nodiscard]] RenderContext& Context() const noexcept { return m_context; }
|
||||
[[nodiscard]] bool Active() const noexcept { return m_current >= 0; }
|
||||
void CheckActive() const;
|
||||
RenderCommandBuffer& Current() const;
|
||||
[[nodiscard]] uint64_t CurrentTick() const noexcept { return m_master.CurrentTick(); }
|
||||
[[nodiscard]] bool IsFree(uint64_t tick);
|
||||
[[nodiscard]] RenderContext& Context() const noexcept { return m_context; }
|
||||
[[nodiscard]] GraphicContext& Graphics() const noexcept { return m_graphics; }
|
||||
|
||||
private:
|
||||
@@ -91,11 +91,11 @@ private:
|
||||
uint64_t tick = 0;
|
||||
};
|
||||
|
||||
void BindCurrent() const;
|
||||
CommandBuffer& SubmitCurrent(SubmitInfo& submit);
|
||||
void BeginNext();
|
||||
void PriorityOperationsThread(std::stop_token stop);
|
||||
void RunOperation(Common::UniqueFunction<void>&& operation);
|
||||
void BindCurrent() const;
|
||||
CommandBuffer& SubmitCurrent(SubmitInfo& submit);
|
||||
void BeginNext();
|
||||
void PriorityOperationsThread(std::stop_token stop);
|
||||
void RunOperation(Common::UniqueFunction<void>&& operation);
|
||||
[[nodiscard]] CommandSlot* AllocateCommandBuffer();
|
||||
[[nodiscard]] uint64_t NextSubmitSequence() noexcept;
|
||||
|
||||
@@ -109,14 +109,14 @@ private:
|
||||
std::mutex m_operation_mutex;
|
||||
std::condition_variable m_operation_available;
|
||||
std::jthread m_priority_thread;
|
||||
bool m_priority_active = false;
|
||||
bool m_priority_active = false;
|
||||
uint64_t m_priority_active_tick = 0;
|
||||
OperationState m_operation_state = OperationState::Open;
|
||||
int m_current = -1;
|
||||
bool m_recording = false;
|
||||
HW::Context* m_registers = nullptr;
|
||||
HW::UserConfig* m_user_config = nullptr;
|
||||
HW::Shader* m_shaders = nullptr;
|
||||
OperationState m_operation_state = OperationState::Open;
|
||||
int m_current = -1;
|
||||
bool m_recording = false;
|
||||
HW::Context* m_registers = nullptr;
|
||||
HW::UserConfig* m_user_config = nullptr;
|
||||
HW::Shader* m_shaders = nullptr;
|
||||
std::atomic<uint64_t> m_submit_sequence = 0;
|
||||
|
||||
friend class CommandBuffer;
|
||||
|
||||
@@ -8,8 +8,8 @@
|
||||
#include "graphics/host_gpu/renderer/colorRenderTarget.h"
|
||||
#include "graphics/host_gpu/renderer/debug.h"
|
||||
#include "graphics/host_gpu/renderer/depthRenderTarget.h"
|
||||
#include "graphics/host_gpu/renderer/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/renderContext.h"
|
||||
#include "graphics/host_gpu/vma.h"
|
||||
@@ -270,30 +270,30 @@ void CommandBuffer::BeginRendering(const RenderState& state) const {
|
||||
colors[i].sType = vk::StructureType::eRenderingAttachmentInfo;
|
||||
colors[i].imageView = attachment.image_view;
|
||||
colors[i].imageLayout = attachment.image_layout;
|
||||
colors[i].loadOp = attachment.is_clear ? vk::AttachmentLoadOp::eClear
|
||||
: vk::AttachmentLoadOp::eLoad;
|
||||
colors[i].storeOp = vk::AttachmentStoreOp::eStore;
|
||||
colors[i].clearValue.color.uint32 = attachment.clear_value;
|
||||
colors[i].loadOp =
|
||||
attachment.is_clear ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad;
|
||||
colors[i].storeOp = vk::AttachmentStoreOp::eStore;
|
||||
colors[i].clearValue.color.uint32 = attachment.clear_value;
|
||||
}
|
||||
|
||||
const auto& depth_stencil = state.depth_stencil_attachment;
|
||||
const auto& depth_stencil = state.depth_stencil_attachment;
|
||||
vk::RenderingAttachmentInfo depth {};
|
||||
depth.sType = vk::StructureType::eRenderingAttachmentInfo;
|
||||
depth.imageView = depth_stencil.image_view;
|
||||
depth.imageLayout = depth_stencil.image_layout;
|
||||
depth.loadOp = depth_stencil.depth_clear ? vk::AttachmentLoadOp::eClear
|
||||
: vk::AttachmentLoadOp::eLoad;
|
||||
depth.storeOp = vk::AttachmentStoreOp::eStore;
|
||||
depth.loadOp =
|
||||
depth_stencil.depth_clear ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad;
|
||||
depth.storeOp = vk::AttachmentStoreOp::eStore;
|
||||
depth.clearValue.depthStencil.depth = std::bit_cast<float>(depth_stencil.clear_value[0]);
|
||||
|
||||
vk::RenderingAttachmentInfo stencil {};
|
||||
stencil.sType = vk::StructureType::eRenderingAttachmentInfo;
|
||||
stencil.imageView = depth_stencil.image_view;
|
||||
stencil.imageLayout = depth_stencil.image_layout;
|
||||
stencil.loadOp = depth_stencil.stencil_clear ? vk::AttachmentLoadOp::eClear
|
||||
: vk::AttachmentLoadOp::eLoad;
|
||||
stencil.storeOp = vk::AttachmentStoreOp::eStore;
|
||||
stencil.clearValue.depthStencil.stencil = depth_stencil.clear_value[1];
|
||||
stencil.loadOp =
|
||||
depth_stencil.stencil_clear ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad;
|
||||
stencil.storeOp = vk::AttachmentStoreOp::eStore;
|
||||
stencil.clearValue.depthStencil.stencil = depth_stencil.clear_value[1];
|
||||
|
||||
vk::RenderingInfo rendering {};
|
||||
rendering.sType = vk::StructureType::eRenderingInfo;
|
||||
|
||||
@@ -497,7 +497,15 @@ static void ZPrint(const char* func, const HW::DepthRenderTarget& z) {
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
|
||||
static void ZCheck(const HW::DepthRenderTarget& z) {
|
||||
static void ZCheck(const HW::DepthRenderTarget& z, const HW::DepthControl& dc,
|
||||
const HW::RenderControl& rc) {
|
||||
const bool depth_active =
|
||||
dc.z_enable || dc.z_write_enable || dc.depth_bounds_enable || rc.depth_clear_enable;
|
||||
const bool stencil_active = dc.stencil_enable || rc.stencil_clear_enable;
|
||||
if (!depth_active && !stencil_active) {
|
||||
return;
|
||||
}
|
||||
|
||||
EXIT_NOT_IMPLEMENTED(!z.z_info.HasValidTextureCompatibility());
|
||||
EXIT_NOT_IMPLEMENTED(!z.stencil_info.HasValidTextureCompatibility());
|
||||
if (z.z_info.format == 0) {
|
||||
@@ -548,14 +556,6 @@ static void ZCheck(const HW::DepthRenderTarget& z) {
|
||||
EXIT_NOT_IMPLEMENTED(z.htile_surface.prefetch_height != 0x00000000);
|
||||
EXIT_NOT_IMPLEMENTED(z.htile_surface.dst_outside_zero_to_one != 0x00000000);
|
||||
|
||||
if (z.depth_view.slice_start != 0x00000000 || z.depth_view.slice_max != 0x00000000) {
|
||||
static std::atomic<uint32_t> log_count {0};
|
||||
if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
|
||||
LOGF("DepthTarget: temporary: ignoring PS5 array slice view start=0x%08" PRIx32
|
||||
", max=0x%08" PRIx32 "\n",
|
||||
z.depth_view.slice_start, z.depth_view.slice_max);
|
||||
}
|
||||
}
|
||||
if (z.depth_view.current_mip_level != 0x00000000) {
|
||||
static std::atomic<uint32_t> log_count {0};
|
||||
if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
|
||||
@@ -1214,7 +1214,7 @@ void hw_check(const RenderCommandBuffer& buffer) {
|
||||
log_phase("vp");
|
||||
VpCheck(vp, smc);
|
||||
log_phase("z");
|
||||
ZCheck(z);
|
||||
ZCheck(z, d, rc);
|
||||
log_phase("clip");
|
||||
ClipCheck(c);
|
||||
log_phase("rc");
|
||||
|
||||
@@ -10,10 +10,10 @@
|
||||
#include "graphics/guest_gpu/hardwareContext.h"
|
||||
#include "graphics/guest_gpu/tile.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/objects/textureCommon.h"
|
||||
#include "graphics/host_gpu/renderer/debug.h"
|
||||
#include "graphics/host_gpu/renderer/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/image/textureCommon.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/renderContext.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
@@ -150,10 +150,8 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
|
||||
has_stencil, has_htile, z.stencil_info.htile_stencil_disabled);
|
||||
const auto view = ResolveTargetViewInfo(z.depth_view.slice_start, z.depth_view.slice_max);
|
||||
switch (view.type) {
|
||||
case TargetViewType::Image2D: break;
|
||||
case TargetViewType::Image2DArray:
|
||||
DepthFatal("layered depth views are unsupported: base=%u count=%u", view.base_layer,
|
||||
view.layer_count);
|
||||
case TargetViewType::Image2D:
|
||||
case TargetViewType::Image2DArray: break;
|
||||
case TargetViewType::Unsupported:
|
||||
DepthFatal("invalid depth view: base=%u last=%u", z.depth_view.slice_start,
|
||||
z.depth_view.slice_max);
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DEPTHRENDERTARGET_H_
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "graphics/host_gpu/renderer/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/cache/textureCache.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/renderTarget.h"
|
||||
#include "graphics/host_gpu/renderer/textureCache.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
+4
-4
@@ -1,11 +1,11 @@
|
||||
#include "graphics/host_gpu/renderer/blitHelper.h"
|
||||
#include "graphics/host_gpu/renderer/image/blitHelper.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "gpu_blit_shaders/gpu_blit_color_to_ms_depth_spv.h"
|
||||
#include "gpu_blit_shaders/gpu_blit_fs_triangle_spv.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/renderer/commandScheduler.h"
|
||||
#include "graphics/host_gpu/renderer/image.h"
|
||||
#include "graphics/host_gpu/renderer/image/image.h"
|
||||
#include "graphics/host_gpu/renderer/renderTarget.h"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -182,8 +182,8 @@ void BlitHelper::ReinterpretColorAsMsDepth(Image& source, Image& destination) {
|
||||
auto command = command_buffer.Handle();
|
||||
source.Transit(vk::ImageLayout::eShaderReadOnlyOptimal, vk::AccessFlagBits2::eShaderRead, {},
|
||||
command);
|
||||
destination.Transit(ColorToMsDepthLayout,
|
||||
vk::AccessFlagBits2::eDepthStencilAttachmentWrite, {}, command);
|
||||
destination.Transit(ColorToMsDepthLayout, vk::AccessFlagBits2::eDepthStencilAttachmentWrite, {},
|
||||
command);
|
||||
|
||||
vk::RenderingAttachmentInfo depth_attachment {};
|
||||
depth_attachment.sType = vk::StructureType::eRenderingAttachmentInfo;
|
||||
+138
-171
@@ -1,11 +1,11 @@
|
||||
#include "graphics/host_gpu/renderer/image.h"
|
||||
#include "graphics/host_gpu/renderer/image/image.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/profiler.h"
|
||||
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
|
||||
#include "graphics/host_gpu/renderer/commandScheduler.h"
|
||||
#include "graphics/host_gpu/renderer/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/renderTarget.h"
|
||||
#include "graphics/host_gpu/renderer/streamBuffer.h"
|
||||
#include "kernel/memory.h"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -99,9 +99,9 @@ vk::ImageAspectFlags Image::FullAspectMask(vk::Format format) noexcept {
|
||||
}
|
||||
}
|
||||
|
||||
Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
|
||||
vk::AccessFlags2 destination_access,
|
||||
vk::PipelineStageFlags2 destination_stage,
|
||||
Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
|
||||
vk::AccessFlags2 destination_access,
|
||||
vk::PipelineStageFlags2 destination_stage,
|
||||
std::optional<ImageSubresourceRange> range) {
|
||||
auto& state = backing.state;
|
||||
auto& subresource_states = backing.subresource_states;
|
||||
@@ -130,25 +130,25 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
|
||||
constexpr auto write_access = vk::AccessFlagBits2::eTransferWrite |
|
||||
vk::AccessFlagBits2::eShaderWrite |
|
||||
vk::AccessFlagBits2::eMemoryWrite;
|
||||
const bool repeated_write =
|
||||
const bool repeated_write =
|
||||
static_cast<bool>(subresource_state.access_mask & write_access);
|
||||
if (subresource_state.layout != destination_layout ||
|
||||
subresource_state.access_mask != destination_access || repeated_write) {
|
||||
vk::ImageMemoryBarrier2 barrier {};
|
||||
barrier.srcStageMask = subresource_state.pl_stage;
|
||||
barrier.srcAccessMask = subresource_state.access_mask;
|
||||
barrier.dstStageMask = destination_stage;
|
||||
barrier.dstAccessMask = destination_access;
|
||||
barrier.oldLayout = subresource_state.layout;
|
||||
barrier.newLayout = destination_layout;
|
||||
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
barrier.image = backing.image;
|
||||
barrier.subresourceRange.aspectMask = FullAspectMask(backing.format);
|
||||
barrier.subresourceRange.baseMipLevel = level;
|
||||
barrier.subresourceRange.levelCount = 1;
|
||||
barrier.subresourceRange.baseArrayLayer = layer;
|
||||
barrier.subresourceRange.layerCount = 1;
|
||||
barrier.srcStageMask = subresource_state.pl_stage;
|
||||
barrier.srcAccessMask = subresource_state.access_mask;
|
||||
barrier.dstStageMask = destination_stage;
|
||||
barrier.dstAccessMask = destination_access;
|
||||
barrier.oldLayout = subresource_state.layout;
|
||||
barrier.newLayout = destination_layout;
|
||||
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
barrier.image = backing.image;
|
||||
barrier.subresourceRange.aspectMask = FullAspectMask(backing.format);
|
||||
barrier.subresourceRange.baseMipLevel = level;
|
||||
barrier.subresourceRange.levelCount = 1;
|
||||
barrier.subresourceRange.baseArrayLayer = layer;
|
||||
barrier.subresourceRange.layerCount = 1;
|
||||
barriers.push_back(barrier);
|
||||
subresource_state = {destination_stage, destination_access, destination_layout};
|
||||
}
|
||||
@@ -159,10 +159,10 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
|
||||
subresource_states.clear();
|
||||
}
|
||||
} else {
|
||||
constexpr auto write_access = vk::AccessFlagBits2::eTransferWrite |
|
||||
vk::AccessFlagBits2::eShaderWrite |
|
||||
vk::AccessFlagBits2::eMemoryWrite;
|
||||
const bool repeated_write = static_cast<bool>(state.access_mask & write_access);
|
||||
constexpr auto write_access = vk::AccessFlagBits2::eTransferWrite |
|
||||
vk::AccessFlagBits2::eShaderWrite |
|
||||
vk::AccessFlagBits2::eMemoryWrite;
|
||||
const bool repeated_write = static_cast<bool>(state.access_mask & write_access);
|
||||
if (state.layout == destination_layout && state.access_mask == destination_access &&
|
||||
!repeated_write) {
|
||||
return {};
|
||||
@@ -191,8 +191,7 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
|
||||
}
|
||||
|
||||
void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
|
||||
std::optional<ImageSubresourceRange> range,
|
||||
vk::CommandBuffer command_buffer) {
|
||||
std::optional<ImageSubresourceRange> range, vk::CommandBuffer command_buffer) {
|
||||
const auto transfer_access =
|
||||
vk::AccessFlagBits2::eTransferRead | vk::AccessFlagBits2::eTransferWrite;
|
||||
vk::PipelineStageFlags2 destination_stage {};
|
||||
@@ -201,8 +200,8 @@ void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destina
|
||||
}
|
||||
if (!destination_access ||
|
||||
static_cast<bool>(destination_access & ~vk::AccessFlags2 {transfer_access})) {
|
||||
destination_stage |= vk::PipelineStageFlagBits2::eAllGraphics |
|
||||
vk::PipelineStageFlagBits2::eComputeShader;
|
||||
destination_stage |=
|
||||
vk::PipelineStageFlagBits2::eAllGraphics | vk::PipelineStageFlagBits2::eComputeShader;
|
||||
}
|
||||
const auto barriers =
|
||||
GetBarriers(destination_layout, destination_access, destination_stage, range);
|
||||
@@ -218,10 +217,9 @@ void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destina
|
||||
command_buffer.pipelineBarrier2(dependency);
|
||||
}
|
||||
|
||||
void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
|
||||
uint64_t offset, uint64_t size) {
|
||||
EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr ||
|
||||
size == 0);
|
||||
void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
|
||||
uint64_t size) {
|
||||
EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr || size == 0);
|
||||
m_scheduler->EndRendering();
|
||||
vk::BufferMemoryBarrier2 buffer_barrier {};
|
||||
buffer_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands;
|
||||
@@ -234,16 +232,15 @@ void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffe
|
||||
buffer_barrier.offset = offset;
|
||||
buffer_barrier.size = size;
|
||||
const auto image_barriers =
|
||||
GetBarriers(vk::ImageLayout::eTransferDstOptimal,
|
||||
vk::AccessFlagBits2::eTransferWrite,
|
||||
GetBarriers(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
|
||||
vk::PipelineStageFlagBits2::eCopy, {});
|
||||
vk::DependencyInfo dependency {};
|
||||
dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion;
|
||||
dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion;
|
||||
dependency.bufferMemoryBarrierCount = 1;
|
||||
dependency.pBufferMemoryBarriers = &buffer_barrier;
|
||||
dependency.imageMemoryBarrierCount = static_cast<uint32_t>(image_barriers.size());
|
||||
dependency.pImageMemoryBarriers = image_barriers.data();
|
||||
auto command = m_scheduler->Current().Handle();
|
||||
dependency.imageMemoryBarrierCount = static_cast<uint32_t>(image_barriers.size());
|
||||
dependency.pImageMemoryBarriers = image_barriers.data();
|
||||
auto command = m_scheduler->Current().Handle();
|
||||
command.pipelineBarrier2(dependency);
|
||||
command.copyBufferToImage(buffer, backing.image, vk::ImageLayout::eTransferDstOptimal,
|
||||
static_cast<uint32_t>(copies.size()), copies.data());
|
||||
@@ -256,8 +253,7 @@ void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffe
|
||||
dependency.pImageMemoryBarriers = nullptr;
|
||||
command.pipelineBarrier2(dependency);
|
||||
Transit(vk::ImageLayout::eGeneral,
|
||||
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
|
||||
command);
|
||||
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
|
||||
}
|
||||
|
||||
void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
|
||||
@@ -265,7 +261,7 @@ void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buf
|
||||
EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr || size == 0);
|
||||
m_scheduler->EndRendering();
|
||||
vk::BufferMemoryBarrier2 buffer_barrier {};
|
||||
buffer_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands;
|
||||
buffer_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands;
|
||||
buffer_barrier.srcAccessMask =
|
||||
vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite;
|
||||
buffer_barrier.dstStageMask = vk::PipelineStageFlagBits2::eCopy;
|
||||
@@ -276,16 +272,15 @@ void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buf
|
||||
buffer_barrier.offset = offset;
|
||||
buffer_barrier.size = size;
|
||||
const auto image_barriers =
|
||||
GetBarriers(vk::ImageLayout::eTransferSrcOptimal,
|
||||
vk::AccessFlagBits2::eTransferRead,
|
||||
GetBarriers(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead,
|
||||
vk::PipelineStageFlagBits2::eCopy, {});
|
||||
vk::DependencyInfo dependency {};
|
||||
dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion;
|
||||
dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion;
|
||||
dependency.bufferMemoryBarrierCount = 1;
|
||||
dependency.pBufferMemoryBarriers = &buffer_barrier;
|
||||
dependency.imageMemoryBarrierCount = static_cast<uint32_t>(image_barriers.size());
|
||||
dependency.pImageMemoryBarriers = image_barriers.data();
|
||||
auto command = m_scheduler->Current().Handle();
|
||||
dependency.imageMemoryBarrierCount = static_cast<uint32_t>(image_barriers.size());
|
||||
dependency.pImageMemoryBarriers = image_barriers.data();
|
||||
auto command = m_scheduler->Current().Handle();
|
||||
command.pipelineBarrier2(dependency);
|
||||
command.copyImageToBuffer(backing.image, vk::ImageLayout::eTransferSrcOptimal, buffer,
|
||||
static_cast<uint32_t>(copies.size()), copies.data());
|
||||
@@ -299,11 +294,11 @@ void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buf
|
||||
command.pipelineBarrier2(dependency);
|
||||
}
|
||||
|
||||
std::pair<uint32_t, uint32_t>
|
||||
Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_t depth) {
|
||||
const auto source_type = source.backing.image_type;
|
||||
const auto destination_type = destination.backing.image_type;
|
||||
uint32_t source_layers = source.backing.layers;
|
||||
std::pair<uint32_t, uint32_t> Image::SanitizeCopyLayers(const Image& source,
|
||||
const Image& destination, uint32_t depth) {
|
||||
const auto source_type = source.backing.image_type;
|
||||
const auto destination_type = destination.backing.image_type;
|
||||
uint32_t source_layers = source.backing.layers;
|
||||
uint32_t destination_layers = destination.backing.layers;
|
||||
if (source_type == vk::ImageType::e3D) {
|
||||
source_layers = 1;
|
||||
@@ -312,13 +307,10 @@ Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_
|
||||
destination_layers = 1;
|
||||
}
|
||||
if (source_type == destination_type) {
|
||||
source_layers = destination_layers =
|
||||
std::min(source_layers, destination_layers);
|
||||
} else if (source_type == vk::ImageType::e2D &&
|
||||
destination_type == vk::ImageType::e3D) {
|
||||
source_layers = destination_layers = std::min(source_layers, destination_layers);
|
||||
} else if (source_type == vk::ImageType::e2D && destination_type == vk::ImageType::e3D) {
|
||||
source_layers = depth;
|
||||
} else if (source_type == vk::ImageType::e3D &&
|
||||
destination_type == vk::ImageType::e2D) {
|
||||
} else if (source_type == vk::ImageType::e3D && destination_type == vk::ImageType::e2D) {
|
||||
destination_layers = depth;
|
||||
}
|
||||
return {source_layers, destination_layers};
|
||||
@@ -327,12 +319,11 @@ Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_
|
||||
void Image::CopyImage(Image& source) {
|
||||
EXIT_IF(m_scheduler == nullptr || source.backing.samples != backing.samples);
|
||||
m_scheduler->EndRendering();
|
||||
const uint32_t levels =
|
||||
std::min(source.backing.mip_levels, backing.mip_levels);
|
||||
const uint32_t levels = std::min(source.backing.mip_levels, backing.mip_levels);
|
||||
const uint32_t base_depth = backing.image_type == vk::ImageType::e3D
|
||||
? backing.extent.depth
|
||||
: source.backing.extent.depth;
|
||||
const auto source_aspect =
|
||||
const auto source_aspect =
|
||||
FullAspectMask(source.backing.format) & ~vk::ImageAspectFlagBits::eStencil;
|
||||
const auto destination_aspect =
|
||||
FullAspectMask(backing.format) & ~vk::ImageAspectFlagBits::eStencil;
|
||||
@@ -342,8 +333,7 @@ void Image::CopyImage(Image& source) {
|
||||
const auto width = std::max(source.backing.extent.width >> level, 1u);
|
||||
const auto height = std::max(source.backing.extent.height >> level, 1u);
|
||||
const auto depth = std::max(base_depth >> level, 1u);
|
||||
const auto [source_layers, destination_layers] =
|
||||
SanitizeCopyLayers(source, *this, depth);
|
||||
const auto [source_layers, destination_layers] = SanitizeCopyLayers(source, *this, depth);
|
||||
vk::ImageCopy copy {};
|
||||
copy.srcSubresource = {source_aspect, level, 0, 1};
|
||||
copy.dstSubresource = {destination_aspect, level, 0, 1};
|
||||
@@ -351,8 +341,7 @@ void Image::CopyImage(Image& source) {
|
||||
if (source.backing.image_type == vk::ImageType::e3D) {
|
||||
copy.extent = {width, height, depth};
|
||||
} else {
|
||||
copy.srcSubresource.layerCount =
|
||||
std::min(source_layers, destination_layers);
|
||||
copy.srcSubresource.layerCount = std::min(source_layers, destination_layers);
|
||||
copy.dstSubresource.layerCount = copy.srcSubresource.layerCount;
|
||||
copy.extent = {width, height, 1};
|
||||
}
|
||||
@@ -369,34 +358,30 @@ void Image::CopyImage(Image& source) {
|
||||
return;
|
||||
}
|
||||
auto command = m_scheduler->Current().Handle();
|
||||
source.Transit(vk::ImageLayout::eTransferSrcOptimal,
|
||||
vk::AccessFlagBits2::eTransferRead, {}, command);
|
||||
Transit(vk::ImageLayout::eTransferDstOptimal,
|
||||
vk::AccessFlagBits2::eTransferWrite, {}, command);
|
||||
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
|
||||
backing.image, vk::ImageLayout::eTransferDstOptimal,
|
||||
static_cast<uint32_t>(copies.size()), copies.data());
|
||||
source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
|
||||
command);
|
||||
Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
|
||||
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
|
||||
vk::ImageLayout::eTransferDstOptimal, static_cast<uint32_t>(copies.size()),
|
||||
copies.data());
|
||||
Transit(vk::ImageLayout::eGeneral,
|
||||
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
|
||||
command);
|
||||
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
|
||||
}
|
||||
|
||||
void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
|
||||
const ImageSubresourceRange& destination_range) {
|
||||
EXIT_IF(m_scheduler == nullptr || backing.samples != 1 ||
|
||||
source.backing.image_type != vk::ImageType::e2D ||
|
||||
backing.image_type != vk::ImageType::e2D ||
|
||||
source_range.level_count != 1 || destination_range.level_count != 1 ||
|
||||
backing.image_type != vk::ImageType::e2D || source_range.level_count != 1 ||
|
||||
destination_range.level_count != 1 ||
|
||||
source_range.base_level >= source.backing.mip_levels ||
|
||||
destination_range.base_level >= backing.mip_levels ||
|
||||
source_range.base_layer >= source.backing.layers ||
|
||||
destination_range.base_layer >= backing.layers);
|
||||
const auto layers = std::min(
|
||||
{source_range.layer_count, destination_range.layer_count,
|
||||
source.backing.layers - source_range.base_layer,
|
||||
backing.layers - destination_range.base_layer});
|
||||
const auto source_width =
|
||||
std::max(source.backing.extent.width >> source_range.base_level, 1u);
|
||||
const auto layers = std::min({source_range.layer_count, destination_range.layer_count,
|
||||
source.backing.layers - source_range.base_layer,
|
||||
backing.layers - destination_range.base_layer});
|
||||
const auto source_width = std::max(source.backing.extent.width >> source_range.base_level, 1u);
|
||||
const auto source_height =
|
||||
std::max(source.backing.extent.height >> source_range.base_level, 1u);
|
||||
const auto destination_width =
|
||||
@@ -404,43 +389,40 @@ void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
|
||||
const auto destination_height =
|
||||
std::max(backing.extent.height >> destination_range.base_level, 1u);
|
||||
const bool copy = source.backing.samples == 1;
|
||||
EXIT_IF(layers == 0 || info.extent.width > source_width ||
|
||||
info.extent.height > source_height || info.extent.width > destination_width ||
|
||||
info.extent.height > destination_height ||
|
||||
EXIT_IF(layers == 0 || info.extent.width > source_width || info.extent.height > source_height ||
|
||||
info.extent.width > destination_width || info.extent.height > destination_height ||
|
||||
(copy ? !ImageViewOps::FormatsCompatible(source.backing.format, backing.format)
|
||||
: source.backing.format != backing.format));
|
||||
auto resolved_source_range = source_range;
|
||||
auto resolved_destination_range = destination_range;
|
||||
auto resolved_source_range = source_range;
|
||||
auto resolved_destination_range = destination_range;
|
||||
resolved_source_range.layer_count = layers;
|
||||
resolved_destination_range.layer_count = layers;
|
||||
const vk::Extent3D resolve_extent {info.extent.width, info.extent.height, 1};
|
||||
|
||||
m_scheduler->EndRendering();
|
||||
auto command = m_scheduler->Current().Handle();
|
||||
source.Transit(vk::ImageLayout::eTransferSrcOptimal,
|
||||
vk::AccessFlagBits2::eTransferRead, resolved_source_range, command);
|
||||
Transit(vk::ImageLayout::eTransferDstOptimal,
|
||||
vk::AccessFlagBits2::eTransferWrite, resolved_destination_range, command);
|
||||
source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead,
|
||||
resolved_source_range, command);
|
||||
Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
|
||||
resolved_destination_range, command);
|
||||
if (copy) {
|
||||
vk::ImageCopy region {};
|
||||
region.srcSubresource = {vk::ImageAspectFlagBits::eColor,
|
||||
resolved_source_range.base_level,
|
||||
region.srcSubresource = {vk::ImageAspectFlagBits::eColor, resolved_source_range.base_level,
|
||||
resolved_source_range.base_layer, layers};
|
||||
region.dstSubresource = {vk::ImageAspectFlagBits::eColor,
|
||||
resolved_destination_range.base_level,
|
||||
resolved_destination_range.base_layer, layers};
|
||||
region.extent = resolve_extent;
|
||||
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
|
||||
backing.image, vk::ImageLayout::eTransferDstOptimal, region);
|
||||
region.extent = resolve_extent;
|
||||
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
|
||||
vk::ImageLayout::eTransferDstOptimal, region);
|
||||
} else {
|
||||
vk::ImageResolve region {};
|
||||
region.srcSubresource = {vk::ImageAspectFlagBits::eColor,
|
||||
resolved_source_range.base_level,
|
||||
region.srcSubresource = {vk::ImageAspectFlagBits::eColor, resolved_source_range.base_level,
|
||||
resolved_source_range.base_layer, layers};
|
||||
region.dstSubresource = {vk::ImageAspectFlagBits::eColor,
|
||||
resolved_destination_range.base_level,
|
||||
resolved_destination_range.base_layer, layers};
|
||||
region.extent = resolve_extent;
|
||||
region.extent = resolve_extent;
|
||||
command.resolveImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
|
||||
backing.image, vk::ImageLayout::eTransferDstOptimal, region);
|
||||
}
|
||||
@@ -454,22 +436,21 @@ uint32_t Image::CopyRows(uint64_t row_size, uint32_t rows, uint64_t capacity) no
|
||||
}
|
||||
|
||||
void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
|
||||
EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr ||
|
||||
source.backing.samples != 1 || backing.samples != 1);
|
||||
EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr || source.backing.samples != 1 ||
|
||||
backing.samples != 1);
|
||||
m_scheduler->EndRendering();
|
||||
const uint32_t levels =
|
||||
std::min(source.backing.mip_levels, backing.mip_levels);
|
||||
const auto source_aspect =
|
||||
const uint32_t levels = std::min(source.backing.mip_levels, backing.mip_levels);
|
||||
const auto source_aspect =
|
||||
FullAspectMask(source.backing.format) & ~vk::ImageAspectFlagBits::eStencil;
|
||||
const auto destination_aspect =
|
||||
FullAspectMask(backing.format) & ~vk::ImageAspectFlagBits::eStencil;
|
||||
const auto source_bytes = DepthAspectTransferBytes(source.backing.format) != 0
|
||||
? DepthAspectTransferBytes(source.backing.format)
|
||||
: source.info.bytes_per_block;
|
||||
const auto destination_bytes = DepthAspectTransferBytes(backing.format) != 0
|
||||
? DepthAspectTransferBytes(backing.format)
|
||||
: info.bytes_per_block;
|
||||
const uint32_t source_block = source.info.IsBlock() ? 4u : 1u;
|
||||
const auto source_bytes = DepthAspectTransferBytes(source.backing.format) != 0
|
||||
? DepthAspectTransferBytes(source.backing.format)
|
||||
: source.info.bytes_per_block;
|
||||
const auto destination_bytes = DepthAspectTransferBytes(backing.format) != 0
|
||||
? DepthAspectTransferBytes(backing.format)
|
||||
: info.bytes_per_block;
|
||||
const uint32_t source_block = source.info.IsBlock() ? 4u : 1u;
|
||||
const uint32_t destination_block = info.IsBlock() ? 4u : 1u;
|
||||
EXIT_IF(levels == 0 || source_bytes == 0 || source_bytes != destination_bytes ||
|
||||
source_block != destination_block);
|
||||
@@ -484,74 +465,66 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
|
||||
barrier.buffer = buffer.Handle();
|
||||
barrier.offset = 0;
|
||||
vk::DependencyInfo dependency {};
|
||||
dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion;
|
||||
dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion;
|
||||
dependency.bufferMemoryBarrierCount = 1;
|
||||
dependency.pBufferMemoryBarriers = &barrier;
|
||||
auto command = m_scheduler->Current().Handle();
|
||||
source.Transit(vk::ImageLayout::eTransferSrcOptimal,
|
||||
vk::AccessFlagBits2::eTransferRead, {}, command);
|
||||
Transit(vk::ImageLayout::eTransferDstOptimal,
|
||||
vk::AccessFlagBits2::eTransferWrite, {}, command);
|
||||
auto command = m_scheduler->Current().Handle();
|
||||
source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
|
||||
command);
|
||||
Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
|
||||
for (uint32_t level = 0; level < levels; level++) {
|
||||
const auto width = std::max(source.backing.extent.width >> level, 1u);
|
||||
const auto height = std::max(source.backing.extent.height >> level, 1u);
|
||||
const auto source_depth = source.backing.image_type == vk::ImageType::e3D
|
||||
? std::max(source.backing.extent.depth >> level, 1u)
|
||||
: source.backing.layers;
|
||||
const auto width = std::max(source.backing.extent.width >> level, 1u);
|
||||
const auto height = std::max(source.backing.extent.height >> level, 1u);
|
||||
const auto source_depth = source.backing.image_type == vk::ImageType::e3D
|
||||
? std::max(source.backing.extent.depth >> level, 1u)
|
||||
: source.backing.layers;
|
||||
const auto destination_depth = backing.image_type == vk::ImageType::e3D
|
||||
? std::max(backing.extent.depth >> level, 1u)
|
||||
: backing.layers;
|
||||
const auto slices = std::min(source_depth, destination_depth);
|
||||
const auto block_rows = (height + source_block - 1) / source_block;
|
||||
const auto slices = std::min(source_depth, destination_depth);
|
||||
const auto block_rows = (height + source_block - 1) / source_block;
|
||||
const auto row_size =
|
||||
static_cast<uint64_t>((width + source_block - 1) / source_block) * source_bytes;
|
||||
const auto rows_per_copy = CopyRows(row_size, block_rows, buffer.Size());
|
||||
EXIT_IF(slices == 0 || rows_per_copy == 0);
|
||||
for (uint32_t slice = 0; slice < slices; slice++) {
|
||||
for (uint32_t block_row = 0; block_row < block_rows;
|
||||
block_row += rows_per_copy) {
|
||||
const auto copy_rows = std::min(rows_per_copy, block_rows - block_row);
|
||||
const auto y = block_row * source_block;
|
||||
const auto copy_height =
|
||||
std::min(copy_rows * source_block, height - y);
|
||||
const auto copy_size = row_size * copy_rows;
|
||||
for (uint32_t block_row = 0; block_row < block_rows; block_row += rows_per_copy) {
|
||||
const auto copy_rows = std::min(rows_per_copy, block_rows - block_row);
|
||||
const auto y = block_row * source_block;
|
||||
const auto copy_height = std::min(copy_rows * source_block, height - y);
|
||||
const auto copy_size = row_size * copy_rows;
|
||||
vk::BufferImageCopy source_copy {};
|
||||
source_copy.imageSubresource = {
|
||||
source_aspect, level,
|
||||
source.backing.image_type == vk::ImageType::e3D ? 0u : slice, 1};
|
||||
source_copy.imageOffset = {
|
||||
0, static_cast<int32_t>(y),
|
||||
source.backing.image_type == vk::ImageType::e3D
|
||||
? static_cast<int32_t>(slice)
|
||||
: 0};
|
||||
source_copy.imageExtent = {width, copy_height, 1};
|
||||
auto destination_copy = source_copy;
|
||||
source_copy.imageOffset = {0, static_cast<int32_t>(y),
|
||||
source.backing.image_type == vk::ImageType::e3D
|
||||
? static_cast<int32_t>(slice)
|
||||
: 0};
|
||||
source_copy.imageExtent = {width, copy_height, 1};
|
||||
auto destination_copy = source_copy;
|
||||
destination_copy.imageSubresource = {
|
||||
destination_aspect, level,
|
||||
backing.image_type == vk::ImageType::e3D ? 0u : slice, 1};
|
||||
destination_copy.imageOffset.z =
|
||||
backing.image_type == vk::ImageType::e3D
|
||||
? static_cast<int32_t>(slice)
|
||||
: 0;
|
||||
backing.image_type == vk::ImageType::e3D ? static_cast<int32_t>(slice) : 0;
|
||||
barrier.size = copy_size;
|
||||
barrier.srcAccessMask = vk::AccessFlagBits2::eTransferRead;
|
||||
barrier.dstAccessMask = vk::AccessFlagBits2::eTransferWrite;
|
||||
command.pipelineBarrier2(dependency);
|
||||
command.copyImageToBuffer(source.backing.image,
|
||||
vk::ImageLayout::eTransferSrcOptimal,
|
||||
buffer.Handle(), source_copy);
|
||||
vk::ImageLayout::eTransferSrcOptimal, buffer.Handle(),
|
||||
source_copy);
|
||||
barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
|
||||
barrier.dstAccessMask = vk::AccessFlagBits2::eTransferRead;
|
||||
command.pipelineBarrier2(dependency);
|
||||
command.copyBufferToImage(buffer.Handle(), backing.image,
|
||||
vk::ImageLayout::eTransferDstOptimal,
|
||||
destination_copy);
|
||||
vk::ImageLayout::eTransferDstOptimal, destination_copy);
|
||||
}
|
||||
}
|
||||
}
|
||||
Transit(vk::ImageLayout::eGeneral,
|
||||
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
|
||||
command);
|
||||
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
|
||||
}
|
||||
|
||||
void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
|
||||
@@ -561,11 +534,9 @@ void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
|
||||
const auto width = std::max(backing.extent.width >> mip, 1u);
|
||||
const auto height = std::max(backing.extent.height >> mip, 1u);
|
||||
const auto depth = std::max(backing.extent.depth >> mip, 1u);
|
||||
EXIT_IF(width != source.backing.extent.width ||
|
||||
height != source.backing.extent.height);
|
||||
const auto [source_layers, destination_layers] =
|
||||
SanitizeCopyLayers(source, *this, depth);
|
||||
const auto aspects = FullAspectMask(source.backing.format);
|
||||
EXIT_IF(width != source.backing.extent.width || height != source.backing.extent.height);
|
||||
const auto [source_layers, destination_layers] = SanitizeCopyLayers(source, *this, depth);
|
||||
const auto aspects = FullAspectMask(source.backing.format);
|
||||
EXIT_IF(aspects != FullAspectMask(backing.format));
|
||||
std::array<vk::ImageCopy, 2> copies {};
|
||||
uint32_t copy_count = 0;
|
||||
@@ -580,16 +551,13 @@ void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
|
||||
copy.extent = {width, height, depth};
|
||||
}
|
||||
auto command = m_scheduler->Current().Handle();
|
||||
Transit(vk::ImageLayout::eTransferDstOptimal,
|
||||
vk::AccessFlagBits2::eTransferWrite, {}, command);
|
||||
source.Transit(vk::ImageLayout::eTransferSrcOptimal,
|
||||
vk::AccessFlagBits2::eTransferRead, {}, command);
|
||||
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
|
||||
backing.image, vk::ImageLayout::eTransferDstOptimal, copy_count,
|
||||
copies.data());
|
||||
Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
|
||||
source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
|
||||
command);
|
||||
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
|
||||
vk::ImageLayout::eTransferDstOptimal, copy_count, copies.data());
|
||||
Transit(vk::ImageLayout::eGeneral,
|
||||
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
|
||||
command);
|
||||
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
|
||||
}
|
||||
|
||||
namespace ImageOps {
|
||||
@@ -601,8 +569,7 @@ void Validate(const ImageInfo& info) {
|
||||
if (info.pixel_format == vk::Format::eUndefined) {
|
||||
const bool metadata_empty =
|
||||
info.metadata.range.address == 0 && info.metadata.range.size == 0 &&
|
||||
info.metadata.kind == ImageMetadataKind::None &&
|
||||
info.metadata.control == 0 &&
|
||||
info.metadata.kind == ImageMetadataKind::None && info.metadata.control == 0 &&
|
||||
info.metadata.compression == VideoOutCompression::Uncompressed &&
|
||||
!info.metadata.stencil_compressed;
|
||||
if (info.data.Empty() || info.HasStencil() || !metadata_empty || info.extent.width == 0 ||
|
||||
@@ -615,9 +582,9 @@ void Validate(const ImageInfo& info) {
|
||||
}
|
||||
|
||||
if (info.extent.width == 0 || info.extent.height == 0 || info.extent.depth == 0 ||
|
||||
info.resources.levels == 0 ||
|
||||
info.resources.levels > info.mip_layout.size() || info.resources.layers == 0 ||
|
||||
info.samples == 0 || vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} ||
|
||||
info.resources.levels == 0 || info.resources.levels > info.mip_layout.size() ||
|
||||
info.resources.layers == 0 || info.samples == 0 ||
|
||||
vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} ||
|
||||
info.bytes_per_block == 0 || (info.data.address != 0 && info.pitch == 0)) {
|
||||
EXIT("invalid image geometry or format\n");
|
||||
}
|
||||
@@ -688,11 +655,11 @@ uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element) {
|
||||
|
||||
} // namespace ImageOps
|
||||
|
||||
Image::Image(GraphicContext& graphics, CommandScheduler& scheduler,
|
||||
const ImageInfo& image_info)
|
||||
Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageInfo& image_info)
|
||||
: info(image_info), m_graphics(&graphics), m_scheduler(&scheduler) {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
ImageOps::Validate(info);
|
||||
m_cpu_dirty = !info.data.Empty();
|
||||
if (info.pixel_format == vk::Format::eUndefined) {
|
||||
return;
|
||||
}
|
||||
@@ -742,9 +709,9 @@ Image::Image(GraphicContext& graphics, CommandScheduler& scheduler,
|
||||
}
|
||||
|
||||
uint64_t Image::HashGuestEdges() const {
|
||||
constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1;
|
||||
constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1;
|
||||
std::array<uint8_t, TRACKER_PAGE_SIZE * 2> bytes {};
|
||||
const auto range = info.data;
|
||||
const auto range = info.data;
|
||||
const uint64_t head_end = std::min(range.End(), (range.address + page_mask) & ~page_mask);
|
||||
const uint64_t tail_begin = std::max(range.address, range.End() & ~page_mask);
|
||||
const uint64_t head_size = head_end - range.address;
|
||||
+46
-37
@@ -3,7 +3,7 @@
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/renderer/imageInfo.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageInfo.h"
|
||||
|
||||
#include <compare>
|
||||
#include <limits>
|
||||
@@ -66,16 +66,16 @@ public:
|
||||
[[nodiscard]] vk::ImageView FindView(const ImageViewInfo& view_info);
|
||||
void AssociateDepth(ImageId image_id) { depth_id = image_id; }
|
||||
using Barriers = std::vector<vk::ImageMemoryBarrier2>;
|
||||
[[nodiscard]] Barriers
|
||||
GetBarriers(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
|
||||
vk::PipelineStageFlags2 destination_stage,
|
||||
std::optional<ImageSubresourceRange> range);
|
||||
[[nodiscard]] Barriers GetBarriers(vk::ImageLayout destination_layout,
|
||||
vk::AccessFlags2 destination_access,
|
||||
vk::PipelineStageFlags2 destination_stage,
|
||||
std::optional<ImageSubresourceRange> range);
|
||||
void Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
|
||||
std::optional<ImageSubresourceRange> range, vk::CommandBuffer command_buffer);
|
||||
void Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
|
||||
uint64_t offset, uint64_t size);
|
||||
void Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
|
||||
uint64_t offset, uint64_t size);
|
||||
void Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
|
||||
uint64_t size);
|
||||
void Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
|
||||
uint64_t size);
|
||||
void CopyImage(Image& source);
|
||||
void Resolve(Image& source, const ImageSubresourceRange& source_range,
|
||||
const ImageSubresourceRange& destination_range);
|
||||
@@ -84,8 +84,8 @@ public:
|
||||
|
||||
void InvalidateCpuWrite(uint64_t vaddr, uint64_t size) {
|
||||
if (ImageRangeOverlaps(info.data.address, info.data.size, vaddr, size)) {
|
||||
m_cpu_dirty = true;
|
||||
m_maybe_cpu_dirty = false;
|
||||
m_cpu_dirty = true;
|
||||
m_maybe_cpu_dirty = false;
|
||||
m_maybe_hash_valid = false;
|
||||
} else if (ImagePageRangesOverlap(info.data.address, info.data.size, vaddr, size)) {
|
||||
m_maybe_cpu_dirty = true;
|
||||
@@ -95,6 +95,11 @@ public:
|
||||
[[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty || m_maybe_cpu_dirty; }
|
||||
[[nodiscard]] bool IsDefinitelyCpuDirty() const { return m_cpu_dirty; }
|
||||
[[nodiscard]] bool IsMaybeCpuDirty() const { return m_maybe_cpu_dirty; }
|
||||
void MarkMaybeCpuDirty() {
|
||||
if (!m_cpu_dirty) {
|
||||
m_maybe_cpu_dirty = true;
|
||||
}
|
||||
}
|
||||
[[nodiscard]] bool NeedsMaybeCpuHash() const {
|
||||
return m_maybe_cpu_dirty && !m_maybe_hash_valid;
|
||||
}
|
||||
@@ -102,14 +107,14 @@ public:
|
||||
if (!NeedsMaybeCpuHash()) {
|
||||
EXIT("image cannot initialize maybe-dirty hash\n");
|
||||
}
|
||||
m_maybe_cpu_hash = hash;
|
||||
m_maybe_cpu_hash = hash;
|
||||
m_maybe_hash_valid = true;
|
||||
}
|
||||
[[nodiscard]] bool ResolveMaybeCpuHash(uint64_t hash) {
|
||||
if (!m_maybe_cpu_dirty || !m_maybe_hash_valid || m_cpu_dirty) {
|
||||
EXIT("image cannot resolve maybe-dirty hash\n");
|
||||
}
|
||||
m_maybe_cpu_dirty = false;
|
||||
m_maybe_cpu_dirty = false;
|
||||
m_maybe_hash_valid = false;
|
||||
m_cpu_dirty |= hash != m_maybe_cpu_hash;
|
||||
return m_cpu_dirty;
|
||||
@@ -125,14 +130,15 @@ public:
|
||||
}
|
||||
|
||||
[[nodiscard]] bool IsGpuModified() const noexcept { return m_gpu_modified; }
|
||||
void MarkGpuModified() noexcept { m_gpu_modified = true; }
|
||||
void ClearGpuModified() noexcept { m_gpu_modified = false; }
|
||||
void MarkGpuModified() noexcept { m_gpu_modified = true; }
|
||||
void ClearGpuModified() noexcept { m_gpu_modified = false; }
|
||||
|
||||
[[nodiscard]] bool IsBufferModified() const noexcept { return m_buffer_modified; }
|
||||
void MarkBufferModified() noexcept { m_buffer_modified = true; }
|
||||
void ClearBufferModified() noexcept { m_buffer_modified = false; }
|
||||
void MarkBufferModified() noexcept { m_buffer_modified = true; }
|
||||
void ClearBufferModified() noexcept { m_buffer_modified = false; }
|
||||
|
||||
[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size, bool pages = false) const noexcept {
|
||||
[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size,
|
||||
bool pages = false) const noexcept {
|
||||
return pages ? ImagePageRangesOverlap(info.data.address, info.data.size, address, size)
|
||||
: ImageRangeOverlaps(info.data.address, info.data.size, address, size);
|
||||
}
|
||||
@@ -142,38 +148,41 @@ public:
|
||||
[[nodiscard]] bool SafeToDownload() const noexcept {
|
||||
return IsGpuModified() && !IsBufferModified() && !IsCpuDirty();
|
||||
}
|
||||
[[nodiscard]] bool IsTracked() const noexcept { return track_addr != 0 && track_addr_end != 0; }
|
||||
[[nodiscard]] uint64_t AccountedSize() const noexcept {
|
||||
return backing.image == nullptr ? 0 : (info.data.size + 1023) & ~uint64_t {1023};
|
||||
}
|
||||
[[nodiscard]] uint64_t HashGuestEdges() const;
|
||||
|
||||
ImageInfo info;
|
||||
VulkanImage backing;
|
||||
ImageViewCache views;
|
||||
ImageUsage usage;
|
||||
ImageBinding binding;
|
||||
bool registered = false;
|
||||
ImageId depth_id {};
|
||||
uint64_t tick_accessed_last = 0;
|
||||
size_t lru_id = 0;
|
||||
ImageInfo info;
|
||||
VulkanImage backing;
|
||||
ImageViewCache views;
|
||||
ImageUsage usage;
|
||||
ImageBinding binding;
|
||||
bool registered = false;
|
||||
uint64_t track_addr = 0;
|
||||
uint64_t track_addr_end = 0;
|
||||
ImageId depth_id {};
|
||||
uint64_t tick_accessed_last = 0;
|
||||
size_t lru_id = 0;
|
||||
|
||||
private:
|
||||
friend struct ImageTestAccess;
|
||||
|
||||
[[nodiscard]] static vk::ImageAspectFlags FullAspectMask(vk::Format format) noexcept;
|
||||
[[nodiscard]] static uint32_t CopyRows(uint64_t row_size, uint32_t rows,
|
||||
uint64_t capacity) noexcept;
|
||||
[[nodiscard]] static uint32_t CopyRows(uint64_t row_size, uint32_t rows,
|
||||
uint64_t capacity) noexcept;
|
||||
[[nodiscard]] static std::pair<uint32_t, uint32_t>
|
||||
SanitizeCopyLayers(const Image& source, const Image& destination, uint32_t depth);
|
||||
|
||||
GraphicContext* m_graphics = nullptr;
|
||||
CommandScheduler* m_scheduler = nullptr;
|
||||
uint64_t m_maybe_cpu_hash = 0;
|
||||
bool m_cpu_dirty = false;
|
||||
bool m_maybe_cpu_dirty = false;
|
||||
bool m_maybe_hash_valid = false;
|
||||
bool m_gpu_modified = false;
|
||||
bool m_buffer_modified = false;
|
||||
GraphicContext* m_graphics = nullptr;
|
||||
CommandScheduler* m_scheduler = nullptr;
|
||||
uint64_t m_maybe_cpu_hash = 0;
|
||||
bool m_cpu_dirty = false;
|
||||
bool m_maybe_cpu_dirty = false;
|
||||
bool m_maybe_hash_valid = false;
|
||||
bool m_gpu_modified = false;
|
||||
bool m_buffer_modified = false;
|
||||
};
|
||||
|
||||
namespace ImageOps {
|
||||
+16
-23
@@ -19,10 +19,9 @@ struct GuestRange {
|
||||
uint64_t address = 0;
|
||||
uint64_t size = 0;
|
||||
|
||||
[[nodiscard]] constexpr bool Empty() const noexcept { return address == 0 || size == 0; }
|
||||
[[nodiscard]] constexpr bool Valid() const noexcept {
|
||||
return !Empty() && address < TRACKER_ADDRESS_SIZE &&
|
||||
size <= TRACKER_ADDRESS_SIZE - address;
|
||||
[[nodiscard]] constexpr bool Empty() const noexcept { return address == 0 || size == 0; }
|
||||
[[nodiscard]] constexpr bool Valid() const noexcept {
|
||||
return !Empty() && address < TRACKER_ADDRESS_SIZE && size <= TRACKER_ADDRESS_SIZE - address;
|
||||
}
|
||||
[[nodiscard]] constexpr uint64_t End() const noexcept { return address + size; }
|
||||
auto operator<=>(const GuestRange&) const = default;
|
||||
@@ -47,10 +46,10 @@ struct ImageSubresources {
|
||||
};
|
||||
|
||||
struct ImageSubresourceRange {
|
||||
uint32_t base_level = 0;
|
||||
uint32_t level_count = 1;
|
||||
uint32_t base_layer = 0;
|
||||
uint32_t layer_count = 1;
|
||||
uint32_t base_level = 0;
|
||||
uint32_t level_count = 1;
|
||||
uint32_t base_layer = 0;
|
||||
uint32_t layer_count = 1;
|
||||
auto operator<=>(const ImageSubresourceRange&) const = default;
|
||||
};
|
||||
|
||||
@@ -67,10 +66,10 @@ struct ImageInfo {
|
||||
GuestRange stencil;
|
||||
ImageMetadataInfo metadata;
|
||||
uint32_t htile_clear_mask = UINT32_MAX;
|
||||
vk::Format pixel_format = vk::Format::eUndefined;
|
||||
uint32_t guest_format = 0;
|
||||
Prospero::ImageType type = Prospero::ImageType::kColor2D;
|
||||
vk::Extent3D extent = {1, 1, 1};
|
||||
vk::Format pixel_format = vk::Format::eUndefined;
|
||||
uint32_t guest_format = 0;
|
||||
Prospero::ImageType type = Prospero::ImageType::kColor2D;
|
||||
vk::Extent3D extent = {1, 1, 1};
|
||||
ImageSubresources resources;
|
||||
uint32_t pitch = 0;
|
||||
uint32_t bytes_per_block = 0;
|
||||
@@ -352,8 +351,7 @@ inline bool ImageInfo::IsDepth() const noexcept {
|
||||
}
|
||||
const auto transfer_bytes = DepthAspectTransferBytes(info.pixel_format);
|
||||
return transfer_bytes == info.bytes_per_block ||
|
||||
(info.bytes_per_block == sizeof(uint16_t) &&
|
||||
transfer_bytes == sizeof(uint32_t));
|
||||
(info.bytes_per_block == sizeof(uint16_t) && transfer_bytes == sizeof(uint32_t));
|
||||
}
|
||||
|
||||
[[nodiscard]] inline VideoOutCompression
|
||||
@@ -470,18 +468,13 @@ IsSupportedDisplayRenderTargetTileMode(uint32_t tile_mode) noexcept {
|
||||
vk::ClearColorValue& clear) {
|
||||
vk::ClearColorValue next {};
|
||||
const auto unorm8 = [](uint32_t value) { return static_cast<float>(value & 0xffu) / 255.0f; };
|
||||
const auto srgb8 = [](uint32_t value) {
|
||||
const auto srgb8 = [](uint32_t value) {
|
||||
const auto encoded = static_cast<float>(value & 0xffu) / 255.0f;
|
||||
return encoded <= 0.04045f ? encoded / 12.92f
|
||||
: std::pow((encoded + 0.055f) / 1.055f, 2.4f);
|
||||
return encoded <= 0.04045f ? encoded / 12.92f : std::pow((encoded + 0.055f) / 1.055f, 2.4f);
|
||||
};
|
||||
switch (format) {
|
||||
case vk::Format::eR32Uint:
|
||||
next.uint32[0] = packed;
|
||||
break;
|
||||
case vk::Format::eR32Sint:
|
||||
next.int32[0] = static_cast<int32_t>(packed);
|
||||
break;
|
||||
case vk::Format::eR32Uint: next.uint32[0] = packed; break;
|
||||
case vk::Format::eR32Sint: next.int32[0] = static_cast<int32_t>(packed); break;
|
||||
case vk::Format::eR8G8B8A8Srgb:
|
||||
next.float32[0] = srgb8(packed);
|
||||
next.float32[1] = srgb8(packed >> 8u);
|
||||
+20
-24
@@ -1,8 +1,8 @@
|
||||
#include "graphics/host_gpu/renderer/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageView.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/renderer/image.h"
|
||||
#include "graphics/host_gpu/renderer/image/image.h"
|
||||
|
||||
#include <mutex>
|
||||
|
||||
@@ -70,15 +70,14 @@ namespace {
|
||||
}
|
||||
case vk::ImageType::e3D:
|
||||
switch (info.type) {
|
||||
case vk::ImageViewType::e3D:
|
||||
return info.base_layer == 0 && info.layer_count == 1;
|
||||
case vk::ImageViewType::e3D: return info.base_layer == 0 && info.layer_count == 1;
|
||||
case vk::ImageViewType::e2D:
|
||||
return static_cast<bool>(
|
||||
image.flags & vk::ImageCreateFlagBits::e2DArrayCompatible) &&
|
||||
return static_cast<bool>(image.flags &
|
||||
vk::ImageCreateFlagBits::e2DArrayCompatible) &&
|
||||
info.level_count == 1 && info.layer_count == 1;
|
||||
case vk::ImageViewType::e2DArray:
|
||||
return static_cast<bool>(
|
||||
image.flags & vk::ImageCreateFlagBits::e2DArrayCompatible) &&
|
||||
return static_cast<bool>(image.flags &
|
||||
vk::ImageCreateFlagBits::e2DArrayCompatible) &&
|
||||
info.level_count == 1;
|
||||
default: return false;
|
||||
}
|
||||
@@ -325,11 +324,10 @@ bool FormatsCompatible(vk::Format base, vk::Format view) noexcept {
|
||||
} // namespace ImageViewOps
|
||||
|
||||
vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
|
||||
const auto& image = backing;
|
||||
const auto& image = backing;
|
||||
auto normalized = view_info;
|
||||
const bool is_storage =
|
||||
static_cast<bool>(normalized.usage & vk::ImageUsageFlagBits::eStorage);
|
||||
normalized.aspect = FullAspectMask(image.format);
|
||||
const bool is_storage = static_cast<bool>(normalized.usage & vk::ImageUsageFlagBits::eStorage);
|
||||
normalized.aspect = FullAspectMask(image.format);
|
||||
if (normalized.aspect & vk::ImageAspectFlagBits::eDepth &&
|
||||
IsDepthViewFormat(normalized.format)) {
|
||||
normalized.format = image.format;
|
||||
@@ -340,28 +338,26 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
|
||||
normalized.format = image.format;
|
||||
normalized.aspect = vk::ImageAspectFlagBits::eStencil;
|
||||
}
|
||||
normalized.usage =
|
||||
is_storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlags {};
|
||||
normalized.usage = is_storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlags {};
|
||||
const bool format_compatible = normalized.format != vk::Format::eUndefined &&
|
||||
IsCompatibleViewFormat(image.format, normalized.format);
|
||||
const bool slice_view = image.image_type == vk::ImageType::e3D &&
|
||||
(normalized.type == vk::ImageViewType::e2D ||
|
||||
normalized.type == vk::ImageViewType::e2DArray);
|
||||
const bool slice_view =
|
||||
image.image_type == vk::ImageType::e3D && (normalized.type == vk::ImageViewType::e2D ||
|
||||
normalized.type == vk::ImageViewType::e2DArray);
|
||||
const bool levels_valid = normalized.level_count != 0 &&
|
||||
normalized.base_level < image.mip_levels &&
|
||||
normalized.level_count <= image.mip_levels - normalized.base_level;
|
||||
const auto view_layers = slice_view && levels_valid
|
||||
? std::max(image.extent.depth >> normalized.base_level, 1u)
|
||||
: image.layers;
|
||||
const bool ranges_valid = levels_valid &&
|
||||
normalized.layer_count != 0 && normalized.base_layer < view_layers &&
|
||||
const auto view_layers = slice_view && levels_valid
|
||||
? std::max(image.extent.depth >> normalized.base_level, 1u)
|
||||
: image.layers;
|
||||
const bool ranges_valid = levels_valid && normalized.layer_count != 0 &&
|
||||
normalized.base_layer < view_layers &&
|
||||
normalized.layer_count <= view_layers - normalized.base_layer;
|
||||
const bool mapping_valid =
|
||||
IsComponentSwizzle(normalized.mapping.r) && IsComponentSwizzle(normalized.mapping.g) &&
|
||||
IsComponentSwizzle(normalized.mapping.b) && IsComponentSwizzle(normalized.mapping.a);
|
||||
if (image.image == nullptr || !format_compatible || !ranges_valid || !mapping_valid ||
|
||||
!IsValidViewType(image, normalized) ||
|
||||
!IsValidAspect(image, normalized.aspect)) {
|
||||
!IsValidViewType(image, normalized) || !IsValidAspect(image, normalized.aspect)) {
|
||||
EXIT("invalid image view: image_format=%d view_format=%d type=%d aspect=0x%x "
|
||||
"mip=%u+%u layer=%u+%u usage=0x%x image_levels=%u image_layers=%u\n",
|
||||
static_cast<int>(image.format), static_cast<int>(normalized.format),
|
||||
+10
-8
@@ -2,8 +2,8 @@
|
||||
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEVIEW_H_
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "graphics/host_gpu/renderer/imageInfo.h"
|
||||
#include "graphics/shader/recompiler/ShaderIR.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageInfo.h"
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
#include "graphics/shader/shader.h"
|
||||
|
||||
namespace Libs::Graphics {
|
||||
@@ -88,7 +88,9 @@ SelectSampledDepthView(vk::Format image_format, vk::Format view_format, uint32_t
|
||||
IsSupportedSampledDepthResource(const ShaderRecompiler::IR::ImageResource& resource) noexcept {
|
||||
return resource.kind == ShaderRecompiler::IR::ResourceKind::Image &&
|
||||
(resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray ||
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa ||
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray) &&
|
||||
resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.read &&
|
||||
!resource.written && !resource.atomic;
|
||||
}
|
||||
@@ -103,10 +105,7 @@ IsSupportedSampledDepthUintResource(const ShaderRecompiler::IR::ImageResource& r
|
||||
|
||||
inline void ValidateStorageColorView(vk::Format image_format, vk::Format view_format,
|
||||
uint32_t swizzle) noexcept {
|
||||
const auto srgb_view = SrgbStorageViewFormat(image_format);
|
||||
const bool srgb_storage_view =
|
||||
srgb_view != vk::Format::eUndefined && view_format == srgb_view;
|
||||
if ((image_format != view_format && !srgb_storage_view) ||
|
||||
if (!ImageViewOps::FormatsCompatible(image_format, view_format) ||
|
||||
!IsValidImageSwizzle(swizzle)) {
|
||||
UnsupportedColorView("storage", image_format, view_format, swizzle);
|
||||
}
|
||||
@@ -122,7 +121,10 @@ IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resou
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D ||
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
|
||||
resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.written &&
|
||||
!resource.atomic && !resource.depth_compare;
|
||||
(!resource.atomic ||
|
||||
(resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint &&
|
||||
resource.read)) &&
|
||||
!resource.depth_compare;
|
||||
}
|
||||
|
||||
inline void
|
||||
+49
-51
@@ -1,9 +1,9 @@
|
||||
#include "graphics/host_gpu/objects/textureCommon.h"
|
||||
#include "graphics/host_gpu/renderer/image/textureCommon.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "graphics/guest_gpu/gpu_defs.h"
|
||||
#include "graphics/guest_gpu/gpu_format.h"
|
||||
#include "graphics/host_gpu/renderer/tiler.h"
|
||||
#include "graphics/host_gpu/renderer/image/tiler.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -70,6 +70,10 @@ constexpr RenderTargetFormatMapping kRenderTargetFormats[] = {
|
||||
Prospero::ChannelType::kFloat,
|
||||
Prospero::ChannelOrder::kStandard,
|
||||
{vk::Format::eB10G11R11UfloatPack32, 4}},
|
||||
{Prospero::ChannelLayout::k5_6_5,
|
||||
Prospero::ChannelType::kUNorm,
|
||||
Prospero::ChannelOrder::kStandard,
|
||||
{vk::Format::eB5G6R5UnormPack16, 2}},
|
||||
{Prospero::ChannelLayout::k16,
|
||||
Prospero::ChannelType::kUNorm,
|
||||
Prospero::ChannelOrder::kStandard,
|
||||
@@ -102,6 +106,10 @@ constexpr RenderTargetFormatMapping kRenderTargetFormats[] = {
|
||||
Prospero::ChannelType::kUNorm,
|
||||
Prospero::ChannelOrder::kStandard,
|
||||
{vk::Format::eR16G16B16A16Unorm, 8}},
|
||||
{Prospero::ChannelLayout::k16_16_16_16,
|
||||
Prospero::ChannelType::kUInt,
|
||||
Prospero::ChannelOrder::kStandard,
|
||||
{vk::Format::eR16G16B16A16Uint, 8}},
|
||||
{Prospero::ChannelLayout::k16_16_16_16,
|
||||
Prospero::ChannelType::kFloat,
|
||||
Prospero::ChannelOrder::kStandard,
|
||||
@@ -393,10 +401,10 @@ TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64
|
||||
return layout;
|
||||
}
|
||||
|
||||
std::vector<vk::BufferImageCopy>
|
||||
TextureBuildImageCopies(const TextureUploadLayout& layout, uint32_t width, uint32_t height,
|
||||
uint32_t depth, uint64_t levels, bool array_texture,
|
||||
bool volume_texture) {
|
||||
std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayout& layout,
|
||||
uint32_t width, uint32_t height,
|
||||
uint32_t depth, uint64_t levels,
|
||||
bool array_texture, bool volume_texture) {
|
||||
uint32_t mip_width = width;
|
||||
uint32_t mip_height = height;
|
||||
uint32_t mip_pitch = volume_texture && static_cast<Prospero::TileMode>(layout.tile) !=
|
||||
@@ -412,14 +420,13 @@ TextureBuildImageCopies(const TextureUploadLayout& layout, uint32_t width, uint3
|
||||
const auto mip_depth = GetTextureLevelDepth(depth, i, volume_texture);
|
||||
|
||||
for (uint32_t z = 0; z < mip_depth; z++) {
|
||||
const auto slice_offset = z * layout.slice_stride;
|
||||
const auto slice_offset = z * layout.slice_stride;
|
||||
vk::BufferImageCopy region {};
|
||||
region.bufferOffset =
|
||||
layout.level_sizes[i].offset + slice_offset;
|
||||
region.imageSubresource = {vk::ImageAspectFlagBits::eColor, i,
|
||||
array_texture ? z : 0, 1};
|
||||
region.imageOffset.z = volume_texture ? static_cast<int>(z) : 0;
|
||||
region.imageExtent = {mip_width, mip_height, 1};
|
||||
region.bufferOffset = layout.level_sizes[i].offset + slice_offset;
|
||||
region.imageSubresource = {vk::ImageAspectFlagBits::eColor, i, array_texture ? z : 0,
|
||||
1};
|
||||
region.imageOffset.z = volume_texture ? static_cast<int>(z) : 0;
|
||||
region.imageExtent = {mip_width, mip_height, 1};
|
||||
const bool linear =
|
||||
static_cast<Prospero::TileMode>(layout.tile) == Prospero::TileMode::kLinear;
|
||||
if (linear) {
|
||||
@@ -429,9 +436,8 @@ TextureBuildImageCopies(const TextureUploadLayout& layout, uint32_t width, uint3
|
||||
const auto align = [](uint32_t value, uint32_t block) {
|
||||
return ((value + block - 1u) / block) * block;
|
||||
};
|
||||
const auto pitch = align(mip_pitch, layout.texel_block);
|
||||
region.bufferRowLength =
|
||||
pitch > align(mip_width, layout.texel_block) ? pitch : 0;
|
||||
const auto pitch = align(mip_pitch, layout.texel_block);
|
||||
region.bufferRowLength = pitch > align(mip_width, layout.texel_block) ? pitch : 0;
|
||||
}
|
||||
regions.push_back(region);
|
||||
}
|
||||
@@ -476,8 +482,7 @@ static bool SetGpuTileSize(uint64_t offset, uint64_t length, uint64_t capacity,
|
||||
return true;
|
||||
}
|
||||
|
||||
bool TextureBuildGpuTileInfos(uint64_t size,
|
||||
const std::vector<vk::BufferImageCopy>& regions,
|
||||
bool TextureBuildGpuTileInfos(uint64_t size, const std::vector<vk::BufferImageCopy>& regions,
|
||||
const TextureUploadLayout& layout, uint32_t fmt, uint32_t depth,
|
||||
uint64_t levels, std::vector<GpuTileInfo>& out_infos) {
|
||||
if (size == 0 || levels == 0 || levels > 16 || depth == 0 ||
|
||||
@@ -518,13 +523,12 @@ bool TextureBuildGpuTileInfos(uint64_t size,
|
||||
for (uint32_t z = 0; z < mip_depth; z += block.block_depth) {
|
||||
const uint32_t copy_depth = std::min(block.block_depth, mip_depth - z);
|
||||
const auto& region = regions[region_base + z];
|
||||
const auto pitch = region.bufferRowLength != 0
|
||||
? region.bufferRowLength
|
||||
: region.imageExtent.width;
|
||||
const auto logical_height = region.bufferImageHeight != 0
|
||||
? region.bufferImageHeight
|
||||
: region.imageExtent.height;
|
||||
GpuTileInfo info {};
|
||||
const auto pitch =
|
||||
region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
|
||||
const auto logical_height = region.bufferImageHeight != 0
|
||||
? region.bufferImageHeight
|
||||
: region.imageExtent.height;
|
||||
GpuTileInfo info {};
|
||||
info.family = block.family;
|
||||
info.bytes_per_element = block.bytes_per_element;
|
||||
info.linear_offset = region.bufferOffset;
|
||||
@@ -540,20 +544,17 @@ bool TextureBuildGpuTileInfos(uint64_t size,
|
||||
return false;
|
||||
}
|
||||
info.linear_slice_stride = linear_stride;
|
||||
info.width = std::max(
|
||||
(region.imageExtent.width + element.wide - 1u) / element.wide, 1u);
|
||||
info.height = std::max(
|
||||
(logical_height + element.tall - 1u) / element.tall, 1u);
|
||||
info.depth = copy_depth;
|
||||
info.surface_z = block.block_depth == 1
|
||||
? static_cast<uint32_t>(region.imageOffset.z)
|
||||
: 0;
|
||||
info.pitch =
|
||||
std::max((pitch + element.wide - 1u) / element.wide, 1u);
|
||||
info.tail_x = tail ? volume.tail_x[level] : 0;
|
||||
info.tail_y = tail ? volume.tail_y[level] : 0;
|
||||
info.tail = tail;
|
||||
info.tiled_width = volume.level_widths[level];
|
||||
info.width =
|
||||
std::max((region.imageExtent.width + element.wide - 1u) / element.wide, 1u);
|
||||
info.height = std::max((logical_height + element.tall - 1u) / element.tall, 1u);
|
||||
info.depth = copy_depth;
|
||||
info.surface_z =
|
||||
block.block_depth == 1 ? static_cast<uint32_t>(region.imageOffset.z) : 0;
|
||||
info.pitch = std::max((pitch + element.wide - 1u) / element.wide, 1u);
|
||||
info.tail_x = tail ? volume.tail_x[level] : 0;
|
||||
info.tail_y = tail ? volume.tail_y[level] : 0;
|
||||
info.tail = tail;
|
||||
info.tiled_width = volume.level_widths[level];
|
||||
info.tiled_height = volume.level_heights[level];
|
||||
infos.push_back(info);
|
||||
}
|
||||
@@ -577,12 +578,11 @@ bool TextureBuildGpuTileInfos(uint64_t size,
|
||||
const auto level_depth = GetTextureLevelDepth(depth, level, layout.volume_texture);
|
||||
for (uint32_t z = 0; z < level_depth; z++) {
|
||||
const auto& region = regions[region_index++];
|
||||
const auto pitch = region.bufferRowLength != 0
|
||||
? region.bufferRowLength
|
||||
: region.imageExtent.width;
|
||||
const auto logical_height = region.bufferImageHeight != 0
|
||||
? region.bufferImageHeight
|
||||
: region.imageExtent.height;
|
||||
const auto pitch =
|
||||
region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
|
||||
const auto logical_height = region.bufferImageHeight != 0
|
||||
? region.bufferImageHeight
|
||||
: region.imageExtent.height;
|
||||
GpuTileInfo info {};
|
||||
info.family = block.family;
|
||||
info.bytes_per_element = block.bytes_per_element;
|
||||
@@ -593,16 +593,14 @@ bool TextureBuildGpuTileInfos(uint64_t size,
|
||||
info.tiled_size)) {
|
||||
return false;
|
||||
}
|
||||
info.width = std::max(
|
||||
(region.imageExtent.width + element.wide - 1u) / element.wide, 1u);
|
||||
info.height = std::max(
|
||||
(logical_height + element.tall - 1u) / element.tall, 1u);
|
||||
info.width =
|
||||
std::max((region.imageExtent.width + element.wide - 1u) / element.wide, 1u);
|
||||
info.height = std::max((logical_height + element.tall - 1u) / element.tall, 1u);
|
||||
info.surface_z = base_family == TileBlockFamily::RenderTarget64KB ||
|
||||
base_family == TileBlockFamily::Depth64KB
|
||||
? region.imageSubresource.baseArrayLayer
|
||||
: 0;
|
||||
info.pitch =
|
||||
std::max((pitch + element.wide - 1u) / element.wide, 1u);
|
||||
info.pitch = std::max((pitch + element.wide - 1u) / element.wide, 1u);
|
||||
info.tail = tail;
|
||||
info.tail_x = tail ? level_size.x : 0;
|
||||
info.tail_y = tail ? level_size.y : 0;
|
||||
+14
-15
@@ -1,5 +1,5 @@
|
||||
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_TEXTURECOMMON_H_
|
||||
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_TEXTURECOMMON_H_
|
||||
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_
|
||||
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_
|
||||
|
||||
#include "common/abi.h"
|
||||
#include "common/common.h"
|
||||
@@ -32,23 +32,22 @@ struct TextureUploadLayout {
|
||||
TilePaddedSize padded_sizes[16] = {};
|
||||
};
|
||||
|
||||
vk::ComponentMapping TextureGetComponentMapping(uint32_t swizzle);
|
||||
vk::ComponentMapping TextureGetComponentMapping(uint32_t swizzle);
|
||||
vk::Format TextureGetFormat(uint32_t fmt);
|
||||
RenderTargetFormatInfo TextureGetRenderTargetFormat(uint32_t layout, uint32_t type, uint32_t order);
|
||||
TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64_t height,
|
||||
uint64_t levels, uint32_t depth, uint64_t pitch,
|
||||
uint64_t tile, uint64_t upload_size,
|
||||
bool allow_depth_tile, bool volume_texture,
|
||||
const char* owner);
|
||||
std::vector<vk::BufferImageCopy>
|
||||
TextureBuildImageCopies(const TextureUploadLayout& layout, uint32_t width, uint32_t height,
|
||||
uint32_t depth, uint64_t levels, bool array_texture,
|
||||
bool volume_texture);
|
||||
bool TextureBuildGpuTileInfos(uint64_t size,
|
||||
const std::vector<vk::BufferImageCopy>& regions,
|
||||
TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64_t height,
|
||||
uint64_t levels, uint32_t depth, uint64_t pitch,
|
||||
uint64_t tile, uint64_t upload_size,
|
||||
bool allow_depth_tile, bool volume_texture,
|
||||
const char* owner);
|
||||
std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayout& layout,
|
||||
uint32_t width, uint32_t height,
|
||||
uint32_t depth, uint64_t levels,
|
||||
bool array_texture, bool volume_texture);
|
||||
bool TextureBuildGpuTileInfos(uint64_t size, const std::vector<vk::BufferImageCopy>& regions,
|
||||
const TextureUploadLayout& layout, uint32_t fmt, uint32_t depth,
|
||||
uint64_t levels, std::vector<GpuTileInfo>& infos);
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
|
||||
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_OBJECTS_TEXTURECOMMON_H_ */
|
||||
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_ */
|
||||
+3
-3
@@ -1,4 +1,4 @@
|
||||
#include "graphics/host_gpu/renderer/tiler.h"
|
||||
#include "graphics/host_gpu/renderer/image/tiler.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "gpu_tiler_shaders/gpu_tiler_demote_d16_spv.h"
|
||||
@@ -14,9 +14,9 @@
|
||||
#include "gpu_tiler_shaders/gpu_tiler_standard64_spv.h"
|
||||
#include "gpu_tiler_shaders/gpu_tiler_swap_bgra16_spv.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
|
||||
#include "graphics/host_gpu/renderer/commandScheduler.h"
|
||||
#include "graphics/host_gpu/renderer/image.h"
|
||||
#include "graphics/host_gpu/renderer/streamBuffer.h"
|
||||
#include "graphics/host_gpu/renderer/image/image.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
@@ -26,8 +26,8 @@ MasterSemaphore::~MasterSemaphore() {
|
||||
}
|
||||
|
||||
void MasterSemaphore::Refresh() {
|
||||
uint64_t counter = 0;
|
||||
const auto result = m_graphics.device.getSemaphoreCounterValue(m_semaphore, &counter);
|
||||
uint64_t counter = 0;
|
||||
const auto result = m_graphics.device.getSemaphoreCounterValue(m_semaphore, &counter);
|
||||
EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
|
||||
|
||||
auto known = m_gpu_tick.load(std::memory_order_acquire);
|
||||
|
||||
@@ -22,7 +22,7 @@ public:
|
||||
[[nodiscard]] uint64_t KnownGpuTick() const noexcept {
|
||||
return m_gpu_tick.load(std::memory_order_acquire);
|
||||
}
|
||||
[[nodiscard]] bool IsFree(uint64_t tick) const noexcept { return KnownGpuTick() >= tick; }
|
||||
[[nodiscard]] bool IsFree(uint64_t tick) const noexcept { return KnownGpuTick() >= tick; }
|
||||
[[nodiscard]] uint64_t NextTick() noexcept {
|
||||
return m_current_tick.fetch_add(1, std::memory_order_release);
|
||||
}
|
||||
|
||||
+6
-2
@@ -1,8 +1,8 @@
|
||||
#include "graphics/host_gpu/renderer/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/profiler.h"
|
||||
#include "graphics/shader/recompiler/ShaderIR.h"
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
|
||||
#include <array>
|
||||
|
||||
@@ -26,11 +26,15 @@ bool IsSampledImage(BindingKind kind) {
|
||||
case BindingKind::Sampled1DArray:
|
||||
case BindingKind::Sampled2D:
|
||||
case BindingKind::Sampled2DArray:
|
||||
case BindingKind::Sampled2DMsaa:
|
||||
case BindingKind::Sampled2DMsaaArray:
|
||||
case BindingKind::Sampled3D:
|
||||
case BindingKind::SampledUint1D:
|
||||
case BindingKind::SampledUint1DArray:
|
||||
case BindingKind::SampledUint2D:
|
||||
case BindingKind::SampledUint2DArray:
|
||||
case BindingKind::SampledUint2DMsaa:
|
||||
case BindingKind::SampledUint2DMsaaArray:
|
||||
case BindingKind::SampledUint3D: return true;
|
||||
default: return false;
|
||||
}
|
||||
+2
-2
@@ -6,7 +6,7 @@
|
||||
#include "common/common.h"
|
||||
#include "common/threads.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/renderer/textureCache.h"
|
||||
#include "graphics/host_gpu/renderer/cache/textureCache.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
#include "graphics/shader/shaderBindings.h"
|
||||
|
||||
@@ -95,7 +95,7 @@ private:
|
||||
};
|
||||
|
||||
static vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture);
|
||||
void CreatePool();
|
||||
void CreatePool();
|
||||
VulkanDescriptorSet* Allocate(Stage stage, const ShaderRecompiler::IR::Program& program);
|
||||
vk::DescriptorSetLayout
|
||||
GetDescriptorSetLayoutInternal(Stage stage, const ShaderRecompiler::IR::Program& program);
|
||||
+126
-60
@@ -1,4 +1,4 @@
|
||||
#include "graphics/host_gpu/renderer/descriptors.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/common.h"
|
||||
@@ -14,18 +14,18 @@
|
||||
#include "graphics/guest_gpu/tile.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/hostMemory.h"
|
||||
#include "graphics/host_gpu/objects/textureCommon.h"
|
||||
#include "graphics/host_gpu/renderer/debug.h"
|
||||
#include "graphics/host_gpu/renderer/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/image/textureCommon.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/renderContext.h"
|
||||
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
|
||||
#include "graphics/host_gpu/vma.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
#include "graphics/shader/recompiler/BindingLayout.h"
|
||||
#include "graphics/shader/recompiler/ResourceMaterialization.h"
|
||||
#include "graphics/shader/recompiler/ShaderIR.h"
|
||||
#include "graphics/shader/recompiler/ir/BindingLayout.h"
|
||||
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
#include "graphics/shader/shader.h"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -73,6 +73,11 @@ static Prospero::ImageType TextureBaseType(Prospero::ImageType type) {
|
||||
}
|
||||
}
|
||||
|
||||
static bool IsMultisampledTexture(Prospero::ImageType type) {
|
||||
return type == Prospero::ImageType::kColor2DMsaa ||
|
||||
type == Prospero::ImageType::kColor2DMsaaArray;
|
||||
}
|
||||
|
||||
static BufferView NativeStorageBuffer(RenderContext& context, CommandBuffer& command_buffer,
|
||||
const ShaderBufferResource& descriptor,
|
||||
const ShaderRecompiler::IR::BufferResource& resource,
|
||||
@@ -159,6 +164,8 @@ static bool IsSupportedSampledColorResource(const ShaderRecompiler::IR::ImageRes
|
||||
case ShaderRecompiler::Decoder::ImageDimension::Dim1DArray:
|
||||
case ShaderRecompiler::Decoder::ImageDimension::Dim2D:
|
||||
case ShaderRecompiler::Decoder::ImageDimension::Dim2DArray:
|
||||
case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa:
|
||||
case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray:
|
||||
supported_dimension = true;
|
||||
break;
|
||||
default: break;
|
||||
@@ -195,6 +202,22 @@ TargetTextureViewInfo ResolveTargetTextureView(const ShaderRecompiler::IR::Image
|
||||
? TargetTextureViewInfo {vk::ImageViewType::e2DArray, base_layer,
|
||||
image_layers - base_layer}
|
||||
: TargetTextureViewInfo {};
|
||||
case Prospero::ImageType::kColor2DMsaa:
|
||||
return resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa &&
|
||||
base_layer == 0 && image_layers == 1
|
||||
? TargetTextureViewInfo {vk::ImageViewType::e2D, 0, 1}
|
||||
: TargetTextureViewInfo {};
|
||||
case Prospero::ImageType::kColor2DMsaaArray:
|
||||
if (resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa &&
|
||||
base_layer == 0 && image_layers == 1) {
|
||||
return {vk::ImageViewType::e2D, 0, 1};
|
||||
}
|
||||
return resource.dimension ==
|
||||
ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray &&
|
||||
base_layer < image_layers
|
||||
? TargetTextureViewInfo {vk::ImageViewType::e2DArray, base_layer,
|
||||
image_layers - base_layer}
|
||||
: TargetTextureViewInfo {};
|
||||
default: return {};
|
||||
}
|
||||
}
|
||||
@@ -209,36 +232,59 @@ bool IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& r
|
||||
}
|
||||
|
||||
bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, const Image& image) {
|
||||
const auto width = static_cast<uint32_t>(descriptor.Width5()) + 1u;
|
||||
const auto height = static_cast<uint32_t>(descriptor.Height5()) + 1u;
|
||||
const auto pitch = TileGetTexturePitch(descriptor.Format(), width, 1, descriptor.TileMode());
|
||||
const auto type = static_cast<Prospero::ImageType>(descriptor.Type());
|
||||
const bool supported_single_layer =
|
||||
image.info.resources.layers == 1 && descriptor.Depth() == 0 &&
|
||||
descriptor.BaseArray5() == 0 &&
|
||||
(type == Prospero::ImageType::kColor2D || type == Prospero::ImageType::kColor2DArray);
|
||||
const auto width = static_cast<uint32_t>(descriptor.Width5()) + 1u;
|
||||
const auto height = static_cast<uint32_t>(descriptor.Height5()) + 1u;
|
||||
const auto type = static_cast<Prospero::ImageType>(descriptor.Type());
|
||||
const bool multisampled = IsMultisampledTexture(type);
|
||||
const auto samples = multisampled ? 1u << descriptor.LastLevel() : 1u;
|
||||
const auto pitch =
|
||||
multisampled ? TileGetDepthPitch(width, image.info.bytes_per_block, descriptor.LastLevel())
|
||||
: TileGetTexturePitch(descriptor.Format(), width, 1, descriptor.TileMode());
|
||||
const bool supported_2d = type == Prospero::ImageType::kColor2D &&
|
||||
image.info.resources.layers == 1 && descriptor.Depth() == 0 &&
|
||||
descriptor.BaseArray5() == 0;
|
||||
const bool supported_array = type == Prospero::ImageType::kColor2DArray &&
|
||||
descriptor.BaseArray5() <= descriptor.Depth() &&
|
||||
descriptor.Depth() < image.info.resources.layers;
|
||||
const bool supported_cube =
|
||||
type == Prospero::ImageType::kCube && width == height && image.info.resources.layers >= 6 &&
|
||||
image.info.resources.layers % 6u == 0 &&
|
||||
static_cast<uint32_t>(descriptor.Depth()) + 1u == image.info.resources.layers &&
|
||||
descriptor.BaseArray5() == 0;
|
||||
const bool supported_msaa_2d = type == Prospero::ImageType::kColor2DMsaa &&
|
||||
image.info.resources.layers == 1 && descriptor.Depth() == 0 &&
|
||||
descriptor.BaseArray5() == 0;
|
||||
const bool supported_msaa_array = type == Prospero::ImageType::kColor2DMsaaArray &&
|
||||
descriptor.BaseArray5() <= descriptor.Depth() &&
|
||||
descriptor.Depth() < image.info.resources.layers;
|
||||
const bool levels_ok =
|
||||
multisampled
|
||||
? descriptor.BaseLevel() == 0 && descriptor.LastLevel() >= 1 &&
|
||||
descriptor.LastLevel() <= 3 && descriptor.MaxMip() == descriptor.LastLevel() &&
|
||||
image.info.resources.levels == 1 && image.info.samples == samples
|
||||
: descriptor.BaseLevel() == 0 && descriptor.LastLevel() == 0 &&
|
||||
descriptor.MaxMip() == 0 && image.info.samples == 1;
|
||||
return image.info.IsDepth() && width == image.info.extent.width &&
|
||||
height == image.info.extent.height && (supported_single_layer || supported_cube) &&
|
||||
descriptor.BaseLevel() == 0 && descriptor.LastLevel() == 0 && descriptor.MaxMip() == 0 &&
|
||||
descriptor.MinLod() == 0 && descriptor.BaseArray5() == 0 &&
|
||||
height == image.info.extent.height &&
|
||||
(supported_2d || supported_array || supported_cube || supported_msaa_2d ||
|
||||
supported_msaa_array) &&
|
||||
levels_ok && descriptor.MinLod() == 0 &&
|
||||
descriptor.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) &&
|
||||
descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth() && pitch >= width &&
|
||||
pitch == image.info.pitch;
|
||||
descriptor.BCSwizzle() == 0 && (!descriptor.MsaaDepth() || multisampled) &&
|
||||
pitch >= width && pitch == image.info.pitch;
|
||||
}
|
||||
|
||||
bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, const Image& image) {
|
||||
constexpr uint32_t field1_reserved_mask = 0x200fff00u;
|
||||
constexpr uint32_t field2_reserved_mask = 0xf0003000u;
|
||||
constexpr uint32_t field3_common = 0x01800000u;
|
||||
constexpr uint32_t field5_expected = 0x00700000u;
|
||||
const uint32_t field3_expected =
|
||||
(descriptor.Type() << 28u) | field3_common | descriptor.DstSelXYZW();
|
||||
const uint32_t field4_expected = descriptor.Depth() | (descriptor.BaseArray5() << 16u);
|
||||
const uint32_t field3_expected = descriptor.DstSelXYZW() |
|
||||
(static_cast<uint32_t>(descriptor.BaseLevel()) << 12u) |
|
||||
(static_cast<uint32_t>(descriptor.LastLevel()) << 16u) |
|
||||
(static_cast<uint32_t>(descriptor.TileMode()) << 20u) |
|
||||
(static_cast<uint32_t>(descriptor.Type()) << 28u);
|
||||
const uint32_t field4_expected = descriptor.Depth() | (descriptor.BaseArray5() << 16u);
|
||||
const uint32_t field5_expected =
|
||||
0x00700000u | (static_cast<uint32_t>(descriptor.MaxMip()) << 4u);
|
||||
const bool common = (descriptor.fields[1] & field1_reserved_mask) == 0 &&
|
||||
(descriptor.fields[2] & field2_reserved_mask) == 0 &&
|
||||
descriptor.fields[3] == field3_expected &&
|
||||
@@ -251,8 +297,9 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, co
|
||||
return true;
|
||||
}
|
||||
constexpr uint32_t htile_control = 0x00280000u;
|
||||
const auto metadata_addr = descriptor.MetaAddr() << 8u;
|
||||
return (descriptor.fields[6] & 0x00ffffffu) == htile_control && metadata_addr != 0 &&
|
||||
const uint32_t expected_control = htile_control | (descriptor.MsaaDepth() ? (1u << 10u) : 0u);
|
||||
const auto metadata_addr = descriptor.MetaAddr() << 8u;
|
||||
return (descriptor.fields[6] & 0x00ffffffu) == expected_control && metadata_addr != 0 &&
|
||||
descriptor.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) &&
|
||||
image.info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kDepth) &&
|
||||
image.info.metadata.kind == ImageMetadataKind::Htile &&
|
||||
@@ -318,8 +365,8 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
|
||||
const bool valid_2d_slice =
|
||||
(is_color_2d && descriptor.Depth() == 0 && descriptor.BaseArray5() == 0) ||
|
||||
(is_color_2d_array && descriptor.BaseArray5() <= descriptor.Depth());
|
||||
const bool is_2d = resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D &&
|
||||
valid_2d_slice;
|
||||
const bool is_2d =
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && valid_2d_slice;
|
||||
const bool is_2d_array =
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray &&
|
||||
is_color_2d_array && descriptor.BaseArray5() <= descriptor.Depth();
|
||||
@@ -333,18 +380,20 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
|
||||
!Prospero::IsFmaskTextureFormat(descriptor.Format()) && (is_2d || is_2d_array) &&
|
||||
TileGetBlockLayout(TileBlockFamily::Depth64KB, depth_bpe, depth_block);
|
||||
const bool supported_standard_tile =
|
||||
tile == Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB) &&
|
||||
TileIsStandard4KBTextureSupported(descriptor.Format());
|
||||
(tile == Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB) &&
|
||||
TileIsStandard4KBTextureSupported(descriptor.Format())) ||
|
||||
(tile == Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB) &&
|
||||
TileIsStandard64KBTextureSupported(descriptor.Format()));
|
||||
const bool supported_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kLinear) ||
|
||||
tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) ||
|
||||
supported_depth_tile || supported_standard_tile;
|
||||
const auto swizzle = descriptor.DstSelXYZW();
|
||||
const bool supported_swizzle =
|
||||
IsValidImageSwizzle(descriptor.DstSelXYZW()) &&
|
||||
(descriptor.DstSelXYZW() == DstSel(4, 5, 6, 7) || !resource.read);
|
||||
IsValidImageSwizzle(swizzle) &&
|
||||
(swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
|
||||
const bool supported_mip_view = descriptor.BaseLevel() == 0 || is_1d || is_2d;
|
||||
return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
|
||||
supported_mip_view &&
|
||||
descriptor.BaseLevel() == descriptor.LastLevel() &&
|
||||
supported_mip_view && descriptor.BaseLevel() == descriptor.LastLevel() &&
|
||||
descriptor.LastLevel() <= descriptor.MaxMip() && descriptor.MinLod() == 0 &&
|
||||
supported_swizzle && descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth();
|
||||
}
|
||||
@@ -375,8 +424,14 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
|
||||
const bool encoding_ok = IsSupportedStorageTextureEncoding(descriptor);
|
||||
const bool uint_resource =
|
||||
resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint;
|
||||
const bool format_ok = Prospero::IsSupportedTextureFormat(format) &&
|
||||
uint_resource == Prospero::IsUintTextureFormat(format);
|
||||
const bool raw_sint_storage =
|
||||
format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32SInt) && uint_resource &&
|
||||
resource.written && !resource.read && !resource.atomic;
|
||||
const bool format_ok =
|
||||
raw_sint_storage ||
|
||||
(Prospero::IsSupportedTextureFormat(format) &&
|
||||
uint_resource == Prospero::IsUintTextureFormat(format) &&
|
||||
(!resource.atomic || format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt)));
|
||||
if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
|
||||
return;
|
||||
}
|
||||
@@ -514,6 +569,7 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource&
|
||||
view.layer_count = 1;
|
||||
break;
|
||||
case ShaderRecompiler::Decoder::ImageDimension::Dim2DArray:
|
||||
case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray:
|
||||
view.type = vk::ImageViewType::e2DArray;
|
||||
view.base_layer = descriptor.BaseArray5();
|
||||
if (view.base_layer >= image_layers) {
|
||||
@@ -522,6 +578,7 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource&
|
||||
view.layer_count = image_layers - view.base_layer;
|
||||
break;
|
||||
case ShaderRecompiler::Decoder::ImageDimension::Dim2D:
|
||||
case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa:
|
||||
view.type = vk::ImageViewType::e2D;
|
||||
view.base_layer = descriptor.BaseArray5();
|
||||
if (view.base_layer >= image_layers) {
|
||||
@@ -552,25 +609,32 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& reso
|
||||
return {id, nullptr, std::move(desc)};
|
||||
}
|
||||
|
||||
const auto address = descriptor.Base40();
|
||||
const auto width = static_cast<uint32_t>(descriptor.Width5()) + 1u;
|
||||
const auto height = static_cast<uint32_t>(descriptor.Height5()) + 1u;
|
||||
const auto base_level = descriptor.BaseLevel();
|
||||
const auto last_level = descriptor.LastLevel();
|
||||
const auto type = TextureType(descriptor);
|
||||
const bool multisampled =
|
||||
type == Prospero::ImageType::kColor2DMsaa || type == Prospero::ImageType::kColor2DMsaaArray;
|
||||
const auto levels = multisampled ? 1u : static_cast<uint32_t>(descriptor.MaxMip()) + 1u;
|
||||
const auto tile = descriptor.TileMode();
|
||||
const bool msaa_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget);
|
||||
const auto address = descriptor.Base40();
|
||||
const auto width = static_cast<uint32_t>(descriptor.Width5()) + 1u;
|
||||
const auto height = static_cast<uint32_t>(descriptor.Height5()) + 1u;
|
||||
const auto base_level = descriptor.BaseLevel();
|
||||
const auto last_level = descriptor.LastLevel();
|
||||
const auto type = TextureType(descriptor);
|
||||
const bool multisampled = IsMultisampledTexture(type);
|
||||
const auto levels = multisampled ? 1u : static_cast<uint32_t>(descriptor.MaxMip()) + 1u;
|
||||
const auto tile = descriptor.TileMode();
|
||||
const bool depth_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kDepth);
|
||||
const bool msaa_tile =
|
||||
depth_tile || tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget);
|
||||
const bool msaa_array = type == Prospero::ImageType::kColor2DMsaaArray;
|
||||
if ((!multisampled && (base_level > last_level || last_level >= levels)) ||
|
||||
(multisampled &&
|
||||
(base_level != 0 || last_level == 0 || last_level > 3 ||
|
||||
descriptor.MaxMip() != last_level || !msaa_tile || descriptor.MsaaDepth() ||
|
||||
descriptor.MaxMip() != last_level || !msaa_tile || (descriptor.MsaaDepth() && !depth_tile) ||
|
||||
(!msaa_array && (descriptor.Depth() != 0 || descriptor.BaseArray5() != 0))))) {
|
||||
EXIT("unsupported texture mip view: base=%u last=%u levels=%u\n", base_level, last_level,
|
||||
levels);
|
||||
EXIT("unsupported texture mip view: base=%u last=%u levels=%u max=%u type=%u tile=%u "
|
||||
"kind=%u dimension=%u mip_mode=%u read=%d written=%d "
|
||||
"dwords=%08x,%08x,%08x,%08x,%08x,%08x,%08x,%08x\n",
|
||||
base_level, last_level, levels, descriptor.MaxMip(), descriptor.Type(), tile,
|
||||
static_cast<uint32_t>(resource.kind), static_cast<uint32_t>(resource.dimension),
|
||||
static_cast<uint32_t>(resource.mip_mode), resource.read, resource.written,
|
||||
descriptor.fields[0], descriptor.fields[1], descriptor.fields[2], descriptor.fields[3],
|
||||
descriptor.fields[4], descriptor.fields[5], descriptor.fields[6], descriptor.fields[7]);
|
||||
}
|
||||
const auto samples = multisampled ? 1u << last_level : 1u;
|
||||
const auto view_levels =
|
||||
@@ -597,7 +661,8 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& reso
|
||||
TileSizeAlign size {};
|
||||
if (multisampled) {
|
||||
const auto bytes = Prospero::NumBytesPerElement(format);
|
||||
pitch = TileGetRenderTargetPitch(width, bytes, last_level);
|
||||
pitch = depth_tile ? TileGetDepthPitch(width, bytes, last_level)
|
||||
: TileGetRenderTargetPitch(width, bytes, last_level);
|
||||
if (pitch == 0 || !TileGetRenderTargetSize(width, height, pitch, bytes, size, last_level) ||
|
||||
size.size > UINT32_MAX / image_layers) {
|
||||
EXIT("unsupported multisample texture layout\n");
|
||||
@@ -612,16 +677,17 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& reso
|
||||
(address & (static_cast<uint64_t>(size.align) - 1u)) != 0);
|
||||
if (storage) {
|
||||
ValidateStorageTexture(resource, descriptor, size.size);
|
||||
m_context.GetBufferCache().ValidateGpuAccess(address, size.size, resource.read,
|
||||
resource.written);
|
||||
}
|
||||
|
||||
const auto pixel_format = TextureGetFormat(format);
|
||||
const auto storage_view_format = SrgbStorageViewFormat(pixel_format);
|
||||
const auto view_format =
|
||||
storage && storage_view_format != vk::Format::eUndefined ? storage_view_format
|
||||
: pixel_format;
|
||||
const auto block_bytes = Prospero::BlockCompressedBytesPerBlock(format);
|
||||
const auto storage_view_format =
|
||||
storage && format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32SInt)
|
||||
? vk::Format::eR32Uint
|
||||
: SrgbStorageViewFormat(pixel_format);
|
||||
const auto view_format = storage && storage_view_format != vk::Format::eUndefined
|
||||
? storage_view_format
|
||||
: pixel_format;
|
||||
const auto block_bytes = Prospero::BlockCompressedBytesPerBlock(format);
|
||||
TextureCache::ImageDesc desc {};
|
||||
desc.info.data = {address, size.size};
|
||||
desc.info.pixel_format = pixel_format;
|
||||
+3
-3
@@ -2,9 +2,9 @@
|
||||
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORS_H_
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "graphics/host_gpu/renderer/image.h"
|
||||
#include "graphics/host_gpu/renderer/image/image.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
#include "graphics/shader/recompiler/ShaderIR.h"
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
#include "graphics/shader/shaderBindings.h"
|
||||
|
||||
#include <cstdint>
|
||||
@@ -36,7 +36,7 @@ ResolveTargetTextureView(const ShaderRecompiler::IR::ImageResource& resource,
|
||||
[[nodiscard]] bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor,
|
||||
const Image& image);
|
||||
[[nodiscard]] bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor,
|
||||
const Image& image);
|
||||
const Image& image);
|
||||
[[nodiscard]] bool
|
||||
IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource,
|
||||
const ShaderTextureResource& descriptor, const Image& image);
|
||||
+13
-14
@@ -1,4 +1,4 @@
|
||||
#include "graphics/host_gpu/renderer/pipelineCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/logging/log.h"
|
||||
@@ -7,7 +7,7 @@
|
||||
#include "graphics/host_gpu/renderer/colorRenderTarget.h"
|
||||
#include "graphics/host_gpu/renderer/debug.h"
|
||||
#include "graphics/host_gpu/renderer/depthRenderTarget.h"
|
||||
#include "graphics/host_gpu/renderer/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageView.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/renderContext.h"
|
||||
|
||||
@@ -88,12 +88,12 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
|
||||
|
||||
PipelineStaticParameters static_params {};
|
||||
GraphicsPipeline p {};
|
||||
p.ps_shader_id = ps_id;
|
||||
p.vs_shader_id = vs_id;
|
||||
p.ps_shader_id = ps_id;
|
||||
p.vs_shader_id = vs_id;
|
||||
|
||||
static_params.color_count = color_count;
|
||||
PipelineRenderingState rendering {};
|
||||
rendering.color_count = color_count;
|
||||
rendering.color_count = color_count;
|
||||
uint32_t attachment_samples = 0;
|
||||
for (uint32_t i = 0; i < color_count; i++) {
|
||||
EXIT_IF(!colors[i].image_id || colors[i].format == vk::Format::eUndefined);
|
||||
@@ -116,8 +116,8 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
|
||||
if (attachment_samples == 0) {
|
||||
attachment_samples = depth.samples;
|
||||
} else if (attachment_samples != depth.samples) {
|
||||
EXIT("mixed color/depth sample counts are unsupported: %u and %u\n",
|
||||
attachment_samples, depth.samples);
|
||||
EXIT("mixed color/depth sample counts are unsupported: %u and %u\n", attachment_samples,
|
||||
depth.samples);
|
||||
}
|
||||
}
|
||||
EXIT_IF(attachment_samples == 0 ||
|
||||
@@ -179,10 +179,10 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
|
||||
NormalizeStaticParamsForDynamicState(static_params);
|
||||
|
||||
GraphicsPipelineKey key {};
|
||||
key.rendering = rendering;
|
||||
key.vs_shader_id = p.vs_shader_id;
|
||||
key.ps_shader_id = p.ps_shader_id;
|
||||
key.static_params = static_params;
|
||||
key.rendering = rendering;
|
||||
key.vs_shader_id = p.vs_shader_id;
|
||||
key.ps_shader_id = p.ps_shader_id;
|
||||
key.static_params = static_params;
|
||||
|
||||
if (auto iter = m_graphics_pipelines.find(key); iter != m_graphics_pipelines.end()) {
|
||||
return *iter->second;
|
||||
@@ -203,9 +203,8 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
|
||||
LogPipelineTrace("CreatePipelineInternal begin", vs_id.hash0, vs_id.crc32, ps_id.hash0,
|
||||
ps_id.crc32);
|
||||
CreatePipelineInternal(m_graphics, m_descriptor_cache, *cached, rendering, vs_input_info,
|
||||
vs_spirv, ps_input_info,
|
||||
ps_spirv, static_params, vs_id.hash0, vs_id.crc32, ps_id.hash0,
|
||||
ps_id.crc32, ps_active);
|
||||
vs_spirv, ps_input_info, ps_spirv, static_params, vs_id.hash0,
|
||||
vs_id.crc32, ps_id.hash0, ps_id.crc32, ps_active);
|
||||
LogPipelineTrace("CreatePipelineInternal done", vs_id.hash0, vs_id.crc32, ps_id.hash0,
|
||||
ps_id.crc32);
|
||||
|
||||
+17
-19
@@ -88,9 +88,9 @@ static_assert(sizeof(PipelineStaticParameters) ==
|
||||
|
||||
struct PipelineRenderingState {
|
||||
std::array<vk::Format, RENDER_COLOR_ATTACHMENTS_MAX> color_formats {};
|
||||
vk::Format depth_format = vk::Format::eUndefined;
|
||||
vk::Format stencil_format = vk::Format::eUndefined;
|
||||
uint32_t color_count = 0;
|
||||
vk::Format depth_format = vk::Format::eUndefined;
|
||||
vk::Format stencil_format = vk::Format::eUndefined;
|
||||
uint32_t color_count = 0;
|
||||
|
||||
bool operator==(const PipelineRenderingState&) const = default;
|
||||
};
|
||||
@@ -118,11 +118,12 @@ public:
|
||||
ShaderId cs_shader_id;
|
||||
};
|
||||
|
||||
GraphicsPipeline& CreateGraphicsPipeline(
|
||||
RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth,
|
||||
ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command,
|
||||
ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology, bool ps_active,
|
||||
std::span<const uint32_t> vs_spirv, std::span<const uint32_t> ps_spirv);
|
||||
GraphicsPipeline&
|
||||
CreateGraphicsPipeline(RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth,
|
||||
ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command,
|
||||
ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
|
||||
bool ps_active, std::span<const uint32_t> vs_spirv,
|
||||
std::span<const uint32_t> ps_spirv);
|
||||
ComputePipeline& CreateComputePipeline(ShaderComputeInputInfo& input_info,
|
||||
const HW::ComputeShaderInfo& cs_regs,
|
||||
std::span<const uint32_t> cs_spirv);
|
||||
@@ -199,7 +200,7 @@ private:
|
||||
}
|
||||
};
|
||||
|
||||
GraphicContext& m_graphics;
|
||||
GraphicContext& m_graphics;
|
||||
DescriptorCache& m_descriptor_cache;
|
||||
std::unordered_map<GraphicsPipelineKey, std::unique_ptr<GraphicsPipeline>,
|
||||
GraphicsPipelineKeyHash>
|
||||
@@ -211,16 +212,13 @@ private:
|
||||
|
||||
void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0,
|
||||
uint32_t ps_crc32);
|
||||
void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache,
|
||||
PipelineCache::GraphicsPipeline& pipeline,
|
||||
const PipelineRenderingState& rendering,
|
||||
const ShaderVertexInputInfo& vs_input_info,
|
||||
std::span<const uint32_t> vs_shader,
|
||||
const ShaderPixelInputInfo* ps_input_info,
|
||||
std::span<const uint32_t> ps_shader,
|
||||
const PipelineStaticParameters& static_params, uint32_t vs_hash0,
|
||||
uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32,
|
||||
bool ps_active);
|
||||
void CreatePipelineInternal(
|
||||
GraphicContext& graphics, DescriptorCache& descriptor_cache,
|
||||
PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering,
|
||||
const ShaderVertexInputInfo& vs_input_info, std::span<const uint32_t> vs_shader,
|
||||
const ShaderPixelInputInfo* ps_input_info, std::span<const uint32_t> ps_shader,
|
||||
const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32,
|
||||
uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active);
|
||||
void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache,
|
||||
PipelineCache::ComputePipeline& pipeline,
|
||||
const ShaderComputeInputInfo& input_info,
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "graphics/shader/shader.h"
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERRESOURCEBARRIER_H_
|
||||
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/shader/recompiler/ResourceMaterialization.h"
|
||||
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
#include "graphics/host_gpu/renderer/shaderSubgroup.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
|
||||
|
||||
#include "graphics/shader/recompiler/SpirvEmitter.h"
|
||||
#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
|
||||
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/shader/recompiler/ShaderIR.h"
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
+68
-60
@@ -6,14 +6,14 @@
|
||||
#include "graphics/guest_gpu/gpu_defs.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/renderer/debug.h"
|
||||
#include "graphics/host_gpu/renderer/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipelineCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/renderContext.h"
|
||||
#include "graphics/host_gpu/renderer/renderTarget.h"
|
||||
#include "graphics/host_gpu/renderer/shaderSubgroup.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
#include "graphics/shader/recompiler/ShaderIR.h"
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
#include "graphics/shader/shader.h"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -385,9 +385,8 @@ static vk::BlendOp GetBlendOp(uint32_t op) {
|
||||
return vk::BlendOp::eAdd;
|
||||
}
|
||||
|
||||
static void CreateLayout(DescriptorCache& descriptor_cache,
|
||||
std::span<vk::DescriptorSetLayout> set_layouts,
|
||||
uint32_t& set_layouts_num,
|
||||
static void CreateLayout(DescriptorCache& descriptor_cache,
|
||||
std::span<vk::DescriptorSetLayout> set_layouts, uint32_t& set_layouts_num,
|
||||
std::span<vk::PushConstantRange> push_constant_info,
|
||||
uint32_t& push_constant_info_num,
|
||||
const ShaderRecompiler::IR::Program& program,
|
||||
@@ -412,12 +411,11 @@ static void CreateLayout(DescriptorCache& descriptor_cache,
|
||||
}
|
||||
}
|
||||
|
||||
static void ConfigureSubgroupSize(const GraphicContext& graphics,
|
||||
vk::ShaderStageFlagBits vk_stage,
|
||||
static void ConfigureSubgroupSize(const GraphicContext& graphics, vk::ShaderStageFlagBits vk_stage,
|
||||
const ShaderRecompiler::IR::Program& program,
|
||||
vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo& required,
|
||||
vk::PipelineShaderStageCreateInfo& stage) {
|
||||
const auto config =
|
||||
const auto config =
|
||||
ConfigureShaderSubgroup(ShaderSubgroupCapabilities {graphics}, vk_stage, program);
|
||||
switch (config.mode) {
|
||||
case ShaderSubgroupMode::Natural: return;
|
||||
@@ -456,16 +454,13 @@ static void ConfigureSubgroupSize(const GraphicContext&
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
|
||||
void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache,
|
||||
PipelineCache::GraphicsPipeline& pipeline,
|
||||
const PipelineRenderingState& rendering,
|
||||
const ShaderVertexInputInfo& vs_input_info,
|
||||
std::span<const uint32_t> vs_shader,
|
||||
const ShaderPixelInputInfo* ps_input_info,
|
||||
std::span<const uint32_t> ps_shader,
|
||||
const PipelineStaticParameters& static_params, uint32_t vs_hash0,
|
||||
uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32,
|
||||
bool ps_active) {
|
||||
void CreatePipelineInternal(
|
||||
GraphicContext& graphics, DescriptorCache& descriptor_cache,
|
||||
PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering,
|
||||
const ShaderVertexInputInfo& vs_input_info, std::span<const uint32_t> vs_shader,
|
||||
const ShaderPixelInputInfo* ps_input_info, std::span<const uint32_t> ps_shader,
|
||||
const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32,
|
||||
uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active) {
|
||||
EXIT_IF(ps_active && ps_input_info == nullptr);
|
||||
|
||||
vk::ShaderModule vert_shader_module = nullptr;
|
||||
@@ -511,8 +506,7 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
|
||||
vert_shader_stage_info.pName = "main";
|
||||
vert_shader_stage_info.pSpecializationInfo = nullptr;
|
||||
EXIT_IF(!vs_input_info.stage);
|
||||
ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eVertex,
|
||||
*vs_input_info.stage.program,
|
||||
ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eVertex, *vs_input_info.stage.program,
|
||||
vert_subgroup_size, vert_shader_stage_info);
|
||||
|
||||
vk::PipelineShaderStageCreateInfo frag_shader_stage_info {};
|
||||
@@ -527,8 +521,8 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
|
||||
if (ps_active) {
|
||||
EXIT_IF(!ps_input_info->stage);
|
||||
ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eFragment,
|
||||
*ps_input_info->stage.program,
|
||||
frag_subgroup_size, frag_shader_stage_info);
|
||||
*ps_input_info->stage.program, frag_subgroup_size,
|
||||
frag_shader_stage_info);
|
||||
}
|
||||
|
||||
vk::PipelineShaderStageCreateInfo shader_stages[] = {vert_shader_stage_info,
|
||||
@@ -728,8 +722,14 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
|
||||
clip_ext.depthClipEnable = static_params.depth_clip_enable ? VK_TRUE : VK_FALSE;
|
||||
|
||||
vk::PipelineRasterizationStateCreateInfo rasterizer {};
|
||||
rasterizer.sType = vk::StructureType::ePipelineRasterizationStateCreateInfo;
|
||||
rasterizer.pNext = &clip_ext;
|
||||
rasterizer.sType = vk::StructureType::ePipelineRasterizationStateCreateInfo;
|
||||
// MoltenVK lacks VK_EXT_depth_clip_enable; omit the depth-clip struct on macOS and accept
|
||||
// Vulkan's default depth clipping (enabled) instead of the PS5's clamp behavior.
|
||||
#if defined(__APPLE__)
|
||||
rasterizer.pNext = nullptr;
|
||||
#else
|
||||
rasterizer.pNext = &clip_ext;
|
||||
#endif
|
||||
rasterizer.flags = {};
|
||||
rasterizer.depthClampEnable = VK_FALSE;
|
||||
rasterizer.rasterizerDiscardEnable = VK_FALSE;
|
||||
@@ -806,8 +806,14 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
|
||||
color_write.pColorWriteEnables = color_write_enable;
|
||||
|
||||
vk::PipelineColorBlendStateCreateInfo color_blending {};
|
||||
color_blending.sType = vk::StructureType::ePipelineColorBlendStateCreateInfo;
|
||||
color_blending.pNext = &color_write;
|
||||
color_blending.sType = vk::StructureType::ePipelineColorBlendStateCreateInfo;
|
||||
// MoltenVK lacks VK_EXT_color_write_enable; drop the dynamic color-write struct on macOS
|
||||
// and rely on each attachment's static colorWriteMask (all channels enabled by default).
|
||||
#if defined(__APPLE__)
|
||||
color_blending.pNext = nullptr;
|
||||
#else
|
||||
color_blending.pNext = &color_write;
|
||||
#endif
|
||||
color_blending.flags = {};
|
||||
color_blending.logicOpEnable = VK_FALSE;
|
||||
color_blending.logicOp = vk::LogicOp::eCopy;
|
||||
@@ -826,15 +832,13 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
|
||||
|
||||
EXIT_IF(!vs_input_info.stage);
|
||||
CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
|
||||
push_constant_info_num,
|
||||
*vs_input_info.stage.program, vk::ShaderStageFlagBits::eVertex,
|
||||
DescriptorCache::Stage::Vertex);
|
||||
push_constant_info_num, *vs_input_info.stage.program,
|
||||
vk::ShaderStageFlagBits::eVertex, DescriptorCache::Stage::Vertex);
|
||||
if (ps_active) {
|
||||
EXIT_IF(!ps_input_info->stage);
|
||||
CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
|
||||
push_constant_info_num,
|
||||
*ps_input_info->stage.program, vk::ShaderStageFlagBits::eFragment,
|
||||
DescriptorCache::Stage::Pixel);
|
||||
push_constant_info_num, *ps_input_info->stage.program,
|
||||
vk::ShaderStageFlagBits::eFragment, DescriptorCache::Stage::Pixel);
|
||||
}
|
||||
|
||||
vk::PipelineLayoutCreateInfo pipeline_layout_info {};
|
||||
@@ -873,7 +877,11 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
|
||||
depth_stencil_info.depthWriteEnable = (static_params.depth_write_enable ? VK_TRUE : VK_FALSE);
|
||||
depth_stencil_info.depthCompareOp = static_params.depth_compare_op;
|
||||
depth_stencil_info.depthBoundsTestEnable =
|
||||
#if defined(__APPLE__)
|
||||
VK_FALSE; // MoltenVK lacks the depthBounds feature; depth-bounds testing is disabled
|
||||
#else
|
||||
(static_params.depth_bounds_test_enable ? VK_TRUE : VK_FALSE);
|
||||
#endif
|
||||
depth_stencil_info.stencilTestEnable = (static_params.stencil_test_enable ? VK_TRUE : VK_FALSE);
|
||||
depth_stencil_info.front.failOp = static_params.stencil_front.failOp;
|
||||
depth_stencil_info.front.passOp = static_params.stencil_front.passOp;
|
||||
@@ -893,7 +901,9 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
|
||||
vk::DynamicState::eStencilCompareMask,
|
||||
vk::DynamicState::eStencilReference,
|
||||
vk::DynamicState::eStencilWriteMask,
|
||||
vk::DynamicState::eColorWriteEnableEXT,
|
||||
#if !defined(__APPLE__)
|
||||
vk::DynamicState::eColorWriteEnableEXT, // unsupported by MoltenVK; static mask instead
|
||||
#endif
|
||||
};
|
||||
const auto dynamic_states_count =
|
||||
static_cast<uint32_t>(sizeof(dynamic_states) / sizeof(dynamic_states[0]));
|
||||
@@ -905,32 +915,32 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
|
||||
dynamic_state.dynamicStateCount = dynamic_states_count;
|
||||
dynamic_state.pDynamicStates = dynamic_states;
|
||||
|
||||
vk::GraphicsPipelineCreateInfo pipeline_info {};
|
||||
vk::GraphicsPipelineCreateInfo pipeline_info {};
|
||||
vk::PipelineRenderingCreateInfo rendering_info {};
|
||||
rendering_info.sType = vk::StructureType::ePipelineRenderingCreateInfo;
|
||||
rendering_info.colorAttachmentCount = rendering.color_count;
|
||||
rendering_info.pColorAttachmentFormats = rendering.color_formats.data();
|
||||
rendering_info.depthAttachmentFormat = rendering.depth_format;
|
||||
rendering_info.stencilAttachmentFormat = rendering.stencil_format;
|
||||
pipeline_info.sType = vk::StructureType::eGraphicsPipelineCreateInfo;
|
||||
pipeline_info.pNext = &rendering_info;
|
||||
pipeline_info.flags = {};
|
||||
pipeline_info.stageCount = shader_stage_count;
|
||||
pipeline_info.pStages = shader_stages;
|
||||
pipeline_info.pVertexInputState = &vertex_input_info;
|
||||
pipeline_info.pInputAssemblyState = &input_assembly;
|
||||
pipeline_info.pTessellationState = nullptr;
|
||||
pipeline_info.pViewportState = &viewport_state;
|
||||
pipeline_info.pRasterizationState = &rasterizer;
|
||||
pipeline_info.pMultisampleState = &multisampling;
|
||||
pipeline_info.pDepthStencilState = (static_params.with_depth ? &depth_stencil_info : nullptr);
|
||||
pipeline_info.pColorBlendState = &color_blending;
|
||||
pipeline_info.pDynamicState = &dynamic_state;
|
||||
pipeline_info.layout = pipeline.pipeline_layout;
|
||||
pipeline_info.renderPass = nullptr;
|
||||
pipeline_info.subpass = 0;
|
||||
pipeline_info.basePipelineHandle = nullptr;
|
||||
pipeline_info.basePipelineIndex = -1;
|
||||
pipeline_info.sType = vk::StructureType::eGraphicsPipelineCreateInfo;
|
||||
pipeline_info.pNext = &rendering_info;
|
||||
pipeline_info.flags = {};
|
||||
pipeline_info.stageCount = shader_stage_count;
|
||||
pipeline_info.pStages = shader_stages;
|
||||
pipeline_info.pVertexInputState = &vertex_input_info;
|
||||
pipeline_info.pInputAssemblyState = &input_assembly;
|
||||
pipeline_info.pTessellationState = nullptr;
|
||||
pipeline_info.pViewportState = &viewport_state;
|
||||
pipeline_info.pRasterizationState = &rasterizer;
|
||||
pipeline_info.pMultisampleState = &multisampling;
|
||||
pipeline_info.pDepthStencilState = (static_params.with_depth ? &depth_stencil_info : nullptr);
|
||||
pipeline_info.pColorBlendState = &color_blending;
|
||||
pipeline_info.pDynamicState = &dynamic_state;
|
||||
pipeline_info.layout = pipeline.pipeline_layout;
|
||||
pipeline_info.renderPass = nullptr;
|
||||
pipeline_info.subpass = 0;
|
||||
pipeline_info.basePipelineHandle = nullptr;
|
||||
pipeline_info.basePipelineIndex = -1;
|
||||
|
||||
EXIT_IF(pipeline.pipeline != nullptr);
|
||||
|
||||
@@ -994,8 +1004,7 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
|
||||
comp_shader_stage_info.pName = "main";
|
||||
comp_shader_stage_info.pSpecializationInfo = nullptr;
|
||||
EXIT_IF(!input_info.stage);
|
||||
ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eCompute,
|
||||
*input_info.stage.program,
|
||||
ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eCompute, *input_info.stage.program,
|
||||
comp_subgroup_size, comp_shader_stage_info);
|
||||
|
||||
vk::DescriptorSetLayout set_layouts[1] = {};
|
||||
@@ -1006,9 +1015,8 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
|
||||
|
||||
EXIT_IF(!input_info.stage);
|
||||
CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
|
||||
push_constant_info_num,
|
||||
*input_info.stage.program, vk::ShaderStageFlagBits::eCompute,
|
||||
DescriptorCache::Stage::Compute);
|
||||
push_constant_info_num, *input_info.stage.program,
|
||||
vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute);
|
||||
|
||||
vk::PipelineLayoutCreateInfo pipeline_layout_info {};
|
||||
pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
|
||||
@@ -4,7 +4,7 @@
|
||||
#include "common/abi.h"
|
||||
#include "common/assert.h"
|
||||
#include "common/common.h"
|
||||
#include "graphics/host_gpu/renderer/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/renderTarget.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
|
||||
|
||||
@@ -10,17 +10,17 @@
|
||||
#include "graphics/guest_gpu/graphicsRun.h"
|
||||
#include "graphics/guest_gpu/hardwareContext.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/renderer/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/descriptors.h"
|
||||
#include "graphics/host_gpu/renderer/imageInfo.h"
|
||||
#include "graphics/host_gpu/renderer/pipelineCache.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageInfo.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/renderContext.h"
|
||||
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
|
||||
#include "graphics/host_gpu/renderer/shaderSubgroup.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
#include "graphics/shader/recompiler/ResourceMaterialization.h"
|
||||
#include "graphics/shader/recompiler/ShaderIR.h"
|
||||
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
#include "graphics/shader/shader.h"
|
||||
#include "kernel/eventQueue.h"
|
||||
#include "kernel/pthread.h"
|
||||
|
||||
@@ -14,8 +14,7 @@ namespace Libs::Graphics {
|
||||
RenderContext::RenderContext(GraphicContext& graphics)
|
||||
: m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
|
||||
m_descriptor_cache(graphics), m_pipeline_cache(graphics, m_descriptor_cache),
|
||||
m_sampler_cache(graphics),
|
||||
m_gpu_resources(graphics, m_command_scheduler) {
|
||||
m_sampler_cache(graphics), m_gpu_resources(graphics, m_command_scheduler) {
|
||||
EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
|
||||
}
|
||||
|
||||
@@ -27,7 +26,7 @@ RenderContext::~RenderContext() {
|
||||
void RenderContext::InitializeGpu(VideoOut::VideoOutDriver* video_out) {
|
||||
EXIT_IF(m_gpu != nullptr);
|
||||
m_video_out = video_out;
|
||||
m_gpu = std::make_unique<Gpu>(*this);
|
||||
m_gpu = std::make_unique<Gpu>(*this);
|
||||
m_gpu_resources.SetGpu(m_gpu.get());
|
||||
}
|
||||
|
||||
@@ -99,8 +98,7 @@ void RenderContext::TriggerEopEvent(uint32_t context_id) {
|
||||
registration.eq, static_cast<uintptr_t>(registration.id),
|
||||
LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS,
|
||||
reinterpret_cast<void*>(static_cast<uintptr_t>(context_id)));
|
||||
if (result == LibKernel::KERNEL_ERROR_EBADF ||
|
||||
result == LibKernel::KERNEL_ERROR_ENOENT) {
|
||||
if (result == LibKernel::KERNEL_ERROR_EBADF || result == LibKernel::KERNEL_ERROR_ENOENT) {
|
||||
DeleteEopEq(registration.eq, registration.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -5,13 +5,13 @@
|
||||
#include "common/assert.h"
|
||||
#include "common/common.h"
|
||||
#include "common/threads.h"
|
||||
#include "graphics/host_gpu/renderer/bufferCache.h"
|
||||
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
|
||||
#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
|
||||
#include "graphics/host_gpu/renderer/cache/samplerCache.h"
|
||||
#include "graphics/host_gpu/renderer/cache/textureCache.h"
|
||||
#include "graphics/host_gpu/renderer/commandScheduler.h"
|
||||
#include "graphics/host_gpu/renderer/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/gpuResourceManager.h"
|
||||
#include "graphics/host_gpu/renderer/pipelineCache.h"
|
||||
#include "graphics/host_gpu/renderer/samplerCache.h"
|
||||
#include "graphics/host_gpu/renderer/textureCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
|
||||
#include "kernel/eventQueue.h"
|
||||
|
||||
#include <memory>
|
||||
@@ -32,10 +32,10 @@ public:
|
||||
~RenderContext();
|
||||
KYTY_CLASS_NO_COPY(RenderContext);
|
||||
|
||||
[[nodiscard]] GraphicContext& GetGraphics() const noexcept { return m_graphics; }
|
||||
void InitializeGpu(VideoOut::VideoOutDriver* video_out);
|
||||
void ShutdownGpu();
|
||||
[[nodiscard]] Gpu& GetGpu() const;
|
||||
[[nodiscard]] GraphicContext& GetGraphics() const noexcept { return m_graphics; }
|
||||
void InitializeGpu(VideoOut::VideoOutDriver* video_out);
|
||||
void ShutdownGpu();
|
||||
[[nodiscard]] Gpu& GetGpu() const;
|
||||
[[nodiscard]] VideoOut::VideoOutDriver& GetVideoOut() const;
|
||||
|
||||
Common::Mutex& GetMutex() { return m_mutex; }
|
||||
@@ -56,18 +56,18 @@ private:
|
||||
struct EopEqRegistration {
|
||||
LibKernel::EventQueue::KernelEqueue eq = LibKernel::EventQueue::KERNEL_EQUEUE_INVALID;
|
||||
LibKernel::EventQueue::KernelEqueueRef queue;
|
||||
int id = 0;
|
||||
int id = 0;
|
||||
};
|
||||
|
||||
GraphicContext& m_graphics;
|
||||
Common::Mutex m_mutex;
|
||||
RenderExecutor m_render_executor;
|
||||
CommandScheduler m_command_scheduler;
|
||||
DescriptorCache m_descriptor_cache;
|
||||
PipelineCache m_pipeline_cache;
|
||||
SamplerCache m_sampler_cache;
|
||||
GpuResourceManager m_gpu_resources;
|
||||
std::unique_ptr<Gpu> m_gpu;
|
||||
GraphicContext& m_graphics;
|
||||
Common::Mutex m_mutex;
|
||||
RenderExecutor m_render_executor;
|
||||
CommandScheduler m_command_scheduler;
|
||||
DescriptorCache m_descriptor_cache;
|
||||
PipelineCache m_pipeline_cache;
|
||||
SamplerCache m_sampler_cache;
|
||||
GpuResourceManager m_gpu_resources;
|
||||
std::unique_ptr<Gpu> m_gpu;
|
||||
VideoOut::VideoOutDriver* m_video_out = nullptr;
|
||||
|
||||
Common::Mutex m_eop_mutex;
|
||||
|
||||
@@ -16,17 +16,18 @@
|
||||
#include "graphics/host_gpu/renderer/colorRenderTarget.h"
|
||||
#include "graphics/host_gpu/renderer/debug.h"
|
||||
#include "graphics/host_gpu/renderer/depthRenderTarget.h"
|
||||
#include "graphics/host_gpu/renderer/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipelineCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/renderContext.h"
|
||||
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
|
||||
#include "graphics/host_gpu/renderer/shaderSubgroup.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
#include "graphics/shader/recompiler/ResourceMaterialization.h"
|
||||
#include "graphics/shader/recompiler/ShaderIR.h"
|
||||
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
#include "graphics/shader/shader.h"
|
||||
#include "kernel/eventQueue.h"
|
||||
#include "kernel/memory.h"
|
||||
#include "kernel/pthread.h"
|
||||
#include "libs/errno.h"
|
||||
|
||||
@@ -221,8 +222,7 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col
|
||||
LogMrtState(draw_name, buffer, ps_input_info);
|
||||
}
|
||||
|
||||
static void LogDrawInputState(const RenderCommandBuffer& buffer,
|
||||
const RenderColorInfo& color,
|
||||
static void LogDrawInputState(const RenderCommandBuffer& buffer, const RenderColorInfo& color,
|
||||
const ShaderVertexInputInfo& vs_input_info,
|
||||
uint32_t index_type_and_size, uint32_t index_count,
|
||||
const void* index_addr) {
|
||||
@@ -395,6 +395,10 @@ static void SetDynamicParams(const RenderCommandBuffer& buffer, vk::CommandBuffe
|
||||
dynamic_params.stencil_back.reference);
|
||||
}
|
||||
|
||||
#if defined(__APPLE__)
|
||||
// MoltenVK has no VK_EXT_color_write_enable; the pipeline is created without the
|
||||
// eColorWriteEnableEXT dynamic state and relies on the static colorWriteMask instead.
|
||||
#else
|
||||
vk::Bool32 enable[RENDER_COLOR_ATTACHMENTS_MAX] = {};
|
||||
for (uint32_t i = 0; i < dynamic_params.color_write_count; i++) {
|
||||
enable[i] = (dynamic_params.color_write_enable[i] ? VK_TRUE : VK_FALSE);
|
||||
@@ -402,6 +406,7 @@ static void SetDynamicParams(const RenderCommandBuffer& buffer, vk::CommandBuffe
|
||||
if (dynamic_params.color_write_count != 0) {
|
||||
vk_buffer.setColorWriteEnableEXT(dynamic_params.color_write_count, enable);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
static bool DrawHasValidVertexShader(const HW::Shader& sh_ctx) {
|
||||
@@ -494,9 +499,9 @@ struct DrawCallInfo {
|
||||
};
|
||||
|
||||
RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
|
||||
uint32_t color_count, RenderDepthInfo& depth) {
|
||||
uint32_t color_count, RenderDepthInfo& depth) {
|
||||
EXIT_IF(colors == nullptr || color_count > RENDER_COLOR_ATTACHMENTS_MAX);
|
||||
auto& cache = m_context.GetTextureCache();
|
||||
auto& cache = m_context.GetTextureCache();
|
||||
RenderState state {};
|
||||
state.width = std::numeric_limits<uint32_t>::max();
|
||||
state.height = std::numeric_limits<uint32_t>::max();
|
||||
@@ -507,8 +512,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
|
||||
auto& target = colors[i];
|
||||
EXIT_IF(!target.image_id);
|
||||
const auto old_image = cache.ResolveOwner(target.image_id);
|
||||
if (old_image == nullptr ||
|
||||
(!old_image->registered && !old_image->info.data.Empty()) ||
|
||||
if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
|
||||
old_image->binding.needs_rebind) {
|
||||
if (old_image != nullptr) {
|
||||
old_image->binding = {};
|
||||
@@ -517,7 +521,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
|
||||
BindRenderTarget(target.image_id);
|
||||
}
|
||||
target.image_view = cache.FindRenderTarget(target.image_id, target.desc);
|
||||
auto& image = cache.GetImage(target.image_id);
|
||||
auto& image = cache.GetImage(target.image_id);
|
||||
EXIT_IF(image.backing.samples != target.samples || target.image_view == nullptr);
|
||||
if (attachment_samples == 0) {
|
||||
attachment_samples = target.samples;
|
||||
@@ -525,20 +529,19 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
|
||||
EXIT("mixed color attachment sample counts are unsupported: %u and %u\n",
|
||||
attachment_samples, target.samples);
|
||||
}
|
||||
const auto& view = target.desc.view_info;
|
||||
const auto layout =
|
||||
image.binding.is_bound ? vk::ImageLayout::eGeneral
|
||||
: vk::ImageLayout::eColorAttachmentOptimal;
|
||||
const auto& view = target.desc.view_info;
|
||||
const auto layout = image.binding.is_bound ? vk::ImageLayout::eGeneral
|
||||
: vk::ImageLayout::eColorAttachmentOptimal;
|
||||
image.Transit(layout,
|
||||
vk::AccessFlagBits2::eColorAttachmentRead |
|
||||
vk::AccessFlagBits2::eColorAttachmentWrite,
|
||||
ImageSubresourceRange {view.base_level, view.level_count, view.base_layer,
|
||||
view.layer_count},
|
||||
buffer.Handle());
|
||||
state.width = std::min(state.width, target.extent.width);
|
||||
state.height = std::min(state.height, target.extent.height);
|
||||
state.num_layers = std::min(state.num_layers, view.layer_count);
|
||||
auto& attachment = state.color_attachments[i];
|
||||
state.width = std::min(state.width, target.extent.width);
|
||||
state.height = std::min(state.height, target.extent.height);
|
||||
state.num_layers = std::min(state.num_layers, view.layer_count);
|
||||
auto& attachment = state.color_attachments[i];
|
||||
attachment.image_view = target.image_view;
|
||||
attachment.image_layout = layout;
|
||||
attachment.clear_value = target.color_clear_value.uint32;
|
||||
@@ -556,8 +559,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
|
||||
depth.depth_meta_clear_enable =
|
||||
depth.htile &&
|
||||
cache.IsMetaCleared(depth.htile_buffer_vaddr, depth.desc.view_info.base_layer);
|
||||
depth.depth_load_clear_enable =
|
||||
depth.depth_clear_enable || depth.depth_meta_clear_enable;
|
||||
depth.depth_load_clear_enable = depth.depth_clear_enable || depth.depth_meta_clear_enable;
|
||||
if (depth.depth_meta_clear_enable &&
|
||||
!cache.TouchMeta(depth.htile_buffer_vaddr, depth.desc.view_info.base_layer, false)) {
|
||||
EXIT("failed to consume HTile clear state\n");
|
||||
@@ -567,12 +569,12 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
|
||||
if (attachment_samples == 0) {
|
||||
attachment_samples = depth.samples;
|
||||
} else if (attachment_samples != depth.samples) {
|
||||
EXIT("mixed color/depth sample counts are unsupported: %u and %u\n",
|
||||
attachment_samples, depth.samples);
|
||||
EXIT("mixed color/depth sample counts are unsupported: %u and %u\n", attachment_samples,
|
||||
depth.samples);
|
||||
}
|
||||
const auto layout = depth_attachment_layout(depth);
|
||||
const auto writes = depth.AttachmentWriteAspects();
|
||||
auto access = vk::AccessFlags2 {vk::AccessFlagBits2::eDepthStencilAttachmentRead};
|
||||
auto access = vk::AccessFlags2 {vk::AccessFlagBits2::eDepthStencilAttachmentRead};
|
||||
if (writes) {
|
||||
access |= vk::AccessFlagBits2::eDepthStencilAttachmentWrite;
|
||||
}
|
||||
@@ -581,21 +583,19 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
|
||||
ImageSubresourceRange {view.base_level, view.level_count, view.base_layer,
|
||||
view.layer_count},
|
||||
buffer.Handle());
|
||||
state.width = std::min(state.width, depth.width);
|
||||
state.height = std::min(state.height, depth.height);
|
||||
state.num_layers = std::min(state.num_layers, view.layer_count);
|
||||
const auto aspects = ImageViewOps::DepthAspectMask(depth.format);
|
||||
auto& attachment = state.depth_stencil_attachment;
|
||||
state.width = std::min(state.width, depth.width);
|
||||
state.height = std::min(state.height, depth.height);
|
||||
state.num_layers = std::min(state.num_layers, view.layer_count);
|
||||
const auto aspects = ImageViewOps::DepthAspectMask(depth.format);
|
||||
auto& attachment = state.depth_stencil_attachment;
|
||||
attachment.image_view = depth.image_view;
|
||||
attachment.image_layout = layout;
|
||||
attachment.clear_value[0] = std::bit_cast<uint32_t>(depth.depth_clear_value);
|
||||
attachment.clear_value[1] = depth.stencil_clear_value;
|
||||
attachment.has_depth =
|
||||
static_cast<bool>(aspects & vk::ImageAspectFlagBits::eDepth);
|
||||
attachment.depth_clear = depth.depth_load_clear_enable;
|
||||
attachment.has_stencil =
|
||||
static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
|
||||
attachment.stencil_clear = depth.stencil_clear_enable;
|
||||
attachment.has_depth = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eDepth);
|
||||
attachment.depth_clear = depth.depth_load_clear_enable;
|
||||
attachment.has_stencil = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
|
||||
attachment.stencil_clear = depth.stencil_clear_enable;
|
||||
}
|
||||
if (attachment_samples == 0 ||
|
||||
vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) {
|
||||
@@ -680,6 +680,85 @@ static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer
|
||||
: buffer.num_records);
|
||||
}
|
||||
|
||||
struct VertexBufferRange {
|
||||
uint64_t base_address = 0;
|
||||
uint64_t requested_end = 0;
|
||||
uint64_t acquired_end = 0;
|
||||
BufferBinding binding;
|
||||
|
||||
[[nodiscard]] uint64_t RequestedSize() const { return requested_end - base_address; }
|
||||
};
|
||||
|
||||
static std::vector<BufferBinding> AcquireVertexBuffers(RenderCommandBuffer& buffer,
|
||||
const ShaderVertexInputInfo& vs_input_info) {
|
||||
// Collect the non-empty guest vertex ranges.
|
||||
std::vector<VertexBufferRange> ranges;
|
||||
ranges.reserve(vs_input_info.buffers_num);
|
||||
for (int i = 0; i < vs_input_info.buffers_num; i++) {
|
||||
const auto& vertex = vs_input_info.buffers[i];
|
||||
const auto size = VertexBufferDescriptorSize(vertex);
|
||||
if (size == 0) {
|
||||
continue;
|
||||
}
|
||||
if (vertex.addr == 0 || size > UINT64_MAX - vertex.addr) {
|
||||
EXIT("invalid vertex buffer range: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
|
||||
vertex.addr, size);
|
||||
}
|
||||
ranges.push_back({vertex.addr, vertex.addr + size});
|
||||
}
|
||||
|
||||
std::ranges::sort(ranges, [](const VertexBufferRange& left, const VertexBufferRange& right) {
|
||||
return left.base_address < right.base_address;
|
||||
});
|
||||
|
||||
// Merge overlapping or touching ranges before acquiring host buffers.
|
||||
std::vector<VertexBufferRange> merged_ranges;
|
||||
merged_ranges.reserve(ranges.size());
|
||||
for (const auto& range: ranges) {
|
||||
if (!merged_ranges.empty() && merged_ranges.back().requested_end >= range.base_address) {
|
||||
merged_ranges.back().requested_end =
|
||||
std::max(merged_ranges.back().requested_end, range.requested_end);
|
||||
continue;
|
||||
}
|
||||
merged_ranges.push_back(range);
|
||||
}
|
||||
|
||||
auto& cache = buffer.GetContext().GetBufferCache();
|
||||
for (auto& range: merged_ranges) {
|
||||
// PPSA20298
|
||||
const auto size =
|
||||
Libs::LibKernel::Memory::ClampRangeSize(range.base_address, range.RequestedSize());
|
||||
range.acquired_end = range.base_address + size;
|
||||
range.binding = cache.ObtainBuffer(buffer, range.base_address, size);
|
||||
}
|
||||
|
||||
// Rebuild slot bindings, offsetting non-empty slots into their acquired merged range.
|
||||
std::vector<BufferBinding> bindings;
|
||||
bindings.reserve(vs_input_info.buffers_num);
|
||||
for (int i = 0; i < vs_input_info.buffers_num; i++) {
|
||||
const auto& vertex = vs_input_info.buffers[i];
|
||||
const auto size = VertexBufferDescriptorSize(vertex);
|
||||
if (size == 0) {
|
||||
auto owner = cache.ObtainNullBuffer();
|
||||
bindings.push_back({owner, owner->Handle(), 0});
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto range = std::ranges::find_if(merged_ranges, [&](const VertexBufferRange& value) {
|
||||
return vertex.addr >= value.base_address && vertex.addr < value.acquired_end;
|
||||
});
|
||||
if (range == merged_ranges.end()) {
|
||||
EXIT("vertex buffer address is outside the acquired range: addr=0x%016" PRIx64 "\n",
|
||||
vertex.addr);
|
||||
}
|
||||
|
||||
auto binding = range->binding;
|
||||
binding.offset += vertex.addr - range->base_address;
|
||||
bindings.push_back(std::move(binding));
|
||||
}
|
||||
return bindings;
|
||||
}
|
||||
|
||||
static void SetDrawDebugPhase(RenderCommandBuffer& buffer, uint64_t submit_id,
|
||||
const DrawCallInfo& draw, uint32_t phase) {
|
||||
EXIT_IF(draw.name == nullptr);
|
||||
@@ -731,9 +810,9 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw, bool use
|
||||
}
|
||||
|
||||
bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buffer,
|
||||
const DrawCallInfo& draw,
|
||||
uint32_t render_target_slice_offset,
|
||||
bool log_setup_phases, DrawRenderState& state) {
|
||||
const DrawCallInfo& draw,
|
||||
uint32_t render_target_slice_offset,
|
||||
bool log_setup_phases, DrawRenderState& state) {
|
||||
EXIT_IF(draw.name == nullptr);
|
||||
auto& ctx = buffer.GetRegisters();
|
||||
|
||||
@@ -818,37 +897,13 @@ static std::vector<BufferBinding> PrepareVertexBuffers(uint64_t
|
||||
(void)submit_id;
|
||||
|
||||
LogDrawPhase(draw.name, "PrepareVertexBuffers");
|
||||
std::vector<BufferBinding> bindings;
|
||||
bindings.reserve(vs_input_info.buffers_num);
|
||||
for (int i = 0; i < vs_input_info.buffers_num; i++) {
|
||||
const auto& b = vs_input_info.buffers[i];
|
||||
const auto size = VertexBufferDescriptorSize(b);
|
||||
if (size == 0) {
|
||||
auto owner = buffer.GetContext().GetBufferCache().ObtainNullBuffer();
|
||||
bindings.push_back({owner, owner->Handle(), 0});
|
||||
} else {
|
||||
bindings.push_back(
|
||||
buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, b.addr, size));
|
||||
}
|
||||
}
|
||||
return bindings;
|
||||
return AcquireVertexBuffers(buffer, vs_input_info);
|
||||
}
|
||||
|
||||
static void RebindVertexBuffers(RenderCommandBuffer& buffer,
|
||||
const ShaderVertexInputInfo& vs_input_info,
|
||||
std::vector<BufferBinding>& bindings) {
|
||||
EXIT_IF(bindings.size() != static_cast<size_t>(vs_input_info.buffers_num));
|
||||
for (int i = 0; i < vs_input_info.buffers_num; i++) {
|
||||
const auto& vertex = vs_input_info.buffers[i];
|
||||
const auto size = VertexBufferDescriptorSize(vertex);
|
||||
if (size == 0) {
|
||||
auto owner = buffer.GetContext().GetBufferCache().ObtainNullBuffer();
|
||||
bindings[i] = {owner, owner->Handle(), 0};
|
||||
} else {
|
||||
bindings[i] =
|
||||
buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, vertex.addr, size);
|
||||
}
|
||||
}
|
||||
bindings = AcquireVertexBuffers(buffer, vs_input_info);
|
||||
}
|
||||
|
||||
static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer& buffer,
|
||||
@@ -1006,17 +1061,17 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
|
||||
}
|
||||
|
||||
void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer,
|
||||
const DrawCallInfo& draw, DrawRenderState& state,
|
||||
vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
|
||||
const DrawIndexBufferSource& index_source,
|
||||
bool log_pipeline_phase, bool set_bind_debug,
|
||||
bool set_auto_debug) {
|
||||
const DrawCallInfo& draw, DrawRenderState& state,
|
||||
vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
|
||||
const DrawIndexBufferSource& index_source,
|
||||
bool log_pipeline_phase, bool set_bind_debug,
|
||||
bool set_auto_debug) {
|
||||
EXIT_IF(draw.name == nullptr);
|
||||
auto& ucfg = buffer.GetUserConfig();
|
||||
|
||||
LogDrawPhase(draw.name, "PrepareBindings");
|
||||
auto bindings = PrepareGraphicsBindings(buffer, state.vs_input_info.stage,
|
||||
state.ps_input_info.stage, state.ps_active);
|
||||
auto bindings = PrepareGraphicsBindings(buffer, state.vs_input_info.stage,
|
||||
state.ps_input_info.stage, state.ps_active);
|
||||
auto vertex_bindings = PrepareVertexBuffers(submit_id, buffer, draw, state.vs_input_info);
|
||||
auto index_binding = PrepareIndexBuffer(buffer, index_source);
|
||||
RebindVertexBuffers(buffer, state.vs_input_info, vertex_bindings);
|
||||
@@ -1089,10 +1144,10 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
|
||||
}
|
||||
|
||||
void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
|
||||
uint32_t index_type_and_size, uint32_t index_count,
|
||||
const void* index_addr, uint32_t flags, uint32_t type,
|
||||
uint32_t instance_count, uint32_t render_target_slice_offset,
|
||||
int32_t vertex_offset_add, uint32_t first_instance) {
|
||||
uint32_t index_type_and_size, uint32_t index_count,
|
||||
const void* index_addr, uint32_t flags, uint32_t type,
|
||||
uint32_t instance_count, uint32_t render_target_slice_offset,
|
||||
int32_t vertex_offset_add, uint32_t first_instance) {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
|
||||
EXIT_IF(buffer.IsInvalid());
|
||||
@@ -1223,11 +1278,10 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
|
||||
void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
|
||||
uint32_t index_count,
|
||||
uint32_t flags, uint32_t render_target_slice_offset,
|
||||
uint32_t instance_count, uint32_t first_vertex,
|
||||
uint32_t first_instance) {
|
||||
void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_count,
|
||||
uint32_t flags, uint32_t render_target_slice_offset,
|
||||
uint32_t instance_count, uint32_t first_vertex,
|
||||
uint32_t first_instance) {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
|
||||
EXIT_IF(buffer.IsInvalid());
|
||||
@@ -1285,7 +1339,8 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
|
||||
instance_count, first_instance};
|
||||
|
||||
DrawRenderState state {};
|
||||
if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, false, state)) {
|
||||
if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, false,
|
||||
state)) {
|
||||
ResetBindings();
|
||||
return;
|
||||
}
|
||||
@@ -1335,7 +1390,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
|
||||
}
|
||||
|
||||
bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer,
|
||||
uint32_t render_target_slice_offset) {
|
||||
uint32_t render_target_slice_offset) {
|
||||
const auto& hw = buffer.GetRegisters();
|
||||
if (hw.GetColorControl().mode != 3) {
|
||||
return false;
|
||||
@@ -1364,8 +1419,7 @@ bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer
|
||||
cache.MarkGpuWritten(dst.image_id);
|
||||
auto& source = cache.GetImage(src.image_id);
|
||||
auto& destination = cache.GetImage(dst.image_id);
|
||||
destination.Resolve(source,
|
||||
{src.base_mip_level, 1, src.base_array_layer, 1},
|
||||
destination.Resolve(source, {src.base_mip_level, 1, src.base_array_layer, 1},
|
||||
{dst.base_mip_level, 1, dst.base_array_layer, 1});
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -12,13 +12,13 @@ namespace Libs::Graphics {
|
||||
static constexpr uint32_t RENDER_COLOR_ATTACHMENTS_MAX = 8;
|
||||
|
||||
struct RenderAttachment {
|
||||
vk::ImageView image_view = nullptr;
|
||||
vk::ImageLayout image_layout = vk::ImageLayout::eUndefined;
|
||||
std::array<uint32_t, 4> clear_value = {};
|
||||
vk::ImageView image_view = nullptr;
|
||||
vk::ImageLayout image_layout = vk::ImageLayout::eUndefined;
|
||||
std::array<uint32_t, 4> clear_value = {};
|
||||
bool is_clear = false;
|
||||
bool has_depth = false;
|
||||
bool depth_clear = false;
|
||||
bool has_stencil = false;
|
||||
bool has_depth = false;
|
||||
bool depth_clear = false;
|
||||
bool has_stencil = false;
|
||||
bool stencil_clear = false;
|
||||
|
||||
bool operator==(const RenderAttachment&) const = default;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
#include "common/logging/log.h"
|
||||
#include "common/threads.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/renderer/bufferCache.h"
|
||||
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/renderContext.h"
|
||||
#include "graphics/presentation/videoOut.h"
|
||||
@@ -251,9 +251,9 @@ uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index, int f
|
||||
int64_t flip_arg) {
|
||||
for (;;) {
|
||||
uint64_t request_id = 0;
|
||||
auto& video_out = buffer.GetContext().GetVideoOut();
|
||||
const auto result = video_out.SubmitFlipFromGpu(
|
||||
buffer, handle, index, flip_mode, flip_arg, request_id);
|
||||
auto& video_out = buffer.GetContext().GetVideoOut();
|
||||
const auto result =
|
||||
video_out.SubmitFlipFromGpu(buffer, handle, index, flip_mode, flip_arg, request_id);
|
||||
if (result == OK) {
|
||||
EXIT_IF(request_id == 0);
|
||||
return request_id;
|
||||
|
||||
@@ -122,9 +122,9 @@ uint64_t GraphicContext::GetDeviceMemoryUsage() const {
|
||||
physical_device_properties.deviceType == vk::PhysicalDeviceType::eDiscreteGpu;
|
||||
uint64_t usage = 0;
|
||||
for (uint32_t heap = 0; heap < physical_device_memory_properties.memoryHeapCount; heap++) {
|
||||
const bool device_local = static_cast<bool>(
|
||||
physical_device_memory_properties.memoryHeaps[heap].flags &
|
||||
vk::MemoryHeapFlagBits::eDeviceLocal);
|
||||
const bool device_local =
|
||||
static_cast<bool>(physical_device_memory_properties.memoryHeaps[heap].flags &
|
||||
vk::MemoryHeapFlagBits::eDeviceLocal);
|
||||
if (!discrete || device_local) {
|
||||
usage += budgets[heap].usage;
|
||||
}
|
||||
@@ -144,7 +144,7 @@ uint64_t GraphicContext::GetTotalMemoryBudget() const {
|
||||
uint64_t local = 0;
|
||||
uint64_t usage = 0;
|
||||
for (uint32_t heap = 0; heap < physical_device_memory_properties.memoryHeapCount; heap++) {
|
||||
const auto& properties = physical_device_memory_properties.memoryHeaps[heap];
|
||||
const auto& properties = physical_device_memory_properties.memoryHeaps[heap];
|
||||
const bool device_local =
|
||||
static_cast<bool>(properties.flags & vk::MemoryHeapFlagBits::eDeviceLocal);
|
||||
if (device_local) {
|
||||
@@ -159,9 +159,8 @@ uint64_t GraphicContext::GetTotalMemoryBudget() const {
|
||||
return budget - std::min<uint64_t>(budget / 8, 1024ull * 1024 * 1024);
|
||||
}
|
||||
constexpr uint64_t system_reserve = 8ull * 1024 * 1024 * 1024;
|
||||
const auto available = budget > usage ? budget - usage : uint64_t {0};
|
||||
return std::max(local,
|
||||
available > system_reserve ? available - system_reserve : uint64_t {0});
|
||||
const auto available = budget > usage ? budget - usage : uint64_t {0};
|
||||
return std::max(local, available > system_reserve ? available - system_reserve : uint64_t {0});
|
||||
}
|
||||
|
||||
void GraphicContext::CreateBuffer(uint64_t size, VulkanBuffer& buffer) {
|
||||
|
||||
@@ -37,6 +37,7 @@ constexpr FormatMapping kFormatMappings[] = {
|
||||
{Prospero::BufferFormat::k16_16Float, vk::Format::eR16G16Sfloat},
|
||||
{Prospero::BufferFormat::k11_11_10Float, vk::Format::eB10G11R11UfloatPack32},
|
||||
{Prospero::BufferFormat::k10_10_10_2UNorm, vk::Format::eA2B10G10R10UnormPack32},
|
||||
{Prospero::BufferFormat::k10_10_10_2UInt, vk::Format::eA2B10G10R10UintPack32},
|
||||
{Prospero::BufferFormat::k8_8_8_8UNorm, vk::Format::eR8G8B8A8Unorm},
|
||||
{Prospero::BufferFormat::k8_8_8_8SNorm, vk::Format::eR8G8B8A8Snorm},
|
||||
{Prospero::BufferFormat::k8_8_8_8UInt, vk::Format::eR8G8B8A8Uint},
|
||||
@@ -55,6 +56,10 @@ constexpr FormatMapping kFormatMappings[] = {
|
||||
{Prospero::BufferFormat::k32_32_32_32UInt, vk::Format::eR32G32B32A32Uint},
|
||||
{Prospero::BufferFormat::k32_32_32_32SInt, vk::Format::eR32G32B32A32Sint},
|
||||
{Prospero::BufferFormat::k32_32_32_32Float, vk::Format::eR32G32B32A32Sfloat},
|
||||
// Narrow-channel sRGB formats are optional in Vulkan. Keep a same-width fallback until
|
||||
// sampler-aware sRGB emulation is available.
|
||||
{Prospero::BufferFormat::k8Srgb, vk::Format::eR8Unorm},
|
||||
{Prospero::BufferFormat::k8_8Srgb, vk::Format::eR8G8Unorm},
|
||||
{Prospero::BufferFormat::k8_8_8_8Srgb, vk::Format::eR8G8B8A8Srgb},
|
||||
{Prospero::BufferFormat::k9_9_9_5Float, vk::Format::eE5B9G9R9UfloatPack32},
|
||||
{Prospero::BufferFormat::k5_6_5UNorm, vk::Format::eB5G6R5UnormPack16},
|
||||
|
||||
@@ -20,14 +20,14 @@ public:
|
||||
~Presenter();
|
||||
KYTY_CLASS_NO_COPY(Presenter);
|
||||
|
||||
[[nodiscard]] Frame& PrepareFrame(CommandBuffer& command, const ImageInfo& info);
|
||||
[[nodiscard]] Frame& PrepareBlankFrame(uint32_t width, uint32_t height, bool opaque,
|
||||
CommandBuffer* producer = nullptr);
|
||||
[[nodiscard]] Frame* PrepareLastFrame();
|
||||
[[nodiscard]] bool IsGuestPaused() const noexcept;
|
||||
[[nodiscard]] Frame& PrepareFrame(CommandBuffer& command, const ImageInfo& info);
|
||||
[[nodiscard]] Frame& PrepareBlankFrame(uint32_t width, uint32_t height, bool opaque,
|
||||
CommandBuffer* producer = nullptr);
|
||||
[[nodiscard]] Frame* PrepareLastFrame();
|
||||
[[nodiscard]] bool IsGuestPaused() const noexcept;
|
||||
[[nodiscard]] RenderContext& Renderer() const noexcept;
|
||||
void Present(Frame& frame, bool reuse = false);
|
||||
void Discard(Frame& frame);
|
||||
void Present(Frame& frame, bool reuse = false);
|
||||
void Discard(Frame& frame);
|
||||
|
||||
private:
|
||||
struct Impl;
|
||||
|
||||
@@ -19,12 +19,16 @@
|
||||
#include <windows.h>
|
||||
#undef min
|
||||
#undef max
|
||||
#else
|
||||
#include <dlfcn.h>
|
||||
|
||||
// RenderDoc uses Windows-style names in its cross-platform API.
|
||||
#define __cdecl
|
||||
using HMODULE = void*;
|
||||
#endif
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
|
||||
using RenderDocDevicePointer = void*;
|
||||
using RenderDocWindowHandle = void*;
|
||||
|
||||
@@ -107,6 +111,8 @@ static RenderDocDevicePointer GetRenderDocDevicePointer(vk::Instance instance) {
|
||||
return VulkanHandleToPointer(instance);
|
||||
}
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
|
||||
static bool BindRenderDocApi(HMODULE module) {
|
||||
auto* get_api = reinterpret_cast<pRENDERDOC_GetAPI>(GetProcAddress(module, "RENDERDOC_GetAPI"));
|
||||
if (get_api == nullptr) {
|
||||
@@ -147,10 +153,84 @@ static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) {
|
||||
return info.info.win.window;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static bool BindRenderDocApi(HMODULE module) {
|
||||
auto* get_api = reinterpret_cast<pRENDERDOC_GetAPI>(::dlsym(module, "RENDERDOC_GetAPI"));
|
||||
if (get_api == nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void* api = nullptr;
|
||||
if (get_api(eRENDERDOC_API_Version_1_4_2, &api) == 0 || api == nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
g_module = module;
|
||||
g_api = static_cast<RenderDocApi*>(api);
|
||||
|
||||
g_api->SetCaptureFilePathTemplate("_RenderDoc/kyty");
|
||||
static RenderDocInputButton capture_keys[] = {eRENDERDOC_Key_F1};
|
||||
g_api->SetCaptureKeys(capture_keys, 1);
|
||||
g_api->UnloadCrashHandler();
|
||||
|
||||
Dl_info info {};
|
||||
if (::dladdr(reinterpret_cast<void*>(get_api), &info) != 0 && info.dli_fname != nullptr) {
|
||||
LOGF("RenderDoc: bound API from %s\n", info.dli_fname);
|
||||
} else {
|
||||
LOGF("RenderDoc: bound API\n");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) {
|
||||
if (window == nullptr) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
SDL_SysWMinfo info {};
|
||||
SDL_VERSION(&info.version);
|
||||
|
||||
if (SDL_GetWindowWMInfo(window, &info) != SDL_TRUE) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
#if defined(SDL_VIDEO_DRIVER_X11)
|
||||
if (info.subsystem == SDL_SYSWM_X11) {
|
||||
// RenderDoc takes the raw xlib Window id in the pointer slot, not a Display*.
|
||||
return reinterpret_cast<RenderDocWindowHandle>(
|
||||
static_cast<uintptr_t>(info.info.x11.window));
|
||||
}
|
||||
#endif
|
||||
|
||||
// Wayland capture works without an active-window handle.
|
||||
static std::atomic_bool logged = false;
|
||||
if (!logged.exchange(true)) {
|
||||
LOGF("RenderDoc: no native window handle for SDL subsystem %d (Wayland?); the in-app "
|
||||
"overlay is unavailable, but --rd captures still work\n",
|
||||
static_cast<int>(info.subsystem));
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
|
||||
static bool IsAvailable() {
|
||||
return g_api != nullptr && g_device != nullptr && g_window != nullptr;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static bool IsAvailable() {
|
||||
return g_api != nullptr && g_device != nullptr;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
|
||||
void RenderDocInit() {
|
||||
bool expected = false;
|
||||
if (!g_init_done.compare_exchange_strong(expected, true)) {
|
||||
@@ -187,6 +267,33 @@ void RenderDocInit() {
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void RenderDocInit() {
|
||||
bool expected = false;
|
||||
if (!g_init_done.compare_exchange_strong(expected, true)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Prefer an injected RenderDoc instance.
|
||||
auto* module = ::dlopen("librenderdoc.so", RTLD_NOW | RTLD_NOLOAD);
|
||||
if (module == nullptr) {
|
||||
module = ::dlopen("librenderdoc.so", RTLD_NOW);
|
||||
}
|
||||
if (module == nullptr) {
|
||||
LOGF("RenderDoc: librenderdoc.so was not found; in-app capture disabled\n");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!BindRenderDocApi(module)) {
|
||||
LOGF("RenderDoc: API 1.4.2 is not available; in-app capture disabled\n");
|
||||
::dlclose(module);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
void RenderDocSetActiveWindow(vk::Instance instance, SDL_Window* window) {
|
||||
if (g_api == nullptr) {
|
||||
return;
|
||||
@@ -274,13 +381,4 @@ void RenderDocOnPresent() {
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void RenderDocInit() {}
|
||||
void RenderDocSetActiveWindow(vk::Instance /*instance*/, SDL_Window* /*window*/) {}
|
||||
void RenderDocRequestCapture() {}
|
||||
void RenderDocOnPresent() {}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
#include "graphics/guest_gpu/gpu_defs.h"
|
||||
#include "graphics/guest_gpu/graphicsRun.h"
|
||||
#include "graphics/guest_gpu/tile.h"
|
||||
#include "graphics/host_gpu/renderer/imageInfo.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageInfo.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/renderContext.h"
|
||||
#include "graphics/presentation/presenter.h"
|
||||
@@ -69,8 +69,8 @@ enum class FlipRequestSource { Cpu, GpuEop };
|
||||
struct VideoOutEventState;
|
||||
|
||||
struct VideoOutEventRegistration {
|
||||
EventQueue::KernelEqueue handle = EventQueue::KERNEL_EQUEUE_INVALID;
|
||||
std::shared_ptr<VideoOutEventState> state;
|
||||
EventQueue::KernelEqueue handle = EventQueue::KERNEL_EQUEUE_INVALID;
|
||||
std::shared_ptr<VideoOutEventState> state;
|
||||
uint64_t generation = 0;
|
||||
VideoOutEventKind kind = VideoOutEventKind::Flip;
|
||||
};
|
||||
@@ -170,13 +170,13 @@ struct BufferAttributeGroup {
|
||||
struct VideoOutConfig {
|
||||
Common::Mutex mutex;
|
||||
Common::CondVar vblank_cond;
|
||||
std::shared_ptr<VideoOutEventState> events = std::make_shared<VideoOutEventState>();
|
||||
uint32_t width = 0;
|
||||
uint32_t height = 0;
|
||||
uint64_t generation = 0;
|
||||
bool opened = false;
|
||||
bool closing = false;
|
||||
int flip_rate = 0;
|
||||
std::shared_ptr<VideoOutEventState> events = std::make_shared<VideoOutEventState>();
|
||||
uint32_t width = 0;
|
||||
uint32_t height = 0;
|
||||
uint64_t generation = 0;
|
||||
bool opened = false;
|
||||
bool closing = false;
|
||||
int flip_rate = 0;
|
||||
uint64_t output_mode = VIDEO_OUT_OUTPUT_MODE_DEFAULT;
|
||||
float gamma = 1.0f;
|
||||
VideoOutFlipStatus flip_status;
|
||||
@@ -250,8 +250,8 @@ public:
|
||||
VideoOutConfig* Get(int handle, uint64_t& generation);
|
||||
bool IsOpened(int handle);
|
||||
|
||||
void Init(uint32_t width, uint32_t height);
|
||||
FlipQueue& GetFlipQueue() { return m_flip_queue; }
|
||||
void Init(uint32_t width, uint32_t height);
|
||||
FlipQueue& GetFlipQueue() { return m_flip_queue; }
|
||||
Graphics::RenderContext& Renderer() const noexcept { return m_renderer; }
|
||||
|
||||
void VblankBegin();
|
||||
@@ -259,12 +259,12 @@ public:
|
||||
void PresentThread(std::stop_token token);
|
||||
|
||||
private:
|
||||
Common::Mutex m_mutex;
|
||||
VideoOutConfig m_video_out_ctx[VIDEO_OUT_NUM_MAX];
|
||||
Common::Mutex m_mutex;
|
||||
VideoOutConfig m_video_out_ctx[VIDEO_OUT_NUM_MAX];
|
||||
Graphics::RenderContext& m_renderer;
|
||||
Graphics::Presenter& m_presenter;
|
||||
FlipQueue m_flip_queue;
|
||||
std::jthread m_present_thread;
|
||||
Graphics::Presenter& m_presenter;
|
||||
FlipQueue m_flip_queue;
|
||||
std::jthread m_present_thread;
|
||||
};
|
||||
|
||||
static std::unique_ptr<VideoOutDriver> g_video_out_driver;
|
||||
@@ -279,7 +279,7 @@ static uintptr_t VideoOutEventId(VideoOutEventKind kind) {
|
||||
}
|
||||
|
||||
static VideoOutEventQueues& VideoOutEventQueuesFor(VideoOutEventState& state,
|
||||
VideoOutEventKind kind) {
|
||||
VideoOutEventKind kind) {
|
||||
switch (kind) {
|
||||
case VideoOutEventKind::Flip: return state.flip;
|
||||
case VideoOutEventKind::Vblank: return state.vblank;
|
||||
@@ -359,9 +359,9 @@ static void TriggerVideoOutEvents(VideoOutConfig& video_out, VideoOutEventKind k
|
||||
if (!registration || registration->generation != video_out.generation) {
|
||||
continue;
|
||||
}
|
||||
const auto result = EventQueue::KernelTriggerEvent(
|
||||
registration->handle, VideoOutEventId(kind), EventQueue::KERNEL_EVFILT_VIDEO_OUT,
|
||||
trigger_data);
|
||||
const auto result =
|
||||
EventQueue::KernelTriggerEvent(registration->handle, VideoOutEventId(kind),
|
||||
EventQueue::KERNEL_EVFILT_VIDEO_OUT, trigger_data);
|
||||
EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_EBADF &&
|
||||
result != LibKernel::KERNEL_ERROR_ENOENT);
|
||||
}
|
||||
@@ -372,9 +372,8 @@ static void DeleteVideoOutEvents(const VideoOutEventQueues& queues, VideoOutEven
|
||||
if (!registration) {
|
||||
continue;
|
||||
}
|
||||
const auto result =
|
||||
EventQueue::KernelDeleteEvent(registration->handle, VideoOutEventId(kind),
|
||||
EventQueue::KERNEL_EVFILT_VIDEO_OUT);
|
||||
const auto result = EventQueue::KernelDeleteEvent(
|
||||
registration->handle, VideoOutEventId(kind), EventQueue::KERNEL_EVFILT_VIDEO_OUT);
|
||||
EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_EBADF &&
|
||||
result != LibKernel::KERNEL_ERROR_ENOENT);
|
||||
}
|
||||
@@ -383,7 +382,7 @@ static void DeleteVideoOutEvents(const VideoOutEventQueues& queues, VideoOutEven
|
||||
static int RegisterVideoOutEvent(int handle, EventQueue::KernelEqueue eq, VideoOutEventKind kind,
|
||||
void* udata) {
|
||||
uint64_t generation = 0;
|
||||
auto* video_out = DriverState().Get(handle, generation);
|
||||
auto* video_out = DriverState().Get(handle, generation);
|
||||
if (video_out == nullptr) {
|
||||
return VIDEO_OUT_ERROR_INVALID_HANDLE;
|
||||
}
|
||||
@@ -425,27 +424,25 @@ static int RegisterVideoOutEvent(int handle, EventQueue::KernelEqueue eq, VideoO
|
||||
bool add_queue = false;
|
||||
{
|
||||
Common::LockGuard event_lock(event_state->mutex);
|
||||
const auto existing = std::find_if(queues.begin(), queues.end(), [&](const auto& candidate) {
|
||||
return candidate->handle == eq && candidate->generation == generation;
|
||||
});
|
||||
const auto existing =
|
||||
std::find_if(queues.begin(), queues.end(), [&](const auto& candidate) {
|
||||
return candidate->handle == eq && candidate->generation == generation;
|
||||
});
|
||||
if (existing != queues.end()) {
|
||||
registration = *existing;
|
||||
} else {
|
||||
registration = std::make_shared<VideoOutEventRegistration>(
|
||||
VideoOutEventRegistration {.handle = eq,
|
||||
.state = event_state,
|
||||
.generation = generation,
|
||||
.kind = kind});
|
||||
registration = std::make_shared<VideoOutEventRegistration>(VideoOutEventRegistration {
|
||||
.handle = eq, .state = event_state, .generation = generation, .kind = kind});
|
||||
queues.push_back(registration);
|
||||
add_queue = true;
|
||||
}
|
||||
}
|
||||
event.filter.data = registration.get();
|
||||
event.filter.owner = registration;
|
||||
const int result = EventQueue::KernelAddEvent(eq, event);
|
||||
const int result = EventQueue::KernelAddEvent(eq, event);
|
||||
if (result != OK && add_queue) {
|
||||
Common::LockGuard event_lock(event_state->mutex);
|
||||
const auto added = std::find(queues.begin(), queues.end(), registration);
|
||||
const auto added = std::find(queues.begin(), queues.end(), registration);
|
||||
if (added != queues.end()) {
|
||||
queues.erase(added);
|
||||
}
|
||||
@@ -455,7 +452,7 @@ static int RegisterVideoOutEvent(int handle, EventQueue::KernelEqueue eq, VideoO
|
||||
|
||||
static int DeleteVideoOutEvent(int handle, EventQueue::KernelEqueue eq, VideoOutEventKind kind) {
|
||||
uint64_t generation = 0;
|
||||
auto* video_out = DriverState().Get(handle, generation);
|
||||
auto* video_out = DriverState().Get(handle, generation);
|
||||
if (video_out == nullptr) {
|
||||
return VIDEO_OUT_ERROR_INVALID_HANDLE;
|
||||
}
|
||||
@@ -814,8 +811,8 @@ void VideoOutDriver::Impl::PresentThread(std::stop_token token) {
|
||||
m_presenter.Present(*frame, true);
|
||||
}
|
||||
const auto frame_end = Common::Timer::QueryPerformanceCounter();
|
||||
total_wait += static_cast<int64_t>(period) -
|
||||
static_cast<int64_t>(frame_end - frame_begin);
|
||||
total_wait +=
|
||||
static_cast<int64_t>(period) - static_cast<int64_t>(frame_end - frame_begin);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -841,8 +838,7 @@ void VideoOutDriver::Impl::PresentThread(std::stop_token token) {
|
||||
VblankEnd();
|
||||
|
||||
const auto frame_end = Common::Timer::QueryPerformanceCounter();
|
||||
total_wait += static_cast<int64_t>(period) -
|
||||
static_cast<int64_t>(frame_end - frame_begin);
|
||||
total_wait += static_cast<int64_t>(period) - static_cast<int64_t>(frame_end - frame_begin);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1000,8 +996,8 @@ void FlipQueue::Prepare(uint64_t request_id, Graphics::CommandBuffer& buffer) {
|
||||
}
|
||||
Graphics::Presenter::Frame* frame = nullptr;
|
||||
if (special) {
|
||||
frame = &m_presenter.PrepareBlankFrame(width, height,
|
||||
index == VIDEO_OUT_BUFFER_INDEX_BLACK, &buffer);
|
||||
frame = &m_presenter.PrepareBlankFrame(width, height, index == VIDEO_OUT_BUFFER_INDEX_BLACK,
|
||||
&buffer);
|
||||
} else {
|
||||
frame = &m_presenter.PrepareFrame(buffer, source_info);
|
||||
}
|
||||
|
||||
@@ -32,13 +32,13 @@ public:
|
||||
~VideoOutDriver();
|
||||
KYTY_CLASS_NO_COPY(VideoOutDriver);
|
||||
|
||||
int SubmitFlipFromGpu(Graphics::CommandBuffer& buffer, int handle, int index, int flip_mode,
|
||||
int64_t flip_arg, uint64_t& request_id);
|
||||
void PrepareFlip(uint64_t request_id, Graphics::CommandBuffer& buffer);
|
||||
void CompleteFlip(uint64_t request_id);
|
||||
void SubmitFlipPreparation(uint64_t request_id);
|
||||
void WaitForSubmitSlot();
|
||||
void WaitFlipDone(int handle, int index);
|
||||
int SubmitFlipFromGpu(Graphics::CommandBuffer& buffer, int handle, int index, int flip_mode,
|
||||
int64_t flip_arg, uint64_t& request_id);
|
||||
void PrepareFlip(uint64_t request_id, Graphics::CommandBuffer& buffer);
|
||||
void CompleteFlip(uint64_t request_id);
|
||||
void SubmitFlipPreparation(uint64_t request_id);
|
||||
void WaitForSubmitSlot();
|
||||
void WaitFlipDone(int handle, int index);
|
||||
|
||||
[[nodiscard]] Impl& State() noexcept;
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ namespace Libs::Graphics {
|
||||
struct Presenter::Frame {
|
||||
VulkanImage image;
|
||||
std::unique_ptr<CommandBuffer> present_commands;
|
||||
bool busy = false;
|
||||
bool busy = false;
|
||||
bool reusing_last = false;
|
||||
|
||||
void Configure(GraphicContext& graphics, vk::Extent2D extent, vk::Format format);
|
||||
@@ -155,7 +155,7 @@ public:
|
||||
EXIT("last submitted frame is not available for reuse\n");
|
||||
}
|
||||
m_free.erase(free);
|
||||
m_last_frame = nullptr;
|
||||
m_last_frame = nullptr;
|
||||
frame->busy = true;
|
||||
frame->reusing_last = true;
|
||||
m_mutex.Unlock();
|
||||
@@ -197,30 +197,27 @@ private:
|
||||
}
|
||||
}
|
||||
|
||||
WindowContext& m_window;
|
||||
Common::Mutex m_mutex;
|
||||
Common::CondVar m_available;
|
||||
WindowContext& m_window;
|
||||
Common::Mutex m_mutex;
|
||||
Common::CondVar m_available;
|
||||
std::vector<std::unique_ptr<Presenter::Frame>> m_frames;
|
||||
std::deque<Presenter::Frame*> m_free;
|
||||
Presenter::Frame* m_last_frame = nullptr;
|
||||
vk::Format m_format = vk::Format::eUndefined;
|
||||
vk::Format m_format = vk::Format::eUndefined;
|
||||
};
|
||||
|
||||
void Presenter::Frame::Configure(GraphicContext& graphics, vk::Extent2D extent,
|
||||
vk::Format format) {
|
||||
void Presenter::Frame::Configure(GraphicContext& graphics, vk::Extent2D extent, vk::Format format) {
|
||||
if (extent.width == 0 || extent.height == 0 || format == vk::Format::eUndefined) {
|
||||
EXIT("unsupported prepared frame, extent=%ux%u format=%d\n", extent.width, extent.height,
|
||||
static_cast<int>(format));
|
||||
}
|
||||
const auto features = graphics.GetFormatProperties(format).optimalTilingFeatures;
|
||||
const auto required = vk::FormatFeatureFlagBits::eBlitSrc |
|
||||
vk::FormatFeatureFlagBits::eSampledImageFilterLinear |
|
||||
vk::FormatFeatureFlagBits::eTransferSrc |
|
||||
vk::FormatFeatureFlagBits::eTransferDst;
|
||||
const auto required =
|
||||
vk::FormatFeatureFlagBits::eBlitSrc | vk::FormatFeatureFlagBits::eSampledImageFilterLinear |
|
||||
vk::FormatFeatureFlagBits::eTransferSrc | vk::FormatFeatureFlagBits::eTransferDst;
|
||||
if ((features & required) != required) {
|
||||
EXIT("prepared presentation format lacks optimal blit support: format=%d features=0x%x\n",
|
||||
static_cast<int>(format),
|
||||
static_cast<vk::FormatFeatureFlags::MaskType>(features));
|
||||
static_cast<int>(format), static_cast<vk::FormatFeatureFlags::MaskType>(features));
|
||||
}
|
||||
|
||||
auto& dst = image;
|
||||
@@ -234,11 +231,11 @@ void Presenter::Frame::Configure(GraphicContext& graphics, vk::Extent2D extent,
|
||||
dst.memory = {};
|
||||
}
|
||||
|
||||
dst.extent = {extent.width, extent.height, 1};
|
||||
dst.format = format;
|
||||
dst.layers = 1;
|
||||
dst.mip_levels = 1;
|
||||
dst.state = {};
|
||||
dst.extent = {extent.width, extent.height, 1};
|
||||
dst.format = format;
|
||||
dst.layers = 1;
|
||||
dst.mip_levels = 1;
|
||||
dst.state = {};
|
||||
dst.subresource_states.clear();
|
||||
dst.memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal;
|
||||
|
||||
@@ -262,13 +259,12 @@ void Presenter::Frame::Configure(GraphicContext& graphics, vk::Extent2D extent,
|
||||
|
||||
void Presenter::Frame::Transit(vk::CommandBuffer command, vk::ImageLayout layout,
|
||||
vk::AccessFlags2 access) {
|
||||
const auto stage = access == vk::AccessFlagBits2::eTransferRead ||
|
||||
access == vk::AccessFlagBits2::eTransferWrite
|
||||
? vk::PipelineStageFlagBits2::eTransfer
|
||||
: vk::PipelineStageFlagBits2::eAllCommands;
|
||||
const auto stage = access == vk::AccessFlagBits2::eTransferRead ||
|
||||
access == vk::AccessFlagBits2::eTransferWrite
|
||||
? vk::PipelineStageFlagBits2::eTransfer
|
||||
: vk::PipelineStageFlagBits2::eAllCommands;
|
||||
constexpr auto writes = vk::AccessFlagBits2::eTransferWrite |
|
||||
vk::AccessFlagBits2::eShaderWrite |
|
||||
vk::AccessFlagBits2::eMemoryWrite;
|
||||
vk::AccessFlagBits2::eShaderWrite | vk::AccessFlagBits2::eMemoryWrite;
|
||||
if (image.state.layout == layout && image.state.access_mask == access &&
|
||||
!static_cast<bool>(image.state.access_mask & writes)) {
|
||||
return;
|
||||
@@ -299,35 +295,27 @@ void Presenter::Frame::Transit(vk::CommandBuffer command, vk::ImageLayout layout
|
||||
void Presenter::Frame::CopyFrom(CommandBuffer& command_buffer, Image& source) {
|
||||
command_buffer.EndRendering();
|
||||
auto command = command_buffer.Handle();
|
||||
source.Transit(vk::ImageLayout::eTransferSrcOptimal,
|
||||
vk::AccessFlagBits2::eTransferRead, {}, command);
|
||||
Transit(command, vk::ImageLayout::eTransferDstOptimal,
|
||||
vk::AccessFlagBits2::eTransferWrite);
|
||||
source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
|
||||
command);
|
||||
Transit(command, vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite);
|
||||
vk::ImageCopy copy {};
|
||||
copy.srcSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0,
|
||||
source.backing.layers};
|
||||
copy.srcSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, source.backing.layers};
|
||||
copy.dstSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, image.layers};
|
||||
copy.extent = {std::min(source.backing.extent.width, image.extent.width),
|
||||
std::min(source.backing.extent.height, image.extent.height), 1};
|
||||
copy.extent = {std::min(source.backing.extent.width, image.extent.width),
|
||||
std::min(source.backing.extent.height, image.extent.height), 1};
|
||||
EXIT_IF(copy.srcSubresource.layerCount != copy.dstSubresource.layerCount);
|
||||
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
|
||||
image.image, vk::ImageLayout::eTransferDstOptimal, copy);
|
||||
Transit(command, vk::ImageLayout::eTransferSrcOptimal,
|
||||
vk::AccessFlagBits2::eTransferRead);
|
||||
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, image.image,
|
||||
vk::ImageLayout::eTransferDstOptimal, copy);
|
||||
Transit(command, vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead);
|
||||
}
|
||||
|
||||
void Presenter::Frame::Clear(CommandBuffer& command_buffer,
|
||||
const vk::ClearColorValue& color) {
|
||||
void Presenter::Frame::Clear(CommandBuffer& command_buffer, const vk::ClearColorValue& color) {
|
||||
command_buffer.EndRendering();
|
||||
auto command = command_buffer.Handle();
|
||||
Transit(command, vk::ImageLayout::eTransferDstOptimal,
|
||||
vk::AccessFlagBits2::eTransferWrite);
|
||||
const vk::ImageSubresourceRange range {
|
||||
vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
|
||||
command.clearColorImage(image.image, vk::ImageLayout::eTransferDstOptimal, &color, 1,
|
||||
&range);
|
||||
Transit(command, vk::ImageLayout::eTransferSrcOptimal,
|
||||
vk::AccessFlagBits2::eTransferRead);
|
||||
Transit(command, vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite);
|
||||
const vk::ImageSubresourceRange range {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
|
||||
command.clearColorImage(image.image, vk::ImageLayout::eTransferDstOptimal, &color, 1, &range);
|
||||
Transit(command, vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead);
|
||||
}
|
||||
|
||||
class Swapchain final {
|
||||
@@ -338,8 +326,8 @@ public:
|
||||
~Swapchain();
|
||||
KYTY_CLASS_NO_COPY(Swapchain);
|
||||
|
||||
void Create();
|
||||
void Recreate(bool surface_lost = false);
|
||||
void Create();
|
||||
void Recreate(bool surface_lost = false);
|
||||
[[nodiscard]] Status AcquireNextImage();
|
||||
void RecordPresentCommands(CommandBuffer& command, VulkanImage& source);
|
||||
void Submit(CommandBuffer& command);
|
||||
@@ -395,17 +383,17 @@ struct Presenter::Impl {
|
||||
desc.view_info.usage = vk::ImageUsageFlagBits::eTransferSrc;
|
||||
desc.type = TextureCache::BindingType::VideoOut;
|
||||
|
||||
auto& cache = renderer.GetTextureCache();
|
||||
auto& image = cache.GetImage(cache.FindImage(desc));
|
||||
auto& cache = renderer.GetTextureCache();
|
||||
auto& image = cache.GetImage(cache.FindImage(desc));
|
||||
image.usage.video_out = true;
|
||||
return image;
|
||||
}
|
||||
|
||||
RenderContext& renderer;
|
||||
WindowContext& window;
|
||||
Swapchain swapchain;
|
||||
RenderContext& renderer;
|
||||
WindowContext& window;
|
||||
Swapchain swapchain;
|
||||
CommandScheduler present_scheduler;
|
||||
FramePool frames;
|
||||
FramePool frames;
|
||||
};
|
||||
|
||||
void Swapchain::Create() {
|
||||
@@ -441,25 +429,20 @@ void Swapchain::Create() {
|
||||
? vk::CompositeAlphaFlagBitsKHR::eOpaque
|
||||
: vk::CompositeAlphaFlagBitsKHR::eInherit;
|
||||
|
||||
vk::SurfaceFormatKHR format {vk::Format::eR8G8B8A8Unorm,
|
||||
vk::ColorSpaceKHR::eSrgbNonlinear};
|
||||
if (surface.formats.size() != 1 ||
|
||||
surface.formats.front().format != vk::Format::eUndefined) {
|
||||
vk::SurfaceFormatKHR format {vk::Format::eR8G8B8A8Unorm, vk::ColorSpaceKHR::eSrgbNonlinear};
|
||||
if (surface.formats.size() != 1 || surface.formats.front().format != vk::Format::eUndefined) {
|
||||
const auto it = std::find_if(surface.formats.begin(), surface.formats.end(),
|
||||
[](const vk::SurfaceFormatKHR& candidate) {
|
||||
return candidate.format ==
|
||||
vk::Format::eB8G8R8A8Unorm ||
|
||||
candidate.format ==
|
||||
vk::Format::eR8G8B8A8Unorm;
|
||||
return candidate.format == vk::Format::eB8G8R8A8Unorm ||
|
||||
candidate.format == vk::Format::eR8G8B8A8Unorm;
|
||||
});
|
||||
if (it == surface.formats.end()) {
|
||||
EXIT("no supported UNORM swapchain format\n");
|
||||
}
|
||||
format = *it;
|
||||
}
|
||||
m_format = format.format;
|
||||
const auto swapchain_features =
|
||||
graphics.GetFormatProperties(m_format).optimalTilingFeatures;
|
||||
m_format = format.format;
|
||||
const auto swapchain_features = graphics.GetFormatProperties(m_format).optimalTilingFeatures;
|
||||
if (!static_cast<bool>(swapchain_features & vk::FormatFeatureFlagBits::eBlitDst)) {
|
||||
EXIT("swapchain format cannot be a blit destination: format=%d\n",
|
||||
static_cast<int>(m_format));
|
||||
@@ -503,9 +486,8 @@ void Swapchain::Create() {
|
||||
view.subresourceRange.baseMipLevel = 0;
|
||||
view.subresourceRange.layerCount = 1;
|
||||
view.subresourceRange.levelCount = 1;
|
||||
RequireVulkanSuccess(
|
||||
graphics.device.createImageView(&view, nullptr, &m_image_views[i]),
|
||||
"vkCreateImageView");
|
||||
RequireVulkanSuccess(graphics.device.createImageView(&view, nullptr, &m_image_views[i]),
|
||||
"vkCreateImageView");
|
||||
EXIT_IF(m_image_views[i] == nullptr);
|
||||
}
|
||||
|
||||
@@ -586,7 +568,13 @@ void Swapchain::RefreshSurfaceSize() {
|
||||
void Swapchain::Recreate(bool surface_lost) {
|
||||
Destroy();
|
||||
if (surface_lost) {
|
||||
#if defined(__APPLE__)
|
||||
// Surface recreation goes through SDL_Vulkan_CreateSurface, which touches the
|
||||
// window's view/layer and must run on the main thread on macOS.
|
||||
m_window.RunOnMainThread([this] { m_window.RecreateSurface(); }, true);
|
||||
#else
|
||||
m_window.RecreateSurface();
|
||||
#endif
|
||||
}
|
||||
RefreshSurfaceSize();
|
||||
Create();
|
||||
@@ -594,7 +582,7 @@ void Swapchain::Recreate(bool surface_lost) {
|
||||
|
||||
Swapchain::Status Swapchain::AcquireNextImage() {
|
||||
EXIT_IF(m_handle == nullptr || m_frame_index >= m_image_acquired.size());
|
||||
m_image_index = static_cast<uint32_t>(-1);
|
||||
m_image_index = static_cast<uint32_t>(-1);
|
||||
const auto result = m_window.graphic_ctx.device.acquireNextImageKHR(
|
||||
m_handle, std::numeric_limits<uint64_t>::max(), m_image_acquired[m_frame_index], nullptr,
|
||||
&m_image_index);
|
||||
@@ -677,10 +665,9 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, VulkanImage& sourc
|
||||
to_present.subresourceRange.levelCount = 1;
|
||||
to_present.subresourceRange.baseArrayLayer = 0;
|
||||
to_present.subresourceRange.layerCount = 1;
|
||||
vk_command.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
|
||||
vk::PipelineStageFlagBits::eAllCommands,
|
||||
vk::DependencyFlagBits::eByRegion, 0,
|
||||
nullptr, 0, nullptr, 1, &to_present);
|
||||
vk_command.pipelineBarrier(
|
||||
vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eAllCommands,
|
||||
vk::DependencyFlagBits::eByRegion, 0, nullptr, 0, nullptr, 1, &to_present);
|
||||
command.End();
|
||||
}
|
||||
|
||||
@@ -694,7 +681,7 @@ void Swapchain::Submit(CommandBuffer& command) {
|
||||
|
||||
Swapchain::Status Swapchain::Present() {
|
||||
EXIT_IF(m_image_index >= m_render_complete.size());
|
||||
const auto ready = m_render_complete[m_image_index];
|
||||
const auto ready = m_render_complete[m_image_index];
|
||||
vk::PresentInfoKHR present {};
|
||||
present.sType = vk::StructureType::ePresentInfoKHR;
|
||||
present.swapchainCount = 1;
|
||||
@@ -732,7 +719,7 @@ Presenter::~Presenter() = default;
|
||||
Presenter::Frame& Presenter::PrepareFrame(CommandBuffer& buffer, const ImageInfo& info) {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
EXIT_IF(buffer.IsInvalid());
|
||||
auto* frame = m_impl->frames.Acquire();
|
||||
auto* frame = m_impl->frames.Acquire();
|
||||
Common::LockGuard render_lock(m_impl->renderer.GetMutex());
|
||||
auto& image = m_impl->ResolveSurface(info);
|
||||
if (image.backing.format == vk::Format::eUndefined) {
|
||||
@@ -746,14 +733,13 @@ Presenter::Frame& Presenter::PrepareFrame(CommandBuffer& buffer, const ImageInfo
|
||||
default: break;
|
||||
}
|
||||
frame->Configure(m_impl->window.graphic_ctx,
|
||||
{image.backing.extent.width, image.backing.extent.height},
|
||||
frame_format);
|
||||
{image.backing.extent.width, image.backing.extent.height}, frame_format);
|
||||
frame->CopyFrom(buffer, image);
|
||||
return *frame;
|
||||
}
|
||||
|
||||
Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height, bool opaque,
|
||||
CommandBuffer* producer) {
|
||||
CommandBuffer* producer) {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
auto format = m_impl->frames.GetFormat();
|
||||
auto* frame = m_impl->frames.Acquire();
|
||||
@@ -766,8 +752,7 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
|
||||
frame->Clear(*producer, clear);
|
||||
} else {
|
||||
if (frame->present_commands == nullptr) {
|
||||
frame->present_commands =
|
||||
std::make_unique<CommandBuffer>(m_impl->present_scheduler);
|
||||
frame->present_commands = std::make_unique<CommandBuffer>(m_impl->present_scheduler);
|
||||
}
|
||||
auto& command = *frame->present_commands;
|
||||
command.WaitForFenceAndReset();
|
||||
@@ -797,9 +782,20 @@ void Presenter::Present(Frame& frame, bool reuse) {
|
||||
|
||||
auto& window = m_impl->window;
|
||||
if (window.window_hidden) {
|
||||
#if defined(__APPLE__)
|
||||
// AppKit traps if a window is shown off the main thread; marshal and wait so the
|
||||
// swapchain below is recreated against a visible window.
|
||||
window.RunOnMainThread(
|
||||
[&window] {
|
||||
window.UpdateIcon();
|
||||
SDL_ShowWindow(window.window);
|
||||
},
|
||||
true);
|
||||
#else
|
||||
window.UpdateIcon();
|
||||
|
||||
SDL_ShowWindow(window.window);
|
||||
#endif
|
||||
|
||||
window.window_hidden = false;
|
||||
m_impl->RecoverSwapchain(Swapchain::Status::Recreate);
|
||||
@@ -813,8 +809,7 @@ void Presenter::Present(Frame& frame, bool reuse) {
|
||||
continue;
|
||||
}
|
||||
if (frame.present_commands == nullptr) {
|
||||
frame.present_commands =
|
||||
std::make_unique<CommandBuffer>(m_impl->present_scheduler);
|
||||
frame.present_commands = std::make_unique<CommandBuffer>(m_impl->present_scheduler);
|
||||
}
|
||||
{
|
||||
Common::LockGuard render_lock(m_impl->renderer.GetMutex());
|
||||
|
||||
@@ -32,11 +32,11 @@
|
||||
#include "graphics/host_gpu/vma.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
#include "graphics/presentation/presenter.h"
|
||||
#include "kernel/memory.h"
|
||||
#include "graphics/presentation/renderDoc.h"
|
||||
#include "graphics/presentation/videoOut.h"
|
||||
#include "graphics/presentation/window.h"
|
||||
#include "graphics/presentation/window/windowInternal.h"
|
||||
#include "kernel/memory.h"
|
||||
#include "libs/controller.h"
|
||||
#include "loader/systemContent.h"
|
||||
|
||||
@@ -217,7 +217,9 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
|
||||
|
||||
if (color_write_ext.colorWriteEnable != VK_TRUE) {
|
||||
LOGF("colorWriteEnable is not supported\n");
|
||||
#if !defined(__APPLE__)
|
||||
skip_device = true;
|
||||
#endif
|
||||
}
|
||||
|
||||
if (depth_clip_control.depthClipControl != VK_TRUE) {
|
||||
@@ -226,7 +228,9 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
|
||||
}
|
||||
if (depth_clip_enable.depthClipEnable != VK_TRUE) {
|
||||
LOGF("depthClipEnable is not supported\n");
|
||||
#if !defined(__APPLE__)
|
||||
skip_device = true;
|
||||
#endif
|
||||
}
|
||||
|
||||
if (features12.samplerMirrorClampToEdge != VK_TRUE) {
|
||||
@@ -271,7 +275,9 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
|
||||
}
|
||||
if (device_features2.features.depthBounds != VK_TRUE) {
|
||||
LOGF("depthBounds is not supported\n");
|
||||
#if !defined(__APPLE__)
|
||||
skip_device = true;
|
||||
#endif
|
||||
}
|
||||
if (device_features2.features.shaderStorageImageWriteWithoutFormat != VK_TRUE) {
|
||||
LOGF("shaderStorageImageWriteWithoutFormat is not supported\n");
|
||||
@@ -469,9 +475,9 @@ static void VulkanInitSubgroupSizeControl(vk::PhysicalDevice physical_device,
|
||||
}
|
||||
|
||||
static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const VulkanExtensions& r,
|
||||
uint32_t queue_family,
|
||||
uint32_t queue_family,
|
||||
const std::vector<const char*>& device_extensions,
|
||||
GraphicContext& graphics) {
|
||||
GraphicContext& graphics) {
|
||||
EXIT_IF(physical_device == nullptr);
|
||||
EXIT_IF(queue_family == static_cast<uint32_t>(-1));
|
||||
|
||||
@@ -494,7 +500,14 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
|
||||
|
||||
vk::PhysicalDeviceDepthClipControlFeaturesEXT depth_clip_control {};
|
||||
depth_clip_control.sType = vk::StructureType::ePhysicalDeviceDepthClipControlFeaturesEXT;
|
||||
// MoltenVK lacks VK_EXT_depth_clip_enable and VK_EXT_color_write_enable, so drop those
|
||||
// feature structs from the chain on macOS (the renderer falls back to default depth
|
||||
// clipping and static color-write masks).
|
||||
#if defined(__APPLE__)
|
||||
depth_clip_control.pNext = nullptr;
|
||||
#else
|
||||
depth_clip_control.pNext = &depth_clip_enable;
|
||||
#endif
|
||||
depth_clip_control.depthClipControl = VK_TRUE;
|
||||
|
||||
vk::PhysicalDeviceVulkan12Features features12 {};
|
||||
@@ -538,17 +551,19 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
|
||||
features12.timelineSemaphore = VK_TRUE;
|
||||
|
||||
vk::PhysicalDeviceFeatures device_features {};
|
||||
device_features.fragmentStoresAndAtomics = VK_TRUE;
|
||||
device_features.samplerAnisotropy = VK_TRUE;
|
||||
device_features.robustBufferAccess = VK_TRUE;
|
||||
device_features.depthBounds = VK_TRUE;
|
||||
device_features.fragmentStoresAndAtomics = VK_TRUE;
|
||||
device_features.samplerAnisotropy = VK_TRUE;
|
||||
device_features.robustBufferAccess = VK_TRUE;
|
||||
#if !defined(__APPLE__)
|
||||
device_features.depthBounds = VK_TRUE; // unsupported by MoltenVK
|
||||
#endif
|
||||
device_features.shaderStorageImageWriteWithoutFormat = VK_TRUE;
|
||||
device_features.shaderStorageImageReadWithoutFormat = VK_TRUE;
|
||||
device_features.shaderImageGatherExtended = VK_TRUE;
|
||||
device_features.independentBlend = VK_TRUE;
|
||||
device_features.tessellationShader = VK_TRUE;
|
||||
device_features.sampleRateShading = VK_TRUE;
|
||||
graphics.sample_rate_shading_enabled = true;
|
||||
device_features.sampleRateShading = VK_TRUE;
|
||||
graphics.sample_rate_shading_enabled = true;
|
||||
device_features.vertexPipelineStoresAndAtomics =
|
||||
supported_features2.features.vertexPipelineStoresAndAtomics;
|
||||
|
||||
@@ -895,10 +910,19 @@ void WindowContext::CreateVulkan() {
|
||||
surface = native_surface;
|
||||
|
||||
std::vector<const char*> device_extensions = {
|
||||
VK_KHR_SWAPCHAIN_EXTENSION_NAME, VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME,
|
||||
VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME, VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME,
|
||||
VK_KHR_SWAPCHAIN_EXTENSION_NAME, VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME,
|
||||
VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, "VK_KHR_maintenance1"};
|
||||
|
||||
#if defined(__APPLE__)
|
||||
// MoltenVK lacks VK_EXT_depth_clip_enable and VK_EXT_color_write_enable; the renderer
|
||||
// falls back to default depth clipping and static color-write masks on macOS. It also
|
||||
// requires VK_KHR_portability_subset per the Vulkan portability spec.
|
||||
device_extensions.push_back("VK_KHR_portability_subset");
|
||||
#else
|
||||
device_extensions.push_back(VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME);
|
||||
device_extensions.push_back(VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME);
|
||||
#endif
|
||||
|
||||
#ifdef KYTY_ENABLE_DEBUG_PRINTF
|
||||
if (Config::SpirvDebugPrintfEnabled()) {
|
||||
device_extensions.push_back(VK_KHR_SHADER_NON_SEMANTIC_INFO_EXTENSION_NAME);
|
||||
@@ -907,8 +931,8 @@ void WindowContext::CreateVulkan() {
|
||||
|
||||
uint32_t queue_family = static_cast<uint32_t>(-1);
|
||||
|
||||
VulkanFindPhysicalDevice(graphic_ctx.instance, surface, device_extensions,
|
||||
surface_capabilities, graphic_ctx.physical_device, queue_family);
|
||||
VulkanFindPhysicalDevice(graphic_ctx.instance, surface, device_extensions, surface_capabilities,
|
||||
graphic_ctx.physical_device, queue_family);
|
||||
|
||||
if (graphic_ctx.physical_device == nullptr) {
|
||||
EXIT("Could not find suitable device");
|
||||
@@ -924,9 +948,8 @@ void WindowContext::CreateVulkan() {
|
||||
auto available_extensions = EnumerateVulkan<vk::ExtensionProperties>(
|
||||
"vkEnumerateDeviceExtensionProperties",
|
||||
[&](uint32_t* count, vk::ExtensionProperties* values) {
|
||||
return graphic_ctx.physical_device.enumerateDeviceExtensionProperties(nullptr,
|
||||
count,
|
||||
values);
|
||||
return graphic_ctx.physical_device.enumerateDeviceExtensionProperties(
|
||||
nullptr, count, values);
|
||||
});
|
||||
|
||||
if (HasExtension(available_extensions, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) {
|
||||
@@ -960,7 +983,7 @@ void WindowContext::CreateVulkan() {
|
||||
|
||||
render_context = std::make_unique<RenderContext>(graphic_ctx);
|
||||
LibKernel::Memory::InstallGpuResources(&render_context->GetGpuResources());
|
||||
presenter = std::make_unique<Presenter>(*this);
|
||||
presenter = std::make_unique<Presenter>(*this);
|
||||
RenderDocSetActiveWindow(graphic_ctx.instance, window);
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
@@ -57,7 +58,7 @@
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
constexpr int KEYBOARD_CONTROLLER_ID = -1000;
|
||||
constexpr int KEYBOARD_CONTROLLER_ID = -1000;
|
||||
|
||||
struct EventKeyboard {
|
||||
bool down;
|
||||
@@ -249,9 +250,7 @@ static void GameEventKeyboard(WindowLoopState& game, const EventKeyboard& key) {
|
||||
if (key.down) {
|
||||
switch (key.key_code) {
|
||||
case SDLK_ESCAPE: game.need_exit = true; break;
|
||||
case SDLK_SPACE:
|
||||
SetPause(game, !game.paused.load(std::memory_order_acquire));
|
||||
break;
|
||||
case SDLK_SPACE: SetPause(game, !game.paused.load(std::memory_order_acquire)); break;
|
||||
case SDLK_F1:
|
||||
if (!key.repeat) {
|
||||
RenderDocRequestCapture();
|
||||
@@ -388,7 +387,9 @@ void WindowContext::Resize(uint32_t new_width, uint32_t new_height) {
|
||||
void WindowContext::ProcessWindowEvent(const SDL_WindowEvent& event) {
|
||||
const auto& window_event = event;
|
||||
switch (window_event.event) {
|
||||
case SDL_WINDOWEVENT_SHOWN: LOGF("Window %" PRIu32 " shown\n", window_event.windowID); break;
|
||||
case SDL_WINDOWEVENT_SHOWN:
|
||||
LOGF("Window %" PRIu32 " shown\n", window_event.windowID);
|
||||
break;
|
||||
|
||||
case SDL_WINDOWEVENT_HIDDEN:
|
||||
LOGF("Window %" PRIu32 " hidden\n", window_event.windowID);
|
||||
@@ -399,13 +400,13 @@ void WindowContext::ProcessWindowEvent(const SDL_WindowEvent& event) {
|
||||
break;
|
||||
|
||||
case SDL_WINDOWEVENT_MOVED:
|
||||
LOGF("Window %" PRIu32 " moved to %" PRId32 ",%" PRId32 "\n",
|
||||
window_event.windowID, window_event.data1, window_event.data2);
|
||||
LOGF("Window %" PRIu32 " moved to %" PRId32 ",%" PRId32 "\n", window_event.windowID,
|
||||
window_event.data1, window_event.data2);
|
||||
break;
|
||||
|
||||
case SDL_WINDOWEVENT_RESIZED:
|
||||
LOGF("Window %" PRIu32 " resized to %" PRId32 "x%" PRId32 "\n",
|
||||
window_event.windowID, window_event.data1, window_event.data2);
|
||||
LOGF("Window %" PRIu32 " resized to %" PRId32 "x%" PRId32 "\n", window_event.windowID,
|
||||
window_event.data1, window_event.data2);
|
||||
|
||||
LOGF("m: %d\n", static_cast<int>(SDL_ThreadID()));
|
||||
Resize(window_event.data1, window_event.data2);
|
||||
@@ -697,6 +698,52 @@ void WindowContext::ProcessEvent(double time_s) {
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__APPLE__)
|
||||
void WindowContext::RunOnMainThread(std::function<void()> task, bool wait) {
|
||||
if (Common::Thread::IsMainThread()) {
|
||||
task();
|
||||
return;
|
||||
}
|
||||
|
||||
uint64_t ticket = 0;
|
||||
{
|
||||
Common::LockGuard lock(main_task_mutex);
|
||||
main_tasks.push_back(std::move(task));
|
||||
ticket = ++main_tasks_queued;
|
||||
}
|
||||
|
||||
// Wake the main loop in case it is blocked in SDL_WaitEvent.
|
||||
SDL_Event event {};
|
||||
event.type = SDL_USEREVENT;
|
||||
SDL_PushEvent(&event);
|
||||
|
||||
if (!wait) {
|
||||
return;
|
||||
}
|
||||
Common::LockGuard lock(main_task_mutex);
|
||||
while (main_tasks_run < ticket) {
|
||||
main_task_done.Wait(&main_task_mutex);
|
||||
}
|
||||
}
|
||||
|
||||
void WindowContext::DrainMainThreadTasks() {
|
||||
std::vector<std::function<void()>> tasks;
|
||||
{
|
||||
Common::LockGuard lock(main_task_mutex);
|
||||
tasks.swap(main_tasks);
|
||||
}
|
||||
if (tasks.empty()) {
|
||||
return;
|
||||
}
|
||||
for (auto& task: tasks) {
|
||||
task();
|
||||
}
|
||||
Common::LockGuard lock(main_task_mutex);
|
||||
main_tasks_run += tasks.size();
|
||||
main_task_done.SignalAll();
|
||||
}
|
||||
#endif
|
||||
|
||||
void WindowContext::Run() {
|
||||
Common::Timer timer;
|
||||
timer.Start();
|
||||
@@ -706,6 +753,9 @@ void WindowContext::Run() {
|
||||
loop.paused.store(false, std::memory_order_release);
|
||||
|
||||
while (!loop.need_exit) {
|
||||
#if defined(__APPLE__)
|
||||
DrainMainThreadTasks();
|
||||
#endif
|
||||
if (SDL_PollEvent(&loop.event) != 0) {
|
||||
ProcessEvent(timer.GetTimeS());
|
||||
continue;
|
||||
@@ -747,9 +797,17 @@ static void WindowCreate(WindowContext& context) {
|
||||
|
||||
LOGF("WindowCreate(): width = %d, height = %d\n", width, height);
|
||||
|
||||
context.window =
|
||||
SDL_CreateWindow(KYTY_SDL_WINDOW_CAPTION, KYTY_SDL_WINDOWPOS_CENTERED,
|
||||
KYTY_SDL_WINDOWPOS_CENTERED, width, height, KYTY_SDL_WINDOW_FLAGS);
|
||||
uint32_t window_flags = KYTY_SDL_WINDOW_FLAGS;
|
||||
#if defined(__APPLE__)
|
||||
// macOS 26 window chrome (CoreUI asset decode, SwiftUI titlebar) has been observed
|
||||
// throwing NSExceptions under Rosetta during the first CATransaction commit. A
|
||||
// borderless window skips that machinery entirely.
|
||||
if (std::getenv("KYTY_BORDERLESS") != nullptr) {
|
||||
window_flags |= static_cast<uint32_t>(SDL_WINDOW_BORDERLESS);
|
||||
}
|
||||
#endif
|
||||
context.window = SDL_CreateWindow(KYTY_SDL_WINDOW_CAPTION, KYTY_SDL_WINDOWPOS_CENTERED,
|
||||
KYTY_SDL_WINDOWPOS_CENTERED, width, height, window_flags);
|
||||
|
||||
context.window_hidden = true;
|
||||
|
||||
@@ -772,7 +830,7 @@ Presenter& WindowInit(uint32_t width, uint32_t height) {
|
||||
WindowCreate(*window);
|
||||
window->CreateVulkan();
|
||||
auto& presenter = *window->presenter;
|
||||
g_window = std::move(window);
|
||||
g_window = std::move(window);
|
||||
return presenter;
|
||||
}
|
||||
|
||||
@@ -874,9 +932,9 @@ void WindowContext::UpdateTitle() {
|
||||
Loader::SystemContentParamSfoGetString("TITLE_ID", title_id, sizeof(title_id));
|
||||
static bool has_app_ver =
|
||||
Loader::SystemContentParamSfoGetString("APP_VER", app_ver, sizeof(app_ver));
|
||||
static uint64_t fps_start = Common::Timer::QueryPerformanceCounter();
|
||||
static uint64_t frame_num = 0;
|
||||
static uint64_t fps_frames = 0;
|
||||
static uint64_t fps_start = Common::Timer::QueryPerformanceCounter();
|
||||
static uint64_t frame_num = 0;
|
||||
static uint64_t fps_frames = 0;
|
||||
static double current_fps = 0.0;
|
||||
|
||||
const auto now = Common::Timer::QueryPerformanceCounter();
|
||||
@@ -886,17 +944,23 @@ void WindowContext::UpdateTitle() {
|
||||
if (now - fps_start >= frequency) {
|
||||
current_fps = static_cast<double>(fps_frames) * static_cast<double>(frequency) /
|
||||
static_cast<double>(now - fps_start);
|
||||
fps_start = now;
|
||||
fps_frames = 0;
|
||||
fps_start = now;
|
||||
fps_frames = 0;
|
||||
}
|
||||
|
||||
auto fps = fmt::format("{}{}{}{}{}{}[{}] [{}], frame: {}, fps: {:f}", (has_title ? title : ""),
|
||||
(has_title ? ", " : ""), (has_title_id ? title_id : ""),
|
||||
(has_title_id ? ", " : ""), (has_app_ver ? app_ver : ""),
|
||||
(has_app_ver ? " " : ""), device_name, processor_name,
|
||||
frame_num, current_fps);
|
||||
auto fps =
|
||||
fmt::format("{}{}{}{}{}{}[{}] [{}], frame: {}, fps: {:f}", (has_title ? title : ""),
|
||||
(has_title ? ", " : ""), (has_title_id ? title_id : ""),
|
||||
(has_title_id ? ", " : ""), (has_app_ver ? app_ver : ""),
|
||||
(has_app_ver ? " " : ""), device_name, processor_name, frame_num, current_fps);
|
||||
|
||||
#if defined(__APPLE__)
|
||||
// AppKit traps on title changes off the main thread; fire-and-forget keeps present pacing.
|
||||
RunOnMainThread([this, fps = std::move(fps)] { SDL_SetWindowTitle(window, fps.c_str()); },
|
||||
false);
|
||||
#else
|
||||
SDL_SetWindowTitle(window, fps.c_str());
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
|
||||
@@ -12,6 +12,10 @@
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#if defined(__APPLE__)
|
||||
#include <functional>
|
||||
#endif
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
class Presenter;
|
||||
@@ -24,8 +28,8 @@ struct SurfaceCapabilities {
|
||||
};
|
||||
|
||||
struct WindowLoopState {
|
||||
SDL_Event event {};
|
||||
bool need_exit = false;
|
||||
SDL_Event event {};
|
||||
bool need_exit = false;
|
||||
std::atomic_bool paused = false;
|
||||
};
|
||||
|
||||
@@ -34,32 +38,47 @@ struct WindowContext {
|
||||
~WindowContext();
|
||||
KYTY_CLASS_NO_COPY(WindowContext);
|
||||
|
||||
[[nodiscard]] static vk::PhysicalDeviceVulkan13Features
|
||||
RequiredVulkan13Features() noexcept;
|
||||
void CreateVulkan();
|
||||
void RecreateSurface();
|
||||
void RefreshSurfaceCapabilities();
|
||||
void UpdateIcon();
|
||||
void UpdateTitle();
|
||||
void Resize(uint32_t width, uint32_t height);
|
||||
[[nodiscard]] static vk::PhysicalDeviceVulkan13Features RequiredVulkan13Features() noexcept;
|
||||
void CreateVulkan();
|
||||
void RecreateSurface();
|
||||
void RefreshSurfaceCapabilities();
|
||||
void UpdateIcon();
|
||||
void UpdateTitle();
|
||||
void Resize(uint32_t width, uint32_t height);
|
||||
void ProcessWindowEvent(const SDL_WindowEvent& event);
|
||||
void ProcessDisplayEvent(const SDL_DisplayEvent& event);
|
||||
void ProcessEvent(double time_seconds);
|
||||
void Run();
|
||||
|
||||
GraphicContext graphic_ctx;
|
||||
SDL_Window* window = nullptr;
|
||||
bool window_hidden = true;
|
||||
vk::SurfaceKHR surface = nullptr;
|
||||
SurfaceCapabilities surface_capabilities;
|
||||
#if defined(__APPLE__)
|
||||
// AppKit only allows window operations (show, icon, title, view/layer changes) on the
|
||||
// main thread; the present thread marshals them through the SDL main loop with these.
|
||||
// wait=true blocks until the task has run on the main thread.
|
||||
void RunOnMainThread(std::function<void()> task, bool wait);
|
||||
void DrainMainThreadTasks();
|
||||
#endif
|
||||
|
||||
GraphicContext graphic_ctx;
|
||||
SDL_Window* window = nullptr;
|
||||
bool window_hidden = true;
|
||||
vk::SurfaceKHR surface = nullptr;
|
||||
SurfaceCapabilities surface_capabilities;
|
||||
std::unique_ptr<RenderContext> render_context;
|
||||
std::unique_ptr<Presenter> presenter;
|
||||
WindowLoopState loop;
|
||||
std::unique_ptr<Presenter> presenter;
|
||||
WindowLoopState loop;
|
||||
|
||||
char device_name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE] = {0};
|
||||
char processor_name[64] = {0};
|
||||
|
||||
Common::Mutex mutex;
|
||||
|
||||
#if defined(__APPLE__)
|
||||
Common::Mutex main_task_mutex;
|
||||
Common::CondVar main_task_done;
|
||||
std::vector<std::function<void()>> main_tasks; // guarded by main_task_mutex
|
||||
uint64_t main_tasks_queued = 0; // guarded by main_task_mutex
|
||||
uint64_t main_tasks_run = 0; // guarded by main_task_mutex
|
||||
#endif
|
||||
};
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user