From b51d4322ccc461fb6522f713ea1e9e7757c653bf Mon Sep 17 00:00:00 2001 From: nmzik Date: Mon, 20 Jul 2026 07:03:07 +0200 Subject: [PATCH] Add GPU tilers/tests --- src/CMakeLists.txt | 34 + src/embed_spirv.cmake | 30 + src/graphics/guest_gpu/tile.cpp | 2435 ++++------------- src/graphics/guest_gpu/tile.h | 105 +- src/graphics/host_gpu/gpuTiler.cpp | 547 ++++ src/graphics/host_gpu/gpuTiler.h | 45 + .../host_gpu/objects/textureCommon.cpp | 694 ++--- src/graphics/host_gpu/objects/textureCommon.h | 54 +- src/graphics/host_gpu/renderer/image.cpp | 78 +- src/graphics/host_gpu/renderer/image.h | 1 + .../host_gpu/renderer/textureCache.cpp | 260 +- src/graphics/host_gpu/renderer/tiler.cpp | 296 +- src/graphics/host_gpu/renderer/tiler.h | 4 - .../host_gpu/shaders/gpu_tiler_common.inc | 70 + .../host_gpu/shaders/gpu_tiler_depth.comp | 41 + .../host_gpu/shaders/gpu_tiler_prt.comp | 20 + .../host_gpu/shaders/gpu_tiler_prt_3d.comp | 17 + .../shaders/gpu_tiler_render_target.comp | 42 + .../host_gpu/shaders/gpu_tiler_standard.inc | 18 + .../shaders/gpu_tiler_standard256.comp | 12 + .../host_gpu/shaders/gpu_tiler_standard4.comp | 12 + .../shaders/gpu_tiler_standard4_3d.comp | 15 + .../shaders/gpu_tiler_standard4_3d.inc | 22 + .../shaders/gpu_tiler_standard64.comp | 12 + .../host_gpu/shaders/gpu_tiler_standard64.inc | 29 + .../shaders/gpu_tiler_standard64_3d.comp | 9 + .../shaders/gpu_tiler_standard64_3d.inc | 17 + src/graphics/host_gpu/transfer.cpp | 67 +- src/graphics/host_gpu/transfer.h | 11 +- src/libs/agc.cpp | 1 - tests/ShaderRecompilerComputeTests.cpp | 1433 ++++++---- 31 files changed, 3071 insertions(+), 3360 deletions(-) create mode 100644 src/embed_spirv.cmake create mode 100644 src/graphics/host_gpu/gpuTiler.cpp create mode 100644 src/graphics/host_gpu/gpuTiler.h create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_common.inc create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_depth.comp create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_prt.comp create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_prt_3d.comp create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_render_target.comp create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_standard.inc create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_standard256.comp create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_standard4.comp create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_standard4_3d.comp create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_standard4_3d.inc create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_standard64.comp create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_standard64.inc create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_standard64_3d.comp create mode 100644 src/graphics/host_gpu/shaders/gpu_tiler_standard64_3d.inc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 95485ea..718cc83 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -170,6 +170,40 @@ file(GLOB kyty_emulator_src CONFIGURE_DEPENDS loader/*.h ) +find_program(KYTY_GLSLANG_VALIDATOR glslangValidator REQUIRED) +set(gpu_tiler_shader_dir "${CMAKE_CURRENT_SOURCE_DIR}/graphics/host_gpu/shaders") +set(gpu_tiler_generated_dir "${PROJECT_BINARY_DIR}/gpu_tiler_shaders") +set(gpu_tiler_shader_names + standard256 + standard4 + standard4_3d + standard64 + standard64_3d + prt + prt_3d + render_target + depth +) +file(GLOB gpu_tiler_shader_includes CONFIGURE_DEPENDS "${gpu_tiler_shader_dir}/gpu_tiler_*.inc") +foreach(shader_name IN LISTS gpu_tiler_shader_names) + set(shader_source "${gpu_tiler_shader_dir}/gpu_tiler_${shader_name}.comp") + set(shader_spv "${gpu_tiler_generated_dir}/gpu_tiler_${shader_name}.spv") + set(shader_header "${gpu_tiler_generated_dir}/gpu_tiler_${shader_name}_spv.h") + string(TOUPPER "GPU_TILER_${shader_name}_SPV" shader_symbol) + add_custom_command( + OUTPUT "${shader_header}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_tiler_generated_dir}" + COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os + "-I${gpu_tiler_shader_dir}" -o "${shader_spv}" "${shader_source}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${shader_spv} -DOUTPUT=${shader_header} + -DSYMBOL=${shader_symbol} -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${shader_source}" ${gpu_tiler_shader_includes} + VERBATIM + ) + list(APPEND gpu_tiler_shader_headers "${shader_header}") +endforeach() +list(APPEND kyty_emulator_src ${gpu_tiler_shader_headers}) + list(APPEND kyty_emulator_src emulator.h emulator.cpp diff --git a/src/embed_spirv.cmake b/src/embed_spirv.cmake new file mode 100644 index 0000000..eb0c9b9 --- /dev/null +++ b/src/embed_spirv.cmake @@ -0,0 +1,30 @@ +if(NOT DEFINED INPUT OR NOT DEFINED OUTPUT OR NOT DEFINED SYMBOL) + message(FATAL_ERROR "embed_spirv.cmake requires INPUT, OUTPUT, and SYMBOL") +endif() + +file(READ "${INPUT}" bytes HEX) +string(LENGTH "${bytes}" length) +math(EXPR remainder "${length} % 8") +if(NOT remainder EQUAL 0) + message(FATAL_ERROR "SPIR-V byte count is not a multiple of four: ${INPUT}") +endif() + +set(contents "#pragma once\n#include \ninline constexpr uint32_t ${SYMBOL}[] = {\n") +set(column 0) +while(length GREATER 0) + string(SUBSTRING "${bytes}" 0 8 word) + string(SUBSTRING "${word}" 0 2 b0) + string(SUBSTRING "${word}" 2 2 b1) + string(SUBSTRING "${word}" 4 2 b2) + string(SUBSTRING "${word}" 6 2 b3) + string(APPEND contents "0x${b3}${b2}${b1}${b0}u,") + math(EXPR column "${column} + 1") + if(column EQUAL 8) + string(APPEND contents "\n") + set(column 0) + endif() + string(SUBSTRING "${bytes}" 8 -1 bytes) + math(EXPR length "${length} - 8") +endwhile() +string(APPEND contents "\n};\n") +file(WRITE "${OUTPUT}" "${contents}") diff --git a/src/graphics/guest_gpu/tile.cpp b/src/graphics/guest_gpu/tile.cpp index 744a21c..bea0c64 100644 --- a/src/graphics/guest_gpu/tile.cpp +++ b/src/graphics/guest_gpu/tile.cpp @@ -4,24 +4,17 @@ #include "common/logging/log.h" #include "common/profiler.h" #include "common/stringUtils.h" -#include "common/threads.h" -#include "graphics/asyncJob.h" #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/gpu_format.h" #include #include #include -#include #include #include namespace Libs::Graphics { -static uint32_t IntLog2(uint32_t i) { - return 31 - __builtin_clz(i | 1u); -} - static uint32_t AlignUp(uint32_t value, uint32_t alignment) { return (value + alignment - 1u) & ~(alignment - 1u); } @@ -53,7 +46,7 @@ static uint32_t SetLinearMipChainLayout(uint32_t levels, const uint32_t* mip_pit const uint32_t* mip_height, const uint32_t* mip_size, TileSizeOffset* level_sizes, TilePaddedSize* padded_size) { uint32_t offset = 0; - // AGC linear surfaces store smaller mip records first; mip 0 is last in the block slice. + // Smaller mip records come first; mip 0 is last in the block slice. for (int32_t l = static_cast(levels) - 1; l >= 0; l--) { const auto level = static_cast(l); if (level_sizes != nullptr) { @@ -134,47 +127,6 @@ static bool Gen5Standard4KBLayout(uint32_t format, uint32_t* bytes_per_element, } } -bool TileGetStandard4KBVolumeLayout(uint32_t format, uint32_t* bytes_per_element, - uint32_t* texels_per_element_wide, - uint32_t* texels_per_element_tall, uint32_t* block_width_log2, - uint32_t* block_height_log2, uint32_t* block_depth_log2) { - if (!Gen5Standard4KBLayout(format, bytes_per_element, texels_per_element_wide, - texels_per_element_tall, block_width_log2, block_height_log2)) { - return false; - } - - switch (*bytes_per_element) { - case 1: - *block_width_log2 = 4; - *block_height_log2 = 4; - *block_depth_log2 = 4; - return true; - case 2: - *block_width_log2 = 3; - *block_height_log2 = 4; - *block_depth_log2 = 4; - return true; - case 4: - *block_width_log2 = 3; - *block_height_log2 = 4; - *block_depth_log2 = 3; - return true; - case 8: - *block_width_log2 = 3; - *block_height_log2 = 3; - *block_depth_log2 = 3; - return true; - case 16: - *block_width_log2 = 2; - *block_height_log2 = 3; - *block_depth_log2 = 3; - return true; - default: break; - } - - return false; -} - static bool Gen5Standard256BLayout(uint32_t format, uint32_t* bytes_per_element, uint32_t* texels_per_element_wide, uint32_t* texels_per_element_tall, uint32_t* block_width_log2, @@ -302,7 +254,7 @@ static bool Gen5Standard64KBLayout(uint32_t format, uint32_t* bytes_per_element, static bool Gen5Thin64KBBlockSizeFromElementBytes(uint32_t bytes_per_element, uint32_t* block_width, uint32_t* block_height) { - // AGC thin 64 KiB block table, shared by depth and render-target tiles. + // Thin 64 KiB block dimensions shared by depth and render-target tiles. switch (bytes_per_element) { case 1: *block_width = 256; @@ -340,8 +292,7 @@ static bool Gen5Msaa64KBBlockSizeFromElementBytes(uint32_t bytes_per_element, bytes_per_element > 16) { return false; } - // AGC kLog2BlockSizeMsaa. Row zero is the ordinary thin 64 KiB layout; rows 1..3 - // describe 2x, 4x and 8x depth/render-target blocks respectively. + // Row zero is the ordinary thin layout; rows 1..3 are 2x, 4x and 8x blocks. static constexpr uint8_t LOG2_BLOCK[4][5][2] = { {{8, 8}, {8, 7}, {7, 7}, {7, 6}, {6, 6}}, {{7, 8}, {7, 7}, {6, 7}, {6, 6}, {5, 6}}, @@ -430,6 +381,226 @@ static constexpr Gen5MipTailLocation GEN5_MIP_TAIL_LOCATIONS_THIN_64KB[5][12] = {0, 0}}, }; +static constexpr Gen5MipTailLocation GEN5_MIP_TAIL_LOCATIONS_THICK_64KB[5][10] = { + {{32, 0}, {0, 16}, {16, 0}, {8, 8}, {0, 12}, {0, 8}, {8, 4}, {8, 0}, {0, 4}, {0, 0}}, + {{16, 0}, {0, 16}, {8, 0}, {4, 8}, {0, 12}, {0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}}, + {{16, 0}, {0, 16}, {8, 0}, {4, 8}, {0, 12}, {0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}}, + {{16, 0}, {0, 8}, {8, 0}, {4, 4}, {0, 6}, {0, 4}, {4, 2}, {4, 0}, {0, 2}, {0, 0}}, + {{8, 0}, {0, 8}, {4, 0}, {2, 4}, {0, 6}, {0, 4}, {2, 2}, {2, 0}, {0, 2}, {0, 0}}, +}; + +static constexpr Gen5MipTailLocation GEN5_MIP_TAIL_LOCATIONS_THICK_4KB[5][5] = { + {{0, 8}, {8, 4}, {8, 0}, {0, 4}, {0, 0}}, {{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}}, + {{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}}, {{0, 4}, {4, 2}, {4, 0}, {0, 2}, {0, 0}}, + {{0, 4}, {2, 2}, {2, 0}, {0, 2}, {0, 0}}, +}; + +struct TextureBlockLayout { + uint32_t bytes; + uint32_t texel_width; + uint32_t texel_height; + uint32_t block_width; + uint32_t block_height; + uint32_t block_size; +}; + +static bool GetTextureBlockLayout(uint32_t format, uint32_t tile, TextureBlockLayout* out) { + uint32_t width_log2 = 0, height_log2 = 0; + if (tile == 1 && Gen5Standard256BLayout(format, &out->bytes, &out->texel_width, + &out->texel_height, &width_log2, &height_log2)) { + out->block_size = 256; + } else if (tile == 5 && Gen5Standard4KBLayout(format, &out->bytes, &out->texel_width, + &out->texel_height, &width_log2, &height_log2)) { + out->block_size = 4096; + } else if ((tile == 9 || tile == 17) && + Gen5Standard64KBLayout(format, &out->bytes, &out->texel_width, &out->texel_height, + &width_log2, &height_log2)) { + out->block_size = 65536; + } else if (tile == 24 || tile == 27) { + out->bytes = Prospero::NumBytesPerElement(format); + out->texel_width = out->texel_height = 1; + if (out->bytes == 0 || (tile == 24 && out->bytes > 8) || + !Gen5Thin64KBBlockSizeFromElementBytes(out->bytes, &out->block_width, + &out->block_height)) { + return false; + } + out->block_size = 65536; + return true; + } else { + return false; + } + out->block_width = 1u << width_log2; + out->block_height = 1u << height_log2; + return true; +} + +static void SetMicroMipLayout(const TextureBlockLayout& block, uint32_t width, uint32_t height, + uint32_t levels, TileSizeAlign* total_size, + TileSizeOffset* level_sizes, TilePaddedSize* padded_size) { + const uint32_t width0 = (width + block.texel_width - 1u) / block.texel_width; + const uint32_t height0 = (height + block.texel_height - 1u) / block.texel_height; + uint32_t offset = 0; + for (int32_t l = static_cast(levels) - 1; l >= 0; --l) { + const auto level = static_cast(l); + const uint32_t padded_width = + AlignUp(std::max(ShiftCeil(width0, level), 1u), block.block_width); + const uint32_t padded_height = + AlignUp(std::max(ShiftCeil(height0, level), 1u), block.block_height); + const uint32_t size = padded_width * padded_height * block.bytes; + if (level_sizes != nullptr) level_sizes[level] = {size, offset}; + if (padded_size != nullptr) { + padded_size[level] = {padded_width * block.texel_width, + padded_height * block.texel_height}; + } + offset += size; + } + if (total_size != nullptr) *total_size = {offset, block.block_size}; +} + +static void SetMacroMipLayout(const TextureBlockLayout& block, uint32_t tile, uint32_t width, + uint32_t height, uint32_t levels, TileSizeAlign* total_size, + TileSizeOffset* level_sizes, TilePaddedSize* padded_size) { + const uint32_t width0 = (width + block.texel_width - 1u) / block.texel_width; + const uint32_t height0 = (height + block.texel_height - 1u) / block.texel_height; + const uint32_t bytes_log2 = std::countr_zero(block.bytes); + const uint32_t max_tail = block.block_size == 4096 ? 8u : 12u; + uint32_t tail_width = block.block_width >> 1u, tail_height = block.block_height; + if (tile == 24 && block.bytes < 4) { + tail_width = 64; + tail_height = 128; + } + + uint32_t first_tail = levels; + if (levels > 1) { + for (uint32_t level = 0; level < levels; ++level) { + if (ShiftCeil(width0, level) <= tail_width && + ShiftCeil(height0, level) <= tail_height && levels - level <= max_tail) { + first_tail = level; + break; + } + } + } + + uint32_t offset = first_tail < levels ? block.block_size : 0; + for (int32_t l = static_cast(first_tail) - 1; l >= 0; --l) { + const auto level = static_cast(l); + const uint32_t padded_width = + AlignUp(std::max(ShiftCeil(width0, level), 1u), block.block_width); + const uint32_t padded_height = + AlignUp(std::max(ShiftCeil(height0, level), 1u), block.block_height); + const uint32_t size = padded_width * padded_height * block.bytes; + if (level_sizes != nullptr) level_sizes[level] = {size, offset, size, offset}; + if (padded_size != nullptr) { + padded_size[level] = {padded_width * block.texel_width, + padded_height * block.texel_height}; + } + offset += size; + } + + uint32_t linear_offset = 0; + for (uint32_t level = first_tail; level < levels; ++level) { + const uint32_t mip_width = + std::max(((width >> level) + block.texel_width - 1u) / block.texel_width, 1u); + const uint32_t mip_height = + std::max(((height >> level) + block.texel_height - 1u) / block.texel_height, 1u); + const auto tail = block.block_size == 4096 + ? GEN5_MIP_TAIL_LOCATIONS_THIN_4KB[bytes_log2][level - first_tail] + : GEN5_MIP_TAIL_LOCATIONS_THIN_64KB[bytes_log2][level - first_tail]; + if (level_sizes != nullptr) { + level_sizes[level] = {mip_width * mip_height * block.bytes, + linear_offset, + block.block_size, + 0, + tail.x, + tail.y}; + } + if (padded_size != nullptr) { + padded_size[level] = {block.block_width * block.texel_width, + block.block_height * block.texel_height}; + } + linear_offset += mip_width * mip_height * block.bytes; + } + if (total_size != nullptr) *total_size = {offset, block.block_size}; +} + +bool TileGetTextureVolumeLayout(uint32_t format, uint32_t width, uint32_t height, uint32_t depth, + uint32_t levels, uint32_t tile, TileVolumeLayout* out) { + if (out == nullptr || width == 0 || height == 0 || depth == 0 || levels == 0 || levels > 16) { + return false; + } + TextureBlockLayout element {}; + if (!GetTextureBlockLayout(format, tile, &element)) return false; + + TileBlockFamily family = TileBlockFamily::Count; + switch (tile) { + case 5: family = TileBlockFamily::Standard4KB3D; break; + case 9: family = TileBlockFamily::Standard64KB3D; break; + case 17: family = TileBlockFamily::Prt64KB3D; break; + case 24: family = TileBlockFamily::Depth64KB; break; + case 27: family = TileBlockFamily::RenderTarget64KB; break; + default: return false; + } + TileBlockLayout block {}; + if (!TileGetBlockLayout(family, element.bytes, &block)) return false; + + *out = {}; + out->family = family; + out->bytes_per_element = element.bytes; + out->texel_width = element.texel_width; + out->texel_height = element.texel_height; + out->block_depth = block.block_depth; + out->first_tail_level = levels; + const uint32_t width0 = (width + element.texel_width - 1u) / element.texel_width; + const uint32_t height0 = (height + element.texel_height - 1u) / element.texel_height; + const bool thick4 = family == TileBlockFamily::Standard4KB3D; + const bool thick64 = + family == TileBlockFamily::Standard64KB3D || family == TileBlockFamily::Prt64KB3D; + const uint32_t max_tail = thick4 ? 5u : (thick64 ? 10u : 12u); + uint32_t tail_width = thick4 ? block.block_width : block.block_width >> 1u; + uint32_t tail_height = thick4 ? block.block_height >> 1u : block.block_height; + if (family == TileBlockFamily::Depth64KB && element.bytes < 4) { + tail_width = 64; + tail_height = 128; + } + + for (uint32_t level = 0; level < levels; ++level) { + const uint32_t mip_width = std::max(ShiftCeil(width0, level), 1u); + const uint32_t mip_height = std::max(ShiftCeil(height0, level), 1u); + if (levels > 1 && mip_width <= tail_width && mip_height <= tail_height && + levels - level <= max_tail) { + out->first_tail_level = level; + out->block_slice_size += block.block_size; + break; + } + out->level_widths[level] = AlignUp(mip_width, block.block_width); + out->level_heights[level] = AlignUp(mip_height, block.block_height); + out->level_sizes[level] = static_cast(block.block_depth) * + out->level_widths[level] * out->level_heights[level] * + element.bytes; + out->block_slice_size += out->level_sizes[level]; + } + + const auto bytes_log2 = std::countr_zero(element.bytes); + for (uint32_t level = out->first_tail_level; level < levels; ++level) { + const auto index = level - out->first_tail_level; + const auto tail = thick4 ? GEN5_MIP_TAIL_LOCATIONS_THICK_4KB[bytes_log2][index] + : (thick64 ? GEN5_MIP_TAIL_LOCATIONS_THICK_64KB[bytes_log2][index] + : GEN5_MIP_TAIL_LOCATIONS_THIN_64KB[bytes_log2][index]); + out->level_sizes[level] = block.block_size; + out->level_widths[level] = block.block_width; + out->level_heights[level] = block.block_height; + out->tail_x[level] = tail.x; + out->tail_y[level] = tail.y; + } + uint64_t offset = out->first_tail_level < levels ? block.block_size : 0; + for (int32_t l = static_cast(out->first_tail_level) - 1; l >= 0; --l) { + out->level_offsets[l] = offset; + offset += out->level_sizes[l]; + } + out->total_size = out->block_slice_size * ShiftCeil(depth, std::countr_zero(block.block_depth)); + return offset == out->block_slice_size; +} + static uint32_t Gen5Standard4KBOffsetInBlock(uint32_t x, uint32_t y, uint32_t bytes_per_element) { uint32_t offset = 0; @@ -550,6 +721,19 @@ static uint32_t Gen5Standard4KBVolumeOffsetInBlock(uint32_t x, uint32_t y, uint3 return 0; } +static uint32_t Bit(uint32_t value, uint32_t source, uint32_t destination) { + return ((value >> source) & 1u) << destination; +} + +static uint32_t Gen5Standard64KBVolumeOffsetInBlock(uint32_t x, uint32_t y, uint32_t z, + uint32_t bytes_per_element) { + static constexpr uint8_t SOURCES[5][4] = { + {4, 4, 4, 5}, {3, 4, 4, 4}, {3, 3, 4, 4}, {3, 3, 3, 4}, {2, 3, 3, 3}}; + const auto* bits = SOURCES[std::countr_zero(bytes_per_element)]; + return Gen5Standard4KBVolumeOffsetInBlock(x, y, z, bytes_per_element) ^ Bit(x, bits[0], 12) ^ + Bit(z, bits[1], 13) ^ Bit(y, bits[2], 14) ^ Bit(x, bits[3], 15); +} + bool TileIsStandard4KBTextureSupported(uint32_t format) { uint32_t bytes_per_element = 0; uint32_t texels_per_element_wide = 0; @@ -587,466 +771,12 @@ struct Uint128 { uint64_t n[2]; }; -struct Uint256 { - Uint128 n[2]; -}; - -template -static void DetileStandard4KBTyped(T* dst, const T* src, uint32_t row_elements, - uint32_t width_elements, uint32_t height_elements, - uint32_t padded_width, uint32_t block_width_log2, - uint32_t block_height_log2, uint64_t dst_size, uint64_t src_size, - uint32_t src_x, uint32_t src_y) { - constexpr auto bytes_per_element = static_cast(sizeof(T)); - - std::array x_elements {}; - std::array y_elements {}; - - const uint32_t block_width = 1u << block_width_log2; - const uint32_t block_height = 1u << block_height_log2; - const uint32_t elements_per_block = 4096u / bytes_per_element; - const uint32_t blocks_per_row = padded_width >> block_width_log2; - - for (uint32_t x = 0; x < block_width; x++) { - x_elements[x] = Gen5Standard4KBOffsetInBlock(x, 0, bytes_per_element) / bytes_per_element; - } - - for (uint32_t y = 0; y < block_height; y++) { - y_elements[y] = Gen5Standard4KBOffsetInBlock(0, y, bytes_per_element) / bytes_per_element; - } - - const uint64_t dst_count = dst_size / bytes_per_element; - const uint64_t src_count = src_size / bytes_per_element; - - for (uint32_t block_y = 0; block_y < height_elements; block_y += block_height) { - const uint32_t copy_height = std::min(block_height, height_elements - block_y); - - for (uint32_t block_x = 0; block_x < width_elements; block_x += block_width) { - const uint32_t copy_width = std::min(block_width, width_elements - block_x); - const uint64_t block_index = - (static_cast(block_y >> block_height_log2) * blocks_per_row) + - (block_x >> block_width_log2); - const uint64_t block_base = block_index * elements_per_block; - - for (uint32_t local_y = 0; local_y < copy_height; local_y++) { - const uint64_t dst_row = - (static_cast(block_y + local_y) * row_elements) + block_x; - const uint64_t src_row = block_base + y_elements[src_y + local_y]; - - for (uint32_t local_x = 0; local_x < copy_width; local_x++) { - const uint64_t dst_index = dst_row + local_x; - const uint64_t src_index = src_row + x_elements[src_x + local_x]; - - if (src_index < src_count && dst_index < dst_count) { - dst[dst_index] = src[src_index]; - } - } - } - } - } -} - -template -static void DetileStandard4KBVolumeTyped(T* dst, const T* src, uint32_t row_elements, - uint32_t width_elements, uint32_t height_elements, - uint32_t depth_elements, uint32_t padded_width, - uint32_t padded_height, uint32_t block_width_log2, - uint32_t block_height_log2, uint32_t block_depth_log2, - uint64_t dst_slice_stride, uint64_t dst_size, - uint64_t src_size) { - constexpr auto bytes_per_element = static_cast(sizeof(T)); - - const uint32_t block_width = 1u << block_width_log2; - const uint32_t block_height = 1u << block_height_log2; - const uint32_t block_depth = 1u << block_depth_log2; - const uint32_t elements_per_block = 4096u / bytes_per_element; - const uint32_t blocks_per_row = padded_width >> block_width_log2; - const uint32_t blocks_per_column = padded_height >> block_height_log2; - const uint64_t blocks_per_slice = - static_cast(blocks_per_row) * static_cast(blocks_per_column); - const uint64_t dst_count = dst_size / bytes_per_element; - const uint64_t src_count = src_size / bytes_per_element; - const uint64_t dst_slice_elements = dst_slice_stride / bytes_per_element; - - for (uint32_t block_z = 0; block_z < depth_elements; block_z += block_depth) { - const uint32_t copy_depth = std::min(block_depth, depth_elements - block_z); - - for (uint32_t block_y = 0; block_y < height_elements; block_y += block_height) { - const uint32_t copy_height = std::min(block_height, height_elements - block_y); - - for (uint32_t block_x = 0; block_x < width_elements; block_x += block_width) { - const uint32_t copy_width = std::min(block_width, width_elements - block_x); - const uint64_t block_index = - ((static_cast(block_z >> block_depth_log2) * blocks_per_slice) + - (static_cast(block_y >> block_height_log2) * blocks_per_row) + - (block_x >> block_width_log2)); - const uint64_t block_base = block_index * elements_per_block; - - for (uint32_t local_z = 0; local_z < copy_depth; local_z++) { - for (uint32_t local_y = 0; local_y < copy_height; local_y++) { - const uint64_t dst_row = - (static_cast(block_z + local_z) * dst_slice_elements) + - (static_cast(block_y + local_y) * row_elements) + block_x; - - for (uint32_t local_x = 0; local_x < copy_width; local_x++) { - const uint64_t dst_index = dst_row + local_x; - const uint64_t src_index = - block_base + (Gen5Standard4KBVolumeOffsetInBlock( - local_x, local_y, local_z, bytes_per_element) / - bytes_per_element); - - if (src_index < src_count && dst_index < dst_count) { - dst[dst_index] = src[src_index]; - } - } - } - } - } - } - } -} - static uint32_t Gen5Standard256BOffsetInBlock(uint32_t x, uint32_t y, uint32_t bytes_per_element) { return Gen5Standard4KBOffsetInBlock(x, y, bytes_per_element) & 0xffu; } -template -static void -DetileStandard256BTyped(T* dst, const T* src, uint32_t row_elements, uint32_t width_elements, - uint32_t height_elements, uint32_t padded_width, uint32_t block_width_log2, - uint32_t block_height_log2, uint64_t dst_size, uint64_t src_size) { - constexpr auto bytes_per_element = static_cast(sizeof(T)); - - std::array x_elements {}; - std::array y_elements {}; - - const uint32_t block_width = 1u << block_width_log2; - const uint32_t block_height = 1u << block_height_log2; - const uint32_t elements_per_block = 256u / bytes_per_element; - const uint32_t blocks_per_row = padded_width >> block_width_log2; - - for (uint32_t x = 0; x < block_width; x++) { - x_elements[x] = Gen5Standard256BOffsetInBlock(x, 0, bytes_per_element) / bytes_per_element; - } - - for (uint32_t y = 0; y < block_height; y++) { - y_elements[y] = Gen5Standard256BOffsetInBlock(0, y, bytes_per_element) / bytes_per_element; - } - - const uint64_t dst_count = dst_size / bytes_per_element; - const uint64_t src_count = src_size / bytes_per_element; - - for (uint32_t block_y = 0; block_y < height_elements; block_y += block_height) { - const uint32_t copy_height = std::min(block_height, height_elements - block_y); - - for (uint32_t block_x = 0; block_x < width_elements; block_x += block_width) { - const uint32_t copy_width = std::min(block_width, width_elements - block_x); - const uint64_t block_index = - (static_cast(block_y >> block_height_log2) * blocks_per_row) + - (block_x >> block_width_log2); - const uint64_t block_base = block_index * elements_per_block; - - for (uint32_t local_y = 0; local_y < copy_height; local_y++) { - const uint64_t dst_row = - (static_cast(block_y + local_y) * row_elements) + block_x; - const uint64_t src_row = block_base + y_elements[local_y]; - - for (uint32_t local_x = 0; local_x < copy_width; local_x++) { - const uint64_t dst_index = dst_row + local_x; - const uint64_t src_index = src_row + x_elements[local_x]; - - if (src_index < src_count && dst_index < dst_count) { - dst[dst_index] = src[src_index]; - } - } - } - } - } -} - -class Tiler { -public: - Tiler() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } - ~Tiler() { KYTY_NOT_IMPLEMENTED; } - - KYTY_CLASS_NO_COPY(Tiler); - - Common::Mutex m_mutex; - - AsyncJob m_job1; -}; - -class Tiler32 { -public: - uint32_t m_macro_tile_height = 0; - uint32_t m_bank_height = 0; - uint32_t m_num_banks = 0; - uint32_t m_num_pipes = 0; - uint32_t m_padded_width = 0; - uint32_t m_padded_height = 0; - uint32_t m_pipe_bits = 0; - uint32_t m_bank_bits = 0; - - void Init(uint32_t width, uint32_t height) { - m_macro_tile_height = 128; - m_bank_height = 2; - m_num_banks = 8; - m_num_pipes = 16; - m_padded_width = AlignUp(width, 128); - m_padded_height = AlignUp(height, m_macro_tile_height); - m_pipe_bits = 4; - m_bank_bits = 3; - } - - static uint32_t GetElementIndex(uint32_t x, uint32_t y) { - uint32_t elem = 0; - elem |= ((x >> 0u) & 0x1u) << 0u; - elem |= ((x >> 1u) & 0x1u) << 1u; - elem |= ((y >> 0u) & 0x1u) << 2u; - elem |= ((x >> 2u) & 0x1u) << 3u; - elem |= ((y >> 1u) & 0x1u) << 4u; - elem |= ((y >> 2u) & 0x1u) << 5u; - - return elem; - } - - static uint32_t GetPipeIndex(uint32_t x, uint32_t y) { - uint32_t pipe = 0; - - pipe |= (((x >> 3u) ^ (y >> 3u) ^ (x >> 4u)) & 0x1u) << 0u; - pipe |= (((x >> 4u) ^ (y >> 4u)) & 0x1u) << 1u; - pipe |= (((x >> 5u) ^ (y >> 5u)) & 0x1u) << 2u; - pipe |= (((x >> 6u) ^ (y >> 5u)) & 0x1u) << 3u; - - return pipe; - } - - static uint32_t GetBankIndex(uint32_t x, uint32_t y, uint32_t bank_width, uint32_t bank_height, - uint32_t num_banks, uint32_t num_pipes) { - const uint32_t x_shift_offset = IntLog2(bank_width * num_pipes); - const uint32_t y_shift_offset = IntLog2(bank_height); - const uint32_t xs = x >> x_shift_offset; - const uint32_t ys = y >> y_shift_offset; - uint32_t bank = 0; - switch (num_banks) { - case 8: - bank |= (((xs >> 3u) ^ (ys >> 5u)) & 0x1u) << 0u; - bank |= (((xs >> 4u) ^ (ys >> 4u) ^ (ys >> 5u)) & 0x1u) << 1u; - bank |= (((xs >> 5u) ^ (ys >> 3u)) & 0x1u) << 2u; - break; - case 16: - bank |= (((xs >> 3u) ^ (ys >> 6u)) & 0x1u) << 0u; - bank |= (((xs >> 4u) ^ (ys >> 5u) ^ (ys >> 6u)) & 0x1u) << 1u; - bank |= (((xs >> 5u) ^ (ys >> 4u)) & 0x1u) << 2u; - bank |= (((xs >> 6u) ^ (ys >> 3u)) & 0x1u) << 3u; - break; - default:; - } - - return bank; - } - - [[nodiscard]] uint64_t GetTiledOffset(uint32_t x, uint32_t y) const { - return GetTiledOffset(x, y, 32); - } - - [[nodiscard]] uint64_t GetTiledOffset(uint32_t x, uint32_t y, uint32_t bits_per_element) const { - uint64_t element_index = GetElementIndex(x, y); - - uint32_t xh = x; - uint32_t yh = y; - uint64_t pipe = GetPipeIndex(xh, yh); - uint64_t bank = GetBankIndex(xh, yh, 1, m_bank_height, m_num_banks, m_num_pipes); - uint32_t tile_bytes = (8 * 8 * bits_per_element + 7) / 8; - uint64_t element_offset = (element_index * bits_per_element); - uint64_t tile_split_slice = 0; - - if (tile_bytes > 512) { - tile_split_slice = element_offset / (static_cast(512) * 8); - element_offset %= (static_cast(512) * 8); - tile_bytes = 512; - } - - uint64_t macro_tile_bytes = - (128 / 8) * (m_macro_tile_height / 8) * tile_bytes / (m_num_pipes * m_num_banks); - uint64_t macro_tiles_per_row = m_padded_width / 128; - uint64_t macro_tile_row_index = y / m_macro_tile_height; - uint64_t macro_tile_column_index = x / 128; - uint64_t macro_tile_index = - (macro_tile_row_index * macro_tiles_per_row) + macro_tile_column_index; - uint64_t macro_tile_offset = macro_tile_index * macro_tile_bytes; - uint64_t macro_tiles_per_slice = - macro_tiles_per_row * (m_padded_height / m_macro_tile_height); - uint64_t slice_bytes = macro_tiles_per_slice * macro_tile_bytes; - uint64_t slice_offset = tile_split_slice * slice_bytes; - uint64_t tile_row_index = (y / 8) % m_bank_height; - uint64_t tile_index = tile_row_index; - uint64_t tile_offset = tile_index * tile_bytes; - - uint64_t tile_split_slice_rotation = ((m_num_banks / 2) + 1) * tile_split_slice; - bank ^= tile_split_slice_rotation; - bank &= (m_num_banks - 1); - - uint64_t total_offset = - (slice_offset + macro_tile_offset + tile_offset) * 8 + element_offset; - uint64_t bit_offset = total_offset & 0x7u; - total_offset /= 8; - - uint64_t pipe_interleave_offset = total_offset & 0xffu; - uint64_t offset = total_offset >> 8u; - uint64_t byte_offset = pipe_interleave_offset | (pipe << (8u)) | - (bank << (8u + m_pipe_bits)) | - (offset << (8u + m_pipe_bits + m_bank_bits)); - - return ((byte_offset << 3u) | bit_offset) / 8; - } -}; - -struct FastTiler32XInfo { - uint64_t macro_offset = 0; - uint16_t element_bytes = 0; - uint8_t pipe = 0; - uint8_t bank = 0; -}; - -struct FastTiler32YInfo { - uint64_t macro_offset = 0; - uint16_t tile_offset = 0; - uint16_t element_bytes = 0; - uint8_t pipe = 0; - uint8_t bank = 0; -}; - -class FastTiler32 { -public: - void Init(const Tiler32& t, uint32_t width, uint32_t height) { - constexpr uint32_t bytes_per_element = 4; - const uint32_t tile_bytes = 8u * 8u * bytes_per_element; - const uint64_t macro_tile_bytes = (128u / 8u) * (t.m_macro_tile_height / 8u) * tile_bytes / - (t.m_num_pipes * t.m_num_banks); - const uint64_t macro_tiles_per_row = t.m_padded_width / 128u; - - m_x.resize(width); - m_y.resize(height); - m_pipe_bits = t.m_pipe_bits; - m_bank_bits = t.m_bank_bits; - - for (uint32_t x = 0; x < width; x++) { - auto& info = m_x[x]; - - info.macro_offset = (x / 128u) * macro_tile_bytes; - info.element_bytes = - static_cast(Tiler32::GetElementIndex(x, 0) * bytes_per_element); - info.pipe = static_cast(Tiler32::GetPipeIndex(x, 0)); - info.bank = static_cast( - Tiler32::GetBankIndex(x, 0, 1, t.m_bank_height, t.m_num_banks, t.m_num_pipes)); - } - - for (uint32_t y = 0; y < height; y++) { - auto& info = m_y[y]; - - info.macro_offset = (static_cast(y) / t.m_macro_tile_height) * - macro_tiles_per_row * macro_tile_bytes; - info.tile_offset = static_cast(((y / 8u) % t.m_bank_height) * tile_bytes); - info.element_bytes = - static_cast(Tiler32::GetElementIndex(0, y) * bytes_per_element); - info.pipe = static_cast(Tiler32::GetPipeIndex(0, y)); - info.bank = static_cast( - Tiler32::GetBankIndex(0, y, 1, t.m_bank_height, t.m_num_banks, t.m_num_pipes)); - } - } - - [[nodiscard]] uint64_t GetTiledOffset(uint32_t x, uint32_t y) const { - const auto& x_info = m_x[x]; - const auto& y_info = m_y[y]; - - const uint64_t base = y_info.macro_offset + x_info.macro_offset + y_info.tile_offset + - y_info.element_bytes + x_info.element_bytes; - const uint64_t pipe = x_info.pipe ^ y_info.pipe; - const uint64_t bank = x_info.bank ^ y_info.bank; - - return (base & 0xffu) | (pipe << 8u) | (bank << (8u + m_pipe_bits)) | - ((base >> 8u) << (8u + m_pipe_bits + m_bank_bits)); - } - -private: - std::vector m_x; - std::vector m_y; - uint32_t m_pipe_bits = 0; - uint32_t m_bank_bits = 0; -}; - -static Tiler* g_tiler = nullptr; - -void TileInit() { - EXIT_IF(g_tiler != nullptr); - - g_tiler = new Tiler; -} - -// NOLINTNEXTLINE(readability-non-const-parameter) -static void Detile32(const Tiler32& t, uint32_t width, uint32_t height, uint32_t dst_pitch, - uint8_t* dst, const uint8_t* src) { - EXIT_IF(g_tiler == nullptr); - - Common::LockGuard lock(g_tiler->m_mutex); - - FastTiler32 fast_tiler; - fast_tiler.Init(t, width, height); - - struct DetileParams { - const FastTiler32* t; - uint32_t start_y; - uint32_t width; - uint32_t height; - uint32_t dst_pitch; - uint8_t* dst; - const uint8_t* src; - }; - - auto func = [](void* args) { - auto* p = static_cast(args); - - auto* dst = p->dst; - const auto* src = p->src; - const FastTiler32* t = p->t; - uint32_t start_y = p->start_y; - uint32_t width = p->width; - uint32_t height = p->height; - uint64_t dst_pitch = p->dst_pitch; - - for (uint32_t y = start_y; y < height; y++) { - uint32_t x = 0; - uint64_t linear_offset = y * dst_pitch * 4; - - for (; x + 1 < width; x += 2) { - auto tiled_offset = t->GetTiledOffset(x, y); - - *reinterpret_cast(dst + linear_offset) = - *reinterpret_cast(src + tiled_offset); - linear_offset += 8; - } - if (x < width) { - auto tiled_offset = t->GetTiledOffset(x, y); - - *reinterpret_cast(dst + linear_offset) = - *reinterpret_cast(src + tiled_offset); - } - } - }; - - const uint32_t split = height / 2; - DetileParams p1 {&fast_tiler, 0, width, split, dst_pitch, dst, src}; - DetileParams p2 {&fast_tiler, split, width, height, dst_pitch, dst, src}; - - g_tiler->m_job1.Execute([func, &p2] { func(&p2); }); - func(&p1); - g_tiler->m_job1.Wait(); -} - static constexpr uint32_t GetStandard64KB32XPart(uint32_t x) { - // FIXME: Temporary PS5 AGC TileMode::kStandard64KB 32bpp detiler. - // AGC Standard64KB is block-linear: 32bpp surfaces use 128x128-element + // Standard64KB is block-linear: 32bpp surfaces use 128x128-element // 64KB blocks, with fixed x/y bit interleaving inside each block. uint32_t element_offset = 0; element_offset ^= (x << 2u) & 0x0cu; @@ -1167,16 +897,14 @@ static uint32_t Gen5RenderTargetOffsetInBlock(uint32_t x, uint32_t y) { offset ^= (x << 7u) & 0x2000u; offset ^= (x << 8u) & 0x8000u; } else if constexpr (sizeof(T) == 4) { - offset ^= (y << 3u) & 0x0008u; - offset ^= (y << 4u) & 0x0020u; - offset ^= (y << 5u) & 0x0f80u; + offset ^= (y << 4u) & 0x0070u; + offset ^= (y << 5u) & 0x0f00u; offset ^= (y << 9u) & 0x1000u; offset ^= (y << 8u) & 0x4000u; - offset ^= (x << 2u) & 0x0004u; - offset ^= (x << 3u) & 0x0010u; - offset ^= (x << 4u) & 0x0440u; - offset ^= (x << 5u) & 0x0300u; + offset ^= (x << 2u) & 0x000cu; + offset ^= (x << 5u) & 0x0380u; + offset ^= (x << 4u) & 0x0400u; offset ^= (x << 6u) & 0x0800u; offset ^= (x << 9u) & 0xa000u; } else if constexpr (sizeof(T) == 8) { @@ -1191,6 +919,16 @@ static uint32_t Gen5RenderTargetOffsetInBlock(uint32_t x, uint32_t y) { offset ^= (x << 6u) & 0x0800u; offset ^= (x << 10u) & 0x2000u; offset ^= (x << 9u) & 0x8000u; + } else if constexpr (sizeof(T) == 16) { + offset ^= (x << 4u) & 0x0410u; + offset ^= (x << 5u) & 0x0340u; + offset ^= (x << 6u) & 0x0800u; + offset ^= (x << 11u) & 0xa000u; + + offset ^= (y << 5u) & 0x0f20u; + offset ^= (y << 6u) & 0x0080u; + offset ^= (y << 10u) & 0x1000u; + offset ^= (y << 11u) & 0x4000u; } else { EXIT("unsupported render-target element size: %u\n", static_cast(sizeof(T))); } @@ -1198,124 +936,6 @@ static uint32_t Gen5RenderTargetOffsetInBlock(uint32_t x, uint32_t y) { return offset; } -template -static void ConvertRenderTargetTyped(uint32_t width, uint32_t height, uint32_t pitch, T* dst, - const T* src, uint64_t size) { - EXIT_IF(g_tiler == nullptr); - - uint32_t block_width = 0; - uint32_t block_height = 0; - EXIT_NOT_IMPLEMENTED(!Gen5Thin64KBBlockSizeFromElementBytes(static_cast(sizeof(T)), - &block_width, &block_height)); - - const uint64_t blocks_per_row = (static_cast(pitch) + block_width - 1u) / block_width; - const uint32_t block_columns = (width == 0 ? 0 : 1u + (width - 1u) / block_width); - const uint64_t elements_per_block = 65536u / sizeof(T); - const uint64_t count = size / sizeof(T); - - std::array x_elements {}; - std::array y_elements {}; - for (uint32_t x = 0; x < block_width; x++) { - x_elements[x] = Gen5RenderTargetOffsetInBlock(x, 0) / sizeof(T); - } - for (uint32_t y = 0; y < block_height; y++) { - y_elements[y] = Gen5RenderTargetOffsetInBlock(0, y) / sizeof(T); - } - - struct ConvertParams { - uint32_t start_block_row; - uint32_t end_block_row; - uint32_t width; - uint32_t height; - uint32_t pitch; - uint32_t block_width; - uint32_t block_height; - uint32_t block_columns; - uint64_t blocks_per_row; - uint64_t elements_per_block; - const uint32_t* x_elements; - const uint32_t* y_elements; - T* dst; - const T* src; - uint64_t count; - }; - - auto func = [](void* args) { - auto* p = static_cast(args); - - for (uint32_t block_row = p->start_block_row; block_row < p->end_block_row; block_row++) { - const uint32_t block_y = block_row * p->block_height; - const uint32_t copy_height = std::min(p->block_height, p->height - block_y); - const uint32_t block_y_element = - Gen5RenderTargetOffsetInBlock(0, block_y) / sizeof(T); - - for (uint32_t block_column = 0; block_column < p->block_columns; block_column++) { - const uint32_t block_x = - static_cast(static_cast(block_column) * p->block_width); - const uint32_t copy_width = std::min(p->block_width, p->width - block_x); - const uint64_t block_base = - (static_cast(block_row) * p->blocks_per_row + block_column) * - p->elements_per_block; - - for (uint32_t y = 0; y < copy_height; y++) { - const uint64_t linear_row = - static_cast(block_y + y) * p->pitch + block_x; - for (uint32_t x = 0; x < copy_width; x++) { - const uint64_t linear_index = linear_row + x; - const uint64_t tiled_index = - block_base + (p->x_elements[x] ^ p->y_elements[y] ^ block_y_element); - if (linear_index < p->count && tiled_index < p->count) { - if constexpr (tiled_to_linear) { - p->dst[linear_index] = p->src[tiled_index]; - } else { - p->dst[tiled_index] = p->src[linear_index]; - } - } - } - } - } - } - }; - - const uint32_t block_rows = (height == 0 ? 0 : 1u + (height - 1u) / block_height); - const uint32_t split = block_rows / 2; - ConvertParams p1 {0, - split, - width, - height, - pitch, - block_width, - block_height, - block_columns, - blocks_per_row, - elements_per_block, - x_elements.data(), - y_elements.data(), - dst, - src, - count}; - ConvertParams p2 {split, - block_rows, - width, - height, - pitch, - block_width, - block_height, - block_columns, - blocks_per_row, - elements_per_block, - x_elements.data(), - y_elements.data(), - dst, - src, - count}; - - Common::LockGuard lock(g_tiler->m_mutex); - g_tiler->m_job1.Execute([func, &p2] { func(&p2); }); - func(&p1); - g_tiler->m_job1.Wait(); -} - static const Standard64KB16Tables& GetStandard64KB16Tables() { static constexpr Standard64KB16Tables tables; return tables; @@ -1417,329 +1037,243 @@ static const Standard64KB128Tables& GetStandard64KB128Tables() { return tables; } -template -static void ConvertTiledToLinearStandard64KBElements(void* dst, const void* src, uint32_t width, - uint32_t height, uint32_t pitch, uint64_t size, - uint64_t src_size, uint32_t src_x, - uint32_t src_y, const Tables& tables) { - static_assert(BlockWidth * BlockHeight * sizeof(T) == 64u * 1024u); - static_assert(CopyBatch != 0 && BlockWidth % CopyBatch == 0); +static constexpr uint32_t Depth64KB8XOffsetBytes(uint32_t x); +static constexpr uint32_t Depth64KB8YOffsetBytes(uint32_t y); +static constexpr uint32_t Depth64KB16XOffsetBytes(uint32_t x); +static constexpr uint32_t Depth64KB16YOffsetBytes(uint32_t y); +static constexpr uint32_t Depth64KB32XOffsetBytes(uint32_t x); +static constexpr uint32_t Depth64KB32YOffsetBytes(uint32_t y); +static constexpr uint32_t Depth64KB64XOffsetBytes(uint32_t x); +static constexpr uint32_t Depth64KB64YOffsetBytes(uint32_t y); - EXIT_IF(dst == nullptr || src == nullptr); - EXIT_NOT_IMPLEMENTED(pitch == 0); - const bool tail_block = (src_size != 0 && (src_x != 0 || src_y != 0 || size < src_size)); - EXIT_NOT_IMPLEMENTED(width > pitch); - EXIT_NOT_IMPLEMENTED(tail_block && - (src_x + width > BlockWidth || src_y + height > BlockHeight)); - - auto* dst_elements = static_cast(dst); - const auto* src_elements = static_cast(src); - const auto element_count = size / sizeof(T); - const auto src_element_count = (src_size != 0 ? src_size : size) / sizeof(T); - const auto blocks_per_row = - tail_block ? 1u : static_cast(AlignUp(pitch, BlockWidth) / BlockWidth); - const auto dst_element_count = static_cast(pitch) * height; - const bool exact_surface = !tail_block && (!ExactSurfaceRequiresFullWidth || width == pitch) && - size % sizeof(T) == 0 && element_count == dst_element_count; - const auto* x_elements = tables.x_words.data(); - const auto* y_elements = tables.y_words.data(); - - if (!exact_surface) { - std::memset(dst_elements, 0, static_cast(size)); +bool TileGetBlockLayout(TileBlockFamily family, uint32_t bytes_per_element, + TileBlockLayout* layout) { + if (layout == nullptr || !std::has_single_bit(bytes_per_element) || bytes_per_element > 16) { + return false; } - constexpr uint64_t ElementsPerBlock = BlockWidth * BlockHeight; - for (uint32_t block_y = 0; block_y < height; block_y += BlockHeight) { - const uint32_t copy_height = std::min(BlockHeight, height - block_y); - - for (uint32_t block_x = 0; block_x < width; block_x += BlockWidth) { - const uint32_t copy_width = std::min(BlockWidth, width - block_x); - const uint64_t block_base = - tail_block ? 0u - : ((static_cast(block_y / BlockHeight) * blocks_per_row) + - block_x / BlockWidth) * - ElementsPerBlock; - const bool full_block = !tail_block && copy_width == BlockWidth && - copy_height == BlockHeight && - block_base + ElementsPerBlock <= src_element_count && - static_cast(block_y + BlockHeight - 1u) * pitch + - block_x + BlockWidth - 1u < - element_count; - - if (full_block) { - for (uint32_t local_y = 0; local_y < BlockHeight; local_y++) { - const uint64_t dst_row = - static_cast(block_y + local_y) * pitch + block_x; - const uint64_t src_row = block_base + y_elements[src_y + local_y]; - - for (uint32_t local_x = 0; local_x < BlockWidth; local_x += CopyBatch) { - for (uint32_t i = 0; i < CopyBatch; i++) { - dst_elements[dst_row + local_x + i] = - src_elements[src_row + x_elements[src_x + local_x + i]]; - } - } - } - continue; - } - - for (uint32_t local_y = 0; local_y < copy_height; local_y++) { - const uint64_t dst_row = static_cast(block_y + local_y) * pitch + block_x; - const uint64_t src_row = block_base + y_elements[src_y + local_y]; - - for (uint32_t local_x = 0; local_x < copy_width; local_x++) { - const uint64_t dst_index = dst_row + local_x; - const uint64_t src_index = src_row + x_elements[src_x + local_x]; - - if (src_index < src_element_count && dst_index < element_count) { - dst_elements[dst_index] = src_elements[src_index]; - } - } - } + TileBlockLayout result {family, bytes_per_element, 0, 0, 0, 1}; + auto thin = [&] { + return Gen5Thin64KBBlockSizeFromElementBytes(bytes_per_element, &result.block_width, + &result.block_height); + }; + auto thick64 = [&] { + static constexpr uint8_t LOG2_DIMS[5][3] = { + {6, 5, 5}, {5, 5, 5}, {5, 5, 4}, {5, 4, 4}, {4, 4, 4}}; + const auto index = std::countr_zero(bytes_per_element); + result.block_width = 1u << LOG2_DIMS[index][0]; + result.block_height = 1u << LOG2_DIMS[index][1]; + result.block_depth = 1u << LOG2_DIMS[index][2]; + }; + switch (family) { + case TileBlockFamily::Standard256B: + result.block_size = 256; + result.block_width = bytes_per_element <= 2 ? 16 : (bytes_per_element <= 8 ? 8 : 4); + result.block_height = result.block_size / (result.block_width * bytes_per_element); + break; + case TileBlockFamily::Standard4KB: + result.block_size = 4096; + result.block_width = bytes_per_element <= 2 ? 64 : (bytes_per_element <= 8 ? 32 : 16); + result.block_height = result.block_size / (result.block_width * bytes_per_element); + break; + case TileBlockFamily::Standard4KB3D: { + static constexpr uint8_t LOG2_DIMS[5][3] = { + {4, 4, 4}, {3, 4, 4}, {3, 4, 3}, {3, 3, 3}, {2, 3, 3}}; + const auto index = std::countr_zero(bytes_per_element); + result.block_size = 4096; + result.block_width = 1u << LOG2_DIMS[index][0]; + result.block_height = 1u << LOG2_DIMS[index][1]; + result.block_depth = 1u << LOG2_DIMS[index][2]; + break; } + case TileBlockFamily::Standard64KB: + case TileBlockFamily::Prt64KB: + result.block_size = 65536; + if (!thin()) { + return false; + } + break; + case TileBlockFamily::Standard64KB3D: + case TileBlockFamily::Prt64KB3D: + result.block_size = 65536; + thick64(); + break; + case TileBlockFamily::RenderTarget64KB: + result.block_size = 65536; + if (!thin()) { + return false; + } + break; + case TileBlockFamily::Depth64KB: + if (bytes_per_element > 8) { + return false; + } + result.block_size = 65536; + if (!thin()) { + return false; + } + break; + case TileBlockFamily::Count: return false; } + + if (static_cast(result.block_width) * result.block_height * result.block_depth * + bytes_per_element != + result.block_size) { + return false; + } + *layout = result; + return true; } -void TileConvertTiledToLinearStandard64KB32(void* dst, const void* src, uint32_t width, - uint32_t height, uint32_t pitch, uint64_t size, - uint64_t src_size, uint32_t src_x, uint32_t src_y) { - KYTY_PROFILER_FUNCTION(); - - ConvertTiledToLinearStandard64KBElements( - dst, src, width, height, pitch, size, src_size, src_x, src_y, GetStandard64KB32Tables()); -} - -void TileConvertLinearToTiledStandard64KB32(void* dst, const void* src, uint32_t width, - uint32_t height, uint32_t pitch, uint64_t size) { - KYTY_PROFILER_FUNCTION(); - - const auto rows = static_cast(height == 0 ? 0 : height - 1u); - const auto expected_pitch = (static_cast(width) + 127u) & ~uint64_t {127u}; - if (pitch != expected_pitch || size < (rows * pitch + width) * sizeof(uint32_t)) { - EXIT("invalid linear-to-Standard64KB32 conversion, dst=%p src=%p extent=%ux%u " - "pitch=%u size=0x%016" PRIx64 "\n", - dst, src, width, height, pitch, size); +bool TileGetBlockOffset(const TileBlockLayout& layout, uint32_t x, uint32_t y, uint32_t z, + uint32_t* byte_offset) { + TileBlockLayout expected {}; + if (byte_offset == nullptr || + !TileGetBlockLayout(layout.family, layout.bytes_per_element, &expected) || + layout.block_size != expected.block_size || layout.block_width != expected.block_width || + layout.block_height != expected.block_height || + layout.block_depth != expected.block_depth || x >= layout.block_width || + y >= layout.block_height || z >= layout.block_depth) { + return false; } - auto* dst32 = static_cast(dst); - const auto* src32 = static_cast(src); - const auto& tables = GetStandard64KB32Tables(); - const auto blocks_per_row = (static_cast(pitch) + 127u) >> 7u; - const auto block_rows = (static_cast(height) + 127u) >> 7u; - if (blocks_per_row > UINT64_MAX / block_rows / 65536u || - size != blocks_per_row * block_rows * 65536u) { - EXIT("invalid Standard64KB32 allocation, extent=%ux%u pitch=%u size=0x%016" PRIx64 - " blocks=%" PRIu64 "x%" PRIu64 "\n", - width, height, pitch, size, blocks_per_row, block_rows); - } - const auto* x_words = tables.x_words.data(); - const auto* y_words = tables.y_words.data(); - - std::memset(dst32, 0, static_cast(size)); - for (uint32_t block_y = 0; block_y < height; block_y += 128u) { - const uint32_t block_height = std::min(128u, height - block_y); - for (uint32_t block_x = 0; block_x < width; block_x += 128u) { - const uint32_t block_width = std::min(128u, width - block_x); - const uint64_t block_base = - (((static_cast(block_y) >> 7u) * blocks_per_row) + (block_x >> 7u)) - << 14u; - if (block_width == 128u && block_height == 128u) { - for (uint32_t local_y = 0; local_y < 128u; local_y++) { - const uint64_t src_row = - (static_cast(block_y + local_y) * pitch) + block_x; - const uint64_t dst_row = block_base + y_words[local_y]; - for (uint32_t local_x = 0; local_x < 128u; local_x += 4u) { - dst32[dst_row + x_words[local_x + 0u]] = src32[src_row + local_x + 0u]; - dst32[dst_row + x_words[local_x + 1u]] = src32[src_row + local_x + 1u]; - dst32[dst_row + x_words[local_x + 2u]] = src32[src_row + local_x + 2u]; - dst32[dst_row + x_words[local_x + 3u]] = src32[src_row + local_x + 3u]; - } - } - continue; - } - - for (uint32_t local_y = 0; local_y < block_height; local_y++) { - const uint64_t src_row = - (static_cast(block_y + local_y) * pitch) + block_x; - const uint64_t dst_row = block_base + y_words[local_y]; - for (uint32_t local_x = 0; local_x < block_width; local_x++) { - dst32[dst_row + x_words[local_x]] = src32[src_row + local_x]; - } + uint32_t offset = 0; + switch (layout.family) { + case TileBlockFamily::Standard256B: + offset = Gen5Standard256BOffsetInBlock(x, y, layout.bytes_per_element); + break; + case TileBlockFamily::Standard4KB: + offset = Gen5Standard4KBOffsetInBlock(x, y, layout.bytes_per_element); + break; + case TileBlockFamily::Standard4KB3D: + offset = Gen5Standard4KBVolumeOffsetInBlock(x, y, z, layout.bytes_per_element); + break; + case TileBlockFamily::Standard64KB3D: + case TileBlockFamily::Prt64KB3D: { + offset = Gen5Standard64KBVolumeOffsetInBlock(x, y, z, layout.bytes_per_element); + if (layout.family == TileBlockFamily::Prt64KB3D) { + static constexpr uint8_t SOURCES[5][4] = { + {4, 5, 4, 4}, {4, 4, 3, 4}, {4, 4, 3, 3}, {3, 4, 3, 3}, {3, 3, 2, 3}}; + const auto* bits = SOURCES[std::countr_zero(layout.bytes_per_element)]; + offset ^= Bit(y, bits[0], 10) ^ Bit(x, bits[1], 10) ^ Bit(x, bits[2], 11) ^ + Bit(z, bits[3], 11); } + break; } - } -} - -void TileConvertTiledToLinearStandard64KB16(void* dst, const void* src, uint32_t width, - uint32_t height, uint32_t pitch, uint64_t size, - uint64_t src_size, uint32_t src_x, uint32_t src_y) { - KYTY_PROFILER_FUNCTION(); - - ConvertTiledToLinearStandard64KBElements( - dst, src, width, height, pitch, size, src_size, src_x, src_y, GetStandard64KB16Tables()); -} - -static void TileConvertTiledToLinearStandard64KB8Elements( - void* dst, const void* src, uint32_t width_elements, uint32_t height_elements, - uint32_t pitch_elements, uint64_t size, uint64_t src_size, uint32_t src_x, uint32_t src_y) { - KYTY_PROFILER_FUNCTION(); - - ConvertTiledToLinearStandard64KBElements( - dst, src, width_elements, height_elements, pitch_elements, size, src_size, src_x, src_y, - GetStandard64KB8Tables()); -} - -static void TileConvertTiledToLinearStandard64KB64Elements( - void* dst, const void* src, uint32_t width_elements, uint32_t height_elements, - uint32_t pitch_elements, uint64_t size, uint64_t src_size, uint32_t src_x, uint32_t src_y) { - KYTY_PROFILER_FUNCTION(); - - ConvertTiledToLinearStandard64KBElements( - dst, src, width_elements, height_elements, pitch_elements, size, src_size, src_x, src_y, - GetStandard64KB64Tables()); -} - -static void TileConvertLinearToTiledStandard64KB64Elements( - void* dst, const void* src, uint32_t width_elements, uint32_t height_elements, - uint32_t pitch_elements, uint64_t size, uint64_t dst_size, uint32_t dst_x, uint32_t dst_y) { - if (width_elements > pitch_elements) { - EXIT("invalid linear-to-Standard64KB64 tail conversion, extent=%ux%u pitch=%u " - "size=0x%016" PRIx64 " dst_size=0x%016" PRIx64 " dst=%u,%u\n", - width_elements, height_elements, pitch_elements, size, dst_size, dst_x, dst_y); - } - auto* dst64 = static_cast(dst); - const auto* src64 = static_cast(src); - const auto& tables = GetStandard64KB64Tables(); - const auto size_words = size >> 3u; - const auto dst_size_words = dst_size >> 3u; - for (uint32_t y = 0; y < height_elements; y++) { - const uint64_t src_row = static_cast(y) * pitch_elements; - const uint64_t dst_row = tables.y_words[dst_y + y]; - for (uint32_t x = 0; x < width_elements; x++) { - const uint64_t src_index = src_row + x; - const uint64_t dst_index = dst_row + tables.x_words[dst_x + x]; - if (src_index < size_words && dst_index < dst_size_words) { - dst64[dst_index] = src64[src_index]; + case TileBlockFamily::Standard64KB: + case TileBlockFamily::Prt64KB: + switch (layout.bytes_per_element) { + case 1: + offset = + GetStandard64KB8Tables().x_words[x] ^ GetStandard64KB8Tables().y_words[y]; + break; + case 2: + offset = (GetStandard64KB16Tables().x_words[x] ^ + GetStandard64KB16Tables().y_words[y]) * + 2u; + break; + case 4: + offset = (GetStandard64KB32Tables().x_words[x] ^ + GetStandard64KB32Tables().y_words[y]) * + 4u; + break; + case 8: + offset = (GetStandard64KB64Tables().x_words[x] ^ + GetStandard64KB64Tables().y_words[y]) * + 8u; + break; + case 16: + offset = (GetStandard64KB128Tables().x_words[x] ^ + GetStandard64KB128Tables().y_words[y]) * + 16u; + break; + default: return false; } + if (layout.family == TileBlockFamily::Prt64KB) { + static constexpr uint8_t SOURCES[5][4] = { + {7, 7, 6, 6}, {7, 6, 6, 5}, {6, 6, 5, 5}, {6, 5, 5, 4}, {5, 5, 4, 4}}; + const auto* bits = SOURCES[std::countr_zero(layout.bytes_per_element)]; + offset ^= Bit(x, bits[0], 8) ^ Bit(y, bits[1], 9) ^ Bit(x, bits[2], 10) ^ + Bit(y, bits[3], 11); + } + break; + case TileBlockFamily::RenderTarget64KB: + switch (layout.bytes_per_element) { + case 1: offset = Gen5RenderTargetOffsetInBlock(x, y); break; + case 2: offset = Gen5RenderTargetOffsetInBlock(x, y); break; + case 4: offset = Gen5RenderTargetOffsetInBlock(x, y); break; + case 8: offset = Gen5RenderTargetOffsetInBlock(x, y); break; + case 16: offset = Gen5RenderTargetOffsetInBlock(x, y); break; + default: return false; + } + break; + case TileBlockFamily::Depth64KB: + switch (layout.bytes_per_element) { + case 1: offset = Depth64KB8XOffsetBytes(x) ^ Depth64KB8YOffsetBytes(y); break; + case 2: offset = Depth64KB16XOffsetBytes(x) ^ Depth64KB16YOffsetBytes(y); break; + case 4: offset = Depth64KB32XOffsetBytes(x) ^ Depth64KB32YOffsetBytes(y); break; + case 8: offset = Depth64KB64XOffsetBytes(x) ^ Depth64KB64YOffsetBytes(y); break; + default: return false; + } + break; + case TileBlockFamily::Count: return false; + } + + if (offset >= layout.block_size || offset % layout.bytes_per_element != 0) { + return false; + } + *byte_offset = offset; + return true; +} + +bool TileGetBlockXor(const TileBlockLayout& layout, uint32_t block_x, uint32_t block_y, + uint32_t* byte_offset) { + return TileGetBlockXor(layout, block_x, block_y, 0, byte_offset); +} + +bool TileGetBlockXor(const TileBlockLayout& layout, uint32_t block_x, uint32_t block_y, + uint32_t block_z, uint32_t* byte_offset) { + TileBlockLayout expected {}; + if (byte_offset == nullptr || + !TileGetBlockLayout(layout.family, layout.bytes_per_element, &expected) || + layout.block_size != expected.block_size || layout.block_width != expected.block_width || + layout.block_height != expected.block_height || + layout.block_depth != expected.block_depth) { + return false; + } + if (layout.family == TileBlockFamily::Depth64KB && layout.bytes_per_element == 8) { + if (block_x > UINT32_MAX / layout.block_width || + block_y > UINT32_MAX / layout.block_height) { + return false; } + *byte_offset = Depth64KB64XOffsetBytes(block_x * layout.block_width) ^ + Depth64KB64YOffsetBytes(block_y * layout.block_height); + } else if (layout.family == TileBlockFamily::RenderTarget64KB) { + if (block_x > UINT32_MAX / layout.block_width || + block_y > UINT32_MAX / layout.block_height) { + return false; + } + const auto x = block_x * layout.block_width; + const auto y = block_y * layout.block_height; + switch (layout.bytes_per_element) { + case 1: *byte_offset = Gen5RenderTargetOffsetInBlock(x, y); break; + case 2: *byte_offset = Gen5RenderTargetOffsetInBlock(x, y); break; + case 4: *byte_offset = Gen5RenderTargetOffsetInBlock(x, y); break; + case 8: *byte_offset = Gen5RenderTargetOffsetInBlock(x, y); break; + case 16: *byte_offset = Gen5RenderTargetOffsetInBlock(x, y); break; + default: return false; + } + } else { + *byte_offset = 0; } + if (layout.family == TileBlockFamily::RenderTarget64KB || + layout.family == TileBlockFamily::Depth64KB) { + *byte_offset ^= ((block_z & 8u) << 5u) ^ ((block_z & 4u) << 7u) ^ ((block_z & 2u) << 9u) ^ + ((block_z & 1u) << 11u); + } + return *byte_offset < layout.block_size && *byte_offset % layout.bytes_per_element == 0; } -static void TileConvertTiledToLinearStandard64KB128Elements( - void* dst, const void* src, uint32_t width_elements, uint32_t height_elements, - uint32_t pitch_elements, uint64_t size, uint64_t src_size, uint32_t src_x, uint32_t src_y) { - KYTY_PROFILER_FUNCTION(); - - ConvertTiledToLinearStandard64KBElements( - dst, src, width_elements, height_elements, pitch_elements, size, src_size, src_x, src_y, - GetStandard64KB128Tables()); -} - -void TileConvertTiledToLinearStandard64KB(void* dst, const void* src, uint32_t format, - uint32_t width, uint32_t height, uint32_t pitch, - uint64_t size, uint64_t src_size, uint32_t src_x, - uint32_t src_y) { - uint32_t bytes_per_element = 0; - uint32_t texels_per_element_wide = 0; - uint32_t texels_per_element_tall = 0; - uint32_t block_width_log2 = 0; - uint32_t block_height_log2 = 0; - - EXIT_NOT_IMPLEMENTED(!Gen5Standard64KBLayout(format, &bytes_per_element, - &texels_per_element_wide, &texels_per_element_tall, - &block_width_log2, &block_height_log2)); - - const uint32_t width_elements = - std::max((width + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); - const uint32_t height_elements = - std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); - const uint32_t pitch_elements = - std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); - - switch (bytes_per_element) { - case 1: - TileConvertTiledToLinearStandard64KB8Elements(dst, src, width_elements, height_elements, - pitch_elements, size, src_size, src_x, - src_y); - break; - case 2: - TileConvertTiledToLinearStandard64KB16(dst, src, width_elements, height_elements, - pitch_elements, size, src_size, src_x, src_y); - break; - case 4: - TileConvertTiledToLinearStandard64KB32(dst, src, width_elements, height_elements, - pitch_elements, size, src_size, src_x, src_y); - break; - case 8: - TileConvertTiledToLinearStandard64KB64Elements(dst, src, width_elements, - height_elements, pitch_elements, size, - src_size, src_x, src_y); - break; - case 16: - TileConvertTiledToLinearStandard64KB128Elements(dst, src, width_elements, - height_elements, pitch_elements, size, - src_size, src_x, src_y); - break; - default: EXIT("unsupported Standard64KB element size: %u\n", bytes_per_element); - } -} - -void TileConvertTiledToLinearStandard256B(void* dst, const void* src, uint32_t format, - uint32_t width, uint32_t height, uint32_t pitch, - uint64_t dst_size, uint64_t src_size) { - uint32_t bytes_per_element = 0; - uint32_t texels_per_element_wide = 0; - uint32_t texels_per_element_tall = 0; - uint32_t block_width_log2 = 0; - uint32_t block_height_log2 = 0; - - EXIT_NOT_IMPLEMENTED(!Gen5Standard256BLayout(format, &bytes_per_element, - &texels_per_element_wide, &texels_per_element_tall, - &block_width_log2, &block_height_log2)); - - const uint32_t row_elements = - std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); - const uint32_t width_elements = - std::max((width + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); - const uint32_t height_elements = - std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); - const uint32_t padded_width = AlignUp(row_elements, 1u << block_width_log2); - - if (src_size == 0) { - src_size = dst_size; - } - - switch (bytes_per_element) { - case 1: - DetileStandard256BTyped(static_cast(dst), static_cast(src), - row_elements, width_elements, height_elements, padded_width, - block_width_log2, block_height_log2, dst_size, src_size); - break; - case 2: - DetileStandard256BTyped(static_cast(dst), static_cast(src), - row_elements, width_elements, height_elements, padded_width, - block_width_log2, block_height_log2, dst_size, src_size); - break; - case 4: - DetileStandard256BTyped(static_cast(dst), static_cast(src), - row_elements, width_elements, height_elements, padded_width, - block_width_log2, block_height_log2, dst_size, src_size); - break; - case 8: - DetileStandard256BTyped(static_cast(dst), static_cast(src), - row_elements, width_elements, height_elements, padded_width, - block_width_log2, block_height_log2, dst_size, src_size); - break; - case 16: - DetileStandard256BTyped(static_cast(dst), static_cast(src), - row_elements, width_elements, height_elements, padded_width, - block_width_log2, block_height_log2, dst_size, src_size); - break; - default: EXIT("unsupported Standard256B element size: %u\n", bytes_per_element); - } -} - -// Prospero depth/stencil-tile address table for 2D, 1xAA surfaces by element size. static constexpr uint32_t Depth64KB8XOffsetBytes(uint32_t x) { uint32_t offset = 0; offset ^= x & 0x0001u; @@ -1809,6 +1343,15 @@ static constexpr uint32_t Depth64KB32YOffsetBytes(uint32_t y) { return offset; } +static constexpr uint32_t Depth64KB64XOffsetBytes(uint32_t x) { + return ((x << 3u) & 0x0008u) ^ ((x << 4u) & 0x0420u) ^ ((x << 5u) & 0x0380u) ^ + ((x << 6u) & 0x0800u) ^ ((x << 10u) & 0x2000u) ^ ((x << 9u) & 0x8000u); +} + +static constexpr uint32_t Depth64KB64YOffsetBytes(uint32_t y) { + return ((y << 4u) & 0x0010u) ^ ((y << 5u) & 0x0f40u) ^ ((y << 10u) & 0x5000u); +} + static_assert(Depth64KB8XOffsetBytes(2) == 0x0004u); static_assert(Depth64KB8YOffsetBytes(1) == 0x0002u); static_assert((Depth64KB8XOffsetBytes(3) ^ Depth64KB8YOffsetBytes(5)) == 0x0027u); @@ -1819,474 +1362,6 @@ static_assert(Depth64KB32XOffsetBytes(2) == 0x0010u); static_assert(Depth64KB32YOffsetBytes(1) == 0x0008u); static_assert((Depth64KB32XOffsetBytes(3) ^ Depth64KB32YOffsetBytes(5)) == 0x009cu); -struct Depth64KB8Tables { - std::array x_words {}; - std::array y_words {}; - - constexpr Depth64KB8Tables() { - for (uint32_t x = 0; x < x_words.size(); x++) { - x_words[x] = Depth64KB8XOffsetBytes(x); - } - for (uint32_t y = 0; y < y_words.size(); y++) { - y_words[y] = Depth64KB8YOffsetBytes(y); - } - } -}; - -struct Depth64KB16Tables { - std::array x_words {}; - std::array y_words {}; - - constexpr Depth64KB16Tables() { - for (uint32_t x = 0; x < x_words.size(); x++) { - x_words[x] = Depth64KB16XOffsetBytes(x) / sizeof(uint16_t); - } - for (uint32_t y = 0; y < y_words.size(); y++) { - y_words[y] = Depth64KB16YOffsetBytes(y) / sizeof(uint16_t); - } - } -}; - -struct Depth64KB32Tables { - std::array x_words {}; - std::array y_words {}; - - constexpr Depth64KB32Tables() { - for (uint32_t x = 0; x < x_words.size(); x++) { - x_words[x] = Depth64KB32XOffsetBytes(x) / sizeof(uint32_t); - } - for (uint32_t y = 0; y < y_words.size(); y++) { - y_words[y] = Depth64KB32YOffsetBytes(y) / sizeof(uint32_t); - } - } -}; - -template -static void TileConvertDepthTyped(void* dst, const void* src, uint32_t width, uint32_t height, - uint32_t pitch, uint64_t size, - const std::array& x_words, - const std::array& y_words) { - constexpr uint32_t block_width = static_cast(x_count); - constexpr uint32_t block_height = static_cast(y_count); - const uint32_t block_columns = 1u + (width - 1u) / block_width; - const uint64_t blocks_per_row = pitch / block_width; - const uint32_t block_rows = 1u + (height - 1u) / block_height; - - struct ConvertParams { - uint32_t start_block_row; - uint32_t end_block_row; - uint32_t width; - uint32_t height; - uint32_t pitch; - uint32_t block_columns; - uint64_t blocks_per_row; - const uint32_t* x_words; - const uint32_t* y_words; - T* dst; - const T* src; - }; - - auto convert = [](void* args) { - auto* p = static_cast(args); - for (uint32_t block_row = p->start_block_row; block_row < p->end_block_row; block_row++) { - const uint32_t block_y = block_row * static_cast(y_count); - const uint32_t copy_height = - std::min(static_cast(y_count), p->height - block_y); - for (uint32_t block_column = 0; block_column < p->block_columns; block_column++) { - const uint32_t block_x = block_column * static_cast(x_count); - const uint32_t copy_width = - std::min(static_cast(x_count), p->width - block_x); - const uint64_t block_base = - (static_cast(block_row) * p->blocks_per_row + block_column) * - (65536u / sizeof(T)); - for (uint32_t y = 0; y < copy_height; y++) { - const uint64_t linear_row = - static_cast(block_y + y) * p->pitch + block_x; - const uint64_t tiled_row = block_base + p->y_words[y]; - for (uint32_t x = 0; x < copy_width; x++) { - const uint64_t linear_index = linear_row + x; - const uint64_t tiled_index = tiled_row ^ p->x_words[x]; - if constexpr (tiled_to_linear) { - p->dst[linear_index] = p->src[tiled_index]; - } else { - p->dst[tiled_index] = p->src[linear_index]; - } - } - } - } - } - }; - - auto* dst_typed = static_cast(dst); - const auto* src_typed = static_cast(src); - std::memset(dst_typed, 0, static_cast(size)); - const uint32_t split = block_rows / 2u; - ConvertParams first {0, - split, - width, - height, - pitch, - block_columns, - blocks_per_row, - x_words.data(), - y_words.data(), - dst_typed, - src_typed}; - ConvertParams second {split, block_rows, width, height, - pitch, block_columns, blocks_per_row, x_words.data(), - y_words.data(), dst_typed, src_typed}; - if (split == 0) { - convert(&second); - return; - } - EXIT_IF(g_tiler == nullptr); - Common::LockGuard lock(g_tiler->m_mutex); - g_tiler->m_job1.Execute([convert, &second] { convert(&second); }); - convert(&first); - g_tiler->m_job1.Wait(); -} - -template -static void TileConvertDepth(void* dst, const void* src, uint32_t format, uint32_t width, - uint32_t height, uint32_t pitch, uint64_t size) { - KYTY_PROFILER_FUNCTION(); - const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); - const bool supported_bpe = - bytes_per_element == 1 || bytes_per_element == 2 || bytes_per_element == 4; - const uint32_t block_width = supported_bpe && bytes_per_element <= 2 ? 256u : 128u; - const uint32_t block_height = supported_bpe ? 65536u / (block_width * bytes_per_element) : 0; - if (dst == nullptr || src == nullptr || width == 0 || height == 0 || pitch < width || - !supported_bpe || pitch % block_width != 0 || size == 0 || size % 65536u != 0) { - EXIT("unsupported depth conversion, dst=%p src=%p format=%u " - "extent=%ux%u pitch=%u size=0x%016" PRIx64 "\n", - dst, src, format, width, height, pitch, size); - } - const uint64_t block_rows = (static_cast(height) + block_height - 1u) / block_height; - const uint64_t blocks_per_row = pitch / block_width; - if (blocks_per_row > UINT64_MAX / block_rows || - blocks_per_row * block_rows > UINT64_MAX / 65536u || - size != blocks_per_row * block_rows * 65536u) { - EXIT("depth storage disagrees with 64 KiB block layout, " - "extent=%ux%u pitch=%u bpe=%u size=0x%016" PRIx64 "\n", - width, height, pitch, bytes_per_element, size); - } - if (bytes_per_element == 1) { - static constexpr Depth64KB8Tables tables; - TileConvertDepthTyped(dst, src, width, height, pitch, size, - tables.x_words, tables.y_words); - } else if (bytes_per_element == 2) { - static constexpr Depth64KB16Tables tables; - TileConvertDepthTyped(dst, src, width, height, pitch, size, - tables.x_words, tables.y_words); - } else { - static constexpr Depth64KB32Tables tables; - TileConvertDepthTyped(dst, src, width, height, pitch, size, - tables.x_words, tables.y_words); - } -} - -void TileConvertTiledToLinearDepth(void* dst, const void* src, uint32_t format, uint32_t width, - uint32_t height, uint32_t pitch, uint64_t size) { - TileConvertDepth(dst, src, format, width, height, pitch, size); -} - -void TileConvertLinearToTiledDepth(void* dst, const void* src, uint32_t format, uint32_t width, - uint32_t height, uint32_t pitch, uint64_t size) { - TileConvertDepth(dst, src, format, width, height, pitch, size); -} - -void TileConvertTiledToLinearStandard4KB(void* dst, const void* src, uint32_t format, - uint32_t width, uint32_t height, uint32_t pitch, - uint64_t dst_size, uint64_t src_size, uint32_t src_x, - uint32_t src_y) { - KYTY_PROFILER_FUNCTION(); - - EXIT_IF(dst == nullptr); - EXIT_IF(src == nullptr); - EXIT_NOT_IMPLEMENTED(pitch == 0); - EXIT_NOT_IMPLEMENTED(width > pitch); - - uint32_t bytes_per_element = 0; - uint32_t texels_per_element_wide = 0; - uint32_t texels_per_element_tall = 0; - uint32_t block_width_log2 = 0; - uint32_t block_height_log2 = 0; - - EXIT_NOT_IMPLEMENTED(!Gen5Standard4KBLayout(format, &bytes_per_element, - &texels_per_element_wide, &texels_per_element_tall, - &block_width_log2, &block_height_log2)); - - const uint32_t row_elements = - std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); - const uint32_t width_elements = - std::max((width + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); - const uint32_t height_elements = - std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); - const uint32_t block_width = 1u << block_width_log2; - const uint32_t block_height = 1u << block_height_log2; - const uint32_t padded_width = (row_elements + block_width - 1u) & ~(block_width - 1u); - const uint32_t padded_height = (height_elements + block_height - 1u) & ~(block_height - 1u); - - EXIT_IF((padded_width & (block_width - 1u)) != 0); - EXIT_IF((padded_height & (block_height - 1u)) != 0); - EXIT_NOT_IMPLEMENTED(src_x >= block_width || src_y >= block_height); - EXIT_NOT_IMPLEMENTED(src_x != 0 && src_x + width_elements > block_width); - EXIT_NOT_IMPLEMENTED(src_y != 0 && src_y + height_elements > block_height); - - auto* dst8 = static_cast(dst); - - std::memset(dst8, 0, static_cast(dst_size)); - - switch (bytes_per_element) { - case 1: - DetileStandard4KBTyped(static_cast(dst), static_cast(src), - row_elements, width_elements, height_elements, padded_width, - block_width_log2, block_height_log2, dst_size, src_size, src_x, - src_y); - break; - case 2: - DetileStandard4KBTyped(static_cast(dst), static_cast(src), - row_elements, width_elements, height_elements, padded_width, - block_width_log2, block_height_log2, dst_size, src_size, src_x, - src_y); - break; - case 4: - DetileStandard4KBTyped(static_cast(dst), static_cast(src), - row_elements, width_elements, height_elements, padded_width, - block_width_log2, block_height_log2, dst_size, src_size, src_x, - src_y); - break; - case 8: - DetileStandard4KBTyped(static_cast(dst), static_cast(src), - row_elements, width_elements, height_elements, padded_width, - block_width_log2, block_height_log2, dst_size, src_size, src_x, - src_y); - break; - case 16: - DetileStandard4KBTyped(static_cast(dst), static_cast(src), - row_elements, width_elements, height_elements, padded_width, - block_width_log2, block_height_log2, dst_size, src_size, src_x, - src_y); - break; - default: EXIT("unsupported Standard4KB element size: %u\n", bytes_per_element); - } -} - -void TileConvertTiledToLinearStandard4KB3D(void* dst, const void* src, uint32_t format, - uint32_t width, uint32_t height, uint32_t depth, - uint32_t pitch, uint64_t dst_slice_stride, - uint64_t dst_size, uint64_t src_size, bool clear_dst) { - KYTY_PROFILER_FUNCTION(); - - EXIT_IF(dst == nullptr); - EXIT_IF(src == nullptr); - EXIT_NOT_IMPLEMENTED(pitch == 0); - EXIT_NOT_IMPLEMENTED(width > pitch); - EXIT_NOT_IMPLEMENTED(depth == 0); - - uint32_t bytes_per_element = 0; - uint32_t texels_per_element_wide = 0; - uint32_t texels_per_element_tall = 0; - uint32_t block_width_log2 = 0; - uint32_t block_height_log2 = 0; - uint32_t block_depth_log2 = 0; - - EXIT_NOT_IMPLEMENTED(!TileGetStandard4KBVolumeLayout( - format, &bytes_per_element, &texels_per_element_wide, &texels_per_element_tall, - &block_width_log2, &block_height_log2, &block_depth_log2)); - EXIT_NOT_IMPLEMENTED(dst_slice_stride == 0); - EXIT_NOT_IMPLEMENTED((dst_slice_stride % bytes_per_element) != 0); - - const uint32_t row_elements = - std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); - const uint32_t width_elements = - std::max((width + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); - const uint32_t height_elements = - std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); - const uint32_t block_width = 1u << block_width_log2; - const uint32_t block_height = 1u << block_height_log2; - const uint32_t padded_width = AlignUp(row_elements, block_width); - const uint32_t padded_height = AlignUp(height_elements, block_height); - - src_size = (src_size != 0 ? src_size : dst_size); - if (clear_dst) { - std::memset(dst, 0, static_cast(dst_size)); - } - - switch (bytes_per_element) { - case 1: - DetileStandard4KBVolumeTyped(static_cast(dst), - static_cast(src), row_elements, - width_elements, height_elements, depth, padded_width, - padded_height, block_width_log2, block_height_log2, - block_depth_log2, dst_slice_stride, dst_size, src_size); - break; - case 2: - DetileStandard4KBVolumeTyped(static_cast(dst), - static_cast(src), row_elements, - width_elements, height_elements, depth, padded_width, - padded_height, block_width_log2, block_height_log2, - block_depth_log2, dst_slice_stride, dst_size, src_size); - break; - case 4: - DetileStandard4KBVolumeTyped(static_cast(dst), - static_cast(src), row_elements, - width_elements, height_elements, depth, padded_width, - padded_height, block_width_log2, block_height_log2, - block_depth_log2, dst_slice_stride, dst_size, src_size); - break; - case 8: - DetileStandard4KBVolumeTyped(static_cast(dst), - static_cast(src), row_elements, - width_elements, height_elements, depth, padded_width, - padded_height, block_width_log2, block_height_log2, - block_depth_log2, dst_slice_stride, dst_size, src_size); - break; - case 16: - DetileStandard4KBVolumeTyped(static_cast(dst), - static_cast(src), row_elements, - width_elements, height_elements, depth, padded_width, - padded_height, block_width_log2, block_height_log2, - block_depth_log2, dst_slice_stride, dst_size, src_size); - break; - default: EXIT("unsupported Standard4KB volume element size: %u\n", bytes_per_element); - } -} - -void TileConvertTiledToLinear(void* dst, const void* src, TileMode mode, uint32_t width, - uint32_t height) { - KYTY_PROFILER_FUNCTION(); - - EXIT_NOT_IMPLEMENTED(mode != TileMode::VideoOutTiled); - - Tiler32 t; - t.Init(width, height); - - Detile32(t, width, height, width, static_cast(dst), static_cast(src)); -} - -void TileConvertTiledToLinearRenderTarget(void* dst, const void* src, uint32_t width, - uint32_t height, uint32_t pitch, - uint32_t bytes_per_element, uint64_t size, - uint64_t src_size, uint32_t src_x, uint32_t src_y) { - KYTY_PROFILER_FUNCTION(); - - EXIT_IF(dst == nullptr || src == nullptr); - EXIT_NOT_IMPLEMENTED(width > pitch); - EXIT_NOT_IMPLEMENTED(pitch == 0); - - src_size = (src_size != 0 ? src_size : size); - const bool tail_block = (src_size != 0 && (src_x != 0 || src_y != 0 || size < src_size)); - if (!tail_block && bytes_per_element <= 8) { - switch (bytes_per_element) { - case 1: - ConvertRenderTargetTyped(width, height, pitch, - static_cast(dst), - static_cast(src), size); - break; - case 2: - ConvertRenderTargetTyped(width, height, pitch, - static_cast(dst), - static_cast(src), size); - break; - case 4: - ConvertRenderTargetTyped(width, height, pitch, - static_cast(dst), - static_cast(src), size); - break; - case 8: - ConvertRenderTargetTyped(width, height, pitch, - static_cast(dst), - static_cast(src), size); - break; - default: EXIT("unsupported render-target element size: %u\n", bytes_per_element); - } - return; - } - - switch (bytes_per_element) { - case 1: - TileConvertTiledToLinearStandard64KB8Elements(dst, src, width, height, pitch, size, - src_size, src_x, src_y); - break; - case 2: - TileConvertTiledToLinearStandard64KB16(dst, src, width, height, pitch, size, src_size, - src_x, src_y); - break; - case 4: - TileConvertTiledToLinearStandard64KB32(dst, src, width, height, pitch, size, src_size, - src_x, src_y); - break; - case 8: - TileConvertTiledToLinearStandard64KB64Elements(dst, src, width, height, pitch, size, - src_size, src_x, src_y); - break; - case 16: - TileConvertTiledToLinearStandard64KB128Elements(dst, src, width, height, pitch, size, - src_size, src_x, src_y); - break; - default: EXIT("unsupported render-target element size: %u\n", bytes_per_element); - } -} - -void TileConvertLinearToTiledRenderTarget(void* dst, const void* src, uint32_t width, - uint32_t height, uint32_t pitch, - uint32_t bytes_per_element, uint64_t size, - uint64_t dst_size, uint32_t dst_x, uint32_t dst_y) { - KYTY_PROFILER_FUNCTION(); - const bool tail_block = dst_size != 0 && (dst_x != 0 || dst_y != 0 || size < dst_size); - if (tail_block) { - // Not sure about this at all - if (bytes_per_element != 8) { - EXIT("unsupported linear-to-tiled render-target tail element size: %u\n", - bytes_per_element); - } - TileConvertLinearToTiledStandard64KB64Elements(dst, src, width, height, pitch, size, - dst_size, dst_x, dst_y); - return; - } - - if (dst == nullptr || src == nullptr || width == 0 || height == 0 || pitch == 0 || - width > pitch || bytes_per_element == 0 || size == 0 || size % bytes_per_element != 0) { - EXIT("invalid linear-to-tiled render-target conversion, dst=%p src=%p extent=%ux%u " - "pitch=%u bpe=%u size=0x%016" PRIx64 "\n", - dst, src, width, height, pitch, bytes_per_element, size); - } - const auto rows = static_cast(height - 1); - if (rows > (UINT64_MAX - width) / pitch || - (rows * pitch + width) > UINT64_MAX / bytes_per_element || - size < (rows * pitch + width) * bytes_per_element) { - EXIT("linear-to-tiled render-target storage is too small, extent=%ux%u pitch=%u " - "bpe=%u size=0x%016" PRIx64 "\n", - width, height, pitch, bytes_per_element, size); - } - - std::memset(dst, 0, size); - switch (bytes_per_element) { - case 1: - ConvertRenderTargetTyped(width, height, pitch, - static_cast(dst), - static_cast(src), size); - break; - case 2: - ConvertRenderTargetTyped(width, height, pitch, - static_cast(dst), - static_cast(src), size); - break; - case 4: - ConvertRenderTargetTyped(width, height, pitch, - static_cast(dst), - static_cast(src), size); - break; - case 8: - ConvertRenderTargetTyped(width, height, pitch, - static_cast(dst), - static_cast(src), size); - break; - default: EXIT("unsupported render-target element size: %u\n", bytes_per_element); - } -} - bool TileGetHtileSize(uint32_t width, uint32_t height, TileSizeAlign* htile_size) { *htile_size = {}; if (width == 0 || width > 16384 || height == 0 || height > 16384) { @@ -2453,157 +1528,14 @@ void TileGetTextureSize(uint32_t format, uint32_t width, uint32_t height, uint32 return; } - uint32_t std256_bytes_per_element = 0; - uint32_t std256_texels_per_element_wide = 0; - uint32_t std256_texels_per_element_tall = 0; - uint32_t std256_block_width_log2 = 0; - uint32_t std256_block_height_log2 = 0; - if (tile == 1 && - Gen5Standard256BLayout(format, &std256_bytes_per_element, &std256_texels_per_element_wide, - &std256_texels_per_element_tall, &std256_block_width_log2, - &std256_block_height_log2)) { - uint32_t offset = 0; - uint32_t mip_pitch = pitch; - uint32_t mip_height = height; - for (uint32_t l = 0; l < levels; l++) { - const uint32_t row_elements = std::max( - (mip_pitch + std256_texels_per_element_wide - 1u) / std256_texels_per_element_wide, - 1u); - const uint32_t height_elements = std::max( - (mip_height + std256_texels_per_element_tall - 1u) / std256_texels_per_element_tall, - 1u); - const uint32_t padded_width = AlignUp(row_elements, 1u << std256_block_width_log2); - const uint32_t padded_height = AlignUp(height_elements, 1u << std256_block_height_log2); - const uint32_t size = padded_width * padded_height * std256_bytes_per_element; - - if (level_sizes != nullptr) { - level_sizes[l].size = size; - level_sizes[l].offset = offset; - } - if (padded_size != nullptr) { - padded_size[l].width = padded_width * std256_texels_per_element_wide; - padded_size[l].height = padded_height * std256_texels_per_element_tall; - } - - offset += size; - mip_pitch = std::max(mip_pitch / 2u, 1u); - mip_height = std::max(mip_height / 2u, 1u); + TextureBlockLayout block {}; + if (GetTextureBlockLayout(format, tile, &block)) { + if (tile == 1) { + SetMicroMipLayout(block, width, height, levels, total_size, level_sizes, padded_size); + } else { + SetMacroMipLayout(block, tile, width, height, levels, total_size, level_sizes, + padded_size); } - - if (total_size != nullptr) { - total_size->size = AlignUp(offset, 256u); - total_size->align = 256; - } - - return; - } - - if (const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); - bytes_per_element != 0 && tile == 27 && levels == 1) { - uint32_t block_width = 0; - uint32_t block_height = 0; - EXIT_NOT_IMPLEMENTED( - !Gen5Thin64KBBlockSizeFromElementBytes(bytes_per_element, &block_width, &block_height)); - - const uint32_t padded_width = AlignUp(pitch, block_width); - const uint32_t padded_height = AlignUp(height, block_height); - const uint32_t size = padded_width * padded_height * bytes_per_element; - - if (total_size != nullptr) { - total_size->size = size; - total_size->align = 65536; - } - - if (level_sizes != nullptr) { - level_sizes[0].size = size; - level_sizes[0].offset = 0; - } - - if (padded_size != nullptr) { - padded_size[0].width = padded_width; - padded_size[0].height = padded_height; - } - - return; - } - - if (const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); - bytes_per_element != 0 && tile == 27 && levels > 1) { - uint32_t block_width = 0; - uint32_t block_height = 0; - EXIT_NOT_IMPLEMENTED( - !Gen5Thin64KBBlockSizeFromElementBytes(bytes_per_element, &block_width, &block_height)); - - const uint32_t bytes_log2 = IntLog2(bytes_per_element); - const uint32_t tail_width_limit = block_width >> 1u; - const uint32_t tail_height_limit = block_height; - constexpr uint32_t max_tail_levels = 12u; - - uint32_t first_tail_level = levels; - for (uint32_t l = 0; l < levels; l++) { - const uint32_t mip_pitch = std::max(ShiftCeil(pitch, l), 1u); - const uint32_t mip_height = std::max(ShiftCeil(height, l), 1u); - if (mip_pitch <= tail_width_limit && mip_height <= tail_height_limit && - levels - l <= max_tail_levels) { - first_tail_level = l; - break; - } - } - - uint32_t offset = (first_tail_level < levels ? 65536u : 0u); - - for (int32_t l = static_cast(first_tail_level) - 1; l >= 0; l--) { - const uint32_t level = static_cast(l); - const uint32_t mip_pitch = std::max(ShiftCeil(pitch, level), 1u); - const uint32_t mip_height = std::max(ShiftCeil(height, level), 1u); - const uint32_t padded_width = AlignUp(mip_pitch, block_width); - const uint32_t padded_height = AlignUp(mip_height, block_height); - const uint32_t size = padded_width * padded_height * bytes_per_element; - - if (level_sizes != nullptr) { - level_sizes[level].size = size; - level_sizes[level].offset = offset; - level_sizes[level].src_size = size; - level_sizes[level].src_offset = offset; - level_sizes[level].x = 0; - level_sizes[level].y = 0; - } - if (padded_size != nullptr) { - padded_size[level].width = padded_width; - padded_size[level].height = padded_height; - } - - offset += size; - } - - uint32_t tail_linear_offset = 0; - for (uint32_t l = first_tail_level; l < levels; l++) { - const uint32_t mip_pitch = std::max(ShiftCeil(pitch, l), 1u); - const uint32_t mip_height = std::max(ShiftCeil(height, l), 1u); - const uint32_t linear_size = mip_pitch * mip_height * bytes_per_element; - const auto& tail_location = - GEN5_MIP_TAIL_LOCATIONS_THIN_64KB[bytes_log2][l - first_tail_level]; - if (level_sizes != nullptr) { - level_sizes[l].size = linear_size; - level_sizes[l].offset = tail_linear_offset; - level_sizes[l].src_size = 65536u; - level_sizes[l].src_offset = 0; - level_sizes[l].x = tail_location.x; - level_sizes[l].y = tail_location.y; - } - if (padded_size != nullptr) { - padded_size[l].width = block_width; - padded_size[l].height = block_height; - } - tail_linear_offset += AlignUp(linear_size, bytes_per_element); - } - EXIT_NOT_IMPLEMENTED(first_tail_level < levels && tail_linear_offset > 65536u); - - if (total_size != nullptr) { - total_size->size = AlignUp(offset, 65536u); - total_size->align = 65536; - } - return; } @@ -2634,328 +1566,6 @@ void TileGetTextureSize(uint32_t format, uint32_t width, uint32_t height, uint32 return; } - uint32_t bytes_per_element = 0; - uint32_t texels_per_element_wide = 0; - uint32_t texels_per_element_tall = 0; - uint32_t block_width_log2 = 0; - uint32_t block_height_log2 = 0; - if (tile == 5 && levels == 1 && - Gen5Standard4KBLayout(format, &bytes_per_element, &texels_per_element_wide, - &texels_per_element_tall, &block_width_log2, &block_height_log2)) { - const uint32_t row_elements = - std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); - const uint32_t height_elements = - std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); - const uint32_t block_width = 1u << block_width_log2; - const uint32_t block_height = 1u << block_height_log2; - const uint32_t padded_width = (row_elements + block_width - 1u) & ~(block_width - 1u); - const uint32_t padded_height = (height_elements + block_height - 1u) & ~(block_height - 1u); - const uint32_t size = padded_width * padded_height * bytes_per_element; - - if (total_size != nullptr) { - total_size->size = size; - total_size->align = 4096; - } - - if (level_sizes != nullptr) { - level_sizes[0].size = size; - level_sizes[0].offset = 0; - } - - if (padded_size != nullptr) { - padded_size[0].width = padded_width * texels_per_element_wide; - padded_size[0].height = padded_height * texels_per_element_tall; - } - - return; - } - - if (tile == 5 && levels > 1 && - Gen5Standard4KBLayout(format, &bytes_per_element, &texels_per_element_wide, - &texels_per_element_tall, &block_width_log2, &block_height_log2)) { - const uint32_t row_elements0 = - std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); - const uint32_t height_elements0 = - std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); - const uint32_t block_width = 1u << block_width_log2; - const uint32_t block_height = 1u << block_height_log2; - const uint32_t bytes_log2 = IntLog2(bytes_per_element); - const uint32_t tail_width_limit = block_width >> 1u; - const uint32_t tail_height_limit = block_height; - - uint32_t first_tail_level = levels; - for (uint32_t l = 0; l < levels; l++) { - const uint32_t row_elements = std::max(ShiftCeil(row_elements0, l), 1u); - const uint32_t height_elements = std::max(ShiftCeil(height_elements0, l), 1u); - if (row_elements <= tail_width_limit && height_elements <= tail_height_limit && - levels - l <= 8) { - first_tail_level = l; - break; - } - } - - uint32_t offset = (first_tail_level < levels ? 4096u : 0u); - - for (int32_t l = static_cast(first_tail_level) - 1; l >= 0; l--) { - const uint32_t level = static_cast(l); - const uint32_t row_elements = std::max(ShiftCeil(row_elements0, level), 1u); - const uint32_t height_elements = std::max(ShiftCeil(height_elements0, level), 1u); - const uint32_t padded_width = AlignUp(row_elements, block_width); - const uint32_t padded_height = AlignUp(height_elements, block_height); - const uint32_t size = padded_width * padded_height * bytes_per_element; - - if (level_sizes != nullptr) { - level_sizes[level].size = size; - level_sizes[level].offset = offset; - level_sizes[level].src_size = size; - level_sizes[level].src_offset = offset; - level_sizes[level].x = 0; - level_sizes[level].y = 0; - } - if (padded_size != nullptr) { - padded_size[level].width = padded_width * texels_per_element_wide; - padded_size[level].height = padded_height * texels_per_element_tall; - } - - offset += size; - } - - uint32_t tail_linear_offset = 0; - for (uint32_t l = first_tail_level; l < levels; l++) { - const uint32_t row_elements = std::max(ShiftCeil(row_elements0, l), 1u); - const uint32_t height_elements = std::max(ShiftCeil(height_elements0, l), 1u); - const uint32_t linear_size = row_elements * height_elements * bytes_per_element; - const auto& tail_location = - GEN5_MIP_TAIL_LOCATIONS_THIN_4KB[bytes_log2][l - first_tail_level]; - if (level_sizes != nullptr) { - level_sizes[l].size = linear_size; - level_sizes[l].offset = tail_linear_offset; - level_sizes[l].src_size = 4096u; - level_sizes[l].src_offset = 0; - level_sizes[l].x = tail_location.x; - level_sizes[l].y = tail_location.y; - } - if (padded_size != nullptr) { - padded_size[l].width = block_width * texels_per_element_wide; - padded_size[l].height = block_height * texels_per_element_tall; - } - tail_linear_offset += AlignUp(linear_size, bytes_per_element); - } - EXIT_NOT_IMPLEMENTED(first_tail_level < levels && tail_linear_offset > 4096u); - - if (total_size != nullptr) { - total_size->size = AlignUp(offset, 4096u); - total_size->align = 4096; - } - - return; - } - - if (tile == 24 && levels == 1) { - const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); - if (bytes_per_element != 0) { - uint32_t block_width = 0; - uint32_t block_height = 0; - const bool supported = - bytes_per_element <= 4 && Gen5Thin64KBBlockSizeFromElementBytes( - bytes_per_element, &block_width, &block_height); - - if (supported) { - const uint32_t padded_width = (pitch + block_width - 1u) & ~(block_width - 1u); - const uint32_t padded_height = AlignUp(height, block_height); - const uint32_t size = padded_width * padded_height * bytes_per_element; - - if (total_size != nullptr) { - total_size->size = size; - total_size->align = 65536; - } - - if (level_sizes != nullptr) { - level_sizes[0].size = size; - level_sizes[0].offset = 0; - } - - if (padded_size != nullptr) { - padded_size[0].width = padded_width; - padded_size[0].height = padded_height; - } - - return; - } - } - } - - if (tile == 24 && levels > 1) { - const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); - if (bytes_per_element != 0) { - uint32_t block_width = 0; - uint32_t block_height = 0; - const bool supported = - bytes_per_element <= 4 && Gen5Thin64KBBlockSizeFromElementBytes( - bytes_per_element, &block_width, &block_height); - - if (supported) { - static bool logged = false; - if (!logged) { - LOGF("\t temporary: sizing PS5 depth tiled texture with mips, format = %u, " - "levels = %u\n", - format, levels); - logged = true; - } - - uint32_t offset = 0; - uint32_t mip_pitch = pitch; - uint32_t mip_height = height; - for (uint32_t l = 0; l < levels; l++) { - offset = AlignUp(offset, 65536u); - - const uint32_t padded_width = AlignUp(mip_pitch, block_width); - const uint32_t padded_height = AlignUp(mip_height, block_height); - const uint32_t size = padded_width * padded_height * bytes_per_element; - - if (level_sizes != nullptr) { - level_sizes[l].size = size; - level_sizes[l].offset = offset; - } - if (padded_size != nullptr) { - padded_size[l].width = padded_width; - padded_size[l].height = padded_height; - } - - offset += size; - mip_pitch = std::max(mip_pitch / 2u, 1u); - mip_height = std::max(mip_height / 2u, 1u); - } - - if (total_size != nullptr) { - total_size->size = AlignUp(offset, 65536u); - total_size->align = 65536; - } - - return; - } - } - } - - uint32_t std64_bytes_per_element = 0; - uint32_t std64_texels_per_element_wide = 0; - uint32_t std64_texels_per_element_tall = 0; - uint32_t std64_block_width_log2 = 0; - uint32_t std64_block_height_log2 = 0; - if (tile == 9 && levels > 1 && - Gen5Standard64KBLayout(format, &std64_bytes_per_element, &std64_texels_per_element_wide, - &std64_texels_per_element_tall, &std64_block_width_log2, - &std64_block_height_log2)) { - const uint32_t row_elements0 = std::max( - (pitch + std64_texels_per_element_wide - 1u) / std64_texels_per_element_wide, 1u); - const uint32_t height_elements0 = std::max( - (height + std64_texels_per_element_tall - 1u) / std64_texels_per_element_tall, 1u); - const uint32_t block_width = 1u << std64_block_width_log2; - const uint32_t block_height = 1u << std64_block_height_log2; - const uint32_t bytes_log2 = IntLog2(std64_bytes_per_element); - const uint32_t tail_width_limit = block_width >> 1u; - const uint32_t tail_height_limit = block_height; - constexpr uint32_t max_tail_levels = 12u; - - uint32_t first_tail_level = levels; - for (uint32_t l = 0; l < levels; l++) { - const uint32_t row_elements = std::max(ShiftCeil(row_elements0, l), 1u); - const uint32_t height_elements = std::max(ShiftCeil(height_elements0, l), 1u); - if (row_elements <= tail_width_limit && height_elements <= tail_height_limit && - levels - l <= max_tail_levels) { - first_tail_level = l; - break; - } - } - - uint32_t offset = (first_tail_level < levels ? 65536u : 0u); - - for (int32_t l = static_cast(first_tail_level) - 1; l >= 0; l--) { - const uint32_t level = static_cast(l); - const uint32_t row_elements = std::max(ShiftCeil(row_elements0, level), 1u); - const uint32_t height_elements = std::max(ShiftCeil(height_elements0, level), 1u); - const uint32_t padded_width = AlignUp(row_elements, block_width); - const uint32_t padded_height = AlignUp(height_elements, block_height); - const uint32_t size = padded_width * padded_height * std64_bytes_per_element; - - if (level_sizes != nullptr) { - level_sizes[level].size = size; - level_sizes[level].offset = offset; - level_sizes[level].src_size = size; - level_sizes[level].src_offset = offset; - level_sizes[level].x = 0; - level_sizes[level].y = 0; - } - if (padded_size != nullptr) { - padded_size[level].width = padded_width * std64_texels_per_element_wide; - padded_size[level].height = padded_height * std64_texels_per_element_tall; - } - - offset += size; - } - - uint32_t tail_linear_offset = 0; - for (uint32_t l = first_tail_level; l < levels; l++) { - const uint32_t row_elements = std::max(ShiftCeil(row_elements0, l), 1u); - const uint32_t height_elements = std::max(ShiftCeil(height_elements0, l), 1u); - const uint32_t linear_size = row_elements * height_elements * std64_bytes_per_element; - const auto& tail_location = - GEN5_MIP_TAIL_LOCATIONS_THIN_64KB[bytes_log2][l - first_tail_level]; - if (level_sizes != nullptr) { - level_sizes[l].size = linear_size; - level_sizes[l].offset = tail_linear_offset; - level_sizes[l].src_size = 65536u; - level_sizes[l].src_offset = 0; - level_sizes[l].x = tail_location.x; - level_sizes[l].y = tail_location.y; - } - if (padded_size != nullptr) { - padded_size[l].width = block_width * std64_texels_per_element_wide; - padded_size[l].height = block_height * std64_texels_per_element_tall; - } - tail_linear_offset += AlignUp(linear_size, std64_bytes_per_element); - } - EXIT_NOT_IMPLEMENTED(first_tail_level < levels && tail_linear_offset > 65536u); - - if (total_size != nullptr) { - total_size->size = AlignUp(offset, 65536u); - total_size->align = 65536; - } - - return; - } - if (tile == 9 && levels == 1 && - Gen5Standard64KBLayout(format, &std64_bytes_per_element, &std64_texels_per_element_wide, - &std64_texels_per_element_tall, &std64_block_width_log2, - &std64_block_height_log2)) { - const uint32_t row_elements = std::max( - (pitch + std64_texels_per_element_wide - 1u) / std64_texels_per_element_wide, 1u); - const uint32_t height_elements = std::max( - (height + std64_texels_per_element_tall - 1u) / std64_texels_per_element_tall, 1u); - const uint32_t block_width = 1u << std64_block_width_log2; - const uint32_t block_height = 1u << std64_block_height_log2; - const uint32_t padded_width = AlignUp(row_elements, block_width); - const uint32_t padded_height = AlignUp(height_elements, block_height); - const uint32_t size = padded_width * padded_height * std64_bytes_per_element; - - if (total_size != nullptr) { - total_size->size = size; - total_size->align = 65536; - } - - if (level_sizes != nullptr) { - level_sizes[0].size = size; - level_sizes[0].offset = 0; - } - - if (padded_size != nullptr) { - padded_size[0].width = padded_width * std64_texels_per_element_wide; - padded_size[0].height = padded_height * std64_texels_per_element_tall; - } - - return; - } - if (total_size != nullptr && total_size->size == 0) { std::vector list; list.push_back(fmt::format("format = {}", format)); @@ -2972,61 +1582,20 @@ void TileGetTextureTotalSize(uint32_t format, uint32_t width, uint32_t height, u uint32_t pitch, uint32_t levels, uint32_t tile, bool volume_texture, TileSizeAlign* total_size) { EXIT_NOT_IMPLEMENTED(depth == 0); + if (volume_texture) { + TileVolumeLayout volume {}; + if (TileGetTextureVolumeLayout(format, width, height, depth, levels, tile, &volume)) { + EXIT_NOT_IMPLEMENTED(volume.total_size > UINT32_MAX); + total_size->size = static_cast(volume.total_size); + total_size->align = tile == 5 ? 4096u : 65536u; + return; + } + } TileSizeAlign slice_size {}; TileGetTextureSize(format, width, height, pitch, levels, tile, &slice_size, nullptr, nullptr); - - *total_size = slice_size; - uint64_t total = static_cast(slice_size.size) * depth; - - uint32_t bytes_per_element = 0; - uint32_t texels_per_element_wide = 0; - uint32_t texels_per_element_tall = 0; - uint32_t block_width_log2 = 0; - uint32_t block_height_log2 = 0; - uint32_t block_depth_log2 = 0; - - if (volume_texture && depth > 1 && tile == 5 && - TileGetStandard4KBVolumeLayout(format, &bytes_per_element, &texels_per_element_wide, - &texels_per_element_tall, &block_width_log2, - &block_height_log2, &block_depth_log2)) { - const uint32_t row_elements0 = - std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); - const uint32_t height_elements0 = - std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); - const uint32_t block_width = 1u << block_width_log2; - const uint32_t block_height = 1u << block_height_log2; - const uint32_t block_depth = 1u << block_depth_log2; - uint64_t block_slice_size = 0; - - if (levels > 1) { - const uint32_t tail_width_limit = block_width; - const uint32_t tail_height_limit = block_height >> 1u; - constexpr uint32_t max_tail_levels = 5u; - - for (uint32_t l = 0; l < levels; l++) { - const uint32_t row_elements = std::max(ShiftCeil(row_elements0, l), 1u); - const uint32_t height_elements = std::max(ShiftCeil(height_elements0, l), 1u); - if (row_elements <= tail_width_limit && height_elements <= tail_height_limit && - levels - l <= max_tail_levels) { - block_slice_size += 4096u; - break; - } - - block_slice_size += static_cast(block_depth) * - AlignUp(row_elements, block_width) * - AlignUp(height_elements, block_height) * bytes_per_element; - } - } else { - block_slice_size = static_cast(block_depth) * - AlignUp(row_elements0, block_width) * - AlignUp(height_elements0, block_height) * bytes_per_element; - } - - total = block_slice_size * ShiftCeil(depth, block_depth_log2); - total_size->align = 4096; - } - + *total_size = slice_size; + const uint64_t total = static_cast(slice_size.size) * depth; EXIT_NOT_IMPLEMENTED(total > 0xffffffffull); total_size->size = static_cast(total); } @@ -3056,7 +1625,7 @@ uint32_t TileGetTexturePitch(uint32_t format, uint32_t width, uint32_t levels, u uint32_t texels_per_element_tall = 0; uint32_t block_width_log2 = 0; uint32_t block_height_log2 = 0; - if (tile == 9 && + if ((tile == 9 || tile == 17) && Gen5Standard64KBLayout(format, &bytes_per_element, &texels_per_element_wide, &texels_per_element_tall, &block_width_log2, &block_height_log2)) { pitch = AlignUp(pitch, (1u << block_width_log2) * texels_per_element_wide); @@ -3065,7 +1634,7 @@ uint32_t TileGetTexturePitch(uint32_t format, uint32_t width, uint32_t levels, u const uint32_t bytes_per_element = Prospero::NumBytesPerElement(format); uint32_t block_width = 0; uint32_t block_height = 0; - if (bytes_per_element <= 4 && + if (bytes_per_element <= 8 && Gen5Thin64KBBlockSizeFromElementBytes(bytes_per_element, &block_width, &block_height)) { pitch = AlignUp(pitch, block_width); } diff --git a/src/graphics/guest_gpu/tile.h b/src/graphics/guest_gpu/tile.h index 85016c2..30122f0 100644 --- a/src/graphics/guest_gpu/tile.h +++ b/src/graphics/guest_gpu/tile.h @@ -6,15 +6,6 @@ namespace Libs::Graphics { -enum class TileMode { - VideoOutLinear, - VideoOutTiled, - TextureLinear, - TextureTiled, - // RenderTextureLinear, - // RenderTextureTiled, -}; - struct TileSizeAlign { uint32_t size = 0; uint32_t align = 0; @@ -34,51 +25,58 @@ struct TilePaddedSize { uint32_t height = 0; }; -void TileInit(); -void TileConvertTiledToLinear(void* dst, const void* src, TileMode mode, uint32_t width, - uint32_t height); -void TileConvertTiledToLinearRenderTarget(void* dst, const void* src, uint32_t width, - uint32_t height, uint32_t pitch, - uint32_t bytes_per_element, uint64_t size, - uint64_t src_size = 0, uint32_t src_x = 0, - uint32_t src_y = 0); -void TileConvertLinearToTiledRenderTarget(void* dst, const void* src, uint32_t width, - uint32_t height, uint32_t pitch, - uint32_t bytes_per_element, uint64_t size, - uint64_t dst_size = 0, uint32_t dst_x = 0, - uint32_t dst_y = 0); -void TileConvertTiledToLinearStandard64KB(void* dst, const void* src, uint32_t format, - uint32_t width, uint32_t height, uint32_t pitch, - uint64_t size, uint64_t src_size = 0, uint32_t src_x = 0, - uint32_t src_y = 0); -void TileConvertTiledToLinearStandard64KB32(void* dst, const void* src, uint32_t width, - uint32_t height, uint32_t pitch, uint64_t size, - uint64_t src_size = 0, uint32_t src_x = 0, - uint32_t src_y = 0); -void TileConvertLinearToTiledStandard64KB32(void* dst, const void* src, uint32_t width, - uint32_t height, uint32_t pitch, uint64_t size); -void TileConvertTiledToLinearStandard64KB16(void* dst, const void* src, uint32_t width, - uint32_t height, uint32_t pitch, uint64_t size, - uint64_t src_size = 0, uint32_t src_x = 0, - uint32_t src_y = 0); -void TileConvertTiledToLinearDepth(void* dst, const void* src, uint32_t format, uint32_t width, - uint32_t height, uint32_t pitch, uint64_t size); -void TileConvertLinearToTiledDepth(void* dst, const void* src, uint32_t format, uint32_t width, - uint32_t height, uint32_t pitch, uint64_t size); -void TileConvertTiledToLinearStandard4KB(void* dst, const void* src, uint32_t format, - uint32_t width, uint32_t height, uint32_t pitch, - uint64_t dst_size, uint64_t src_size, uint32_t src_x = 0, - uint32_t src_y = 0); -void TileConvertTiledToLinearStandard256B(void* dst, const void* src, uint32_t format, - uint32_t width, uint32_t height, uint32_t pitch, - uint64_t dst_size, uint64_t src_size); +enum class TileBlockFamily : uint32_t { + Standard256B, + Standard4KB, + Standard4KB3D, + Standard64KB, + Standard64KB3D, + Prt64KB, + Prt64KB3D, + RenderTarget64KB, + Depth64KB, + Count, +}; + +struct TileBlockLayout { + TileBlockFamily family = TileBlockFamily::Standard256B; + uint32_t bytes_per_element = 0; + uint32_t block_size = 0; + uint32_t block_width = 0; + uint32_t block_height = 0; + uint32_t block_depth = 0; +}; + +struct TileVolumeLayout { + TileBlockFamily family = TileBlockFamily::Count; + uint32_t bytes_per_element = 0; + uint32_t texel_width = 1; + uint32_t texel_height = 1; + uint32_t first_tail_level = 0; + uint32_t block_depth = 1; + uint64_t block_slice_size = 0; + uint64_t total_size = 0; + uint64_t level_offsets[16] = {}; + uint64_t level_sizes[16] = {}; + uint32_t tail_x[16] = {}; + uint32_t tail_y[16] = {}; + uint32_t level_widths[16] = {}; + uint32_t level_heights[16] = {}; +}; + +bool TileGetBlockLayout(TileBlockFamily family, uint32_t bytes_per_element, + TileBlockLayout* layout); +bool TileGetBlockOffset(const TileBlockLayout& layout, uint32_t x, uint32_t y, uint32_t z, + uint32_t* byte_offset); +bool TileGetBlockXor(const TileBlockLayout& layout, uint32_t block_x, uint32_t block_y, + uint32_t* byte_offset); +bool TileGetBlockXor(const TileBlockLayout& layout, uint32_t block_x, uint32_t block_y, + uint32_t block_z, uint32_t* byte_offset); bool TileIsStandard256BTextureSupported(uint32_t format); bool TileIsStandard4KBTextureSupported(uint32_t format); bool TileIsStandard64KBTextureSupported(uint32_t format); -bool TileGetStandard4KBVolumeLayout(uint32_t format, uint32_t* bytes_per_element, - uint32_t* texels_per_element_wide, - uint32_t* texels_per_element_tall, uint32_t* block_width_log2, - uint32_t* block_height_log2, uint32_t* block_depth_log2); +bool TileGetTextureVolumeLayout(uint32_t format, uint32_t width, uint32_t height, uint32_t depth, + uint32_t levels, uint32_t tile, TileVolumeLayout* layout); bool TileGetHtileSize(uint32_t width, uint32_t height, TileSizeAlign* htile_size); bool TileGetDepthSize(uint32_t width, uint32_t height, uint32_t pitch, uint32_t z_format, @@ -103,11 +101,6 @@ void TileGetTextureTotalSize(uint32_t format, uint32_t width, uint32_t height, u uint32_t pitch, uint32_t levels, uint32_t tile, bool volume_texture, TileSizeAlign* total_size); uint32_t TileGetTexturePitch(uint32_t format, uint32_t width, uint32_t levels, uint32_t tile); -void TileConvertTiledToLinearStandard4KB3D(void* dst, const void* src, uint32_t format, - uint32_t width, uint32_t height, uint32_t depth, - uint32_t pitch, uint64_t dst_slice_stride, - uint64_t dst_size, uint64_t src_size, - bool clear_dst = true); } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/gpuTiler.cpp b/src/graphics/host_gpu/gpuTiler.cpp new file mode 100644 index 0000000..27ae91e --- /dev/null +++ b/src/graphics/host_gpu/gpuTiler.cpp @@ -0,0 +1,547 @@ +#include "graphics/host_gpu/gpuTiler.h" + +#include "common/assert.h" +#include "common/threads.h" +#include "gpu_tiler_shaders/gpu_tiler_depth_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_prt_3d_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_prt_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_render_target_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_standard256_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_standard4_3d_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_standard4_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_standard64_3d_spv.h" +#include "gpu_tiler_shaders/gpu_tiler_standard64_spv.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/host_gpu/vma.h" +#include "graphics/host_gpu/vulkanCommon.h" + +#include +#include +#include +#include +#include + +namespace Libs::Graphics { +namespace { + +constexpr uint32_t GROUP_SIZE = 64; +constexpr uint32_t FAMILY_COUNT = static_cast(TileBlockFamily::Count); +constexpr uint32_t BYTES_PER_ELEMENT_COUNT = 5; +constexpr uint32_t DIRECTION_COUNT = 2; +constexpr uint32_t PIPELINE_COUNT = FAMILY_COUNT * BYTES_PER_ELEMENT_COUNT * DIRECTION_COUNT; +static_assert(FAMILY_COUNT == 9); + +struct Push { + uint32_t src_base; + uint32_t dst_base; + uint32_t width; + uint32_t height; + uint32_t depth; + uint32_t surface_z; + uint32_t pitch_bytes; + uint32_t slice_bytes; + uint32_t blocks_per_row; + uint32_t blocks_per_slice; + uint32_t tail_x; + uint32_t tail_y; + uint32_t tail; + uint32_t first; + uint32_t count; +}; +static_assert(sizeof(Push) == 60); + +struct Shader { + const uint32_t* code; + size_t words; +}; + +constexpr std::array SHADERS {{ + {GPU_TILER_STANDARD256_SPV, std::size(GPU_TILER_STANDARD256_SPV)}, + {GPU_TILER_STANDARD4_SPV, std::size(GPU_TILER_STANDARD4_SPV)}, + {GPU_TILER_STANDARD4_3D_SPV, std::size(GPU_TILER_STANDARD4_3D_SPV)}, + {GPU_TILER_STANDARD64_SPV, std::size(GPU_TILER_STANDARD64_SPV)}, + {GPU_TILER_STANDARD64_3D_SPV, std::size(GPU_TILER_STANDARD64_3D_SPV)}, + {GPU_TILER_PRT_SPV, std::size(GPU_TILER_PRT_SPV)}, + {GPU_TILER_PRT_3D_SPV, std::size(GPU_TILER_PRT_3D_SPV)}, + {GPU_TILER_RENDER_TARGET_SPV, std::size(GPU_TILER_RENDER_TARGET_SPV)}, + {GPU_TILER_DEPTH_SPV, std::size(GPU_TILER_DEPTH_SPV)}, +}}; + +struct Dispatch { + Push push {}; + uint32_t pipeline_slot = 0; + uint32_t elements = 0; +}; + +struct Resources { + vk::DescriptorSetLayout descriptor_layout = nullptr; + vk::PipelineLayout pipeline_layout = nullptr; + vk::DescriptorPool descriptor_pool = nullptr; + vk::DescriptorSet descriptor_set = nullptr; + std::array pipelines {}; + VulkanBuffer staging; + VulkanBuffer linear; + void* mapped = nullptr; +}; + +bool CheckedAdd(uint64_t a, uint64_t b, uint64_t* result) { + return b <= UINT64_MAX - a && (*result = a + b, true); +} + +bool CheckedMultiply(uint64_t a, uint64_t b, uint64_t* result) { + return (a == 0 || b <= UINT64_MAX / a) && (*result = a * b, true); +} + +bool CheckedAddProduct(uint64_t* value, uint64_t count, uint64_t stride) { + uint64_t bytes = 0; + return CheckedMultiply(count, stride, &bytes) && CheckedAdd(*value, bytes, value); +} + +bool IsRangeValid(uint64_t offset, uint64_t size, uint64_t capacity) { + return size != 0 && offset <= capacity && size <= capacity - offset; +} + +uint64_t AlignToDword(uint64_t value) { + return (value + 3u) & ~uint64_t {3}; +} + +uint32_t GetPipelineSlot(bool to_tiled, TileBlockFamily family, uint32_t bytes_per_element) { + const uint32_t direction_index = to_tiled ? 1u : 0u; + const uint32_t family_index = static_cast(family); + const uint32_t element_size_index = std::countr_zero(bytes_per_element); + return (direction_index * FAMILY_COUNT + family_index) * BYTES_PER_ELEMENT_COUNT + + element_size_index; +} + +void Barrier(vk::CommandBuffer command, vk::Buffer buffer, vk::AccessFlags src_access, + vk::AccessFlags dst_access, vk::PipelineStageFlags src_stage, + vk::PipelineStageFlags dst_stage) { + vk::BufferMemoryBarrier barrier {}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = src_access; + barrier.dstAccessMask = dst_access; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.size = VK_WHOLE_SIZE; + command.pipelineBarrier(src_stage, dst_stage, {}, 0, nullptr, 1, &barrier, 0, nullptr); +} + +class Tiler final { +public: + void Run(bool to_tiled, GraphicContext* context, const void* input, void* output, + uint64_t tiled_capacity, uint64_t linear_capacity, std::span infos, + const GpuTileRecord& record); + void Release(GraphicContext* context); + +private: + void Prepare(bool to_tiled, GraphicContext* context, uint64_t tiled_capacity, + uint64_t linear_capacity, std::span infos, + std::vector* dispatches) const; + void Init(GraphicContext* context); + void CreatePipelines(std::span dispatches); + void CreatePipeline(uint32_t pipeline_slot); + void Resize(uint64_t staging_size, uint64_t linear_size); + void CreateBuffer(uint64_t size, bool mapped, VulkanBuffer* buffer, void** data) const; + void Execute(bool to_tiled, const void* input, void* output, uint64_t tiled_capacity, + uint64_t linear_capacity, std::span dispatches, + const GpuTileRecord& record); + void Destroy(Resources* target) const; + + Common::Mutex mutex; + GraphicContext* ctx = nullptr; + Resources resources; +}; + +Tiler g_tiler; + +void Tiler::Prepare(bool to_tiled, GraphicContext* context, uint64_t tiled_capacity, + uint64_t linear_capacity, std::span infos, + std::vector* dispatches) const { + EXIT_IF(context == nullptr || g_render_ctx == nullptr || + g_render_ctx->GetGraphicCtx() != context || infos.empty() || tiled_capacity == 0 || + linear_capacity == 0); + const auto& limits = context->GetPhysicalDeviceProperties().limits; + EXIT_NOT_IMPLEMENTED(tiled_capacity > UINT32_MAX || linear_capacity > UINT32_MAX || + AlignToDword(tiled_capacity) > limits.maxStorageBufferRange || + AlignToDword(linear_capacity) > limits.maxStorageBufferRange); + + dispatches->clear(); + dispatches->reserve(infos.size()); + for (const auto& info: infos) { + TileBlockLayout block {}; + const uint32_t tiled_width = info.tiled_width != 0 ? info.tiled_width : info.pitch; + const uint32_t tiled_height = info.tiled_height != 0 ? info.tiled_height : info.height; + EXIT_NOT_IMPLEMENTED( + !TileGetBlockLayout(info.family, info.bytes_per_element, &block) || info.width == 0 || + info.height == 0 || info.depth == 0 || info.pitch < info.width || + (!info.tail && (tiled_width < info.width || tiled_height < info.height)) || + !IsRangeValid(info.linear_offset, info.linear_size, linear_capacity) || + !IsRangeValid(info.tiled_offset, info.tiled_size, tiled_capacity) || + (block.block_depth == 1 && info.depth != 1)); + + uint64_t elements = 0, pitch_bytes = 0; + EXIT_NOT_IMPLEMENTED(!CheckedMultiply(info.width, info.height, &elements) || + !CheckedMultiply(elements, info.depth, &elements) || + !CheckedMultiply(info.pitch, info.bytes_per_element, &pitch_bytes) || + elements > UINT32_MAX || pitch_bytes > UINT32_MAX); + uint64_t slice_bytes = info.linear_slice_stride; + EXIT_NOT_IMPLEMENTED(slice_bytes == 0 && + !CheckedMultiply(pitch_bytes, info.height, &slice_bytes)); + uint64_t linear_used = 0, minimum_slice = 0; + EXIT_NOT_IMPLEMENTED(!CheckedMultiply(pitch_bytes, info.height, &minimum_slice) || + (info.depth > 1 && slice_bytes < minimum_slice) || + !CheckedAddProduct(&linear_used, info.depth - 1u, slice_bytes) || + !CheckedAddProduct(&linear_used, info.height - 1u, pitch_bytes) || + !CheckedAddProduct(&linear_used, info.width, info.bytes_per_element) || + linear_used > info.linear_size || slice_bytes > UINT32_MAX); + + const uint64_t columns = + (static_cast(tiled_width) + block.block_width - 1u) / block.block_width; + const uint64_t rows = + (static_cast(tiled_height) + block.block_height - 1u) / block.block_height; + uint64_t blocks_per_slice = 0; + EXIT_NOT_IMPLEMENTED(!CheckedMultiply(columns, rows, &blocks_per_slice) || + columns > UINT32_MAX || blocks_per_slice > UINT32_MAX || + rows * block.block_height > UINT32_MAX); + if (info.tail) { + const bool supported = info.family != TileBlockFamily::Standard256B; + EXIT_NOT_IMPLEMENTED( + !supported || info.depth > block.block_depth || info.tail_x >= block.block_width || + info.width > block.block_width - info.tail_x || info.tail_y >= block.block_height || + info.height > block.block_height - info.tail_y || + info.tiled_size < block.block_size); + } else { + const uint64_t slices = + (static_cast(info.depth) + block.block_depth - 1u) / block.block_depth; + uint64_t tiled_used = 0; + EXIT_NOT_IMPLEMENTED(!CheckedMultiply(blocks_per_slice, slices, &tiled_used) || + !CheckedMultiply(tiled_used, block.block_size, &tiled_used) || + tiled_used > info.tiled_size); + } + const uint32_t alignment = std::min(info.bytes_per_element, 4u); + EXIT_NOT_IMPLEMENTED(((info.linear_offset | info.tiled_offset | pitch_bytes | slice_bytes) & + (alignment - 1u)) != 0); + + Dispatch dispatch {}; + dispatch.elements = static_cast(elements); + dispatch.pipeline_slot = GetPipelineSlot(to_tiled, info.family, info.bytes_per_element); + dispatch.push.src_base = + static_cast(to_tiled ? info.linear_offset : info.tiled_offset); + dispatch.push.dst_base = + static_cast(to_tiled ? info.tiled_offset : info.linear_offset); + dispatch.push.width = info.width; + dispatch.push.height = info.height; + dispatch.push.depth = info.depth; + dispatch.push.surface_z = info.surface_z; + dispatch.push.pitch_bytes = static_cast(pitch_bytes); + dispatch.push.slice_bytes = static_cast(slice_bytes); + dispatch.push.blocks_per_row = static_cast(columns); + dispatch.push.blocks_per_slice = static_cast(blocks_per_slice); + dispatch.push.tail_x = info.tail_x; + dispatch.push.tail_y = info.tail_y; + dispatch.push.tail = info.tail; + dispatches->push_back(dispatch); + } +} + +void Tiler::Destroy(Resources* target) const { + if (ctx == nullptr) { + return; + } + if (target->mapped != nullptr) { + VulkanUnmapMemory(ctx, &target->staging.memory); + } + if (target->staging.buffer != nullptr) { + VulkanDeleteBuffer(ctx, &target->staging); + } + if (target->linear.buffer != nullptr) { + VulkanDeleteBuffer(ctx, &target->linear); + } + for (auto pipeline: target->pipelines) { + if (pipeline != nullptr) { + ctx->device.destroyPipeline(pipeline, nullptr); + } + } + if (target->descriptor_pool != nullptr) { + ctx->device.destroyDescriptorPool(target->descriptor_pool, nullptr); + } + if (target->pipeline_layout != nullptr) { + ctx->device.destroyPipelineLayout(target->pipeline_layout, nullptr); + } + if (target->descriptor_layout != nullptr) { + ctx->device.destroyDescriptorSetLayout(target->descriptor_layout, nullptr); + } + *target = {}; +} + +void Tiler::Init(GraphicContext* context) { + if (resources.pipeline_layout != nullptr) { + EXIT_IF(ctx != context); + return; + } + EXIT_IF(context == nullptr || context->device == nullptr || context->allocator == nullptr); + ctx = context; + std::array bindings {}; + for (uint32_t i = 0; i < bindings.size(); i++) { + bindings[i] = {i, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, + nullptr}; + } + vk::DescriptorSetLayoutCreateInfo descriptor_info {}; + descriptor_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; + descriptor_info.bindingCount = static_cast(bindings.size()); + descriptor_info.pBindings = bindings.data(); + RequireVulkanSuccess(ctx->device.createDescriptorSetLayout(&descriptor_info, nullptr, + &resources.descriptor_layout), + "create GPU tiler descriptor layout"); + + vk::PushConstantRange push_range {vk::ShaderStageFlagBits::eCompute, 0, sizeof(Push)}; + vk::PipelineLayoutCreateInfo layout_info {}; + layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; + layout_info.setLayoutCount = 1; + layout_info.pSetLayouts = &resources.descriptor_layout; + layout_info.pushConstantRangeCount = 1; + layout_info.pPushConstantRanges = &push_range; + RequireVulkanSuccess( + ctx->device.createPipelineLayout(&layout_info, nullptr, &resources.pipeline_layout), + "create GPU tiler pipeline layout"); + vk::DescriptorPoolSize pool_size {vk::DescriptorType::eStorageBuffer, 2}; + vk::DescriptorPoolCreateInfo pool_info {}; + pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo; + pool_info.maxSets = 1; + pool_info.poolSizeCount = 1; + pool_info.pPoolSizes = &pool_size; + RequireVulkanSuccess( + ctx->device.createDescriptorPool(&pool_info, nullptr, &resources.descriptor_pool), + "create GPU tiler descriptor pool"); + vk::DescriptorSetAllocateInfo set_info {}; + set_info.sType = vk::StructureType::eDescriptorSetAllocateInfo; + set_info.descriptorPool = resources.descriptor_pool; + set_info.descriptorSetCount = 1; + set_info.pSetLayouts = &resources.descriptor_layout; + RequireVulkanSuccess(ctx->device.allocateDescriptorSets(&set_info, &resources.descriptor_set), + "allocate GPU tiler descriptor set"); +} + +void Tiler::CreatePipeline(uint32_t pipeline_slot) { + const uint32_t element_size_index = pipeline_slot % BYTES_PER_ELEMENT_COUNT; + const uint32_t family_direction_index = pipeline_slot / BYTES_PER_ELEMENT_COUNT; + const uint32_t family_index = family_direction_index % FAMILY_COUNT; + const uint32_t direction_index = family_direction_index / FAMILY_COUNT; + const uint32_t specialization_values[] {1u << element_size_index, direction_index}; + const vk::SpecializationMapEntry entries[] {{0, 0, 4}, {1, 4, 4}}; + vk::SpecializationInfo specialization {2, entries, sizeof(specialization_values), + specialization_values}; + vk::ShaderModuleCreateInfo module_info {}; + module_info.sType = vk::StructureType::eShaderModuleCreateInfo; + module_info.codeSize = SHADERS[family_index].words * sizeof(uint32_t); + module_info.pCode = SHADERS[family_index].code; + vk::ShaderModule module = nullptr; + RequireVulkanSuccess(ctx->device.createShaderModule(&module_info, nullptr, &module), + "create GPU tiler shader module"); + vk::PipelineShaderStageCreateInfo stage {}; + stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo; + stage.stage = vk::ShaderStageFlagBits::eCompute; + stage.module = module; + stage.pName = "main"; + stage.pSpecializationInfo = &specialization; + vk::ComputePipelineCreateInfo info {}; + info.sType = vk::StructureType::eComputePipelineCreateInfo; + info.stage = stage; + info.layout = resources.pipeline_layout; + vk::Pipeline pipeline = nullptr; + const auto result = ctx->device.createComputePipelines(nullptr, 1, &info, nullptr, &pipeline); + ctx->device.destroyShaderModule(module, nullptr); + RequireVulkanSuccess(result, "create GPU tiler pipeline"); + resources.pipelines[pipeline_slot] = pipeline; +} + +void Tiler::CreatePipelines(std::span dispatches) { + for (const auto& dispatch: dispatches) { + if (resources.pipelines[dispatch.pipeline_slot] == nullptr) { + CreatePipeline(dispatch.pipeline_slot); + } + } +} + +void Tiler::CreateBuffer(uint64_t size, bool mapped, VulkanBuffer* buffer, void** data) const { + buffer->usage = vk::BufferUsageFlagBits::eStorageBuffer | + vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst; + buffer->memory.property = + mapped + ? vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent + : vk::MemoryPropertyFlags(vk::MemoryPropertyFlagBits::eDeviceLocal); + VulkanCreateBuffer(ctx, size, buffer); + if (mapped) VulkanMapMemory(ctx, &buffer->memory, data); +} + +void Tiler::Resize(uint64_t staging_size, uint64_t linear_size) { + if (resources.staging.buffer_size >= staging_size && + resources.linear.buffer_size >= linear_size) { + return; + } + staging_size = std::max(staging_size, resources.staging.buffer_size); + linear_size = std::max(linear_size, resources.linear.buffer_size); + VulkanBuffer staging {}, linear {}; + void* mapped = nullptr; + CreateBuffer(staging_size, true, &staging, &mapped); + CreateBuffer(linear_size, false, &linear, nullptr); + if (resources.mapped != nullptr) VulkanUnmapMemory(ctx, &resources.staging.memory); + if (resources.staging.buffer != nullptr) VulkanDeleteBuffer(ctx, &resources.staging); + if (resources.linear.buffer != nullptr) VulkanDeleteBuffer(ctx, &resources.linear); + resources.staging = staging; + resources.linear = linear; + resources.mapped = mapped; +} + +void Tiler::Execute(bool to_tiled, const void* input, void* output, uint64_t tiled_capacity, + uint64_t linear_capacity, std::span dispatches, + const GpuTileRecord& record) { + const uint64_t tiled_size = AlignToDword(tiled_capacity); + const uint64_t linear_size = AlignToDword(linear_capacity); + const uint64_t input_size = to_tiled ? linear_capacity : tiled_capacity; + if (input != nullptr) { + std::memcpy(resources.mapped, input, static_cast(input_size)); + std::memset(static_cast(resources.mapped) + input_size, 0, + static_cast(AlignToDword(input_size) - input_size)); + } + + std::array buffer_info {{ + {to_tiled ? resources.linear.buffer : resources.staging.buffer, 0, + to_tiled ? linear_size : tiled_size}, + {to_tiled ? resources.staging.buffer : resources.linear.buffer, 0, + to_tiled ? tiled_size : linear_size}, + }}; + std::array writes {}; + for (uint32_t i = 0; i < writes.size(); i++) { + writes[i].sType = vk::StructureType::eWriteDescriptorSet; + writes[i].dstSet = resources.descriptor_set; + writes[i].dstBinding = i; + writes[i].descriptorCount = 1; + writes[i].descriptorType = vk::DescriptorType::eStorageBuffer; + writes[i].pBufferInfo = &buffer_info[i]; + } + ctx->device.updateDescriptorSets(static_cast(writes.size()), writes.data(), 0, + nullptr); + + CommandBuffer command(GraphicContext::QUEUE_UTIL); + command.Begin(); + auto vk_command = command.Handle(); + if (input != nullptr) { + Barrier(vk_command, resources.staging.buffer, vk::AccessFlagBits::eHostWrite, + vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eTransferRead, + vk::PipelineStageFlagBits::eHost, + vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer); + } + if (to_tiled && input != nullptr) { + const vk::BufferCopy copy {0, 0, linear_size}; + vk_command.copyBuffer(resources.staging.buffer, resources.linear.buffer, 1, ©); + Barrier(vk_command, resources.linear.buffer, vk::AccessFlagBits::eTransferWrite, + vk::AccessFlagBits::eShaderRead, vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eComputeShader); + Barrier(vk_command, resources.staging.buffer, vk::AccessFlagBits::eTransferRead, + vk::AccessFlagBits::eTransferWrite, vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eTransfer); + } + if (to_tiled && record) { + record(&command, &resources.linear); + Barrier(vk_command, resources.linear.buffer, + vk::AccessFlagBits::eTransferWrite | vk::AccessFlagBits::eMemoryWrite, + vk::AccessFlagBits::eShaderRead, vk::PipelineStageFlagBits::eAllCommands, + vk::PipelineStageFlagBits::eComputeShader); + } + + const auto output_buffer = to_tiled ? resources.staging.buffer : resources.linear.buffer; + const auto output_size = to_tiled ? tiled_size : linear_size; + vk_command.fillBuffer(output_buffer, 0, output_size, 0); + Barrier(vk_command, output_buffer, vk::AccessFlagBits::eTransferWrite, + vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite, + vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eComputeShader); + vk_command.bindDescriptorSets(vk::PipelineBindPoint::eCompute, resources.pipeline_layout, 0, 1, + &resources.descriptor_set, 0, nullptr); + const uint64_t limit = + static_cast( + ctx->GetPhysicalDeviceProperties().limits.maxComputeWorkGroupCount[0]) * + GROUP_SIZE; + for (const auto& dispatch: dispatches) { + vk_command.bindPipeline(vk::PipelineBindPoint::eCompute, + resources.pipelines[dispatch.pipeline_slot]); + for (uint32_t first = 0; first < dispatch.elements;) { + auto push = dispatch.push; + push.first = first; + push.count = + static_cast(std::min(dispatch.elements - first, limit)); + vk_command.pushConstants(resources.pipeline_layout, vk::ShaderStageFlagBits::eCompute, + 0, sizeof(push), &push); + vk_command.dispatch((push.count - 1u) / GROUP_SIZE + 1u, 1, 1); + first += push.count; + } + } + Barrier(vk_command, output_buffer, vk::AccessFlagBits::eShaderWrite, + vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eHostRead, + vk::PipelineStageFlagBits::eComputeShader, + vk::PipelineStageFlagBits::eTransfer | vk::PipelineStageFlagBits::eHost); + if (!to_tiled && record) { + record(&command, &resources.linear); + } + if (!to_tiled && output != nullptr) { + const vk::BufferCopy copy {0, 0, linear_size}; + vk_command.copyBuffer(resources.linear.buffer, resources.staging.buffer, 1, ©); + Barrier(vk_command, resources.staging.buffer, vk::AccessFlagBits::eTransferWrite, + vk::AccessFlagBits::eHostRead, vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost); + } + command.End(); + command.Execute(); + command.WaitForFence(); + if (output != nullptr) { + std::memcpy(output, resources.mapped, + static_cast(to_tiled ? tiled_capacity : linear_capacity)); + } +} + +void Tiler::Run(bool to_tiled, GraphicContext* context, const void* input, void* output, + uint64_t tiled_capacity, uint64_t linear_capacity, + std::span infos, const GpuTileRecord& record) { + Common::LockGuard lock(mutex); + EXIT_IF((to_tiled && (output == nullptr || (input == nullptr && !record))) || + (!to_tiled && (input == nullptr || (output == nullptr && !record)))); + std::vector dispatches; + Prepare(to_tiled, context, tiled_capacity, linear_capacity, infos, &dispatches); + Init(context); + CreatePipelines(dispatches); + const uint64_t staging_size = + std::max(AlignToDword(tiled_capacity), AlignToDword(linear_capacity)); + const uint64_t linear_size = AlignToDword(linear_capacity); + Resize(staging_size, linear_size); + Execute(to_tiled, input, output, tiled_capacity, linear_capacity, dispatches, record); +} + +void Tiler::Release(GraphicContext* context) { + Common::LockGuard lock(mutex); + EXIT_IF(ctx != nullptr && context != ctx); + Destroy(&resources); + ctx = nullptr; +} + +} // namespace + +void GpuDetile(GraphicContext* ctx, const void* tiled, void* linear, uint64_t tiled_capacity, + uint64_t linear_capacity, std::span infos, + const GpuTileRecord& after) { + g_tiler.Run(false, ctx, tiled, linear, tiled_capacity, linear_capacity, infos, after); +} + +void GpuTile(GraphicContext* ctx, const void* linear, void* tiled, uint64_t tiled_capacity, + uint64_t linear_capacity, std::span infos, + const GpuTileRecord& before) { + g_tiler.Run(true, ctx, linear, tiled, tiled_capacity, linear_capacity, infos, before); +} + +void GpuTileRelease(GraphicContext* ctx) { + g_tiler.Release(ctx); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/gpuTiler.h b/src/graphics/host_gpu/gpuTiler.h new file mode 100644 index 0000000..cadf554 --- /dev/null +++ b/src/graphics/host_gpu/gpuTiler.h @@ -0,0 +1,45 @@ +#pragma once + +#include "graphics/guest_gpu/tile.h" + +#include +#include +#include + +namespace Libs::Graphics { + +struct GraphicContext; +struct VulkanBuffer; +class CommandBuffer; + +struct GpuTileInfo { + TileBlockFamily family = TileBlockFamily::Count; + uint32_t bytes_per_element = 0; + uint64_t linear_offset = 0; + uint64_t linear_size = 0; + uint64_t tiled_offset = 0; + uint64_t tiled_size = 0; + uint64_t linear_slice_stride = 0; + uint32_t width = 0; + uint32_t height = 0; + uint32_t depth = 1; + uint32_t pitch = 0; + uint32_t tail_x = 0; + uint32_t tail_y = 0; + bool tail = false; + uint32_t tiled_width = 0; + uint32_t tiled_height = 0; + uint32_t surface_z = 0; +}; + +using GpuTileRecord = std::function; + +void GpuDetile(GraphicContext* ctx, const void* tiled, void* linear, uint64_t tiled_capacity, + uint64_t linear_capacity, std::span infos, + const GpuTileRecord& after = {}); +void GpuTile(GraphicContext* ctx, const void* linear, void* tiled, uint64_t tiled_capacity, + uint64_t linear_capacity, std::span infos, + const GpuTileRecord& before = {}); +void GpuTileRelease(GraphicContext* ctx); + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/objects/textureCommon.cpp b/src/graphics/host_gpu/objects/textureCommon.cpp index efc4776..dbe695c 100644 --- a/src/graphics/host_gpu/objects/textureCommon.cpp +++ b/src/graphics/host_gpu/objects/textureCommon.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -165,17 +166,26 @@ static uint64_t GetLevelSrcSize(const TileSizeOffset& level_size) { return (level_size.src_size != 0 ? level_size.src_size : level_size.size); } +static uint32_t GetTextureLevelDepth(uint32_t depth, uint32_t level, bool volume_texture) { + return volume_texture ? std::max(depth >> level, 1u) : depth; +} + +static size_t GetTextureRegionCount(uint32_t depth, uint64_t levels, bool volume_texture) { + size_t count = 0; + for (uint32_t level = 0; level < levels; level++) { + count += GetTextureLevelDepth(depth, level, volume_texture); + } + return count; +} + uint64_t TextureUploadSliceSourceOffset(const TextureUploadLayout& layout, uint32_t level, - uint32_t slice, - TextureUploadSliceLayout source_slice_layout) { + uint32_t slice) { if (level >= 16 || layout.level_sizes[level].size == 0) { EXIT("invalid texture upload slice source, level=%u slice=%u\n", level, slice); } const auto level_offset = GetLevelSrcOffset(layout.level_sizes[level]); const auto slice_stride = - source_slice_layout == TextureUploadSliceLayout::MipChainPerSlice - ? (layout.source_slice_stride != 0 ? layout.source_slice_stride : layout.slice_stride) - : GetLevelSrcSize(layout.level_sizes[level]); + layout.source_slice_stride != 0 ? layout.source_slice_stride : layout.slice_stride; if (slice_stride != 0 && slice > (UINT64_MAX - level_offset) / slice_stride) { EXIT("texture upload slice source offset overflow, level=%u slice=%u\n", level, slice); } @@ -184,7 +194,7 @@ uint64_t TextureUploadSliceSourceOffset(const TextureUploadLayout& layout, uint3 uint64_t TextureCalcUploadSize(const TextureUploadLayout& layout, const std::vector& regions, uint64_t levels, - uint32_t depth, TextureUploadSliceLayout source_slice_layout) { + uint32_t depth) { uint64_t size = 0; for (const auto& r: regions) { @@ -193,61 +203,17 @@ uint64_t TextureCalcUploadSize(const TextureUploadLayout& layout, } for (uint32_t level = 0; level < levels; level++) { - const auto src_size = GetLevelSrcSize(layout.level_sizes[level]); - for (uint32_t z = 0; z < depth; z++) { - size = std::max( - size, - TextureUploadSliceSourceOffset(layout, level, z, source_slice_layout) + src_size); + const auto src_size = GetLevelSrcSize(layout.level_sizes[level]); + const auto mip_depth = GetTextureLevelDepth(depth, level, layout.volume_texture); + for (uint32_t z = 0; z < mip_depth; z++) { + size = std::max(size, + TextureUploadSliceSourceOffset(layout, level, z) + src_size); } } return size; } -void TextureCopyBufferBytes(GraphicContext* ctx, VulkanBuffer* src_buffer, - uint64_t src_buffer_offset, uint64_t copy_size, - Transfer::ScratchBuffer* dst) { - EXIT_IF(ctx == nullptr); - EXIT_IF(src_buffer == nullptr); - EXIT_IF(dst == nullptr); - EXIT_IF(dst->Data() == nullptr); - - std::memset(dst->Data(), 0, copy_size); - - if (copy_size == 0 || src_buffer_offset >= src_buffer->buffer_size) { - return; - } - - const auto available = - std::min(copy_size, src_buffer->buffer_size - src_buffer_offset); - if (available == 0) { - return; - } - - if (src_buffer->memory.property & vk::MemoryPropertyFlagBits::eHostVisible) { - void* data = nullptr; - VulkanMapMemory(ctx, &src_buffer->memory, &data); - std::memcpy(dst->Data(), static_cast(data) + src_buffer_offset, available); - VulkanUnmapMemory(ctx, &src_buffer->memory); - return; - } - - VulkanBuffer readback {}; - readback.usage = vk::BufferUsageFlagBits::eTransferDst; - readback.memory.property = vk::MemoryPropertyFlagBits::eHostVisible | - vk::MemoryPropertyFlagBits::eHostCoherent | - vk::MemoryPropertyFlagBits::eHostCached; - VulkanCreateBuffer(ctx, (src_buffer_offset + available + 3u) & ~uint64_t {3}, &readback); - Transfer::CopyBuffer(src_buffer, &readback, src_buffer_offset + available); - - void* data = nullptr; - VulkanMapMemory(ctx, &readback.memory, &data); - std::memcpy(dst->Data(), static_cast(data) + src_buffer_offset, available); - VulkanUnmapMemory(ctx, &readback.memory); - - VulkanDeleteBuffer(ctx, &readback); -} - vk::ComponentSwizzle TextureGetComponentSwizzle(uint8_t s) { switch (static_cast(s)) { case Prospero::CompSwizzle::kZero: return vk::ComponentSwizzle::eZero; @@ -377,77 +343,6 @@ static uint32_t AlignUpU32(uint32_t value, uint32_t alignment) { return (value + alignment - 1u) & ~(alignment - 1u); } -static uint32_t ShiftCeilU32(uint32_t value, uint32_t shift) { - return static_cast((static_cast(value) + (1ull << shift) - 1ull) >> shift); -} - -struct Standard4KBVolumeMipLayout { - uint32_t first_tail_level = 0; - uint32_t block_depth = 1; - uint64_t block_slice_size = 0; - uint64_t level_offsets[16] = {}; - uint64_t level_sizes[16] = {}; -}; - -static bool CalcStandard4kbVolumeMipLayout(uint32_t format, uint32_t pitch, uint32_t height, - uint64_t levels, Standard4KBVolumeMipLayout* out) { - EXIT_IF(out == nullptr); - EXIT_NOT_IMPLEMENTED(levels == 0 || levels > 16); - - uint32_t bytes_per_element = 0; - uint32_t texels_per_element_wide = 0; - uint32_t texels_per_element_tall = 0; - uint32_t block_width_log2 = 0; - uint32_t block_height_log2 = 0; - uint32_t block_depth_log2 = 0; - - if (!TileGetStandard4KBVolumeLayout(format, &bytes_per_element, &texels_per_element_wide, - &texels_per_element_tall, &block_width_log2, - &block_height_log2, &block_depth_log2)) { - return false; - } - - const uint32_t row_elements0 = - std::max((pitch + texels_per_element_wide - 1u) / texels_per_element_wide, 1u); - const uint32_t height_elements0 = - std::max((height + texels_per_element_tall - 1u) / texels_per_element_tall, 1u); - const uint32_t block_width = 1u << block_width_log2; - const uint32_t block_height = 1u << block_height_log2; - const uint32_t block_depth = 1u << block_depth_log2; - const uint32_t tail_width_limit = block_width; - const uint32_t tail_height_limit = block_height >> 1u; - constexpr uint32_t max_tail_levels = 5u; - - out->first_tail_level = static_cast(levels); - out->block_depth = block_depth; - out->block_slice_size = 0; - - for (uint32_t level = 0; level < levels; level++) { - const uint32_t row_elements = std::max(ShiftCeilU32(row_elements0, level), 1u); - const uint32_t height_elements = std::max(ShiftCeilU32(height_elements0, level), 1u); - - out->level_offsets[level] = out->block_slice_size; - if (row_elements <= tail_width_limit && height_elements <= tail_height_limit && - levels - level <= max_tail_levels) { - out->first_tail_level = level; - out->level_sizes[level] = 4096u; - out->block_slice_size += 4096u; - for (uint32_t tail_level = level + 1; tail_level < levels; tail_level++) { - out->level_offsets[tail_level] = out->level_offsets[level]; - out->level_sizes[tail_level] = 4096u; - } - break; - } - - out->level_sizes[level] = static_cast(block_depth) * - AlignUpU32(row_elements, block_width) * - AlignUpU32(height_elements, block_height) * bytes_per_element; - out->block_slice_size += out->level_sizes[level]; - } - - return out->block_slice_size != 0; -} - uint32_t TextureGetAtlasSliceYStride(vk::Format format, uint32_t mip_height, uint32_t depth, uint64_t levels) { return (depth > 1 && levels > 1 && Transfer::IsBlockCompressedFormat(format) @@ -733,9 +628,8 @@ static uint64_t CalcLinearUploadLevelSize(uint32_t fmt, uint32_t pitch, uint32_t return 0; } -static uint64_t FillVolumeLinearUploadLevels(TileSizeOffset* level_sizes, uint32_t fmt, - uint64_t height, uint64_t levels, - uint32_t base_pitch) { +static uint64_t SetLinearUploadLevels(TileSizeOffset* level_sizes, uint32_t fmt, uint64_t height, + uint64_t levels, uint32_t base_pitch) { uint64_t offset = 0; auto pitch = base_pitch; auto h = static_cast(height); @@ -767,9 +661,8 @@ static uint64_t FillVolumeLinearUploadLevels(TileSizeOffset* level_sizes, uint32 TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64_t height, uint64_t levels, uint32_t depth, uint64_t pitch, uint64_t tile, uint64_t upload_size, - bool allow_depth_tile, - bool require_single_mip_small_tiles, - bool volume_texture, const char* owner) { + bool allow_depth_tile, bool volume_texture, + const char* owner) { TextureUploadLayout layout {}; layout.tile = static_cast(tile); layout.pitch = static_cast(pitch); @@ -778,33 +671,45 @@ TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64 if (fmt != 0) { if (layout.tile != 0) { const auto tile_mode = static_cast(layout.tile); - layout.fmt_tiled_render_target = - (Prospero::RenderTargetBytesPerElement(static_cast(fmt)) != 0 && - tile_mode == Prospero::TileMode::kRenderTarget); - layout.fmt_tiled_standard256b = - (TileIsStandard256BTextureSupported(static_cast(fmt)) && - tile_mode == Prospero::TileMode::kStandard256B && - (!require_single_mip_small_tiles || levels == 1)); - layout.fmt_tiled_standard4kb = - (TileIsStandard4KBTextureSupported(static_cast(fmt)) && - tile_mode == Prospero::TileMode::kStandard4KB && - (!require_single_mip_small_tiles || levels == 1)); - layout.fmt_tiled_standard64kb = - (TileIsStandard64KBTextureSupported(static_cast(fmt)) && - tile_mode == Prospero::TileMode::kStandard64KB); - layout.fmt_tiled_depth = - (allow_depth_tile && - Prospero::RenderTargetBytesPerElement(static_cast(fmt)) != 0 && - tile_mode == Prospero::TileMode::kDepth); - if (!layout.fmt_tiled_render_target && !layout.fmt_tiled_standard256b && - !layout.fmt_tiled_standard4kb && !layout.fmt_tiled_standard64kb && - !layout.fmt_tiled_depth) { - EXIT("%s: unsupported typed tiled upload, using linear fallback: fmt=%u tile=%u " + switch (tile_mode) { + case Prospero::TileMode::kStandard256B: + if (TileIsStandard256BTextureSupported(fmt)) { + layout.tile_family = TileBlockFamily::Standard256B; + } + break; + case Prospero::TileMode::kStandard4KB: + if (TileIsStandard4KBTextureSupported(fmt)) { + layout.tile_family = TileBlockFamily::Standard4KB; + } + break; + case Prospero::TileMode::kStandard64KB: + if (TileIsStandard64KBTextureSupported(fmt)) { + layout.tile_family = TileBlockFamily::Standard64KB; + } + break; + case Prospero::TileMode::kPrt: + if (TileIsStandard64KBTextureSupported(fmt)) { + layout.tile_family = TileBlockFamily::Prt64KB; + } + break; + case Prospero::TileMode::kRenderTarget: + if (Prospero::RenderTargetBytesPerElement(fmt) != 0) { + layout.tile_family = TileBlockFamily::RenderTarget64KB; + } + break; + case Prospero::TileMode::kDepth: + if (allow_depth_tile && Prospero::RenderTargetBytesPerElement(fmt) != 0) { + layout.tile_family = TileBlockFamily::Depth64KB; + } + break; + default: break; + } + if (layout.tile_family == TileBlockFamily::Count) { + EXIT("%s: unsupported typed tiled upload: fmt=%u tile=%u " "size=%" PRIu64 " extent=%" PRIu64 "x%" PRIu64 " pitch=%" PRIu64 " levels=%" PRIu64 "\n", owner, static_cast(fmt), layout.tile, upload_size, width, height, pitch, levels); - layout.tile = 0; } } @@ -813,28 +718,29 @@ TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64 TileGetTextureSize(fmt, width, height, layout.pitch, levels, layout.tile, nullptr, layout.level_sizes, layout.padded_sizes); - if (layout.volume_texture) { - TileSizeOffset source_levels[16] {}; - std::copy_n(layout.level_sizes, levels, source_levels); - layout.slice_stride = - FillVolumeLinearUploadLevels(layout.level_sizes, fmt, height, levels, layout.pitch); - if (layout.fmt_tiled_render_target) { + if (static_cast(layout.tile) != Prospero::TileMode::kLinear) { + if (layout.volume_texture) { + layout.slice_stride = SetLinearUploadLevels(layout.level_sizes, fmt, height, levels, + static_cast(width)); + } else { + TileSizeOffset tiled_levels[16] {}; + std::copy_n(layout.level_sizes, levels, tiled_levels); layout.source_slice_stride = - CalcTextureSliceStride(source_levels, levels, upload_size, depth); - for (uint32_t level = 0; level < levels; level++) { - layout.level_sizes[level].src_offset = source_levels[level].offset; - layout.level_sizes[level].src_size = source_levels[level].size; - layout.level_sizes[level].x = source_levels[level].x; - layout.level_sizes[level].y = source_levels[level].y; + CalcTextureSliceStride(tiled_levels, levels, upload_size, depth); + SetLinearUploadLevels(layout.level_sizes, fmt, height, levels, layout.pitch); + for (uint32_t i = 0; i < levels; ++i) { + if (tiled_levels[i].src_size > tiled_levels[i].size) { + layout.first_tail_level = std::min(layout.first_tail_level, i); + } + layout.level_sizes[i].src_offset = GetLevelSrcOffset(tiled_levels[i]); + layout.level_sizes[i].src_size = GetLevelSrcSize(tiled_levels[i]); + layout.level_sizes[i].x = tiled_levels[i].x; + layout.level_sizes[i].y = tiled_levels[i].y; } } - } - - if (layout.fmt_tiled_depth) { - for (uint32_t i = 0; i < levels; i++) { - layout.level_sizes[i].x = layout.padded_sizes[i].width; - layout.level_sizes[i].y = layout.padded_sizes[i].height; - } + } else if (layout.volume_texture) { + layout.slice_stride = + CalcTextureSliceStride(layout.level_sizes, levels, upload_size, depth); } } else { EXIT("%s: legacy texture upload format unsupported: fmt=0 tile=%u size=%" PRIu64 @@ -849,66 +755,68 @@ TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64 return layout; } -std::vector TextureBuildUploadRegions( - const TextureUploadLayout& layout, vk::Format image_format, uint32_t width, uint32_t height, - uint32_t depth, uint64_t levels, bool array_texture, bool volume_texture, - TextureUploadDestination destination, TextureUploadSliceLayout slice_layout) { +std::vector TextureBuildUploadRegions(const TextureUploadLayout& layout, + vk::Format image_format, uint32_t width, + uint32_t height, uint32_t depth, + uint64_t levels, bool array_texture, + bool volume_texture, + TextureUploadDestination destination) { uint32_t mip_width = width; uint32_t mip_height = height; - uint32_t mip_pitch = layout.pitch; + uint32_t mip_pitch = volume_texture && static_cast(layout.tile) != + Prospero::TileMode::kLinear + ? width + : layout.pitch; - std::vector regions(levels * depth); + std::vector regions; + regions.reserve(GetTextureRegionCount(depth, levels, volume_texture)); for (uint32_t i = 0; i < levels; i++) { EXIT_NOT_IMPLEMENTED(layout.level_sizes[i].size == 0); const auto mipmap_offset = Transfer::MipmapAtlasOffset(i, width, height); + const auto mip_depth = GetTextureLevelDepth(depth, i, volume_texture); - for (uint32_t z = 0; z < depth; z++) { - const auto region_index = i * depth + z; - const auto slice_offset = (slice_layout == TextureUploadSliceLayout::MipChainPerSlice - ? z * layout.slice_stride - : z * static_cast(layout.level_sizes[i].size)); + for (uint32_t z = 0; z < mip_depth; z++) { + const auto slice_offset = z * layout.slice_stride; + BufferImageCopy region {}; - regions[region_index].offset = - static_cast(layout.level_sizes[i].offset + slice_offset); - regions[region_index].width = mip_width; - regions[region_index].height = mip_height; - regions[region_index].copy_height = + region.offset = static_cast(layout.level_sizes[i].offset + slice_offset); + region.width = mip_width; + region.height = mip_height; + region.copy_height = (!array_texture && !volume_texture && depth > 1 && levels > 1 && Transfer::IsBlockCompressedFormat(image_format) ? TextureGetAtlasSliceYStride(image_format, mip_height, depth, levels) : 0); - regions[region_index].dst_layer = (array_texture ? z : 0); - regions[region_index].dst_z = (volume_texture ? static_cast(z) : 0); - if (layout.fmt_tiled_depth && layout.level_sizes[i].x != 0) { - regions[region_index].pitch = layout.level_sizes[i].x; - } else if (!layout.volume_texture && - static_cast(layout.tile) == - Prospero::TileMode::kLinear && - layout.padded_sizes[i].width != 0) { - regions[region_index].pitch = layout.padded_sizes[i].width; + region.dst_layer = (array_texture ? z : 0); + region.dst_z = (volume_texture ? static_cast(z) : 0); + if (!layout.volume_texture && + static_cast(layout.tile) == Prospero::TileMode::kLinear && + layout.padded_sizes[i].width != 0) { + region.pitch = layout.padded_sizes[i].width; } else { - regions[region_index].pitch = mip_pitch; + region.pitch = mip_pitch; } if (destination == TextureUploadDestination::MipLevels) { - regions[region_index].dst_level = i; - regions[region_index].dst_x = 0; - regions[region_index].dst_y = + region.dst_level = i; + region.dst_x = 0; + region.dst_y = (array_texture || volume_texture ? 0 : static_cast(z * TextureGetAtlasSliceYStride( image_format, mip_height, depth, levels))); } else { - regions[region_index].dst_level = 0; - regions[region_index].dst_x = mipmap_offset.first; - regions[region_index].dst_y = + region.dst_level = 0; + region.dst_x = mipmap_offset.first; + region.dst_y = (array_texture || volume_texture ? mipmap_offset.second : mipmap_offset.second + static_cast(z * TextureGetAtlasSliceYStride( image_format, mip_height, depth, levels))); } + regions.push_back(region); } if (mip_width > 1) { @@ -925,12 +833,16 @@ std::vector TextureBuildUploadRegions( return regions; } -static uint64_t GetSliceSrcStride(const TextureUploadLayout& layout, uint32_t level, - TextureUploadSliceLayout source_slice_layout) { - return ( - source_slice_layout == TextureUploadSliceLayout::MipChainPerSlice - ? (layout.source_slice_stride != 0 ? layout.source_slice_stride : layout.slice_stride) - : GetLevelSrcSize(layout.level_sizes[level])); +std::vector +TextureBuildDownloadRegions(const std::vector& upload_regions) { + std::vector regions; + regions.reserve(upload_regions.size()); + for (const auto& region: upload_regions) { + regions.push_back({region.offset, region.pitch, region.dst_level, region.width, + region.height, region.copy_height, region.dst_layer, region.dst_x, + region.dst_y, region.dst_z, region.aspect}); + } + return regions; } static uint64_t FmaskRegionCopySize(const BufferImageCopy& region) { @@ -969,193 +881,191 @@ static void UploadFmaskIdentity(GraphicContext* ctx, VulkanImage* vk_obj, Transfer::UploadImage(ctx, vk_obj, temp_buf.Data(), upload_size, upload_regions, dst_layout); } +struct GpuTileElementLayout { + uint32_t bytes = 0; + uint32_t wide = 1; + uint32_t tall = 1; +}; + +static bool GetGpuTileElementLayout(uint32_t fmt, GpuTileElementLayout* out) { + EXIT_IF(out == nullptr); + if (const auto bytes = Prospero::NumBytesPerElement(fmt); bytes != 0) { + *out = {bytes, 1, 1}; + return true; + } + if (const auto bytes = Prospero::BlockCompressedBytesPerBlock(fmt); bytes != 0) { + *out = {bytes, 4, 4}; + return true; + } + return false; +} + +static bool SetGpuTileSize(uint64_t offset, uint64_t length, uint64_t capacity, uint64_t* size) { + if (size == nullptr || offset > capacity || length > capacity - offset) { + return false; + } + *size = length; + return true; +} + +bool TextureBuildGpuTileInfos(uint64_t size, const std::vector& regions, + const TextureUploadLayout& layout, uint32_t fmt, uint32_t depth, + uint64_t levels, std::vector* out_infos) { + if (out_infos == nullptr || size == 0 || levels == 0 || levels > 16 || depth == 0 || + regions.size() != GetTextureRegionCount(depth, levels, layout.volume_texture) || + Prospero::IsFmaskTextureFormat(fmt)) { + return false; + } + + GpuTileElementLayout element {}; + if (layout.tile_family == TileBlockFamily::RenderTarget64KB || + layout.tile_family == TileBlockFamily::Depth64KB) { + element.bytes = Prospero::RenderTargetBytesPerElement(fmt); + } else if (!GetGpuTileElementLayout(fmt, &element)) { + return false; + } + if (element.bytes == 0) { + return false; + } + + std::vector infos; + infos.reserve(regions.size()); + if (layout.volume_texture) { + TileVolumeLayout volume {}; + if (!TileGetTextureVolumeLayout(fmt, regions[0].width, regions[0].height, depth, + static_cast(levels), layout.tile, &volume)) { + return false; + } + element = {volume.bytes_per_element, volume.texel_width, volume.texel_height}; + TileBlockLayout block {}; + if (!TileGetBlockLayout(volume.family, element.bytes, &block)) return false; + + size_t region_base = 0; + for (uint32_t level = 0; level < levels; ++level) { + const uint32_t mip_depth = GetTextureLevelDepth(depth, level, true); + const bool tail = level >= volume.first_tail_level; + const uint64_t linear_stride = layout.slice_stride; + for (uint32_t z = 0; z < mip_depth; z += block.block_depth) { + const uint32_t copy_depth = std::min(block.block_depth, mip_depth - z); + const auto& region = regions[region_base + z]; + GpuTileInfo info {}; + info.family = block.family; + info.bytes_per_element = block.bytes_per_element; + info.linear_offset = region.offset; + info.tiled_offset = + static_cast(z / block.block_depth) * volume.block_slice_size + + volume.level_offsets[level]; + const uint64_t linear_span = + static_cast(copy_depth - 1u) * linear_stride + + layout.level_sizes[level].size; + if (!SetGpuTileSize(info.linear_offset, linear_span, size, &info.linear_size) || + !SetGpuTileSize(info.tiled_offset, volume.level_sizes[level], size, + &info.tiled_size)) { + return false; + } + info.linear_slice_stride = linear_stride; + info.width = std::max((region.width + element.wide - 1u) / element.wide, 1u); + info.height = std::max((region.height + element.tall - 1u) / element.tall, 1u); + info.depth = copy_depth; + info.surface_z = block.block_depth == 1 ? static_cast(region.dst_z) : 0; + info.pitch = std::max((region.pitch + element.wide - 1u) / element.wide, 1u); + info.tail_x = tail ? volume.tail_x[level] : 0; + info.tail_y = tail ? volume.tail_y[level] : 0; + info.tail = tail; + info.tiled_width = volume.level_widths[level]; + info.tiled_height = volume.level_heights[level]; + infos.push_back(info); + } + region_base += mip_depth; + } + } else { + const auto base_family = layout.tile_family; + if (base_family == TileBlockFamily::Count) { + return false; + } + + size_t region_index = 0; + for (uint32_t level = 0; level < levels; level++) { + const auto& level_size = layout.level_sizes[level]; + const bool tail = level >= layout.first_tail_level; + const auto family = base_family; + TileBlockLayout block {}; + if (!TileGetBlockLayout(family, element.bytes, &block)) { + return false; + } + const auto level_depth = GetTextureLevelDepth(depth, level, layout.volume_texture); + for (uint32_t z = 0; z < level_depth; z++) { + const auto& region = regions[region_index++]; + GpuTileInfo info {}; + info.family = block.family; + info.bytes_per_element = block.bytes_per_element; + info.linear_offset = region.offset; + info.tiled_offset = TextureUploadSliceSourceOffset(layout, level, z); + if (!SetGpuTileSize(info.linear_offset, level_size.size, size, &info.linear_size) || + !SetGpuTileSize(info.tiled_offset, GetLevelSrcSize(level_size), size, + &info.tiled_size)) { + return false; + } + info.width = std::max((region.width + element.wide - 1u) / element.wide, 1u); + info.height = std::max((region.height + element.tall - 1u) / element.tall, 1u); + info.surface_z = base_family == TileBlockFamily::RenderTarget64KB || + base_family == TileBlockFamily::Depth64KB + ? region.dst_layer + : 0; + info.pitch = std::max((region.pitch + element.wide - 1u) / element.wide, 1u); + info.tail = tail; + info.tail_x = tail ? level_size.x : 0; + info.tail_y = tail ? level_size.y : 0; + info.tiled_width = + layout.padded_sizes[level].width != 0 + ? std::max((layout.padded_sizes[level].width + element.wide - 1u) / + element.wide, + 1u) + : info.pitch; + info.tiled_height = + layout.padded_sizes[level].height != 0 + ? std::max((layout.padded_sizes[level].height + element.tall - 1u) / + element.tall, + 1u) + : info.height; + infos.push_back(info); + } + } + } + + if (infos.empty()) { + return false; + } + *out_infos = std::move(infos); + return true; +} + void TextureUploadGuestImage(GraphicContext* ctx, VulkanImage* vk_obj, const void* src_data, uint64_t size, const std::vector& regions, const TextureUploadLayout& layout, uint32_t fmt, uint64_t width, - uint64_t height, uint32_t depth, uint64_t levels, - TextureUploadSliceLayout source_slice_layout, const char* owner, + uint64_t height, uint32_t depth, uint64_t levels, const char* owner, vk::ImageLayout dst_layout) { if (fmt == 0) { - EXIT("%s: legacy texture upload format unsupported: fmt=0 tile=%u size=%" PRIu64 - " extent=%" PRIu64 "x%" PRIu64 " depth=%u pitch=%u levels=%" PRIu64 "\n", + EXIT("%s: texture upload format unsupported: fmt=0 tile=%u size=%" PRIu64 " extent=%" PRIu64 + "x%" PRIu64 " depth=%u pitch=%u levels=%" PRIu64 "\n", owner, layout.tile, size, width, height, depth, layout.pitch, levels); - } else if (static_cast(layout.tile) == Prospero::TileMode::kLinear) { - Transfer::UploadImage(ctx, vk_obj, src_data, size, regions, dst_layout); - } else if (layout.fmt_tiled_render_target) { - LOGF("%s: detiling typed render-target texture: fmt=%u tile=%u size=%" PRIu64 - " extent=%" PRIu64 "x%" PRIu64 " depth=%" PRIu64 " pitch=%u levels=%" PRIu64 "\n", - owner, static_cast(fmt), layout.tile, size, width, height, - static_cast(depth), layout.pitch, levels); - Transfer::ScratchBuffer temp_buf(size); - for (uint32_t i = 0; i < levels; i++) { - for (uint32_t z = 0; z < depth; z++) { - const auto region_index = i * depth + z; - auto* dst = static_cast(temp_buf.Data()) + regions[region_index].offset; - const auto* src = static_cast(src_data) + - TextureUploadSliceSourceOffset(layout, i, z, source_slice_layout); - TileConvertTiledToLinearRenderTarget( - dst, src, regions[region_index].width, regions[region_index].height, - regions[region_index].pitch, - Prospero::RenderTargetBytesPerElement(static_cast(fmt)), - layout.level_sizes[i].size, GetLevelSrcSize(layout.level_sizes[i]), - layout.level_sizes[i].x, layout.level_sizes[i].y); - } - } - Transfer::UploadImage(ctx, vk_obj, temp_buf.Data(), size, regions, dst_layout); - } else if (layout.fmt_tiled_depth) { - if (Prospero::IsFmaskTextureFormat(static_cast(fmt))) { - UploadFmaskIdentity(ctx, vk_obj, regions, dst_layout, owner); - return; - } - const uint32_t bytes_per_element = - Prospero::RenderTargetBytesPerElement(static_cast(fmt)); - if (bytes_per_element == 1) { - Transfer::UploadImage(ctx, vk_obj, src_data, size, regions, dst_layout); - } else { - LOGF("%s: detiling typed depth texture: fmt=%u tile=%u size=%" PRIu64 " extent=%" PRIu64 - "x%" PRIu64 " depth=%" PRIu64 " pitch=%u levels=%" PRIu64 "\n", - owner, static_cast(fmt), layout.tile, size, width, height, - static_cast(depth), layout.pitch, levels); - Transfer::ScratchBuffer temp_buf(size); - for (uint32_t i = 0; i < levels; i++) { - for (uint32_t z = 0; z < depth; z++) { - const auto region_index = i * depth + z; - auto* dst = - static_cast(temp_buf.Data()) + regions[region_index].offset; - const auto* src = static_cast(src_data) + - GetLevelSrcOffset(layout.level_sizes[i]) + - z * GetSliceSrcStride(layout, i, source_slice_layout); - TileConvertTiledToLinearDepth( - dst, src, static_cast(fmt), regions[region_index].width, - regions[region_index].height, regions[region_index].pitch, - layout.level_sizes[i].size); - } - } - Transfer::UploadImage(ctx, vk_obj, temp_buf.Data(), size, regions, dst_layout); - } - } else if (layout.fmt_tiled_standard256b) { - Transfer::ScratchBuffer temp_buf(size); - for (uint32_t i = 0; i < levels; i++) { - for (uint32_t z = 0; z < depth; z++) { - const auto region_index = i * depth + z; - auto* dst = static_cast(temp_buf.Data()) + regions[region_index].offset; - const auto* src = static_cast(src_data) + - layout.level_sizes[i].offset + - z * GetSliceSrcStride(layout, i, source_slice_layout); - TileConvertTiledToLinearStandard256B( - dst, src, static_cast(fmt), regions[region_index].width, - regions[region_index].height, regions[region_index].pitch, - layout.level_sizes[i].size, layout.level_sizes[i].size); - } - } - Transfer::UploadImage(ctx, vk_obj, temp_buf.Data(), size, regions, dst_layout); - } else if (layout.fmt_tiled_standard4kb && layout.volume_texture) { - LOGF("%s: detiling typed Standard4KB 3D texture: fmt=%u tile=%u size=%" PRIu64 - " extent=%" PRIu64 "x%" PRIu64 " depth=%" PRIu64 " pitch=%u levels=%" PRIu64 "\n", - owner, static_cast(fmt), layout.tile, size, width, height, - static_cast(depth), layout.pitch, levels); - Transfer::ScratchBuffer temp_buf(size); - if (levels == 1) { - TileConvertTiledToLinearStandard4KB3D( - temp_buf.Data(), src_data, static_cast(fmt), static_cast(width), - static_cast(height), depth, layout.pitch, layout.slice_stride, size, - size); - } else { - std::memset(temp_buf.Data(), 0, static_cast(size)); - - Standard4KBVolumeMipLayout volume_layout {}; - EXIT_NOT_IMPLEMENTED(!CalcStandard4kbVolumeMipLayout( - static_cast(fmt), layout.pitch, static_cast(height), levels, - &volume_layout)); - - auto* dst_base = static_cast(temp_buf.Data()); - const auto* src_base = static_cast(src_data); - uint32_t mip_width = static_cast(width); - uint32_t mip_height = static_cast(height); - uint32_t mip_pitch = layout.pitch; - - for (uint32_t level = 0; level < levels; level++) { - if (level < volume_layout.first_tail_level) { - for (uint32_t z = 0; z < depth; z += volume_layout.block_depth) { - const uint32_t copy_depth = std::min(volume_layout.block_depth, depth - z); - const auto region_index = level * depth + z; - auto* dst = dst_base + regions[region_index].offset; - const auto* src = src_base + - (static_cast(z / volume_layout.block_depth) * - volume_layout.block_slice_size) + - volume_layout.level_offsets[level]; - const uint64_t dst_size = - (static_cast(copy_depth - 1u) * layout.slice_stride) + - layout.level_sizes[level].size; - TileConvertTiledToLinearStandard4KB3D( - dst, src, static_cast(fmt), mip_width, mip_height, copy_depth, - mip_pitch, layout.slice_stride, dst_size, - volume_layout.level_sizes[level], false); - } - } - - if (mip_width > 1) { - mip_width /= 2; - } - if (mip_height > 1) { - mip_height /= 2; - } - if (mip_pitch > 1) { - mip_pitch /= 2; - } - } - } - Transfer::UploadImage(ctx, vk_obj, temp_buf.Data(), size, regions, dst_layout); - } else if (layout.fmt_tiled_standard4kb) { - LOGF("%s: detiling typed Standard4KB texture: fmt=%u tile=%u size=%" PRIu64 - " extent=%" PRIu64 "x%" PRIu64 " depth=%" PRIu64 " pitch=%u\n", - owner, static_cast(fmt), layout.tile, size, width, height, - static_cast(depth), layout.pitch); - Transfer::ScratchBuffer temp_buf(size); - for (uint32_t i = 0; i < levels; i++) { - for (uint32_t z = 0; z < depth; z++) { - const auto region_index = i * depth + z; - auto* dst = static_cast(temp_buf.Data()) + regions[region_index].offset; - const auto* src = static_cast(src_data) + - GetLevelSrcOffset(layout.level_sizes[i]) + - z * GetSliceSrcStride(layout, i, source_slice_layout); - TileConvertTiledToLinearStandard4KB( - dst, src, static_cast(fmt), regions[region_index].width, - regions[region_index].height, regions[region_index].pitch, - layout.level_sizes[i].size, GetLevelSrcSize(layout.level_sizes[i]), - layout.level_sizes[i].x, layout.level_sizes[i].y); - } - } - Transfer::UploadImage(ctx, vk_obj, temp_buf.Data(), size, regions, dst_layout); - } else if (layout.fmt_tiled_standard64kb) { - LOGF("%s: detiling typed Standard64KB texture: fmt=%u tile=%u size=%" PRIu64 - " extent=%" PRIu64 "x%" PRIu64 " depth=%" PRIu64 " pitch=%u\n", - owner, static_cast(fmt), layout.tile, size, width, height, - static_cast(depth), layout.pitch); - Transfer::ScratchBuffer temp_buf(size); - for (uint32_t i = 0; i < levels; i++) { - for (uint32_t z = 0; z < depth; z++) { - const auto region_index = i * depth + z; - auto* dst = static_cast(temp_buf.Data()) + regions[region_index].offset; - const auto* src = static_cast(src_data) + - GetLevelSrcOffset(layout.level_sizes[i]) + - z * GetSliceSrcStride(layout, i, source_slice_layout); - TileConvertTiledToLinearStandard64KB( - dst, src, static_cast(fmt), regions[region_index].width, - regions[region_index].height, regions[region_index].pitch, - layout.level_sizes[i].size, GetLevelSrcSize(layout.level_sizes[i]), - layout.level_sizes[i].x, layout.level_sizes[i].y); - } - } - Transfer::UploadImage(ctx, vk_obj, temp_buf.Data(), size, regions, dst_layout); - } else if (layout.tile != 0) { - EXIT("%s: typed tiled upload still unsupported after sizing, using linear fallback: fmt=%u " - "tile=%u size=%" PRIu64 " extent=%" PRIu64 "x%" PRIu64 " pitch=%u levels=%" PRIu64 - "\n", - owner, static_cast(fmt), layout.tile, size, width, height, layout.pitch, - levels); - Transfer::UploadImage(ctx, vk_obj, src_data, size, regions, dst_layout); } + if (static_cast(layout.tile) == Prospero::TileMode::kLinear) { + Transfer::UploadImage(ctx, vk_obj, src_data, size, regions, dst_layout); + return; + } + if (layout.tile_family == TileBlockFamily::Depth64KB && Prospero::IsFmaskTextureFormat(fmt)) { + UploadFmaskIdentity(ctx, vk_obj, regions, dst_layout, owner); + return; + } + + std::vector infos; + if (!TextureBuildGpuTileInfos(size, regions, layout, fmt, depth, levels, &infos)) { + EXIT("%s: GPU tiled upload unsupported: fmt=%u tile=%u size=%" PRIu64 " extent=%" PRIu64 + "x%" PRIu64 " depth=%u pitch=%u levels=%" PRIu64 "\n", + owner, fmt, layout.tile, size, width, height, depth, layout.pitch, levels); + } + Transfer::UploadTiledImage(ctx, vk_obj, src_data, size, size, infos, regions, dst_layout); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/objects/textureCommon.h b/src/graphics/host_gpu/objects/textureCommon.h index 38e3d68..777a267 100644 --- a/src/graphics/host_gpu/objects/textureCommon.h +++ b/src/graphics/host_gpu/objects/textureCommon.h @@ -44,8 +44,6 @@ constexpr bool TextureHasFormatUsage(TextureFormatUsage usage, TextureFormatUsag enum class TextureUploadDestination { MipLevels, MipAtlas }; -enum class TextureUploadSliceLayout { MipChainPerSlice, MipLevelPerSlice }; - struct RenderTargetFormatInfo { vk::Format format = vk::Format::eUndefined; uint32_t bytes_per_element = 0; @@ -53,18 +51,15 @@ struct RenderTargetFormatInfo { }; struct TextureUploadLayout { - uint32_t tile = 0; - uint32_t pitch = 0; - uint64_t slice_stride = 0; - uint64_t source_slice_stride = 0; - bool fmt_tiled_render_target = false; - bool fmt_tiled_standard256b = false; - bool fmt_tiled_standard4kb = false; - bool fmt_tiled_standard64kb = false; - bool fmt_tiled_depth = false; - bool volume_texture = false; - TileSizeOffset level_sizes[16] = {}; - TilePaddedSize padded_sizes[16] = {}; + uint32_t tile = 0; + uint32_t pitch = 0; + uint64_t slice_stride = 0; + uint64_t source_slice_stride = 0; + uint32_t first_tail_level = 16; + TileBlockFamily tile_family = TileBlockFamily::Count; + bool volume_texture = false; + TileSizeOffset level_sizes[16] = {}; + TilePaddedSize padded_sizes[16] = {}; }; struct TextureImageCreateParams { @@ -113,27 +108,28 @@ void TextureCreateImageViews(GraphicContext* ctx, VulkanImage* vk_obj, TextureUploadLayout TextureCalcUploadLayout(uint32_t fmt, uint64_t width, uint64_t height, uint64_t levels, uint32_t depth, uint64_t pitch, uint64_t tile, uint64_t upload_size, - bool allow_depth_tile, - bool require_single_mip_small_tiles, - bool volume_texture, const char* owner); + bool allow_depth_tile, bool volume_texture, + const char* owner); uint64_t TextureUploadSliceSourceOffset(const TextureUploadLayout& layout, uint32_t level, - uint32_t slice, - TextureUploadSliceLayout source_slice_layout); + uint32_t slice); uint64_t TextureCalcUploadSize(const TextureUploadLayout& layout, const std::vector& regions, uint64_t levels, - uint32_t depth, TextureUploadSliceLayout source_slice_layout); -std::vector TextureBuildUploadRegions( - const TextureUploadLayout& layout, vk::Format image_format, uint32_t width, uint32_t height, - uint32_t depth, uint64_t levels, bool array_texture, bool volume_texture, - TextureUploadDestination destination, TextureUploadSliceLayout slice_layout); -void TextureCopyBufferBytes(GraphicContext* ctx, VulkanBuffer* src_buffer, - uint64_t src_buffer_offset, uint64_t copy_size, - Transfer::ScratchBuffer* dst); + uint32_t depth); +std::vector TextureBuildUploadRegions(const TextureUploadLayout& layout, + vk::Format image_format, uint32_t width, + uint32_t height, uint32_t depth, + uint64_t levels, bool array_texture, + bool volume_texture, + TextureUploadDestination destination); +std::vector +TextureBuildDownloadRegions(const std::vector& upload_regions); +bool TextureBuildGpuTileInfos(uint64_t size, const std::vector& regions, + const TextureUploadLayout& layout, uint32_t fmt, uint32_t depth, + uint64_t levels, std::vector* infos); void TextureUploadGuestImage(GraphicContext* ctx, VulkanImage* vk_obj, const void* src_data, uint64_t size, const std::vector& regions, const TextureUploadLayout& layout, uint32_t fmt, uint64_t width, - uint64_t height, uint32_t depth, uint64_t levels, - TextureUploadSliceLayout source_slice_layout, const char* owner, + uint64_t height, uint32_t depth, uint64_t levels, const char* owner, vk::ImageLayout dst_layout); } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/image.cpp b/src/graphics/host_gpu/renderer/image.cpp index fa7d843..fdd8a22 100644 --- a/src/graphics/host_gpu/renderer/image.cpp +++ b/src/graphics/host_gpu/renderer/image.cpp @@ -4,6 +4,7 @@ #include "common/profiler.h" #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/gpuTiler.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/objects/textureCommon.h" #include "graphics/host_gpu/regionDefinitions.h" @@ -191,36 +192,39 @@ void UploadRenderTargetLayers(GraphicContext* ctx, RenderTextureVulkanImage* ima const auto format = RenderTargetTransferFormat(info.bytes_per_element); auto layout = TextureCalcUploadLayout(format, info.width, info.height, info.levels, layer_count, info.pitch, info.tile_mode, upload_size, - false, false, false, "TextureCache render target"); + false, false, "TextureCache render target"); const bool render_target_tiled = info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - if (!standard64 && ((render_target_tiled && !layout.fmt_tiled_render_target) || - layout.pitch != info.pitch)) { + if (!standard64 && + ((render_target_tiled && layout.tile_family != TileBlockFamily::RenderTarget64KB) || + layout.pitch != info.pitch)) { EXIT("TextureCache: unsupported render-target mip upload layout, pitch=%u/%u tile=%u\n", info.pitch, layout.pitch, info.tile_mode); } - auto regions = TextureBuildUploadRegions( - layout, info.format, info.width, info.height, layer_count, info.levels, true, false, - TextureUploadDestination::MipLevels, TextureUploadSliceLayout::MipChainPerSlice); + auto regions = TextureBuildUploadRegions(layout, info.format, info.width, info.height, + layer_count, info.levels, true, false, + TextureUploadDestination::MipLevels); for (auto& region: regions) { region.dst_layer += base_layer; } const auto source_address = info.address + slice_size * base_layer; TextureUploadGuestImage(ctx, image, reinterpret_cast(source_address), upload_size, regions, layout, format, info.width, info.height, - layer_count, info.levels, - TextureUploadSliceLayout::MipChainPerSlice, - "TextureCache render target", vk::ImageLayout::eGeneral); + layer_count, info.levels, "TextureCache render target", + vk::ImageLayout::eGeneral); return; } if (info.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && Transfer::GuestBufferIsTiled(info.address, slice_size)) { - Transfer::ScratchBuffer scratch(slice_size); - TileConvertTiledToLinearRenderTarget( - scratch.Data(), reinterpret_cast(info.address), info.width, info.height, - info.pitch, info.bytes_per_element, slice_size); - Transfer::UploadImage(ctx, image, scratch.Data(), slice_size, info.pitch, - vk::ImageLayout::eGeneral); + const auto format = RenderTargetTransferFormat(info.bytes_per_element); + auto layout = TextureCalcUploadLayout(format, info.width, info.height, 1, 1, info.pitch, + info.tile_mode, slice_size, false, false, + "TextureCache render target"); + auto regions = TextureBuildUploadRegions(layout, info.format, info.width, info.height, 1, 1, + true, false, TextureUploadDestination::MipLevels); + TextureUploadGuestImage(ctx, image, reinterpret_cast(info.address), slice_size, + regions, layout, format, info.width, info.height, 1, 1, + "TextureCache render target", vk::ImageLayout::eGeneral); } else { Transfer::UploadImage(ctx, image, reinterpret_cast(info.address), slice_size, info.pitch, vk::ImageLayout::eGeneral); @@ -383,20 +387,46 @@ void UploadVideoOut(GraphicContext* ctx, VideoOutVulkanImage* image, const Video Transfer::WaitForGraphicsIdle(ctx); } image->layout = vk::ImageLayout::eUndefined; - Transfer::ScratchBuffer scratch(info.size); - TileConvertTiledToLinearRenderTarget( - scratch.Data(), reinterpret_cast(info.address), info.width, info.height, - info.pitch, info.bytes_per_element, info.size); - if (info.bgra16) { - auto* pixels = static_cast(scratch.Data()); - for (uint64_t i = 0; i < info.size / sizeof(uint16_t); i += 4) { - std::swap(pixels[i], pixels[i + 2]); - } + if (!info.bgra16) { + auto layout = + TextureCalcUploadLayout(info.guest_format, info.width, info.height, 1, 1, info.pitch, + info.tile_mode, info.size, false, false, "VideoOut"); + auto regions = TextureBuildUploadRegions(layout, info.format, info.width, info.height, 1, 1, + false, false, TextureUploadDestination::MipLevels); + TextureUploadGuestImage(ctx, image, reinterpret_cast(info.address), info.size, + regions, layout, info.guest_format, info.width, info.height, 1, 1, + "VideoOut", vk::ImageLayout::eGeneral); + return; } + Transfer::ScratchBuffer scratch(info.size); + TileBlockLayout block {}; + EXIT_NOT_IMPLEMENTED( + !TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, info.bytes_per_element, &block)); + const GpuTileInfo tile_info {block.family, + block.bytes_per_element, + 0, + info.size, + 0, + info.size, + 0, + info.width, + info.height, + 1, + info.pitch}; + GpuDetile(ctx, reinterpret_cast(info.address), scratch.Data(), info.size, + info.size, std::span(&tile_info, 1)); + SwapVideoOutBgra16(scratch.Data(), info.size); Transfer::UploadImage(ctx, image, scratch.Data(), info.size, info.pitch, vk::ImageLayout::eGeneral); } +void SwapVideoOutBgra16(void* data, uint64_t size) { + auto* pixels = static_cast(data); + for (uint64_t i = 0; i < size / sizeof(uint16_t); i += 4) { + std::swap(pixels[i], pixels[i + 2]); + } +} + GpuTextureVulkanImage* CreateDummyTexture(GraphicContext* ctx, bool uint_format, bool image_3d, bool storage) { auto* image = storage ? static_cast(new StorageTextureVulkanImage) diff --git a/src/graphics/host_gpu/renderer/image.h b/src/graphics/host_gpu/renderer/image.h index 05922fe..2b044b6 100644 --- a/src/graphics/host_gpu/renderer/image.h +++ b/src/graphics/host_gpu/renderer/image.h @@ -119,6 +119,7 @@ void UploadRenderTarget(GraphicContext* ctx, RenderTextureVulkanImage* image, void ValidateVideoOut(GraphicContext* ctx, const VideoOutInfo& info); [[nodiscard]] VideoOutVulkanImage* CreateVideoOut(GraphicContext* ctx, const VideoOutInfo& info); +void SwapVideoOutBgra16(void* data, uint64_t size); void UploadVideoOut(GraphicContext* ctx, VideoOutVulkanImage* image, const VideoOutInfo& info, bool refresh); diff --git a/src/graphics/host_gpu/renderer/textureCache.cpp b/src/graphics/host_gpu/renderer/textureCache.cpp index cdc2161..a362888 100644 --- a/src/graphics/host_gpu/renderer/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/textureCache.cpp @@ -7,6 +7,7 @@ #include "graphics/guest_gpu/gpu_format.h" #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/gpuTiler.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/objects/label.h" #include "graphics/host_gpu/objects/textureCommon.h" @@ -587,7 +588,6 @@ struct TextureCache::ReadbackWorker { cached.image->extent.height, info.width, info.height, meta_overlap, buffer_overlap); } - download.resize(transfer_size); auto regions = Transfer::MakeLayeredImageBufferCopies(info.layers, depth_slice_size, info.pitch, info.width, info.height, vk::ImageAspectFlagBits::eDepth); @@ -600,17 +600,55 @@ struct TextureCache::ReadbackWorker { } regions.insert(regions.end(), stencil_regions.begin(), stencil_regions.end()); } - Transfer::DownloadImage(cached.ctx, download.data(), transfer_size, regions, cached.image, - cached.image->layout); - guest.resize(info.size); - cache.m_tiler.TileImage(guest.data(), download.data(), info); + std::vector gpu_infos; + TileBlockLayout depth_block {}; + TileBlockLayout stencil_block {}; + EXIT_NOT_IMPLEMENTED( + !TileGetBlockLayout(TileBlockFamily::Depth64KB, info.bytes_per_element, &depth_block) || + (has_stencil && !TileGetBlockLayout(TileBlockFamily::Depth64KB, 1, &stencil_block))); + gpu_infos.reserve(regions.size()); + for (uint32_t layer = 0; layer < info.layers; layer++) { + const uint64_t offset = depth_slice_size * layer; + GpuTileInfo tile {depth_block.family, + depth_block.bytes_per_element, + offset, + depth_slice_size, + offset, + depth_slice_size, + 0, + info.width, + info.height, + 1, + info.pitch}; + tile.surface_z = layer; + gpu_infos.push_back(tile); + } + if (has_stencil) { + for (uint32_t layer = 0; layer < info.layers; layer++) { + const uint64_t offset = info.size + stencil_slice_size * layer; + GpuTileInfo tile {stencil_block.family, + stencil_block.bytes_per_element, + offset, + stencil_slice_size, + offset, + stencil_slice_size, + 0, + info.width, + info.height, + 1, + expected_stencil_pitch}; + tile.surface_z = layer; + gpu_infos.push_back(tile); + } + } + guest.resize(transfer_size); + Transfer::DownloadTiledImage(cached.ctx, guest.data(), transfer_size, transfer_size, + gpu_infos, regions, cached.image, cached.image->layout); Libs::LibKernel::Memory::WriteBacking(info.address, guest.data(), info.size); ReadbackTransfer transfer; transfer.Add(info.address, info.size); if (has_stencil) { - guest.resize(info.stencil_size); - cache.m_tiler.TileStencil(guest.data(), download.data() + info.size, info); - Libs::LibKernel::Memory::WriteBacking(info.stencil_address, guest.data(), + Libs::LibKernel::Memory::WriteBacking(info.stencil_address, guest.data() + info.size, info.stencil_size); transfer.Add(info.stencil_address, info.stencil_size); } @@ -672,66 +710,48 @@ struct TextureCache::ReadbackWorker { info.address, info.size, meta_overlap, buffer_overlap, static_cast(cached.kind)); } - download.resize(info.size); - std::fill(download.begin(), download.end(), 0); std::vector regions; - if (target_mip_chain) { - const auto format = ImageOps::RenderTargetTransferFormat(info.bytes_per_element); - auto layout = TextureCalcUploadLayout(format, info.width, info.height, info.levels, 1, - info.pitch, info.tile_mode, info.size, false, - false, false, "RenderTargetReadback"); - if (!layout.fmt_tiled_render_target || layout.pitch != info.pitch) { + std::vector tiled_regions; + TextureUploadLayout tiled_layout {}; + const bool gpu_tiled = tiled_target || tiled_storage; + if (gpu_tiled) { + const auto format = storage + ? cached.info.format + : ImageOps::RenderTargetTransferFormat(info.bytes_per_element); + tiled_layout = TextureCalcUploadLayout(format, info.width, info.height, info.levels, + layers, info.pitch, info.tile_mode, info.size, + false, false, "RenderTargetReadback"); + if (target_mip_chain && + (tiled_layout.tile_family != TileBlockFamily::RenderTarget64KB || + tiled_layout.pitch != info.pitch)) { EXIT("TextureCache: inconsistent render-target readback layout, addr=0x%016" PRIx64 " size=0x%016" PRIx64 " pitch=%u/%u levels=%u\n", - info.address, info.size, info.pitch, layout.pitch, info.levels); - } - const auto uploads = TextureBuildUploadRegions( - layout, info.format, info.width, info.height, 1, info.levels, true, false, - TextureUploadDestination::MipLevels, TextureUploadSliceLayout::MipChainPerSlice); - regions.reserve(uploads.size()); - for (const auto& upload: uploads) { - regions.push_back({upload.offset, upload.pitch, upload.dst_level, upload.width, - upload.height, upload.copy_height, upload.dst_layer, - upload.dst_x, upload.dst_y, upload.dst_z, upload.aspect}); + info.address, info.size, info.pitch, tiled_layout.pitch, info.levels); } + tiled_regions = TextureBuildUploadRegions(tiled_layout, info.format, info.width, + info.height, layers, info.levels, layers > 1, + false, TextureUploadDestination::MipLevels); + regions = TextureBuildDownloadRegions(tiled_regions); } else { regions = Transfer::MakeLayeredImageBufferCopies(layers, slice_size, info.pitch, info.width, info.height); } - Transfer::DownloadImage(cached.ctx, download.data(), info.size, regions, cached.image, - cached.image->layout); - if (target_mip_chain) { + if (gpu_tiled) { + std::vector infos; + const auto format = storage + ? cached.info.format + : ImageOps::RenderTargetTransferFormat(info.bytes_per_element); guest.resize(info.size); - ImageInfo layout {}; - layout.address = info.address; - layout.size = info.size; - layout.format = ImageOps::RenderTargetTransferFormat(info.bytes_per_element); - layout.width = info.width; - layout.height = info.height; - layout.pitch = info.pitch; - layout.levels = info.levels; - layout.view_levels = info.levels; - layout.tile = info.tile_mode; - layout.depth = 1; - layout.type = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); - cache.m_tiler.TileImage(guest.data(), download.data(), layout); - Libs::LibKernel::Memory::WriteBacking(info.address, guest.data(), info.size); - } else if (tiled_target || tiled_storage) { - guest.resize(info.size); - const RenderTargetInfo layout = - target ? cached.target : RenderTargetInfo {info.address, - info.size, - info.format, - info.width, - info.height, - info.pitch, - info.bytes_per_element, - info.tile_mode, - info.levels, - 1}; - cache.m_tiler.TileImage(guest.data(), download.data(), layout); + EXIT_NOT_IMPLEMENTED(!TextureBuildGpuTileInfos(info.size, tiled_regions, tiled_layout, + format, layers, info.levels, &infos)); + Transfer::DownloadTiledImage(cached.ctx, guest.data(), info.size, info.size, infos, + regions, cached.image, cached.image->layout); Libs::LibKernel::Memory::WriteBacking(info.address, guest.data(), info.size); } else { + download.resize(info.size); + std::fill(download.begin(), download.end(), 0); + Transfer::DownloadImage(cached.ctx, download.data(), info.size, regions, cached.image, + cached.image->layout); Libs::LibKernel::Memory::WriteBacking(info.address, download.data(), info.size); } ReadbackTransfer transfer; @@ -1061,8 +1081,7 @@ bool Equal(const DepthTargetInfo& left, const DepthTargetInfo& right) { [[nodiscard]] bool IsCoherentGuestImageSource(const BufferImageCopySource& source, uint64_t address, uint64_t size) { - // The current PS5 Tiler consumes coherent guest backing directly. A native buffer is an - // optional future GPU-detiler source or staging fallback. + // The GPU tiler currently consumes coherent guest backing directly. return source.cpu_current && source.address == address && source.size == size && (source.buffer != nullptr || source.offset == 0); } @@ -1384,8 +1403,7 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c } BufferImageCopySource source {nullptr, 0, info.address, info.size, true}; if (m_buffer_cache.HasPageOverlap(info.address, info.size)) { - // ObtainBufferForImage publishes dirty native-buffer bytes when necessary and otherwise - // uses a CPU-current staging fallback through guest backing. + // ObtainBufferForImage publishes dirty native-buffer bytes into coherent guest backing. source = m_buffer_cache.ObtainBufferForImage(info.address, info.size); if (!IsCoherentGuestImageSource(source, info.address, info.size)) { EXIT("TextureCache: sampled-image buffer source is inconsistent, addr=0x%016" PRIx64 @@ -2963,6 +2981,7 @@ void TextureCache::SynchronizeColorImageToBufferLocked(CachedImage& cached, uint } const bool linear = target.tile_mode == Prospero::GpuEnumValue(Prospero::TileMode::kLinear); const bool tiled = IsTiledRenderTarget(target); + const bool bgra16 = video_out && cached.video_out.bgra16; TileSizeAlign exact {}; bool single_slice = false; if (IsSupportedStandard64RenderTarget(target)) { @@ -3014,47 +3033,72 @@ void TextureCache::SynchronizeColorImageToBufferLocked(CachedImage& cached, uint target.address, target.size); } - // This is the CPU Tiler backend for the image-to-buffer synchronization seam. - // The guest vector and Vulkan staging allocation retain capacity; a future PS5 GPU tiler can - // replace this block without changing alias classification or ownership transitions. Transfer::WaitForGraphicsIdle(cached.ctx); std::vector regions; + std::vector tiled_regions; + TextureUploadLayout tiled_layout {}; + uint32_t tiled_format = 0; if (storage) { - auto layout = TextureCalcUploadLayout( + const bool array_texture = TextureIsLayeredTexture(cached.info.type); + const bool volume_texture = TextureIs3DTexture(cached.info.type); + tiled_format = cached.info.format; + tiled_layout = TextureCalcUploadLayout( cached.info.format, cached.info.width, cached.info.height, cached.info.levels, - cached.info.depth, cached.info.pitch, cached.info.tile, cached.info.size, true, false, - false, "StorageTextureReadback"); - auto uploads = TextureBuildUploadRegions( - layout, cached.image->format, cached.info.width, cached.info.height, cached.info.depth, - cached.info.levels, false, false, TextureUploadDestination::MipLevels, - TextureUploadSliceLayout::MipChainPerSlice); - regions.reserve(uploads.size()); - for (const auto& upload: uploads) { - regions.push_back({upload.offset, upload.pitch, upload.dst_level, upload.width, - upload.height, upload.copy_height, upload.dst_layer, upload.dst_x, - upload.dst_y, upload.dst_z, upload.aspect}); - } + cached.info.depth, cached.info.pitch, cached.info.tile, cached.info.size, true, + volume_texture, "StorageTextureReadback"); + tiled_regions = TextureBuildUploadRegions( + tiled_layout, cached.image->format, cached.info.width, cached.info.height, + cached.info.depth, cached.info.levels, array_texture, volume_texture, + TextureUploadDestination::MipLevels); + regions = TextureBuildDownloadRegions(tiled_regions); + } else if (tiled && !bgra16) { + tiled_format = ImageOps::RenderTargetTransferFormat(target.bytes_per_element); + tiled_layout = TextureCalcUploadLayout( + tiled_format, target.width, target.height, target.levels, target.layers, target.pitch, + target.tile_mode, target.size, false, false, "ColorBufferTransition"); + tiled_regions = TextureBuildUploadRegions( + tiled_layout, target.format, target.width, target.height, target.layers, target.levels, + target.layers > 1, false, TextureUploadDestination::MipLevels); + regions = TextureBuildDownloadRegions(tiled_regions); } else { regions = Transfer::MakeLayeredImageBufferCopies(target.layers, slice_size, target.pitch, target.width, target.height); } - Transfer::ProcessDownloadedImage( - cached.ctx, target.size, regions, cached.image, cached.image->layout, - [&](std::span linear) { - if (storage) { - m_buffer_transition_guest.resize(target.size); - m_tiler.TileImage(m_buffer_transition_guest.data(), linear.data(), cached.info); - Libs::LibKernel::Memory::WriteBacking( - target.address, m_buffer_transition_guest.data(), target.size); - } else if (tiled) { - m_buffer_transition_guest.resize(target.size); - m_tiler.TileImage(m_buffer_transition_guest.data(), linear.data(), target); - Libs::LibKernel::Memory::WriteBacking( - target.address, m_buffer_transition_guest.data(), target.size); - } else { - Libs::LibKernel::Memory::WriteBacking(target.address, linear.data(), target.size); - } - }); + if (tiled && !bgra16) { + std::vector infos; + const uint32_t depth = storage ? cached.info.depth : target.layers; + EXIT_NOT_IMPLEMENTED(!TextureBuildGpuTileInfos(target.size, tiled_regions, tiled_layout, + tiled_format, depth, target.levels, &infos)); + m_buffer_transition_guest.resize(target.size); + Transfer::DownloadTiledImage(cached.ctx, m_buffer_transition_guest.data(), target.size, + target.size, infos, regions, cached.image, + cached.image->layout); + Libs::LibKernel::Memory::WriteBacking(target.address, m_buffer_transition_guest.data(), + target.size); + } else if (tiled) { + std::vector linear_data(target.size); + Transfer::DownloadImage(cached.ctx, linear_data.data(), target.size, regions, cached.image, + cached.image->layout); + ImageOps::SwapVideoOutBgra16(linear_data.data(), target.size); + TileBlockLayout block {}; + EXIT_NOT_IMPLEMENTED(!TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, + target.bytes_per_element, &block)); + const GpuTileInfo info { + block.family, block.bytes_per_element, 0, target.size, 0, target.size, 0, + target.width, target.height, 1, target.pitch}; + m_buffer_transition_guest.resize(target.size); + GpuTile(cached.ctx, linear_data.data(), m_buffer_transition_guest.data(), target.size, + target.size, std::span(&info, 1)); + Libs::LibKernel::Memory::WriteBacking(target.address, m_buffer_transition_guest.data(), + target.size); + } else { + Transfer::ProcessDownloadedImage(cached.ctx, target.size, regions, cached.image, + cached.image->layout, + [&](std::span linear_data) { + Libs::LibKernel::Memory::WriteBacking( + target.address, linear_data.data(), target.size); + }); + } m_memory_tracker.ForEachDownloadRange(target.address, target.size, [](uint64_t, uint64_t) noexcept {}); // Only the impending buffer-write range needs publication into BufferCache ownership. The @@ -3103,14 +3147,26 @@ void TextureCache::SynchronizeDepthImageToBufferLocked(CachedImage& cached, uint Transfer::WaitForGraphicsIdle(cached.ctx); const auto regions = Transfer::MakeLayeredImageBufferCopies( 1, info.size, info.pitch, info.width, info.height, vk::ImageAspectFlagBits::eDepth); - Transfer::ProcessDownloadedImage( - cached.ctx, info.size, regions, cached.image, cached.image->layout, - [&](std::span linear) { - m_buffer_transition_guest.resize(info.size); - m_tiler.TileImage(m_buffer_transition_guest.data(), linear.data(), info); - Libs::LibKernel::Memory::WriteBacking(info.address, m_buffer_transition_guest.data(), - info.size); - }); + TileBlockLayout block {}; + EXIT_NOT_IMPLEMENTED( + !TileGetBlockLayout(TileBlockFamily::Depth64KB, info.bytes_per_element, &block)); + const GpuTileInfo tile_info {block.family, + block.bytes_per_element, + 0, + info.size, + 0, + info.size, + 0, + info.width, + info.height, + 1, + info.pitch}; + m_buffer_transition_guest.resize(info.size); + Transfer::DownloadTiledImage(cached.ctx, m_buffer_transition_guest.data(), info.size, info.size, + std::span(&tile_info, 1), regions, cached.image, + cached.image->layout); + Libs::LibKernel::Memory::WriteBacking(info.address, m_buffer_transition_guest.data(), + info.size); m_memory_tracker.ForEachDownloadRange(info.address, info.size, [](uint64_t, uint64_t) noexcept {}); m_buffer_cache.PublishImageBacking(write_address, write_size); diff --git a/src/graphics/host_gpu/renderer/tiler.cpp b/src/graphics/host_gpu/renderer/tiler.cpp index a00048f..c238a0f 100644 --- a/src/graphics/host_gpu/renderer/tiler.cpp +++ b/src/graphics/host_gpu/renderer/tiler.cpp @@ -4,86 +4,132 @@ #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/gpu_format.h" #include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/gpuTiler.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/objects/textureCommon.h" #include "graphics/host_gpu/renderer/image.h" #include "graphics/host_gpu/transfer.h" -#include "graphics/host_gpu/vulkanCommon.h" #include namespace Libs::Graphics { +namespace { + +struct DepthTransfer { + std::vector infos; + std::vector regions; +}; + +DepthTransfer MakeDepthTransfer(uint64_t size, uint32_t layers, uint32_t format, + uint32_t bytes_per_element, uint32_t width, uint32_t height, + uint32_t pitch, uint32_t base_layer, vk::ImageAspectFlags aspect) { + EXIT_IF(size == 0 || layers == 0 || size % layers != 0); + TileBlockLayout block {}; + EXIT_NOT_IMPLEMENTED( + !TileGetBlockLayout(TileBlockFamily::Depth64KB, bytes_per_element, &block) || + Prospero::NumBytesPerElement(format) != bytes_per_element); + + const uint64_t slice_size = size / layers; + DepthTransfer transfer; + transfer.infos.reserve(layers); + transfer.regions.reserve(layers); + for (uint32_t layer = 0; layer < layers; layer++) { + const uint64_t offset = slice_size * layer; + GpuTileInfo info {block.family, + block.bytes_per_element, + offset, + slice_size, + offset, + slice_size, + 0, + width, + height, + 1, + pitch}; + info.surface_z = base_layer + layer; + transfer.infos.push_back(info); + + BufferImageCopy region {}; + region.offset = static_cast(offset); + region.pitch = pitch; + region.width = width; + region.height = height; + region.dst_layer = base_layer + layer; + region.aspect = aspect; + transfer.regions.push_back(region); + } + return transfer; +} + +void UploadDepth(GraphicContext* ctx, DepthStencilVulkanImage* image, uint64_t source_address, + uint64_t size, uint32_t layers, uint32_t format, uint32_t bytes_per_element, + uint32_t width, uint32_t height, uint32_t pitch, uint32_t base_layer, + vk::ImageAspectFlags aspect) { + auto transfer = MakeDepthTransfer(size, layers, format, bytes_per_element, width, height, pitch, + base_layer, aspect); + Transfer::UploadTiledImage(ctx, image, reinterpret_cast(source_address), size, + size, transfer.infos, transfer.regions, + vk::ImageLayout::eDepthStencilAttachmentOptimal); +} template -static void UploadPromotedD16Depth(GraphicContext* ctx, DepthStencilVulkanImage* image, - const DepthTargetInfo& info, const BufferImageCopySource& source, - uint32_t base_layer) { - const uint64_t slice_size = info.size / info.layers; +void UploadPromotedD16Depth(GraphicContext* ctx, DepthStencilVulkanImage* image, + const DepthTargetInfo& info, const BufferImageCopySource& source, + uint32_t base_layer) { + const uint64_t guest_slice_size = info.size / info.layers; const uint64_t texels = static_cast(info.pitch) * info.height; const uint64_t host_slice_size = texels * sizeof(uint32_t); const uint64_t host_upload_size = host_slice_size * info.layers; - if (host_upload_size > UINT32_MAX) { - EXIT("Tiler: invalid D16 host-promotion footprint, guest_slice=0x%016" PRIx64 - " host_slice=0x%016" PRIx64 " layers=%u\n", - slice_size, host_slice_size, info.layers); - } - Transfer::ScratchBuffer host_linear(host_upload_size); - std::vector guest_linear(slice_size / sizeof(uint16_t)); - std::vector regions; - regions.reserve(info.layers); + EXIT_IF(host_upload_size > UINT32_MAX); + + auto transfer = MakeDepthTransfer(info.size, info.layers, info.guest_format, + info.bytes_per_element, info.width, info.height, info.pitch, + base_layer, vk::ImageAspectFlagBits::eDepth); + std::vector guest_linear(info.size / sizeof(uint16_t)); + GpuDetile(ctx, reinterpret_cast(source.address), guest_linear.data(), info.size, + info.size, transfer.infos); + + Transfer::ScratchBuffer host_linear(host_upload_size); for (uint32_t layer = 0; layer < info.layers; layer++) { - auto* guest_slice = reinterpret_cast(source.address) + slice_size * layer; - TileConvertTiledToLinearDepth(guest_linear.data(), guest_slice, info.guest_format, - info.width, info.height, info.pitch, slice_size); - auto* host_slice = reinterpret_cast(static_cast(host_linear.Data()) + - host_slice_size * layer); + const auto* guest = guest_linear.data() + guest_slice_size / sizeof(uint16_t) * layer; + auto* host = reinterpret_cast(static_cast(host_linear.Data()) + + host_slice_size * layer); for (uint64_t texel = 0; texel < texels; texel++) { - host_slice[texel] = Encode(guest_linear[texel]); + host[texel] = Encode(guest[texel]); } - BufferImageCopy region {}; - region.offset = static_cast(host_slice_size * layer); - region.pitch = info.pitch; - region.width = info.width; - region.height = info.height; - region.dst_layer = base_layer + layer; - region.aspect = vk::ImageAspectFlagBits::eDepth; - regions.push_back(region); + transfer.regions[layer].offset = static_cast(host_slice_size * layer); } - Transfer::UploadImage(ctx, image, host_linear.Data(), host_upload_size, regions, + Transfer::UploadImage(ctx, image, host_linear.Data(), host_upload_size, transfer.regions, vk::ImageLayout::eDepthStencilAttachmentOptimal); } +} // namespace + void Tiler::DetileImage(GraphicContext* ctx, GpuTextureVulkanImage* image, const ImageInfo& info, const BufferImageCopySource& source, bool refresh, bool storage) const { - if (refresh) { - Transfer::WaitForGraphicsIdle(ctx); - } + if (refresh) Transfer::WaitForGraphicsIdle(ctx); + const bool array_texture = TextureIsLayeredTexture(info.type); const bool volume_texture = TextureIs3DTexture(info.type); auto layout = TextureCalcUploadLayout( info.format, info.width, info.height, info.levels, info.depth, info.pitch, info.tile, - info.size, true, false, volume_texture, storage ? "StorageTextureCache" : "TextureCache"); - const auto slice_layout = TextureUploadSliceLayout::MipChainPerSlice; + info.size, true, volume_texture, storage ? "StorageTextureCache" : "TextureCache"); auto regions = TextureBuildUploadRegions(layout, image->format, info.width, info.height, info.depth, info.levels, array_texture, volume_texture, - TextureUploadDestination::MipLevels, slice_layout); - TextureUploadGuestImage( - ctx, image, reinterpret_cast(source.address), info.size, regions, layout, - info.format, info.width, info.height, info.depth, info.levels, slice_layout, - storage ? "StorageTextureCache" : "TextureCache", - storage ? vk::ImageLayout::eGeneral : vk::ImageLayout::eShaderReadOnlyOptimal); + TextureUploadDestination::MipLevels); + TextureUploadGuestImage(ctx, image, reinterpret_cast(source.address), info.size, + regions, layout, info.format, info.width, info.height, info.depth, + info.levels, storage ? "StorageTextureCache" : "TextureCache", + storage ? vk::ImageLayout::eGeneral + : vk::ImageLayout::eShaderReadOnlyOptimal); } void Tiler::DetileImage(GraphicContext* ctx, DepthStencilVulkanImage* image, const DepthTargetInfo& info, const BufferImageCopySource& source, bool refresh, uint32_t base_layer) const { - if (info.samples != 1 || image == nullptr || image->samples != 1) { - EXIT("Tiler: multisampled depth upload is unsupported, samples=%u/%u\n", info.samples, - image != nullptr ? image->samples : 0); - } - if (refresh) { - Transfer::WaitForGraphicsIdle(ctx); - } + EXIT_NOT_IMPLEMENTED(info.samples != 1 || image->samples != 1); + if (refresh) Transfer::WaitForGraphicsIdle(ctx); + if (DepthAspectTransferBytes(info.format) != info.bytes_per_element) { switch (info.format) { case vk::Format::eD24UnormS8Uint: @@ -92,165 +138,25 @@ void Tiler::DetileImage(GraphicContext* ctx, DepthStencilVulkanImage* image, case vk::Format::eD32SfloatS8Uint: UploadPromotedD16Depth(ctx, image, info, source, base_layer); return; - default: - EXIT("Tiler: unsupported depth transfer conversion, format=%d guest_bpe=%u\n", - static_cast(info.format), info.bytes_per_element); + default: EXIT_NOT_IMPLEMENTED(true); } } - const auto slice_size = info.size / info.layers; - Transfer::ScratchBuffer linear(info.size); - std::vector regions; - regions.reserve(info.layers); - for (uint32_t layer = 0; layer < info.layers; layer++) { - auto* linear_slice = static_cast(linear.Data()) + slice_size * layer; - auto* guest_slice = reinterpret_cast(source.address) + slice_size * layer; - TileConvertTiledToLinearDepth(linear_slice, guest_slice, info.guest_format, info.width, - info.height, info.pitch, slice_size); - BufferImageCopy region {}; - region.offset = static_cast(slice_size * layer); - region.pitch = info.pitch; - region.width = info.width; - region.height = info.height; - region.dst_layer = base_layer + layer; - region.aspect = vk::ImageAspectFlagBits::eDepth; - regions.push_back(region); - } - Transfer::UploadImage(ctx, image, linear.Data(), info.size, regions, - vk::ImageLayout::eDepthStencilAttachmentOptimal); + UploadDepth(ctx, image, source.address, info.size, info.layers, info.guest_format, + info.bytes_per_element, info.width, info.height, info.pitch, base_layer, + vk::ImageAspectFlagBits::eDepth); } void Tiler::DetileStencil(GraphicContext* ctx, DepthStencilVulkanImage* image, const DepthTargetInfo& info, const BufferImageCopySource& source, bool refresh, uint32_t base_layer) const { - if (info.samples != 1 || image == nullptr || image->samples != 1) { - EXIT("Tiler: multisampled stencil upload is unsupported, samples=%u/%u\n", info.samples, - image != nullptr ? image->samples : 0); - } - const auto stencil_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt); - const auto stencil_pitch = TileGetTexturePitch( - stencil_format, info.width, 1, Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); - if (refresh) { - Transfer::WaitForGraphicsIdle(ctx); - } - const auto slice_size = info.stencil_size / info.layers; - Transfer::ScratchBuffer linear(info.stencil_size); - std::vector regions; - regions.reserve(info.layers); - for (uint32_t layer = 0; layer < info.layers; layer++) { - auto* linear_slice = static_cast(linear.Data()) + slice_size * layer; - auto* guest_slice = reinterpret_cast(source.address) + slice_size * layer; - TileConvertTiledToLinearDepth(linear_slice, guest_slice, stencil_format, info.width, - info.height, stencil_pitch, slice_size); - BufferImageCopy region {}; - region.offset = static_cast(slice_size * layer); - region.pitch = stencil_pitch; - region.width = info.width; - region.height = info.height; - region.dst_layer = base_layer + layer; - region.aspect = vk::ImageAspectFlagBits::eStencil; - regions.push_back(region); - } - Transfer::UploadImage(ctx, image, linear.Data(), info.stencil_size, regions, - vk::ImageLayout::eDepthStencilAttachmentOptimal); -} + EXIT_NOT_IMPLEMENTED(info.samples != 1 || image->samples != 1); + if (refresh) Transfer::WaitForGraphicsIdle(ctx); -void Tiler::TileImage(void* dst, const void* src, const RenderTargetInfo& info) const { - const bool standard64 = IsSupportedStandard64RenderTarget(info); - if ((info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && - !standard64) || - info.levels != 1 || info.samples != 1) { - EXIT("Tiler: unsupported render-target tile, dst=%p src=%p " - "addr=0x%016" PRIx64 "+0x%016" PRIx64 - " extent=%ux%u pitch=%u levels=%u tile=%u bpe=%u\n", - dst, src, info.address, info.size, info.width, info.height, info.pitch, info.levels, - info.tile_mode, info.bytes_per_element); - } - const auto slice_size = info.size / info.layers; - for (uint32_t layer = 0; layer < info.layers; layer++) { - auto* guest_slice = static_cast(dst) + slice_size * layer; - auto* linear_slice = static_cast(src) + slice_size * layer; - if (standard64) { - TileConvertLinearToTiledStandard64KB32(guest_slice, linear_slice, info.width, - info.height, info.pitch, slice_size); - } else { - TileConvertLinearToTiledRenderTarget(guest_slice, linear_slice, info.width, info.height, - info.pitch, info.bytes_per_element, slice_size); - } - } -} - -void Tiler::TileImage(void* dst, const void* src, const ImageInfo& info) const { - const bool image_2d = - info.type == Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) && info.depth == 1; - const auto bytes_per_element = Prospero::RenderTargetBytesPerElement(info.format); - if (!image_2d || info.levels == 0 || info.levels > 16 || - info.tile != Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) || - bytes_per_element == 0) { - EXIT("Tiler: unsupported storage-texture tile, addr=0x%016" PRIx64 "+0x%016" PRIx64 - " extent=%ux%ux%u levels=%u tile=%u format=%u\n", - info.address, info.size, info.width, info.height, info.depth, info.levels, info.tile, - info.format); - } - auto layout = TextureCalcUploadLayout(info.format, info.width, info.height, info.levels, - info.depth, info.pitch, info.tile, info.size, true, false, - false, "StorageTextureReadback"); - auto regions = TextureBuildUploadRegions( - layout, VulkanFormat(info.format), info.width, info.height, info.depth, info.levels, false, - false, TextureUploadDestination::MipLevels, TextureUploadSliceLayout::MipChainPerSlice); - std::memset(dst, 0, info.size); - for (uint32_t level = 0; level < info.levels; level++) { - const auto& level_size = layout.level_sizes[level]; - const auto guest_offset = - level_size.src_size != 0 ? level_size.src_offset : level_size.offset; - auto* guest = static_cast(dst) + guest_offset; - const auto* linear = static_cast(src) + regions[level].offset; - TileConvertLinearToTiledRenderTarget( - guest, linear, regions[level].width, regions[level].height, regions[level].pitch, - bytes_per_element, level_size.size, level_size.src_size, level_size.x, level_size.y); - } -} - -void Tiler::TileImage(void* dst, const void* src, const DepthTargetInfo& info) const { - if (info.samples != 1 || info.tile_mode != Prospero::GpuEnumValue(Prospero::TileMode::kDepth) || - !IsSupportedDepthReadbackFormat(info)) { - EXIT("Tiler: unsupported depth-target tile, dst=%p src=%p " - "depth=0x%016" PRIx64 "+0x%016" PRIx64 " stencil=0x%016" PRIx64 "+0x%016" PRIx64 - " extent=%ux%u pitch=%u tile=%u format=%d guest_format=%u bpe=%u\n", - dst, src, info.address, info.size, info.stencil_address, info.stencil_size, info.width, - info.height, info.pitch, info.tile_mode, static_cast(info.format), - info.guest_format, info.bytes_per_element); - } - const auto slice_size = info.size / info.layers; - std::memset(dst, 0, info.size); - for (uint32_t layer = 0; layer < info.layers; layer++) { - auto* guest_slice = static_cast(dst) + slice_size * layer; - auto* linear_slice = static_cast(src) + slice_size * layer; - TileConvertLinearToTiledDepth(guest_slice, linear_slice, info.guest_format, info.width, - info.height, info.pitch, slice_size); - } -} - -void Tiler::TileStencil(void* dst, const void* src, const DepthTargetInfo& info) const { const auto format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt); const auto pitch = TileGetTexturePitch(format, info.width, 1, Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); - if (info.samples != 1 || info.stencil_address == 0 || info.stencil_size == 0 || - info.layers == 0 || info.stencil_size % info.layers != 0 || - !IsSupportedDepthReadbackFormat(info)) { - EXIT("Tiler: unsupported stencil-target tile, dst=%p src=%p " - "stencil=0x%016" PRIx64 "+0x%016" PRIx64 - " extent=%ux%u pitch=%u layers=%u format=%d compressed=%d\n", - dst, src, info.stencil_address, info.stencil_size, info.width, info.height, pitch, - info.layers, static_cast(info.format), info.stencil_htile_compressed); - } - const auto slice_size = info.stencil_size / info.layers; - std::memset(dst, 0, info.stencil_size); - for (uint32_t layer = 0; layer < info.layers; layer++) { - auto* guest_slice = static_cast(dst) + slice_size * layer; - auto* linear_slice = static_cast(src) + slice_size * layer; - TileConvertLinearToTiledDepth(guest_slice, linear_slice, format, info.width, info.height, - pitch, slice_size); - } + UploadDepth(ctx, image, source.address, info.stencil_size, info.layers, format, 1, info.width, + info.height, pitch, base_layer, vk::ImageAspectFlagBits::eStencil); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/tiler.h b/src/graphics/host_gpu/renderer/tiler.h index 29bfde9..11776e3 100644 --- a/src/graphics/host_gpu/renderer/tiler.h +++ b/src/graphics/host_gpu/renderer/tiler.h @@ -24,10 +24,6 @@ public: void DetileStencil(GraphicContext* ctx, DepthStencilVulkanImage* image, const DepthTargetInfo& info, const BufferImageCopySource& source, bool refresh, uint32_t base_layer = 0) const; - void TileImage(void* dst, const void* src, const RenderTargetInfo& info) const; - void TileImage(void* dst, const void* src, const ImageInfo& info) const; - void TileImage(void* dst, const void* src, const DepthTargetInfo& info) const; - void TileStencil(void* dst, const void* src, const DepthTargetInfo& info) const; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_common.inc b/src/graphics/host_gpu/shaders/gpu_tiler_common.inc new file mode 100644 index 0000000..ad8929d --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_common.inc @@ -0,0 +1,70 @@ +layout(local_size_x = 64) in; + +layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } input_buffer; +layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } output_buffer; + +layout(push_constant) uniform Push { + uint src_base; + uint dst_base; + uint width; + uint height; + uint depth; + uint surface_z; + uint pitch_bytes; + uint slice_bytes; + uint blocks_per_row; + uint blocks_per_slice; + uint tail_x; + uint tail_y; + uint tail; + uint first; + uint count; +} params; + +void copy_element(uint src, uint dst) { + if (ELEMENT_BYTES >= 4) { + for (uint i = 0; i < ELEMENT_BYTES; i += 4) { + output_buffer.data[(dst + i) >> 2] = input_buffer.data[(src + i) >> 2]; + } + } else { + uint mask = ELEMENT_BYTES == 1 ? 0xffu : 0xffffu; + uint value = (input_buffer.data[src >> 2] >> ((src & 3u) * 8u)) & mask; + atomicOr(output_buffer.data[dst >> 2], value << ((dst & 3u) * 8u)); + } +} + +void main() { + uint index = gl_GlobalInvocationID.x; + if (index >= params.count) { + return; + } + + uint element = params.first + index; + uint plane = params.width * params.height; + uvec3 p; + p.z = element / plane; + element -= p.z * plane; + p.y = element / params.width; + p.x = element - p.y * params.width; + + uvec3 extent = block_extent(); + uvec3 swizzle = p; + uvec3 block = uvec3(0); + if (params.tail != 0) { + swizzle.xy += uvec2(params.tail_x, params.tail_y); + } else { + block = p / extent; + } + + uint block_index = block.z * params.blocks_per_slice + + block.y * params.blocks_per_row + block.x; + uint tiled = block_index * BLOCK_BYTES + + block_offset(uvec3(swizzle.xy, swizzle.z + params.surface_z)); + uint linear = p.z * params.slice_bytes + p.y * params.pitch_bytes + + p.x * ELEMENT_BYTES; + if (TILE == 0) { + copy_element(params.src_base + tiled, params.dst_base + linear); + } else { + copy_element(params.src_base + linear, params.dst_base + tiled); + } +} diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_depth.comp b/src/graphics/host_gpu/shaders/gpu_tiler_depth.comp new file mode 100644 index 0000000..72098d9 --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_depth.comp @@ -0,0 +1,41 @@ +#version 450 +#extension GL_GOOGLE_include_directive : require +layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; +layout(constant_id = 1) const uint TILE = 0u; +const uint BLOCK_BYTES = 65536u; +uvec3 block_extent() { + uint width = ELEMENT_BYTES <= 2 ? 256u : 128u; + return uvec3(width, BLOCK_BYTES / (width * ELEMENT_BYTES), 1u); +} +uint block_offset(uvec3 p) { + uint x = p.x, y = p.y; + uint z = ((p.z & 8u) << 5) ^ ((p.z & 4u) << 7) ^ + ((p.z & 2u) << 9) ^ ((p.z & 1u) << 11); + switch (ELEMENT_BYTES) { + case 1: return z ^ (x & 1u) ^ ((x << 1) & 0x004u) ^ ((x << 2) & 0x010u) ^ + ((x << 3) & 0x040u) ^ ((x << 5) & 0x300u) ^ + ((x << 4) & 0x400u) ^ ((x << 6) & 0x800u) ^ + ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^ + ((y << 1) & 0x002u) ^ ((y << 2) & 0x008u) ^ + ((y << 3) & 0x0a0u) ^ ((y << 5) & 0xf00u) ^ + ((y << 6) & 0x1000u) ^ ((y << 7) & 0x4000u); + case 2: return z ^ ((x << 1) & 0x002u) ^ ((x << 2) & 0x008u) ^ + ((x << 3) & 0x020u) ^ ((x << 4) & 0x480u) ^ + ((x << 5) & 0x300u) ^ ((x << 6) & 0x800u) ^ + ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^ + ((y << 2) & 0x004u) ^ ((y << 3) & 0x010u) ^ + ((y << 4) & 0x040u) ^ ((y << 5) & 0xf00u) ^ ((y << 8) & 0x5000u); + case 4: return z ^ ((x << 2) & 0x004u) ^ ((x << 3) & 0x010u) ^ + ((x << 4) & 0x440u) ^ ((x << 5) & 0x300u) ^ + ((x << 6) & 0x800u) ^ ((x << 9) & 0xa000u) ^ + ((y << 3) & 0x008u) ^ ((y << 4) & 0x020u) ^ + ((y << 5) & 0xf80u) ^ ((y << 9) & 0x1000u) ^ + ((y << 8) & 0x4000u); + default: return z ^ ((x << 3) & 0x008u) ^ ((x << 4) & 0x420u) ^ + ((x << 5) & 0x380u) ^ ((x << 6) & 0x800u) ^ + ((x << 10) & 0x2000u) ^ ((x << 9) & 0x8000u) ^ + ((y << 4) & 0x010u) ^ ((y << 5) & 0xf40u) ^ + ((y << 10) & 0x5000u); + } +} +#include "gpu_tiler_common.inc" diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_prt.comp b/src/graphics/host_gpu/shaders/gpu_tiler_prt.comp new file mode 100644 index 0000000..e34dc79 --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_prt.comp @@ -0,0 +1,20 @@ +#version 450 +#extension GL_GOOGLE_include_directive : require +layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; +layout(constant_id = 1) const uint TILE = 0u; +const uint BLOCK_BYTES = 65536u; +#include "gpu_tiler_standard64.inc" +uvec3 block_extent() { + uint width = ELEMENT_BYTES <= 2 ? 256u : (ELEMENT_BYTES <= 8 ? 128u : 64u); + return uvec3(width, BLOCK_BYTES / (width * ELEMENT_BYTES), 1u); +} +uint block_offset(uvec3 p) { + uvec4 s = ELEMENT_BYTES == 1 ? uvec4(7, 7, 6, 6) : + ELEMENT_BYTES == 2 ? uvec4(7, 6, 6, 5) : + ELEMENT_BYTES == 4 ? uvec4(6, 6, 5, 5) : + ELEMENT_BYTES == 8 ? uvec4(6, 5, 5, 4) : uvec4(5, 5, 4, 4); + uint delta = (((p.x >> s.x) & 1u) << 8) ^ (((p.y >> s.y) & 1u) << 9) ^ + (((p.x >> s.z) & 1u) << 10) ^ (((p.y >> s.w) & 1u) << 11); + return standard64_offset(p.xy) ^ delta; +} +#include "gpu_tiler_common.inc" diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_prt_3d.comp b/src/graphics/host_gpu/shaders/gpu_tiler_prt_3d.comp new file mode 100644 index 0000000..6cb079d --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_prt_3d.comp @@ -0,0 +1,17 @@ +#version 450 +#extension GL_GOOGLE_include_directive : require +layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; +layout(constant_id = 1) const uint TILE = 0u; +const uint BLOCK_BYTES = 65536u; +#include "gpu_tiler_standard64_3d.inc" +uvec3 block_extent() { return standard64_3d_extent(); } +uint block_offset(uvec3 p) { + uvec4 s = ELEMENT_BYTES == 1 ? uvec4(4, 5, 4, 4) : + ELEMENT_BYTES == 2 ? uvec4(4, 4, 3, 4) : + ELEMENT_BYTES == 4 ? uvec4(4, 4, 3, 3) : + ELEMENT_BYTES == 8 ? uvec4(3, 4, 3, 3) : uvec4(3, 3, 2, 3); + uint delta = (((p.y >> s.x) & 1u) << 10) ^ (((p.x >> s.y) & 1u) << 10) ^ + (((p.x >> s.z) & 1u) << 11) ^ (((p.z >> s.w) & 1u) << 11); + return standard64_3d_offset(p) ^ delta; +} +#include "gpu_tiler_common.inc" diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_render_target.comp b/src/graphics/host_gpu/shaders/gpu_tiler_render_target.comp new file mode 100644 index 0000000..a710901 --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_render_target.comp @@ -0,0 +1,42 @@ +#version 450 +#extension GL_GOOGLE_include_directive : require +layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; +layout(constant_id = 1) const uint TILE = 0u; +const uint BLOCK_BYTES = 65536u; +uvec3 block_extent() { + uint width = ELEMENT_BYTES <= 2 ? 256u : (ELEMENT_BYTES <= 8 ? 128u : 64u); + return uvec3(width, BLOCK_BYTES / (width * ELEMENT_BYTES), 1u); +} +uint rt_offset(uint x, uint y) { + switch (ELEMENT_BYTES) { + case 1: return ((y << 2) & 0x008u) ^ ((y << 4) & 0x010u) ^ + ((y << 3) & 0x0a0u) ^ ((y << 5) & 0xf00u) ^ + ((y << 6) & 0x1000u) ^ ((y << 7) & 0x4000u) ^ (x & 7u) ^ + ((x << 3) & 0x040u) ^ ((x << 5) & 0x300u) ^ + ((x << 4) & 0x400u) ^ ((x << 6) & 0x800u) ^ + ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u); + case 2: return ((y << 4) & 0x070u) ^ ((y << 5) & 0xf00u) ^ + ((y << 8) & 0x5000u) ^ ((x << 1) & 0x00eu) ^ + ((x << 4) & 0x480u) ^ ((x << 5) & 0x300u) ^ + ((x << 6) & 0x800u) ^ ((x << 7) & 0x2000u) ^ + ((x << 8) & 0x8000u); + case 4: return ((y << 4) & 0x070u) ^ ((y << 5) & 0xf00u) ^ + ((y << 9) & 0x1000u) ^ ((y << 8) & 0x4000u) ^ + ((x << 2) & 0x00cu) ^ ((x << 5) & 0x380u) ^ + ((x << 4) & 0x400u) ^ ((x << 6) & 0x800u) ^ ((x << 9) & 0xa000u); + case 8: return ((y << 4) & 0x010u) ^ ((y << 6) & 0x080u) ^ + ((y << 5) & 0xf00u) ^ ((y << 10) & 0x5000u) ^ + ((x << 3) & 0x008u) ^ ((x << 4) & 0x460u) ^ + ((x << 5) & 0x300u) ^ ((x << 6) & 0x800u) ^ + ((x << 10) & 0x2000u) ^ ((x << 9) & 0x8000u); + default: return ((x << 4) & 0x410u) ^ ((x << 5) & 0x340u) ^ + ((x << 6) & 0x800u) ^ ((x << 11) & 0xa000u) ^ + ((y << 5) & 0xf20u) ^ ((y << 6) & 0x080u) ^ + ((y << 10) & 0x1000u) ^ ((y << 11) & 0x4000u); + } +} +uint volume_z(uint z) { + return ((z & 8u) << 5) ^ ((z & 4u) << 7) ^ ((z & 2u) << 9) ^ ((z & 1u) << 11); +} +uint block_offset(uvec3 p) { return rt_offset(p.x, p.y) ^ volume_z(p.z); } +#include "gpu_tiler_common.inc" diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_standard.inc b/src/graphics/host_gpu/shaders/gpu_tiler_standard.inc new file mode 100644 index 0000000..9e64656 --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_standard.inc @@ -0,0 +1,18 @@ +uint standard_offset(uvec2 p) { + uint x = p.x, y = p.y; + switch (ELEMENT_BYTES) { + case 1: return ((y << 4) & 0x1f0u) ^ ((y << 5) & 0x400u) ^ (x & 0x00fu) ^ + ((x << 5) & 0x200u) ^ ((x << 6) & 0x800u); + case 2: return ((y << 4) & 0x070u) ^ ((y << 5) & 0x100u) ^ ((y << 6) & 0x400u) ^ + ((x << 1) & 0x00eu) ^ ((x << 4) & 0x080u) ^ + ((x << 5) & 0x200u) ^ ((x << 6) & 0x800u); + case 4: return ((y << 4) & 0x070u) ^ ((y << 5) & 0x100u) ^ ((y << 6) & 0x400u) ^ + ((x << 2) & 0x00cu) ^ ((x << 5) & 0x080u) ^ + ((x << 6) & 0x200u) ^ ((x << 7) & 0x800u); + case 8: return ((y << 4) & 0x030u) ^ ((y << 6) & 0x100u) ^ ((y << 7) & 0x400u) ^ + ((x << 3) & 0x008u) ^ ((x << 5) & 0x0c0u) ^ + ((x << 6) & 0x200u) ^ ((x << 7) & 0x800u); + default: return ((y << 4) & 0x030u) ^ ((y << 6) & 0x100u) ^ ((y << 7) & 0x400u) ^ + ((x << 6) & 0x0c0u) ^ ((x << 7) & 0x200u) ^ ((x << 8) & 0x800u); + } +} diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_standard256.comp b/src/graphics/host_gpu/shaders/gpu_tiler_standard256.comp new file mode 100644 index 0000000..70ab0d8 --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_standard256.comp @@ -0,0 +1,12 @@ +#version 450 +#extension GL_GOOGLE_include_directive : require +layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; +layout(constant_id = 1) const uint TILE = 0u; +const uint BLOCK_BYTES = 256u; +#include "gpu_tiler_standard.inc" +uvec3 block_extent() { + uint width = ELEMENT_BYTES <= 2 ? 16u : (ELEMENT_BYTES <= 8 ? 8u : 4u); + return uvec3(width, BLOCK_BYTES / (width * ELEMENT_BYTES), 1u); +} +uint block_offset(uvec3 p) { return standard_offset(p.xy) & 0xffu; } +#include "gpu_tiler_common.inc" diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_standard4.comp b/src/graphics/host_gpu/shaders/gpu_tiler_standard4.comp new file mode 100644 index 0000000..1bf3a27 --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_standard4.comp @@ -0,0 +1,12 @@ +#version 450 +#extension GL_GOOGLE_include_directive : require +layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; +layout(constant_id = 1) const uint TILE = 0u; +const uint BLOCK_BYTES = 4096u; +#include "gpu_tiler_standard.inc" +uvec3 block_extent() { + uint width = ELEMENT_BYTES <= 2 ? 64u : (ELEMENT_BYTES <= 8 ? 32u : 16u); + return uvec3(width, BLOCK_BYTES / (width * ELEMENT_BYTES), 1u); +} +uint block_offset(uvec3 p) { return standard_offset(p.xy); } +#include "gpu_tiler_common.inc" diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_standard4_3d.comp b/src/graphics/host_gpu/shaders/gpu_tiler_standard4_3d.comp new file mode 100644 index 0000000..64a46b2 --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_standard4_3d.comp @@ -0,0 +1,15 @@ +#version 450 +#extension GL_GOOGLE_include_directive : require +layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; +layout(constant_id = 1) const uint TILE = 0u; +const uint BLOCK_BYTES = 4096u; +#include "gpu_tiler_standard4_3d.inc" +uvec3 block_extent() { + if (ELEMENT_BYTES == 1) return uvec3(16, 16, 16); + if (ELEMENT_BYTES == 2) return uvec3(8, 16, 16); + if (ELEMENT_BYTES == 4) return uvec3(8, 16, 8); + if (ELEMENT_BYTES == 8) return uvec3(8, 8, 8); + return uvec3(4, 8, 8); +} +uint block_offset(uvec3 p) { return standard3d_offset(p); } +#include "gpu_tiler_common.inc" diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_standard4_3d.inc b/src/graphics/host_gpu/shaders/gpu_tiler_standard4_3d.inc new file mode 100644 index 0000000..fa60bf8 --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_standard4_3d.inc @@ -0,0 +1,22 @@ +uint standard3d_offset(uvec3 p) { + uint x = p.x, y = p.y, z = p.z; + if (ELEMENT_BYTES == 1) + return (x & 3u) ^ ((x << 4) & 0x40u) ^ ((x << 6) & 0x200u) ^ + ((y << 3) & 8u) ^ ((y << 4) & 0x20u) ^ ((y << 6) & 0x100u) ^ + ((y << 8) & 0x800u) ^ ((z << 2) & 4u) ^ ((z << 3) & 0x10u) ^ + ((z << 5) & 0x80u) ^ ((z << 7) & 0x400u); + if (ELEMENT_BYTES == 2) + return ((x << 1) & 2u) ^ ((x << 5) & 0x40u) ^ ((x << 7) & 0x200u) ^ + ((y << 3) & 8u) ^ ((y << 4) & 0x20u) ^ ((y << 6) & 0x100u) ^ + ((y << 8) & 0x800u) ^ ((z << 2) & 4u) ^ ((z << 3) & 0x10u) ^ + ((z << 5) & 0x80u) ^ ((z << 7) & 0x400u); + uint z_part = ((z << 4) & 0x10u) ^ ((z << 6) & 0x80u) ^ ((z << 8) & 0x400u); + if (ELEMENT_BYTES == 4) + return ((x << 2) & 4u) ^ ((x << 5) & 0x40u) ^ ((x << 7) & 0x200u) ^ + ((y << 3) & 8u) ^ ((y << 4) & 0x20u) ^ ((y << 6) & 0x100u) ^ + ((y << 8) & 0x800u) ^ z_part; + uint y_part = ((y << 5) & 0x20u) ^ ((y << 7) & 0x100u) ^ ((y << 9) & 0x800u); + if (ELEMENT_BYTES == 8) + return ((x << 3) & 8u) ^ ((x << 5) & 0x40u) ^ ((x << 7) & 0x200u) ^ y_part ^ z_part; + return ((x << 6) & 0x40u) ^ ((x << 8) & 0x200u) ^ y_part ^ z_part; +} diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_standard64.comp b/src/graphics/host_gpu/shaders/gpu_tiler_standard64.comp new file mode 100644 index 0000000..9ff97ad --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_standard64.comp @@ -0,0 +1,12 @@ +#version 450 +#extension GL_GOOGLE_include_directive : require +layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; +layout(constant_id = 1) const uint TILE = 0u; +const uint BLOCK_BYTES = 65536u; +#include "gpu_tiler_standard64.inc" +uvec3 block_extent() { + uint width = ELEMENT_BYTES <= 2 ? 256u : (ELEMENT_BYTES <= 8 ? 128u : 64u); + return uvec3(width, BLOCK_BYTES / (width * ELEMENT_BYTES), 1u); +} +uint block_offset(uvec3 p) { return standard64_offset(p.xy); } +#include "gpu_tiler_common.inc" diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_standard64.inc b/src/graphics/host_gpu/shaders/gpu_tiler_standard64.inc new file mode 100644 index 0000000..1e2cfb0 --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_standard64.inc @@ -0,0 +1,29 @@ +uint standard64_offset(uvec2 p) { + uint x = p.x, y = p.y; + switch (ELEMENT_BYTES) { + case 1: return (x & 0x0fu) ^ ((x << 5) & 0x200u) ^ ((x << 6) & 0x800u) ^ + ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^ + ((y << 4) & 0x1f0u) ^ ((y << 5) & 0x400u) ^ + ((y << 6) & 0x1000u) ^ ((y << 7) & 0x4000u); + case 2: return ((x << 1) & 0x00eu) ^ ((x << 4) & 0x080u) ^ + ((x << 5) & 0x200u) ^ ((x << 6) & 0x800u) ^ + ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^ + ((y << 4) & 0x070u) ^ ((y << 5) & 0x100u) ^ + ((y << 6) & 0x400u) ^ ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u); + case 4: return ((x << 2) & 0x00cu) ^ ((x << 5) & 0x080u) ^ + ((x << 6) & 0x200u) ^ ((x << 7) & 0x800u) ^ + ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^ + ((y << 4) & 0x070u) ^ ((y << 5) & 0x100u) ^ + ((y << 6) & 0x400u) ^ ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u); + case 8: return ((x << 3) & 0x008u) ^ ((x << 5) & 0x0c0u) ^ + ((x << 6) & 0x200u) ^ ((x << 7) & 0x800u) ^ + ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^ + ((y << 4) & 0x030u) ^ ((y << 6) & 0x100u) ^ + ((y << 7) & 0x400u) ^ ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u); + default: return ((x << 6) & 0x0c0u) ^ ((x << 7) & 0x200u) ^ + ((x << 8) & 0x800u) ^ ((x << 9) & 0x2000u) ^ + ((x << 10) & 0x8000u) ^ ((y << 4) & 0x030u) ^ + ((y << 6) & 0x100u) ^ ((y << 7) & 0x400u) ^ + ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u); + } +} diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_standard64_3d.comp b/src/graphics/host_gpu/shaders/gpu_tiler_standard64_3d.comp new file mode 100644 index 0000000..bd0334c --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_standard64_3d.comp @@ -0,0 +1,9 @@ +#version 450 +#extension GL_GOOGLE_include_directive : require +layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; +layout(constant_id = 1) const uint TILE = 0u; +const uint BLOCK_BYTES = 65536u; +#include "gpu_tiler_standard64_3d.inc" +uvec3 block_extent() { return standard64_3d_extent(); } +uint block_offset(uvec3 p) { return standard64_3d_offset(p); } +#include "gpu_tiler_common.inc" diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_standard64_3d.inc b/src/graphics/host_gpu/shaders/gpu_tiler_standard64_3d.inc new file mode 100644 index 0000000..c35851c --- /dev/null +++ b/src/graphics/host_gpu/shaders/gpu_tiler_standard64_3d.inc @@ -0,0 +1,17 @@ +#include "gpu_tiler_standard4_3d.inc" +uvec3 standard64_3d_extent() { + if (ELEMENT_BYTES == 1) return uvec3(64, 32, 32); + if (ELEMENT_BYTES == 2) return uvec3(32, 32, 32); + if (ELEMENT_BYTES == 4) return uvec3(32, 32, 16); + if (ELEMENT_BYTES == 8) return uvec3(32, 16, 16); + return uvec3(16, 16, 16); +} +uint standard64_3d_offset(uvec3 p) { + uvec4 s = ELEMENT_BYTES == 1 ? uvec4(4, 4, 4, 5) : + ELEMENT_BYTES == 2 ? uvec4(3, 4, 4, 4) : + ELEMENT_BYTES == 4 ? uvec4(3, 3, 4, 4) : + ELEMENT_BYTES == 8 ? uvec4(3, 3, 3, 4) : uvec4(2, 3, 3, 3); + return standard3d_offset(p) ^ (((p.x >> s.x) & 1u) << 12) ^ + (((p.z >> s.y) & 1u) << 13) ^ (((p.y >> s.z) & 1u) << 14) ^ + (((p.x >> s.w) & 1u) << 15); +} diff --git a/src/graphics/host_gpu/transfer.cpp b/src/graphics/host_gpu/transfer.cpp index 948f3e3..9f3933b 100644 --- a/src/graphics/host_gpu/transfer.cpp +++ b/src/graphics/host_gpu/transfer.cpp @@ -4,6 +4,7 @@ #include "common/logging/log.h" #include "common/profiler.h" #include "common/threads.h" +#include "graphics/host_gpu/gpuTiler.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/imageView.h" #include "graphics/host_gpu/renderer/render.h" @@ -240,9 +241,11 @@ ConvertImageBufferCopies(std::span regions) { return vk_regions; } -static void RecordImageToBuffer(CommandBuffer& command, VulkanImage& src_image, - VulkanBuffer& dst_buffer, std::span regions, - vk::ImageLayout final_layout) { +static void +RecordImageToBuffer(CommandBuffer& command, VulkanImage& src_image, VulkanBuffer& dst_buffer, + std::span regions, vk::ImageLayout final_layout, + vk::AccessFlags final_access = vk::AccessFlagBits::eHostRead, + vk::PipelineStageFlags final_stage = vk::PipelineStageFlagBits::eHost) { auto vk_command = command.Handle(); vk::ImageAspectFlags aspects = {}; for (const auto& region: regions) { @@ -259,8 +262,8 @@ static void RecordImageToBuffer(CommandBuffer& command, VulkanImage& src_image, dst_buffer.buffer, static_cast(copies.size()), copies.data()); SetBufferMemoryBarrier(vk_command, dst_buffer.buffer, 0, VK_WHOLE_SIZE, - vk::AccessFlagBits::eTransferWrite, vk::AccessFlagBits::eHostRead, - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost); + vk::AccessFlagBits::eTransferWrite, final_access, + vk::PipelineStageFlagBits::eTransfer, final_stage); SetImageLayout(vk_command, &src_image, 0, VK_REMAINING_MIP_LEVELS, aspects, vk::ImageLayout::eTransferSrcOptimal, final_layout); } @@ -358,8 +361,8 @@ public: } void ProcessDownloadedImage(GraphicContext* ctx, uint64_t size, - std::span regions, - VulkanImage* src_image, vk::ImageLayout src_layout, + std::span regions, VulkanImage* src_image, + vk::ImageLayout src_layout, const DownloadedImageConsumer& consumer) { WithDownloadedImage(ctx, size, regions, src_image, src_layout, [&](std::span data) { @@ -367,8 +370,8 @@ public: consumer(data); return; } - // Scattered CPU tiler reads can be much slower from uncached mapped - // memory. Preserve the original sequential-copy path as a fallback. + // Read uncached mappings sequentially before CPU consumers inspect + // them. m_cached_readback.resize(data.size()); std::memcpy(m_cached_readback.data(), data.data(), data.size()); consumer(m_cached_readback); @@ -382,17 +385,16 @@ public: VulkanUnmapMemory(ctx, &m_buffer.memory); VulkanDeleteBuffer(ctx, &m_buffer); } - m_capacity = 0; - m_mapped_data = nullptr; - m_host_cached = false; + m_capacity = 0; + m_mapped_data = nullptr; + m_host_cached = false; } private: template void WithDownloadedImage(GraphicContext* ctx, uint64_t size, - std::span regions, - VulkanImage* src_image, vk::ImageLayout src_layout, - const Consumer& consumer) { + std::span regions, VulkanImage* src_image, + vk::ImageLayout src_layout, const Consumer& consumer) { Common::LockGuard lock(m_mutex); EnsureBuffer(ctx, size, vk::BufferUsageFlagBits::eTransferDst); ExecuteImmediateCommands([&](CommandBuffer* command, vk::CommandBuffer) { @@ -438,7 +440,7 @@ private: ? vk::MemoryPropertyFlags(vk::MemoryPropertyFlagBits::eHostCached) : vk::MemoryPropertyFlags {}; VulkanCreateBuffer(ctx, size, &m_buffer); - m_capacity = size; + m_capacity = size; const auto& properties = ctx->GetPhysicalDeviceMemoryProperties(); EXIT_IF(m_buffer.memory.type >= properties.memoryTypeCount); m_host_cached = @@ -518,11 +520,41 @@ static ReusableStagingBuffer* GetStagingBuffer(StagingBufferType type) { void ReleaseCachedResources(GraphicContext* ctx) { EXIT_IF(ctx == nullptr); + GpuTileRelease(ctx); g_texture_staging_buffer.Release(ctx); g_vertex_staging_buffer.Release(ctx); g_readback_staging_buffer.Release(ctx); } +void UploadTiledImage(GraphicContext* ctx, VulkanImage* dst_image, const void* tiled_data, + uint64_t tiled_size, uint64_t linear_size, std::span infos, + std::span regions, vk::ImageLayout dst_layout) { + EXIT_IF(ctx == nullptr || dst_image == nullptr || tiled_data == nullptr || regions.empty()); + GpuDetile(ctx, tiled_data, nullptr, tiled_size, linear_size, infos, + [&](CommandBuffer* command, VulkanBuffer* linear) { + vk::ImageAspectFlags aspects {}; + for (const auto& region: regions) { + aspects |= GetTransferAspects(*dst_image, region.aspect); + } + RecordBufferToImageCopy(*command, *linear, *dst_image, + ConvertBufferImageCopies(regions), aspects, + dst_image->layout, dst_layout); + }); +} + +void DownloadTiledImage(GraphicContext* ctx, void* tiled_data, uint64_t tiled_size, + uint64_t linear_size, std::span infos, + std::span regions, VulkanImage* src_image, + vk::ImageLayout src_layout) { + EXIT_IF(ctx == nullptr || tiled_data == nullptr || src_image == nullptr || regions.empty()); + GpuTile(ctx, nullptr, tiled_data, tiled_size, linear_size, infos, + [&](CommandBuffer* command, VulkanBuffer* linear) { + RecordImageToBuffer(*command, *src_image, *linear, regions, src_layout, + vk::AccessFlagBits::eShaderRead, + vk::PipelineStageFlagBits::eComputeShader); + }); +} + static void SetImageLayout(vk::CommandBuffer buffer, VulkanImage* dst_image, uint32_t base_level, uint32_t levels, vk::ImageAspectFlags aspect_mask, vk::ImageLayout old_image_layout, vk::ImageLayout new_image_layout) { @@ -973,8 +1005,7 @@ void DownloadImage(GraphicContext* ctx, void* dst_data, uint64_t size, void ProcessDownloadedImage(GraphicContext* ctx, uint64_t size, std::span regions, VulkanImage* src_image, - vk::ImageLayout src_layout, - const DownloadedImageConsumer& consumer) { + vk::ImageLayout src_layout, const DownloadedImageConsumer& consumer) { KYTY_PROFILER_FUNCTION(); EXIT_IF(size == 0 || regions.empty() || !consumer); GetStagingBuffer(StagingBufferType::ReadBack) diff --git a/src/graphics/host_gpu/transfer.h b/src/graphics/host_gpu/transfer.h index 4fb4dc4..0a278e5 100644 --- a/src/graphics/host_gpu/transfer.h +++ b/src/graphics/host_gpu/transfer.h @@ -3,6 +3,7 @@ #include "common/abi.h" #include "common/common.h" +#include "graphics/host_gpu/gpuTiler.h" #include "graphics/host_gpu/vulkanCommon.h" #include @@ -102,6 +103,9 @@ void UploadImage(GraphicContext* ctx, DepthStencilVulkanImage* dst_image, const uint64_t size, uint32_t src_pitch, vk::ImageAspectFlags aspect); void UploadImage(GraphicContext* ctx, VulkanImage* dst_image, const void* src_data, uint64_t size, std::span regions, vk::ImageLayout dst_layout); +void UploadTiledImage(GraphicContext* ctx, VulkanImage* dst_image, const void* tiled_data, + uint64_t tiled_size, uint64_t linear_size, std::span infos, + std::span regions, vk::ImageLayout dst_layout); void CopyImageImmediate(GraphicContext* ctx, std::span regions, VulkanImage* dst_image, vk::ImageLayout dst_layout); void DownloadImage(GraphicContext* ctx, void* dst_data, uint64_t size, uint32_t dst_pitch, @@ -115,8 +119,11 @@ void DownloadImage(GraphicContext* ctx, void* dst_data, uint64_t size, // re-enter Transfer. void ProcessDownloadedImage(GraphicContext* ctx, uint64_t size, std::span regions, VulkanImage* src_image, - vk::ImageLayout src_layout, - const DownloadedImageConsumer& consumer); + vk::ImageLayout src_layout, const DownloadedImageConsumer& consumer); +void DownloadTiledImage(GraphicContext* ctx, void* tiled_data, uint64_t tiled_size, + uint64_t linear_size, std::span infos, + std::span regions, VulkanImage* src_image, + vk::ImageLayout src_layout); void UploadBuffer(GraphicContext* ctx, StagingBufferType type, VulkanBuffer* dst_buffer, uint64_t dst_offset, const void* src_data, uint64_t size); void CopyBuffer(VulkanBuffer* src_buffer, VulkanBuffer* dst_buffer, uint64_t size); diff --git a/src/libs/agc.cpp b/src/libs/agc.cpp index c52dd20..903c04e 100644 --- a/src/libs/agc.cpp +++ b/src/libs/agc.cpp @@ -52,7 +52,6 @@ KYTY_SUBSYSTEM_INIT(Graphics) { GraphicsRenderInit(); GraphicsRunInit(); LabelInit(); - TileInit(); ShaderInit(); } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 2f975a1..00ec326 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -4,8 +4,10 @@ #include "common/emulatorConfig.h" #include "common/logging/log.h" #include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/guest_gpu/gpu_format.h" #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/guest_gpu/tile.h" +#include "graphics/host_gpu/gpuTiler.h" #include "graphics/host_gpu/hostMemory.h" #include "graphics/host_gpu/memoryTracker.h" #include "graphics/host_gpu/objects/textureCommon.h" @@ -36,6 +38,7 @@ #include #include +#include #include #include #include @@ -2343,6 +2346,716 @@ public: return pixel; } + void CheckGpuTilerCpuParity() { + constexpr const char *name = "GpuTilerCpuParity"; + EnsureRuntimeContext(); + + auto fill = [](std::vector *bytes, u32 salt) { + for (size_t i = 0; i < bytes->size(); i++) { + uint64_t value = + i + static_cast(salt) * 0x9e3779b97f4a7c15ull; + value = (value ^ (value >> 30u)) * 0xbf58476d1ce4e5b9ull; + value = (value ^ (value >> 27u)) * 0x94d049bb133111ebull; + (*bytes)[i] = static_cast((value ^ (value >> 31u)) >> 56u); + } + }; + auto compare = [&](const char *stage, const std::vector &expected, + const std::vector &actual) { + if (expected != actual) { + const auto mismatch = static_cast( + std::mismatch(expected.begin(), expected.end(), actual.begin()) + .first - + expected.begin()); + std::ostringstream out; + out << "first mismatch at " << mismatch << " of " << expected.size(); + Fail(name, stage, out.str()); + } + }; + auto convert_reference = [&](bool to_tiled, std::vector *dst, + const std::vector &src, + const GpuTileInfo &info) { + TileBlockLayout block{}; + Require(name, "reference layout", + TileGetBlockLayout(info.family, info.bytes_per_element, &block), + "CPU reference rejected GPU tile info"); + const u32 tiled_width = + info.tiled_width != 0 ? info.tiled_width : info.pitch; + const u32 tiled_height = + info.tiled_height != 0 ? info.tiled_height : info.height; + const uint64_t columns = + (tiled_width + block.block_width - 1u) / block.block_width; + const uint64_t rows = + (tiled_height + block.block_height - 1u) / block.block_height; + const uint64_t slice = info.linear_slice_stride != 0 + ? info.linear_slice_stride + : static_cast(info.pitch) * + info.height * info.bytes_per_element; + for (u32 z = 0; z < info.depth; ++z) { + for (u32 y = 0; y < info.height; ++y) { + for (u32 x = 0; x < info.width; ++x) { + const u32 bx = info.tail ? 0 : x / block.block_width; + const u32 by = info.tail ? 0 : y / block.block_height; + const u32 bz = info.tail ? 0 : z / block.block_depth; + const u32 lx = info.tail ? x + info.tail_x : x % block.block_width; + const u32 ly = info.tail ? y + info.tail_y : y % block.block_height; + const u32 lz = z % block.block_depth; + u32 local = 0, block_xor = 0; + Require(name, "reference offset", + TileGetBlockOffset(block, lx, ly, lz, &local) && + TileGetBlockXor(block, bx, by, info.surface_z + bz, + &block_xor), + "CPU reference address lookup failed"); + const uint64_t block_index = + static_cast(bz) * columns * rows + by * columns + bx; + const uint64_t tiled = info.tiled_offset + + block_index * block.block_size + + (local ^ block_xor); + const uint64_t linear = + info.linear_offset + static_cast(z) * slice + + static_cast(y) * info.pitch * info.bytes_per_element + + static_cast(x) * info.bytes_per_element; + const uint64_t dst_offset = to_tiled ? tiled : linear; + const uint64_t src_offset = to_tiled ? linear : tiled; + Require(name, "reference range", + dst_offset + info.bytes_per_element <= dst->size() && + src_offset + info.bytes_per_element <= src.size(), + "CPU reference address escaped storage"); + std::memcpy(dst->data() + dst_offset, src.data() + src_offset, + info.bytes_per_element); + } + } + } + }; + struct FamilyCase { + TileBlockFamily family; + u32 max_bpe; + }; + constexpr FamilyCase families[] = { + {TileBlockFamily::Standard256B, 16}, + {TileBlockFamily::Standard4KB, 16}, + {TileBlockFamily::Standard4KB3D, 16}, + {TileBlockFamily::Standard64KB, 16}, + {TileBlockFamily::Standard64KB3D, 16}, + {TileBlockFamily::Prt64KB, 16}, + {TileBlockFamily::Prt64KB3D, 16}, + {TileBlockFamily::RenderTarget64KB, 16}, + {TileBlockFamily::Depth64KB, 8}, + }; + + { + TileBlockLayout standard{}, prt{}, standard_3d{}, prt_3d{}, color{}, + depth{}; + u32 standard_offset = 0, prt_offset = 0, standard_3d_offset = 0, + prt_3d_offset = 0, color_z = 0, depth_z = 0; + bool fixed_addresses = + TileGetBlockLayout(TileBlockFamily::Standard64KB, 4, &standard) && + TileGetBlockLayout(TileBlockFamily::Prt64KB, 4, &prt) && + TileGetBlockLayout(TileBlockFamily::Standard64KB3D, 4, + &standard_3d) && + TileGetBlockLayout(TileBlockFamily::Prt64KB3D, 4, &prt_3d) && + TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, 4, &color) && + TileGetBlockLayout(TileBlockFamily::Depth64KB, 4, &depth) && + TileGetBlockOffset(standard, 64, 0, 0, &standard_offset) && + TileGetBlockOffset(prt, 64, 0, 0, &prt_offset) && + TileGetBlockOffset(standard_3d, 16, 0, 0, &standard_3d_offset) && + TileGetBlockOffset(prt_3d, 16, 0, 0, &prt_3d_offset) && + TileGetBlockXor(color, 0, 0, 1, &color_z) && + TileGetBlockXor(depth, 0, 0, 15, &depth_z); + Require(name, "fixed address vectors", + fixed_addresses && standard_offset == 0x8000 && + prt_offset == 0x8100 && standard_3d_offset == 0x8000 && + prt_3d_offset == 0x8400 && standard_3d.block_width == 32 && + standard_3d.block_height == 32 && + standard_3d.block_depth == 16 && color_z == 0x800 && + depth_z == 0xf00, + "a fixed block address changed"); + + constexpr u32 format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + constexpr u32 levels = 7; + TileSizeAlign total{}; + TileSizeOffset mip[levels]{}; + TilePaddedSize padded[levels]{}; + TileGetTextureSize( + format, 65, 33, 72, levels, + Prospero::GpuEnumValue(Prospero::TileMode::kStandard256B), &total, + mip, padded); + constexpr u32 offsets[levels] = {0x1a00, 0xb00, 0x500, 0x300, + 0x200, 0x100, 0}; + constexpr u32 sizes[levels] = {0x2d00, 0xf00, 0x600, 0x200, + 0x100, 0x100, 0x100}; + bool layout_matches = total.size == 0x4700 && total.align == 0x100 && + padded[0].width == 72 && padded[0].height == 40; + for (u32 level = 0; level < levels; ++level) { + layout_matches &= mip[level].offset == offsets[level] && + mip[level].size == sizes[level]; + } + Require(name, "fixed mip vector", layout_matches, + "the reverse-packed small-block mip layout changed"); + } + + u32 case_index = 0; + auto check_round_trip = [&](const char *stage, uint64_t size, + std::span infos) { + std::vector tiled(size); + std::vector cpu(size, 0); + std::vector gpu(size, 0xab); + fill(&tiled, ++case_index); + for (const auto &info : infos) { + convert_reference(false, &cpu, tiled, info); + } + GpuDetile(&m_runtime_context, tiled.data(), gpu.data(), size, size, + infos); + compare((std::string(stage) + " detile bytes").c_str(), cpu, gpu); + + std::vector linear(size); + std::vector cpu_tiled(size, 0); + std::vector gpu_tiled(size, 0xab); + fill(&linear, 0x280u + case_index); + for (const auto &info : infos) { + convert_reference(true, &cpu_tiled, linear, info); + } + GpuTile(&m_runtime_context, linear.data(), gpu_tiled.data(), size, size, + infos); + compare((std::string(stage) + " tile bytes").c_str(), cpu_tiled, + gpu_tiled); + }; + for (const auto family : families) { + for (u32 bpe = 1; bpe <= family.max_bpe; bpe <<= 1u) { + TileBlockLayout block{}; + Require(name, "block layout", + TileGetBlockLayout(family.family, bpe, &block), + "admitted family/BPE has no block layout"); + const bool volume = block.block_depth > 1; + const u32 width = + block.block_width * 3u + std::min(block.block_width, 3u); + const u32 height = + block.block_height * 3u + std::min(block.block_height, 3u); + const u32 depth = volume ? block.block_depth + 1u : 1u; + const u32 pitch = block.block_width * 4u; + const uint64_t block_columns = + (pitch + block.block_width - 1u) / block.block_width; + const uint64_t block_rows = + (height + block.block_height - 1u) / block.block_height; + const uint64_t block_slices = + (depth + block.block_depth - 1u) / block.block_depth; + const uint64_t storage_size = + block_columns * block_rows * block_slices * block.block_size; + const uint64_t slice_stride = + static_cast(pitch) * height * bpe; + + std::vector tiled(storage_size); + std::vector cpu(storage_size, 0); + std::vector gpu(storage_size, 0xab); + fill(&tiled, ++case_index); + + GpuTileInfo info{}; + info.family = block.family; + info.bytes_per_element = block.bytes_per_element; + info.linear_size = storage_size; + info.tiled_size = storage_size; + info.linear_slice_stride = volume ? slice_stride : 0; + info.width = width; + info.height = height; + info.depth = depth; + info.pitch = pitch; + info.surface_z = family.family == TileBlockFamily::RenderTarget64KB || + family.family == TileBlockFamily::Depth64KB + ? 3 + : 0; + convert_reference(false, &cpu, tiled, info); + GpuDetile(&m_runtime_context, tiled.data(), gpu.data(), storage_size, + storage_size, std::span(&info, 1)); + const auto family_label = [&](const char *operation) { + std::ostringstream out; + out << operation << " family=" << static_cast(family.family) + << " bpe=" << bpe; + return out.str(); + }; + compare(family_label("detile bytes").c_str(), cpu, gpu); + + { + std::vector linear(storage_size); + std::vector cpu_tiled(storage_size, 0); + std::vector gpu_tiled(storage_size, 0xab); + fill(&linear, 0x80u + case_index); + convert_reference(true, &cpu_tiled, linear, info); + GpuTile(&m_runtime_context, linear.data(), gpu_tiled.data(), + storage_size, storage_size, + std::span(&info, 1)); + compare(family_label("tile bytes").c_str(), cpu_tiled, gpu_tiled); + } + } + } + + // Exercise the real texture-layout/info-building seam for every format + // admitted by each standard tile mode. Formats sharing a byte/block width + // intentionally share a shader, but this loop still validates their + // texel-to-element conversion (notably every BCn format). + struct StandardMode { + u32 tile; + TileBlockFamily family; + bool (*supported)(u32); + }; + constexpr StandardMode standard_modes[] = { + {Prospero::GpuEnumValue(Prospero::TileMode::kStandard256B), + TileBlockFamily::Standard256B, TileIsStandard256BTextureSupported}, + {Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB), + TileBlockFamily::Standard4KB, TileIsStandard4KBTextureSupported}, + {Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB), + TileBlockFamily::Standard64KB, TileIsStandard64KBTextureSupported}, + {Prospero::GpuEnumValue(Prospero::TileMode::kPrt), + TileBlockFamily::Prt64KB, TileIsStandard64KBTextureSupported}, + }; + u32 format_cases = 0; + for (u32 format = 1; + format <= Prospero::GpuEnumValue(Prospero::BufferFormat::kBc7Srgb); + ++format) { + // FMASK is synthesized as identity metadata by TextureUploadFmask; it is + // deliberately not a texel surface and must never enter the GPU tiler. + if (Prospero::IsFmaskTextureFormat(format)) { + continue; + } + for (const auto &mode : standard_modes) { + if (!mode.supported(format)) { + continue; + } + const u32 bpe = + std::max(Prospero::NumBytesPerElement(format), + Prospero::BlockCompressedBytesPerBlock(format)); + TileBlockLayout block{}; + Require(name, "format block", + TileGetBlockLayout(mode.family, bpe, &block), + "CPU-supported format has no GPU family/BPE mapping"); + + constexpr u32 width = 67; + constexpr u32 height = 51; + const u32 pitch = TileGetTexturePitch(format, width, 1, mode.tile); + TileSizeAlign total{}; + TileGetTextureSize(format, width, height, pitch, 1, mode.tile, &total, + nullptr, nullptr); + Require(name, "format size", total.size != 0, + "supported format has an empty layout"); + + const auto layout = + TextureCalcUploadLayout(format, width, height, 1, 1, pitch, + mode.tile, total.size, false, false, name); + const auto regions = TextureBuildUploadRegions( + layout, TextureGetFormat(format), width, height, 1, 1, false, false, + TextureUploadDestination::MipLevels); + std::vector infos; + if (!TextureBuildGpuTileInfos(total.size, regions, layout, format, 1, 1, + &infos)) { + std::ostringstream out; + out << "format=" << format << " tile=" << mode.tile + << " size=" << total.size << " pitch=" << pitch; + Fail(name, "format infos", out.str()); + } + Require(name, "format family", + infos.size() == 1 && infos[0].family == mode.family && + infos[0].bytes_per_element == bpe, + "texture info selected the wrong shader family"); + + std::vector tiled(total.size); + std::vector cpu(total.size, 0); + std::vector gpu(total.size, 0xab); + fill(&tiled, ++case_index); + convert_reference(false, &cpu, tiled, infos[0]); + GpuDetile(&m_runtime_context, tiled.data(), gpu.data(), total.size, + total.size, infos); + compare("format bytes", cpu, gpu); + ++format_cases; + } + } + Require(name, "format coverage", format_cases != 0, + "no CPU-supported standard formats were tested"); + + { + constexpr u32 format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + constexpr u32 tile = + Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); + constexpr u32 width = 129, height = 65, layers = 3; + const u32 pitch = TileGetTexturePitch(format, width, 1, tile); + TileSizeAlign total{}; + TileGetTextureTotalSize(format, width, height, layers, pitch, 1, tile, + false, &total); + const auto layout = + TextureCalcUploadLayout(format, width, height, 1, layers, pitch, tile, + total.size, true, false, name); + const auto regions = TextureBuildUploadRegions( + layout, vk::Format::eR32Sfloat, width, height, layers, 1, true, false, + TextureUploadDestination::MipLevels); + std::vector infos; + const bool built = TextureBuildGpuTileInfos(total.size, regions, layout, + format, layers, 1, &infos); + Require(name, "array infos", + built && infos.size() == layers && infos[0].surface_z == 0 && + infos[1].surface_z == 1 && infos[2].surface_z == 2, + "array slices lost their absolute surface Z"); + check_round_trip("array", total.size, infos); + } + + for (const auto &mode : standard_modes) { + constexpr u32 format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + constexpr u32 levels = 2; + TileBlockLayout block{}; + Require(name, "odd mip block", TileGetBlockLayout(mode.family, 4, &block), + "odd multi-mip format has no block layout"); + const u32 width = block.block_width * 2u + 1u; + const u32 height = block.block_height * 2u + 1u; + const u32 pitch = TileGetTexturePitch(format, width, levels, mode.tile); + TileSizeAlign total{}; + TileGetTextureSize(format, width, height, pitch, levels, mode.tile, + &total, nullptr, nullptr); + const auto layout = + TextureCalcUploadLayout(format, width, height, levels, 1, pitch, + mode.tile, total.size, false, false, name); + const auto regions = TextureBuildUploadRegions( + layout, vk::Format::eR32Sfloat, width, height, 1, levels, false, + false, TextureUploadDestination::MipLevels); + std::vector infos; + Require(name, "odd mip infos", + TextureBuildGpuTileInfos(total.size, regions, layout, format, 1, + levels, &infos) && + infos.size() == 2 && infos[1].tiled_width >= infos[1].pitch && + infos[1].tiled_height >= infos[1].height && + (mode.family == TileBlockFamily::Standard256B || + infos[1].tiled_height > infos[1].height), + "odd multi-mip physical stride collapsed to the active linear " + "extent"); + check_round_trip("odd multi-mip", total.size, infos); + } + + struct TailMode { + u32 tile; + TileBlockFamily family; + }; + constexpr TailMode tail_modes[] = { + {Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB), + TileBlockFamily::Standard4KB}, + {Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB), + TileBlockFamily::Standard64KB}, + {Prospero::GpuEnumValue(Prospero::TileMode::kPrt), + TileBlockFamily::Prt64KB}, + {Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget), + TileBlockFamily::RenderTarget64KB}, + {Prospero::GpuEnumValue(Prospero::TileMode::kDepth), + TileBlockFamily::Depth64KB}, + }; + for (const auto &mode : tail_modes) { + constexpr u32 format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + constexpr u32 levels = 7; + TileBlockLayout block{}; + Require(name, "2D tail block", TileGetBlockLayout(mode.family, 4, &block), + "2D tail mode has no block layout"); + const u32 width = block.block_width * 2u; + const u32 height = block.block_height * 2u; + const u32 pitch = TileGetTexturePitch(format, width, levels, mode.tile); + TileSizeAlign total{}; + TileGetTextureSize(format, width, height, pitch, levels, mode.tile, + &total, nullptr, nullptr); + const auto layout = + TextureCalcUploadLayout(format, width, height, levels, 1, pitch, + mode.tile, total.size, true, false, name); + const auto regions = TextureBuildUploadRegions( + layout, vk::Format::eR32Sfloat, width, height, 1, levels, false, + false, TextureUploadDestination::MipLevels); + std::vector infos; + Require(name, "2D mip tail seam", + layout.first_tail_level == 2 && + TextureBuildGpuTileInfos(total.size, regions, layout, format, + 1, levels, &infos) && + infos.size() == levels && !infos[0].tail && !infos[1].tail && + std::all_of(infos.begin() + 2, infos.end(), + [](const auto &info) { return info.tail; }), + "2D mip chain lost its linear/tiled tail boundary"); + check_round_trip("2D mip tail seam", total.size, infos); + } + + { + constexpr u32 format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); + constexpr u32 tile = + Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB); + constexpr u32 width = 65; + constexpr u32 height = 129; + constexpr u32 depth = 17; + constexpr u32 levels = 2; + const u32 pitch = TileGetTexturePitch(format, width, levels, tile); + TileSizeAlign total{}; + TileGetTextureTotalSize(format, width, height, depth, pitch, levels, tile, + true, &total); + const auto layout = + TextureCalcUploadLayout(format, width, height, levels, depth, pitch, + tile, total.size, false, true, name); + const auto regions = TextureBuildUploadRegions( + layout, vk::Format::eR32Sfloat, width, height, depth, levels, false, + true, TextureUploadDestination::MipLevels); + std::vector infos; + const bool built = TextureBuildGpuTileInfos( + total.size, regions, layout, format, depth, levels, &infos); + Require(name, "3D mip infos", + regions.size() == depth + (depth >> 1u) && built && + infos.size() == 4 && infos[0].depth == 8 && + infos[1].depth == 8 && infos[2].depth == 1 && + infos[3].depth == 8 && infos[0].tiled_offset == 0x19000 && + infos[1].tiled_offset == 0x83000 && + infos[2].tiled_offset == 0xed000 && + infos[3].tiled_offset == 0 && infos[3].pitch == 32 && + infos[3].height == 64 && infos[3].tiled_width == 40 && + infos[3].tiled_height == 80, + "Standard4KB3D mip depth, packing, or physical stride changed"); + check_round_trip("3D mip", total.size, infos); + } + + { + constexpr u32 format = + Prospero::GpuEnumValue(Prospero::BufferFormat::kBc1UNorm); + constexpr u32 tile = + Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB); + constexpr u32 width = 65; + constexpr u32 height = 129; + constexpr u32 depth = 17; + constexpr u32 levels = 2; + const u32 pitch = TileGetTexturePitch(format, width, levels, tile); + TileSizeAlign total{}; + TileGetTextureTotalSize(format, width, height, depth, pitch, levels, tile, + true, &total); + const auto layout = + TextureCalcUploadLayout(format, width, height, levels, depth, pitch, + tile, total.size, false, true, name); + const auto regions = TextureBuildUploadRegions( + layout, TextureGetFormat(format), width, height, depth, levels, false, + true, TextureUploadDestination::MipLevels); + std::vector infos; + Require(name, "3D BC mip infos", + TextureBuildGpuTileInfos(total.size, regions, layout, format, + depth, levels, &infos) && + infos.size() == 4 && infos[3].pitch == 8 && + infos[3].height == 16 && infos[3].tiled_width == 16 && + infos[3].tiled_height == 24, + "block-compressed 3D mip lost its physical row or slice stride"); + check_round_trip("3D BC mip", total.size, infos); + } + + struct VolumeModeCase { + u32 tile; + TileBlockFamily family; + u32 format; + }; + constexpr VolumeModeCase volume_modes[] = { + {Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB), + TileBlockFamily::Standard64KB3D, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float)}, + {Prospero::GpuEnumValue(Prospero::TileMode::kPrt), + TileBlockFamily::Prt64KB3D, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float)}, + {Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget), + TileBlockFamily::RenderTarget64KB, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float)}, + {Prospero::GpuEnumValue(Prospero::TileMode::kDepth), + TileBlockFamily::Depth64KB, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float)}, + {Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB), + TileBlockFamily::Standard64KB3D, + Prospero::GpuEnumValue(Prospero::BufferFormat::kBc1UNorm)}, + {Prospero::GpuEnumValue(Prospero::TileMode::kPrt), + TileBlockFamily::Prt64KB3D, + Prospero::GpuEnumValue(Prospero::BufferFormat::kBc3UNorm)}, + }; + for (const auto test : volume_modes) { + constexpr u32 width = 65, height = 33, depth = 37, levels = 6; + const u32 pitch = + TileGetTexturePitch(test.format, width, levels, test.tile); + TileSizeAlign total{}; + TileGetTextureTotalSize(test.format, width, height, depth, pitch, levels, + test.tile, true, &total); + const auto layout = TextureCalcUploadLayout( + test.format, width, height, levels, depth, pitch, test.tile, + total.size, true, true, name); + const auto regions = TextureBuildUploadRegions( + layout, TextureGetFormat(test.format), width, height, depth, levels, + false, true, TextureUploadDestination::MipLevels); + std::vector infos; + const bool built = TextureBuildGpuTileInfos( + total.size, regions, layout, test.format, depth, levels, &infos); + const bool uses_z = test.family == TileBlockFamily::RenderTarget64KB || + test.family == TileBlockFamily::Depth64KB; + Require(name, "volume family infos", + built && !infos.empty() && + std::all_of(infos.begin(), infos.end(), + [&](const auto &info) { + return info.family == test.family; + }) && + (!uses_z || std::any_of(infos.begin(), infos.end(), + [](const auto &info) { + return info.surface_z != 0; + })), + "volume mode selected the wrong family or lost its surface Z"); + check_round_trip("volume family", total.size, infos); + } + + struct VolumeTailCase { + u32 format; + u32 bytes; + }; + constexpr VolumeTailCase volume_tails[] = { + {Prospero::GpuEnumValue(Prospero::BufferFormat::k8UNorm), 1}, + {Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), 2}, + {Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), 4}, + {Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float), 8}, + {Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32_32_32Float), 16}, + {Prospero::GpuEnumValue(Prospero::BufferFormat::kBc1UNorm), 8}, + {Prospero::GpuEnumValue(Prospero::BufferFormat::kBc3UNorm), 16}, + }; + constexpr u32 volume_tail_xy[5][5][2] = { + {{0, 8}, {8, 4}, {8, 0}, {0, 4}, {0, 0}}, + {{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}}, + {{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}}, + {{0, 4}, {4, 2}, {4, 0}, {0, 2}, {0, 0}}, + {{0, 4}, {2, 2}, {2, 0}, {0, 2}, {0, 0}}, + }; + for (const auto &tail : volume_tails) { + constexpr u32 tile = + Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB); + constexpr u32 levels = 5; + TileBlockLayout block{}; + Require(name, "3D tail block", + TileGetBlockLayout(TileBlockFamily::Standard4KB3D, tail.bytes, + &block), + "Standard4KB3D tail format has no block layout"); + const bool compressed = + Prospero::BlockCompressedBytesPerBlock(tail.format) != 0; + const u32 width = block.block_width * (compressed ? 4u : 1u); + const u32 height = block.block_height / 2u * (compressed ? 4u : 1u); + const u32 depth = block.block_depth + 1u; + const u32 pitch = TileGetTexturePitch(tail.format, width, levels, tile); + TileSizeAlign total{}; + TileGetTextureTotalSize(tail.format, width, height, depth, pitch, levels, + tile, true, &total); + const auto layout = + TextureCalcUploadLayout(tail.format, width, height, levels, depth, + pitch, tile, total.size, false, true, name); + const auto regions = TextureBuildUploadRegions( + layout, TextureGetFormat(tail.format), width, height, depth, levels, + false, true, TextureUploadDestination::MipLevels); + std::vector infos; + bool valid = + total.size == 8192 && + regions.size() == + depth + std::max(depth >> 1u, 1u) + std::max(depth >> 2u, 1u) + + std::max(depth >> 3u, 1u) + std::max(depth >> 4u, 1u) && + TextureBuildGpuTileInfos(total.size, regions, layout, tail.format, + depth, levels, &infos) && + infos.size() == 6; + const u32 table = std::countr_zero(tail.bytes); + for (u32 level = 0; level < levels && valid; level++) { + const auto &info = infos[level == 0 ? 0 : level + 1]; + valid &= info.tail && info.tail_x == volume_tail_xy[table][level][0] && + info.tail_y == volume_tail_xy[table][level][1] && + info.tiled_offset == 0; + } + if (valid) { + valid = infos[1].tail && + infos[1].tail_x == volume_tail_xy[table][0][0] && + infos[1].tail_y == volume_tail_xy[table][0][1] && + infos[1].tiled_offset == 4096; + } + Require( + name, "3D mip tail infos", valid, + "Standard4KB3D mip tail coordinates or block-slice packing changed"); + check_round_trip("3D mip tail", total.size, infos); + } + + for (const auto family : + {TileBlockFamily::Standard4KB, TileBlockFamily::Standard64KB, + TileBlockFamily::Prt64KB, TileBlockFamily::RenderTarget64KB, + TileBlockFamily::Depth64KB}) { + for (u32 bpe = 1; bpe <= 16; bpe <<= 1u) { + if (family == TileBlockFamily::Depth64KB && bpe == 16) + continue; + TileBlockLayout block{}; + Require(name, "tail layout", TileGetBlockLayout(family, bpe, &block), + "tail family/BPE has no block layout"); + const u32 width = std::max(block.block_width / 4u, 1u); + const u32 height = std::max(block.block_height / 4u, 1u); + const u32 x = block.block_width / 2u; + const u32 y = block.block_height / 2u; + const u32 pitch = width; + const uint64_t linear_size = + static_cast(pitch) * height * bpe; + std::vector tiled(block.block_size); + std::vector cpu(linear_size, 0xcd); + std::vector gpu(linear_size, 0xab); + fill(&tiled, ++case_index); + GpuTileInfo info{}; + info.family = block.family; + info.bytes_per_element = block.bytes_per_element; + info.linear_size = linear_size; + info.tiled_size = block.block_size; + info.width = width; + info.height = height; + info.pitch = pitch; + info.tail = true; + info.tail_x = x; + info.tail_y = y; + info.surface_z = family == TileBlockFamily::RenderTarget64KB || + family == TileBlockFamily::Depth64KB + ? 2 + : 0; + convert_reference(false, &cpu, tiled, info); + GpuDetile(&m_runtime_context, tiled.data(), gpu.data(), + block.block_size, linear_size, + std::span(&info, 1)); + compare("tail bytes", cpu, gpu); + + std::vector linear(linear_size); + std::vector cpu_tiled(block.block_size, 0); + std::vector gpu_tiled(block.block_size, 0xab); + fill(&linear, 0x400u + case_index); + convert_reference(true, &cpu_tiled, linear, info); + GpuTile(&m_runtime_context, linear.data(), gpu_tiled.data(), + block.block_size, linear_size, + std::span(&info, 1)); + compare("tail tile bytes", cpu_tiled, gpu_tiled); + } + } + + TileBlockLayout small_block{}; + Require(name, "small layout", + TileGetBlockLayout(TileBlockFamily::Standard256B, 4, &small_block), + "small fixture layout is unavailable"); + std::vector small_input(small_block.block_size); + std::vector small_expected(small_block.block_size, 0); + std::vector small_output(small_block.block_size, 0xab); + fill(&small_input, 0xee); + GpuTileInfo small_info{}; + small_info.family = small_block.family; + small_info.bytes_per_element = small_block.bytes_per_element; + small_info.linear_size = small_output.size(); + small_info.tiled_size = small_input.size(); + small_info.width = 1; + small_info.height = 1; + small_info.pitch = small_block.block_width; + convert_reference(false, &small_expected, small_input, small_info); + GpuDetile(&m_runtime_context, small_input.data(), small_output.data(), + small_input.size(), small_output.size(), + std::span(&small_info, 1)); + compare("small detile", small_expected, small_output); + + GpuTileRelease(&m_runtime_context); + std::fill(small_output.begin(), small_output.end(), 0xab); + GpuDetile(&m_runtime_context, small_input.data(), small_output.data(), + small_input.size(), small_output.size(), + std::span(&small_info, 1)); + compare("release recovery", small_expected, small_output); + std::printf("[gpu] %-32s ok (%u cases, %u format/mode pairs)\n", name, + case_index, format_cases); + } + private: void EnsureRuntimeContext() { if (m_runtime_context.allocator != nullptr) { @@ -9095,8 +9808,7 @@ void CheckEmbeddedFetchVertexOffset() { Require("EmbeddedFetchNggVertexOffset", "encoding", ngg_code[3] == 0xd55d0005u && ngg_code[4] == 0x04150012u, "test does not encode the PS5 V_SAD_U32 vertex-offset prolog"); - const auto ngg = - Compile("EmbeddedFetchNggVertexOffset", ngg_code, 8); + const auto ngg = Compile("EmbeddedFetchNggVertexOffset", ngg_code, 8); Require("EmbeddedFetchNggVertexOffset", "parse", ngg.program.info.vertex_offset_sgpr == 18 && Resolve(ngg, 0) == 8 && Resolve(ngg, 5) == 5, @@ -10284,20 +10996,17 @@ void CheckStorageTextureLinearUploadLayout() { &total); const auto layout = TextureCalcUploadLayout( format, width, height, 1, depth, pitch, tile, total.size, true, false, - false, "StorageTextureLinearTest"); + "StorageTextureLinearTest"); const auto regions = TextureBuildUploadRegions( layout, vk::Format::eR8G8B8A8Unorm, width, height, depth, 1, false, false, - TextureUploadDestination::MipLevels, - TextureUploadSliceLayout::MipChainPerSlice); + TextureUploadDestination::MipLevels); Require("StorageTextureLinearUpload", "layout", pitch == width && total.size == 0x1fa4000 && total.align == 256 && layout.tile == tile && layout.pitch == width && layout.slice_stride == total.size && regions.size() == 1 && regions[0].offset == 0 && regions[0].width == width && regions[0].height == height && regions[0].pitch == width && - TextureCalcUploadSize( - layout, regions, 1, depth, - TextureUploadSliceLayout::MipChainPerSlice) == total.size, + TextureCalcUploadSize(layout, regions, 1, depth) == total.size, "linear RGBA8 storage upload lost Prospero pitch or allocation size"); std::printf("[host] %-32s ok\n", "StorageTextureLinearUpload"); } @@ -10320,25 +11029,26 @@ void CheckStorageTextureDepthTileUploadLayout() { &total); const auto layout = TextureCalcUploadLayout( format, width, height, 1, depth, pitch, tile, total.size, true, false, - false, "StorageTextureDepthTileTest"); + "StorageTextureDepthTileTest"); const auto regions = TextureBuildUploadRegions( layout, vk::Format::eR8Uint, width, height, depth, 1, true, false, - TextureUploadDestination::MipLevels, - TextureUploadSliceLayout::MipChainPerSlice); + TextureUploadDestination::MipLevels); Require("StorageTextureDepthTileUpload", "PPSA14053 layout", pitch == 256 && padded.width == 256 && padded.height == 256 && slice.size == 0x10000 && slice.align == 0x10000 && level.size == slice.size && level.offset == 0 && total.size == slice.size && total.align == slice.align && - layout.tile == tile && layout.fmt_tiled_depth && - !layout.fmt_tiled_render_target && layout.pitch == pitch && - layout.slice_stride == total.size && regions.size() == 1 && - regions[0].offset == 0 && regions[0].width == width && - regions[0].height == height && regions[0].pitch == pitch && - TextureCalcUploadSize( - layout, regions, 1, depth, - TextureUploadSliceLayout::MipChainPerSlice) == total.size, - "1x1 R8_UINT depth tile lost its PS5 64 KiB block footprint"); + layout.tile == tile && + layout.tile_family == TileBlockFamily::Depth64KB && + layout.pitch == pitch && layout.slice_stride == pitch && + layout.source_slice_stride == total.size && + layout.level_sizes[0].size == pitch && + layout.level_sizes[0].src_size == total.size && + regions.size() == 1 && regions[0].offset == 0 && + regions[0].width == width && regions[0].height == height && + regions[0].pitch == pitch && + TextureCalcUploadSize(layout, regions, 1, depth) == total.size, + "1x1 R8_UINT depth tile lost its 64 KiB source footprint"); std::printf("[host] %-32s ok\n", "StorageTextureDepthTileUpload"); } @@ -10410,7 +11120,6 @@ void CheckStorageTextureVolumeUploadLayout() { constexpr uint32_t width = 33; constexpr uint32_t height = 33; constexpr uint32_t depth = 33; - constexpr uint32_t bytes_per_element = 8; const auto pitch = TileGetTexturePitch( format, width, 1, Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget)); @@ -10421,55 +11130,90 @@ void CheckStorageTextureVolumeUploadLayout() { const auto layout = TextureCalcUploadLayout( format, width, height, 1, depth, pitch, Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget), total.size, - true, false, true, "StorageTextureVolumeTest"); + true, true, "StorageTextureVolumeTest"); const auto regions = TextureBuildUploadRegions( layout, vk::Format::eR16G16B16A16Sfloat, width, height, depth, 1, false, - true, TextureUploadDestination::MipLevels, - TextureUploadSliceLayout::MipChainPerSlice); - Require( - "StorageTextureVolumeUpload", "layout", - pitch == 128 && total.size == 0x210000 && layout.slice_stride == 0x8400 && - layout.source_slice_stride == 0x10000 && - layout.level_sizes[0].size == 0x8400 && - layout.level_sizes[0].src_size == 0x10000 && - TextureCalcUploadSize(layout, regions, 1, depth, - TextureUploadSliceLayout::MipChainPerSlice) == - total.size, - "3D render-target upload did not preserve distinct linear and guest " - "strides"); + true, TextureUploadDestination::MipLevels); + Require("StorageTextureVolumeUpload", "layout", + pitch == 128 && total.size == 0x210000 && + layout.slice_stride == 0x2208 && + layout.source_slice_stride == 0 && + layout.level_sizes[0].size == 0x2208 && + layout.level_sizes[0].src_size == 0 && + TextureCalcUploadSize(layout, regions, 1, depth) == + layout.slice_stride * depth, + "3D render-target upload did not preserve its compact linear layout"); - std::vector source(total.size); - for (uint32_t z = 0; z < depth; z++) { - std::fill_n(source.data() + z * layout.source_slice_stride, - layout.source_slice_stride, static_cast(z + 1)); - } - std::vector linear(layout.slice_stride * depth, 0); + std::vector infos; + Require("StorageTextureVolumeUpload", "GPU records", + TextureBuildGpuTileInfos(total.size, regions, layout, format, depth, + 1, &infos) && + infos.size() == depth, + "3D render-target GPU records were not built"); for (const uint32_t z : {0u, 1u, depth - 1u}) { - const auto src_offset = TextureUploadSliceSourceOffset( - layout, 0, z, TextureUploadSliceLayout::MipChainPerSlice); - Require("StorageTextureVolumeUpload", "source offset", - src_offset == static_cast(z) * 0x10000, - "volume slice selected a compact linear source offset"); - TileConvertTiledToLinearRenderTarget( - linear.data() + regions[z].offset, source.data() + src_offset, width, - height, regions[z].pitch, bytes_per_element, layout.level_sizes[0].size, - layout.level_sizes[0].src_size, layout.level_sizes[0].x, - layout.level_sizes[0].y); - const auto value = static_cast(z + 1); - for (uint32_t y = 0; y < height; y++) { - const auto row = regions[z].offset + static_cast(y) * - regions[z].pitch * - bytes_per_element; - Require("StorageTextureVolumeUpload", "contents", - std::all_of(linear.begin() + row, - linear.begin() + row + width * bytes_per_element, - [value](uint8_t byte) { return byte == value; }), - "volume slice detiled from another slice's padded allocation"); - } + Require("StorageTextureVolumeUpload", "slice offsets", + infos[z].linear_offset == static_cast(z) * 0x2208 && + infos[z].tiled_offset == static_cast(z) * 0x10000 && + infos[z].surface_z == z && infos[z].pitch == width, + "volume slice lost its linear stride, block slice, or Z swizzle"); } std::printf("[host] %-32s ok\n", "StorageTextureVolumeUpload"); } +void CheckStorageTextureVolumeMipRegions() { + constexpr uint32_t format = + Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm); + constexpr uint32_t width = 8; + constexpr uint32_t height = 4; + constexpr uint32_t depth = 5; + constexpr uint32_t levels = 3; + constexpr uint32_t tile = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + const auto pitch = TileGetTexturePitch(format, width, levels, tile); + TileSizeAlign total{}; + TileGetTextureTotalSize(format, width, height, depth, pitch, levels, tile, + true, &total); + const auto layout = TextureCalcUploadLayout( + format, width, height, levels, depth, pitch, tile, total.size, true, true, + "StorageTextureVolumeMipTest"); + const auto uploads = TextureBuildUploadRegions( + layout, vk::Format::eR8G8B8A8Unorm, width, height, depth, levels, false, + true, TextureUploadDestination::MipLevels); + const auto downloads = TextureBuildDownloadRegions(uploads); + + bool valid = uploads.size() == 8 && downloads.size() == uploads.size(); + size_t index = 0; + for (uint32_t level = 0; level < levels && valid; level++) { + const uint32_t mip_depth = std::max(depth >> level, 1u); + const uint32_t mip_width = std::max(width >> level, 1u); + const uint32_t mip_height = std::max(height >> level, 1u); + for (uint32_t z = 0; z < mip_depth; z++, index++) { + const auto &upload = uploads[index]; + const auto &download = downloads[index]; + valid &= + upload.dst_level == level && upload.dst_z == static_cast(z) && + upload.width == mip_width && upload.height == mip_height && + upload.offset == + layout.level_sizes[level].offset + z * layout.slice_stride && + download.src_level == upload.dst_level && + download.src_z == upload.dst_z && download.offset == upload.offset && + download.pitch == upload.pitch; + } + } + valid &= index == uploads.size(); + Require("StorageTextureVolumeMipRegions", "per-mip depth", valid, + "3D upload/readback regions did not shrink depth or preserve Vulkan " + "Z coordinates"); + const auto upload_size = + TextureCalcUploadSize(layout, uploads, levels, depth); + if (upload_size != total.size) { + std::ostringstream out; + out << "calculated=" << upload_size << " total=" << total.size + << " stride=" << layout.slice_stride; + Fail("StorageTextureVolumeMipRegions", "allocation size", out.str()); + } + std::printf("[host] %-32s ok\n", "StorageTextureVolumeMipRegions"); +} + void CheckColorResolveLayers() { RenderColorInfo src{}; RenderColorInfo dst{}; @@ -10496,23 +11240,6 @@ void CheckColorResolveLayers() { std::printf("[host] %-32s ok\n", "ColorResolveLayers"); } -[[noreturn]] void RunStandard64RenderTargetDeathCase() { - RenderTargetInfo info{}; - info.address = 0x10000; - info.size = 0x10000; - info.format = vk::Format::eR16G16B16A16Sfloat; - info.width = 128; - info.height = 64; - info.pitch = 128; - info.bytes_per_element = 8; - info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kStandard64KB); - std::vector linear(info.size, 0); - std::vector tiled(info.size, 0); - Tiler tiler; - tiler.TileImage(tiled.data(), linear.data(), info); - std::_Exit(0x7f); -} - void CheckStandard64RenderTargetTileRoundTrip() { constexpr uint32_t format = Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float); @@ -10589,508 +11316,9 @@ void CheckStandard64RenderTargetTileRoundTrip() { !IsSupportedStandard64RenderTarget(unsupported), "unimplemented Standard64KB array was accepted"); - std::vector linear(storage.size / sizeof(uint32_t), 0); - for (uint32_t y = 0; y < height; y++) { - for (uint32_t x = 0; x < width; x++) { - linear[static_cast(y) * pitch + x] = - (y * 0x1f123bb5u) ^ (x * 0x9e3779b9u) ^ 0xa55a3cc3u; - } - } - std::vector tiled(linear.size(), 0xcdcdcdcdu); - std::vector restored(linear.size(), 0xa5a5a5a5u); - Tiler tiler; - tiler.TileImage(tiled.data(), linear.data(), info); - TileConvertTiledToLinearStandard64KB32(restored.data(), tiled.data(), width, - height, pitch, storage.size); - for (uint32_t y = 0; y < height; y++) { - const auto row = static_cast(y) * pitch; - Require("Standard64RenderTarget", "round trip", - std::memcmp(linear.data() + row, restored.data() + row, - width * sizeof(uint32_t)) == 0, - "Standard64KB render-target conversion did not round-trip"); - } - Require("Standard64RenderTarget", "padding", tiled.back() == 0, - "Standard64KB render-target conversion retained stale padding"); - - constexpr uint32_t block_width = 128; - constexpr uint32_t block_height = 128; - constexpr uint64_t block_size = 0x10000; - std::vector known(block_size / sizeof(uint32_t)); - for (uint32_t i = 0; i < known.size(); i++) { - known[i] = i; - } - std::vector standard(known.size(), 0); - std::vector render_target(known.size(), 0); - TileConvertLinearToTiledStandard64KB32(standard.data(), known.data(), - block_width, block_height, block_width, - block_size); - Require("Standard64RenderTarget", "SDK byte mapping", - standard[0] == 0 && standard[1] == 1 && standard[2] == 2 && - standard[4] == 128 && standard[8] == 256 && standard[16] == 512 && - standard[32] == 4 && standard[16383] == 16383, - "Standard64KB reciprocal mapping disagreed with the PS5 SDK"); - - constexpr uint32_t block_width_16 = 256; - constexpr uint32_t block_height_16 = 128; - std::vector tiled_16(block_size / sizeof(uint16_t)); - for (uint32_t i = 0; i < tiled_16.size(); i++) { - tiled_16[i] = static_cast(i); - } - std::vector linear_16(tiled_16.size(), 0xffffu); - TileConvertTiledToLinearStandard64KB16(linear_16.data(), tiled_16.data(), - block_width_16, block_height_16, - block_width_16, block_size); - const auto sample_16 = [&](uint32_t x, uint32_t y) { - return linear_16[static_cast(y) * block_width_16 + x]; - }; - Require( - "Standard64RenderTarget", "16-bit SDK byte mapping", - sample_16(0, 0) == 0 && sample_16(1, 0) == 1 && sample_16(2, 0) == 2 && - sample_16(4, 0) == 4 && sample_16(8, 0) == 64 && - sample_16(0, 1) == 8 && sample_16(0, 2) == 16 && - sample_16(0, 4) == 32, - "Standard64KB 16-bit mapping interleaved X/Y address bits incorrectly"); - TileConvertLinearToTiledRenderTarget(render_target.data(), known.data(), - block_width, block_height, block_width, - sizeof(uint32_t), block_size); - Require("Standard64RenderTarget", "mode identity", standard != render_target, - "Standard64KB was accidentally treated as RenderTarget/XOR tiling"); - - char path[MAX_PATH]{}; - Require("Standard64RenderTarget", "host", - GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, - "GetModuleFileName failed"); - std::string command = std::string("\"") + path + "\" --standard64-rt-death"; - std::vector mutable_command(command.begin(), command.end()); - mutable_command.push_back('\0'); - STARTUPINFOA startup{sizeof(startup)}; - PROCESS_INFORMATION process{}; - Require("Standard64RenderTarget", "host", - CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, - FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup, - &process) != 0, - "CreateProcess failed"); - Require("Standard64RenderTarget", "host", - WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, - "unsupported Standard64KB render-target case timed out"); - DWORD exit_code = 0; - const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; - CloseHandle(process.hThread); - CloseHandle(process.hProcess); - Require("Standard64RenderTarget", "hard failure", exited && exit_code == 321, - "unsupported Standard64KB render-target shape lost its fatal guard"); std::printf("[host] %-32s ok\n", "Standard64RenderTarget"); } -void CheckRenderTargetTileRoundTrip() { - struct Case { - uint32_t width; - uint32_t height; - uint32_t bytes_per_element; - }; - constexpr std::array cases{Case{129, 65, 1}, Case{257, 131, 2}, - Case{3840, 2160, 4}, Case{65, 67, 8}}; - for (const auto test : cases) { - const auto pitch = - TileGetRenderTargetPitch(test.width, test.bytes_per_element); - TileSizeAlign storage{}; - Require("RenderTargetTileRoundTrip", "layout", - pitch >= test.width && - TileGetRenderTargetSize(test.width, test.height, pitch, - test.bytes_per_element, &storage) && - storage.size != 0, - "render-target test layout was rejected"); - std::vector linear(storage.size, 0); - for (uint32_t y = 0; y < test.height; y++) { - for (uint32_t x = 0; x < test.width * test.bytes_per_element; x++) { - const auto offset = - static_cast(y) * pitch * test.bytes_per_element + x; - linear[offset] = static_cast( - (x * 37u + y * 101u + test.bytes_per_element * 13u) & 0xffu); - } - } - std::vector tiled(storage.size, 0xcd); - std::vector restored(storage.size, 0xa5); - TileConvertLinearToTiledRenderTarget(tiled.data(), linear.data(), - test.width, test.height, pitch, - test.bytes_per_element, storage.size); - TileConvertTiledToLinearRenderTarget(restored.data(), tiled.data(), - test.width, test.height, pitch, - test.bytes_per_element, storage.size); - for (uint32_t y = 0; y < test.height; y++) { - const auto row = - static_cast(y) * pitch * test.bytes_per_element; - Require("RenderTargetTileRoundTrip", "contents", - std::memcmp(linear.data() + row, restored.data() + row, - test.width * test.bytes_per_element) == 0, - "linear-to-tiled render-target conversion did not round-trip"); - } - } - constexpr uint32_t width = 257; - constexpr uint32_t height = 131; - constexpr uint32_t bytes_per_element = 4; - constexpr uint32_t layers = 3; - const auto pitch = TileGetRenderTargetPitch(width, bytes_per_element); - TileSizeAlign storage{}; - Require("RenderTargetTileRoundTrip", "layered layout", - TileGetRenderTargetSize(width, height, pitch, bytes_per_element, - &storage) && - storage.size <= UINT32_MAX / layers, - "layered render-target test layout was rejected"); - RenderTargetInfo info{}; - info.address = 1; - info.size = storage.size * layers; - info.format = vk::Format::eR8G8B8A8Unorm; - info.width = width; - info.height = height; - info.pitch = pitch; - info.bytes_per_element = bytes_per_element; - info.tile_mode = Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - info.layers = layers; - std::vector linear(info.size, 0); - for (uint32_t layer = 0; layer < layers; layer++) { - for (uint32_t y = 0; y < height; y++) { - const auto row = storage.size * layer + - static_cast(y) * pitch * bytes_per_element; - std::fill_n(linear.data() + row, width * bytes_per_element, - static_cast(0x31 + layer * 0x27)); - } - } - std::vector tiled(info.size, 0xcd); - std::vector restored(info.size, 0xa5); - Tiler tiler; - tiler.TileImage(tiled.data(), linear.data(), info); - for (uint32_t layer = 0; layer < layers; layer++) { - TileConvertTiledToLinearRenderTarget(restored.data() + storage.size * layer, - tiled.data() + storage.size * layer, - width, height, pitch, - bytes_per_element, storage.size); - for (uint32_t y = 0; y < height; y++) { - const auto row = storage.size * layer + - static_cast(y) * pitch * bytes_per_element; - Require("RenderTargetTileRoundTrip", "layered contents", - std::memcmp(linear.data() + row, restored.data() + row, - width * bytes_per_element) == 0, - "layered render-target conversion crossed array slices"); - } - } - const auto regions = Transfer::MakeLayeredImageBufferCopies( - layers, storage.size, pitch, width, height); - Require("RenderTargetTileRoundTrip", "layered regions", - regions.size() == layers && regions[0].offset == 0 && - regions[1].offset == storage.size && - regions[2].offset == storage.size * 2 && - regions[0].src_layer == 0 && regions[1].src_layer == 1 && - regions[2].src_layer == 2 && regions[2].pitch == pitch && - regions[2].width == width && regions[2].height == height, - "layered image-buffer regions did not preserve slice offsets"); - - constexpr uint32_t mip_format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k16_16_16_16Float); - ImageInfo mip_info{}; - mip_info.address = 0x10e3d0000ull; - mip_info.width = 512; - mip_info.height = 512; - mip_info.pitch = TileGetRenderTargetPitch(mip_info.width, 8); - mip_info.levels = 10; - mip_info.view_levels = mip_info.levels; - mip_info.tile = Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - mip_info.depth = 1; - mip_info.type = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); - mip_info.format = mip_format; - TileSizeAlign mip_storage{}; - Require("RenderTargetTileRoundTrip", "PPSA06084 mip layout", - mip_info.pitch == 512 && - TileGetRenderTargetMipLayout(mip_info.width, mip_info.height, - mip_info.pitch, 8, mip_info.levels, - &mip_storage, nullptr, nullptr) && - mip_storage.align == 0x10000 && mip_storage.size == 0x2b0000, - "captured render-target mip-chain layout regressed"); - mip_info.size = mip_storage.size; - const auto mip_layout = TextureCalcUploadLayout( - mip_info.format, mip_info.width, mip_info.height, mip_info.levels, - mip_info.depth, mip_info.pitch, mip_info.tile, mip_info.size, false, - false, false, "RenderTargetMipReadbackTest"); - const auto mip_regions = TextureBuildUploadRegions( - mip_layout, vk::Format::eR16G16B16A16Sfloat, mip_info.width, - mip_info.height, mip_info.depth, mip_info.levels, true, false, - TextureUploadDestination::MipLevels, - TextureUploadSliceLayout::MipChainPerSlice); - Require( - "RenderTargetTileRoundTrip", "PPSA06084 mip regions", - mip_regions.size() == mip_info.levels && - mip_layout.fmt_tiled_render_target && - mip_layout.pitch == mip_info.pitch, - "captured render-target mip-chain did not produce exact subresources"); - std::vector mip_linear(mip_info.size, 0); - for (uint32_t level = 0; level < mip_info.levels; level++) { - const auto ®ion = mip_regions[level]; - for (uint32_t y = 0; y < region.height; y++) { - auto *row = mip_linear.data() + region.offset + - static_cast(y) * region.pitch * 8u; - for (uint32_t x = 0; x < region.width * 8u; x++) { - row[x] = - static_cast((level * 47u + y * 19u + x * 11u) & 0xffu); - } - } - } - std::vector mip_guest(mip_info.size, 0xcd); - tiler.TileImage(mip_guest.data(), mip_linear.data(), mip_info); - std::vector mip_restored(mip_info.size, 0); - for (uint32_t level = 0; level < mip_info.levels; level++) { - const auto ®ion = mip_regions[level]; - const auto &level_size = mip_layout.level_sizes[level]; - const auto guest_offset = - level_size.src_size != 0 ? level_size.src_offset : level_size.offset; - TileConvertTiledToLinearRenderTarget( - mip_restored.data() + region.offset, mip_guest.data() + guest_offset, - region.width, region.height, region.pitch, 8u, level_size.size, - level_size.src_size, level_size.x, level_size.y); - for (uint32_t y = 0; y < region.height; y++) { - const auto row = - region.offset + static_cast(y) * region.pitch * 8u; - Require("RenderTargetTileRoundTrip", "PPSA06084 mip contents", - std::memcmp(mip_linear.data() + row, mip_restored.data() + row, - region.width * 8u) == 0, - "render-target mip readback conversion did not round-trip"); - } - } - std::printf("[host] %-32s ok\n", "RenderTargetTileRoundTrip"); -} - -void CheckStorageTextureMipTailRoundTrip() { - constexpr uint32_t format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k32_32UInt); - ImageInfo info{}; - info.address = 0x4a4290000ull; - info.format = format; - info.width = 64; - info.height = 64; - info.pitch = TileGetTexturePitch( - format, info.width, 6, - Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget)); - info.levels = 6; - info.view_levels = 1; - info.tile = Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget); - info.depth = 1; - info.type = Prospero::GpuEnumValue(Prospero::ImageType::kColor2D); - TileSizeAlign total{}; - TileSizeOffset levels[16]{}; - TilePaddedSize padded[16]{}; - TileGetTextureSize(format, info.width, info.height, info.pitch, info.levels, - info.tile, &total, levels, padded); - info.size = total.size; - Require("StorageTextureMipTail", "layout", - info.pitch == 128 && info.size == 0x20000 && - levels[0].src_offset == 0x10000 && - levels[0].src_size == 0x10000 && levels[1].src_offset == 0 && - levels[1].src_size == 0x10000, - "PPSA01530 mip-tail fixture disagrees with the PS5 layout"); - auto layout = TextureCalcUploadLayout( - format, info.width, info.height, info.levels, info.depth, info.pitch, - info.tile, info.size, true, false, false, "StorageTextureMipTailTest"); - auto regions = TextureBuildUploadRegions( - layout, vk::Format::eR32G32Uint, info.width, info.height, info.depth, - info.levels, false, false, TextureUploadDestination::MipLevels, - TextureUploadSliceLayout::MipChainPerSlice); - std::vector linear(info.size, 0); - for (uint32_t level = 0; level < info.levels; level++) { - auto ®ion = regions[level]; - for (uint32_t y = 0; y < region.height; y++) { - auto *row = linear.data() + region.offset + - static_cast(y) * region.pitch * 8u; - for (uint32_t x = 0; x < region.width * 8u; x++) { - row[x] = static_cast((level * 43u + y * 17u + x * 7u) & 0xffu); - } - } - } - std::vector guest(info.size, 0xcd); - Tiler tiler; - tiler.TileImage(guest.data(), linear.data(), info); - std::vector restored(info.size, 0); - for (uint32_t level = 0; level < info.levels; level++) { - const auto ®ion = regions[level]; - const auto &level_size = layout.level_sizes[level]; - const auto guest_offset = - level_size.src_size != 0 ? level_size.src_offset : level_size.offset; - TileConvertTiledToLinearRenderTarget( - restored.data() + region.offset, guest.data() + guest_offset, - region.width, region.height, region.pitch, 8u, level_size.size, - level_size.src_size, level_size.x, level_size.y); - for (uint32_t y = 0; y < region.height; y++) { - const auto row = - region.offset + static_cast(y) * region.pitch * 8u; - Require("StorageTextureMipTail", "contents", - std::memcmp(linear.data() + row, restored.data() + row, - region.width * 8u) == 0, - "multi-mip storage texture did not round-trip through its packed " - "tail"); - } - } - std::printf("[host] %-32s ok\n", "StorageTextureMipTail"); -} - -void CheckDepthTargetTileRoundTrip() { - struct Case { - uint32_t width; - uint32_t height; - uint32_t guest_format; - uint32_t depth_format; - uint32_t bytes_per_element; - }; - constexpr std::array cases{ - Case{8, 8, Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), - Prospero::GpuEnumValue(Prospero::DepthFormat::kZ16), 2}, - Case{129, 130, Prospero::GpuEnumValue(Prospero::BufferFormat::k16UNorm), - Prospero::GpuEnumValue(Prospero::DepthFormat::kZ16), 2}, - Case{8, 8, Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), - Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), 4}, - Case{640, 360, Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), - Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), 4}}; - for (const auto test : cases) { - const auto pitch = - TileGetTexturePitch(test.guest_format, test.width, 1, - Prospero::GpuEnumValue(Prospero::TileMode::kDepth)); - TileSizeAlign stencil{}; - TileSizeAlign htile{}; - TileSizeAlign depth{}; - Require("DepthTargetTileRoundTrip", "layout", - pitch >= test.width && - TileGetDepthSize( - test.width, test.height, 0, test.depth_format, - Prospero::GpuEnumValue(Prospero::StencilFormat::kInvalid), - false, &stencil, &htile, &depth) && - depth.size != 0 && depth.align == 0x10000, - "Prospero depth layout was rejected"); - std::vector linear(depth.size, 0); - for (uint32_t y = 0; y < test.height; y++) { - for (uint32_t x = 0; x < test.width * test.bytes_per_element; x++) { - const auto offset = - static_cast(y) * pitch * test.bytes_per_element + x; - linear[offset] = static_cast( - (x * 29u + y * 83u + test.bytes_per_element * 17u) & 0xffu); - } - } - std::vector tiled(depth.size, 0xcd); - std::vector restored(depth.size, 0xa5); - TileConvertLinearToTiledDepth(tiled.data(), linear.data(), - test.guest_format, test.width, test.height, - pitch, depth.size); - TileConvertTiledToLinearDepth(restored.data(), tiled.data(), - test.guest_format, test.width, test.height, - pitch, depth.size); - for (uint32_t y = 0; y < test.height; y++) { - const auto row = - static_cast(y) * pitch * test.bytes_per_element; - Require("DepthTargetTileRoundTrip", "contents", - std::memcmp(linear.data() + row, restored.data() + row, - test.width * test.bytes_per_element) == 0, - "linear-to-tiled PS5 depth conversion did not round-trip"); - } - if (test.width == 8 && test.height == 8 && test.bytes_per_element == 2) { - const auto *linear16 = reinterpret_cast(linear.data()); - const auto *tiled16 = reinterpret_cast(tiled.data()); - Require("DepthTargetTileRoundTrip", "Prospero D16 SW_64KB_Z_X anchors", - tiled16[0] == linear16[0] && tiled16[1] == linear16[1] && - tiled16[4] == linear16[2] && tiled16[2] == linear16[pitch] && - tiled16[0x27] == linear16[5 * pitch + 3], - "Prospero D16 depth address anchors changed"); - } - if (test.width == 8 && test.height == 8 && test.bytes_per_element == 4) { - const auto *linear32 = reinterpret_cast(linear.data()); - const auto *tiled32 = reinterpret_cast(tiled.data()); - Require("DepthTargetTileRoundTrip", "Prospero D32 SW_64KB_Z_X anchors", - tiled32[0] == linear32[0] && tiled32[1] == linear32[1] && - tiled32[4] == linear32[2] && tiled32[2] == linear32[pitch] && - tiled32[0x27] == linear32[5 * pitch + 3], - "Prospero D32 depth address anchors changed"); - } - if (test.width == 129 && test.height == 130 && - test.bytes_per_element == 2) { - const auto *linear16 = reinterpret_cast(linear.data()); - const auto *tiled16 = reinterpret_cast(tiled.data()); - Require("DepthTargetTileRoundTrip", "Prospero D16 block row", - tiled16[0x10000 / sizeof(uint16_t)] == linear16[128 * pitch], - "Prospero D16 depth block-row order changed"); - } - if (test.width == 640 && test.height == 360 && - test.bytes_per_element == 4) { - const auto *linear32 = reinterpret_cast(linear.data()); - const auto *tiled32 = reinterpret_cast(tiled.data()); - Require("DepthTargetTileRoundTrip", "Prospero D32 block order", - tiled32[0x10000 / sizeof(uint32_t)] == linear32[128] && - tiled32[0x50000 / sizeof(uint32_t)] == linear32[128 * pitch], - "Prospero D32 depth block order changed"); - } - } - std::printf("[host] %-32s ok\n", "DepthTargetTileRoundTrip"); -} - -void CheckStencilTargetTileRoundTrip() { - struct Case { - uint32_t width; - uint32_t height; - }; - constexpr std::array cases{Case{8, 8}, Case{257, 259}, Case{1920, 1080}}; - constexpr auto format = - Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt); - constexpr auto tile = Prospero::GpuEnumValue(Prospero::TileMode::kDepth); - for (const auto test : cases) { - const auto pitch = TileGetTexturePitch(format, test.width, 1, tile); - TileSizeAlign stencil{}; - TileSizeAlign htile{}; - TileSizeAlign depth{}; - Require("StencilTargetTileRoundTrip", "layout", - pitch >= test.width && pitch % 256 == 0 && - TileGetDepthSize( - test.width, test.height, 0, - Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), - Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), - false, &stencil, &htile, &depth) && - stencil.size != 0 && stencil.align == 0x10000, - "Prospero stencil layout was rejected"); - std::vector linear(stencil.size, 0); - for (uint32_t y = 0; y < test.height; y++) { - for (uint32_t x = 0; x < test.width; x++) { - linear[static_cast(y) * pitch + x] = - static_cast((x * 43u + y * 71u + 19u) & 0xffu); - } - } - std::vector tiled(stencil.size, 0xcd); - std::vector restored(stencil.size, 0xa5); - TileConvertLinearToTiledDepth(tiled.data(), linear.data(), format, - test.width, test.height, pitch, stencil.size); - TileConvertTiledToLinearDepth(restored.data(), tiled.data(), format, - test.width, test.height, pitch, stencil.size); - for (uint32_t y = 0; y < test.height; y++) { - const auto row = static_cast(y) * pitch; - Require("StencilTargetTileRoundTrip", "contents", - std::memcmp(linear.data() + row, restored.data() + row, - test.width) == 0, - "linear-to-tiled PS5 stencil conversion did not round-trip"); - } - if (test.width == 8 && test.height == 8) { - Require("StencilTargetTileRoundTrip", "Prospero S8 SW_64KB_Z_X anchors", - tiled[0] == linear[0] && tiled[1] == linear[1] && - tiled[4] == linear[2] && tiled[2] == linear[pitch] && - tiled[0x27] == linear[5 * pitch + 3], - "Prospero S8 stencil address anchors changed"); - } - if (test.width == 257 && test.height == 259) { - Require("StencilTargetTileRoundTrip", "Prospero S8 block order", - tiled[0x10000] == linear[256] && - tiled[0x20000] == linear[256 * pitch], - "Prospero S8 stencil block order changed"); - } - if (test.width == 1920 && test.height == 1080) { - Require("StencilTargetTileRoundTrip", "PPSA01880 footprint", - pitch == 2048 && stencil.size == 0x280000, - "captured PPSA01880 stencil footprint changed"); - } - } - std::printf("[host] %-32s ok\n", "StencilTargetTileRoundTrip"); -} - void CheckStorageTextureGpuOwnedRebindState() { constexpr uintptr_t base = 0x0000000200200000ull; constexpr uint64_t size = 0x10000; @@ -12271,6 +12499,15 @@ void CheckImageOverlapResolution() { "ImageOverlapResolution", "video-out format policy", IsSupportedVideoOutFormat(supported_video_out), "decoded PS5 video-out format was rejected by the centralized policy"); + std::array bgra16_pixels{1, 2, 3, 4, 5, 6, 7, 8}; + ImageOps::SwapVideoOutBgra16(bgra16_pixels.data(), sizeof(bgra16_pixels)); + Require("ImageOverlapResolution", "BGRA16 channel conversion", + bgra16_pixels == std::array{3, 2, 1, 4, 7, 6, 5, 8}, + "BGRA16 video-out conversion did not swap red and blue"); + ImageOps::SwapVideoOutBgra16(bgra16_pixels.data(), sizeof(bgra16_pixels)); + Require("ImageOverlapResolution", "BGRA16 channel round trip", + bgra16_pixels == std::array{1, 2, 3, 4, 5, 6, 7, 8}, + "BGRA16 video-out conversion was not reversible for readback"); Require( "ImageOverlapResolution", "video-out byte-size guards", ([&] { auto mismatched = supported_video_out; @@ -13329,23 +13566,22 @@ void CheckNativeMsaaState() { color_pitch == 1920 && TileGetRenderTargetSize(1920, 1080, color_pitch, 8, &color, 3) && color.align == 0x10000 && color.size == 0x07f80000, - "AGC 8x R16G16B16A16 color footprint was not preserved"); + "8x R16G16B16A16 color footprint was not preserved"); TileSizeAlign depth{}; TileSizeAlign stencil{}; TileSizeAlign htile{}; - Require( - "NativeMsaaState", "8x depth/stencil footprint", - TileGetDepthPitch(1920, 4, 3) == 1920 && - TileGetDepthSize( - 1920, 1080, 0, - Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), - Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, - &stencil, &htile, &depth, 3) && - depth.align == 0x10000 && depth.size == 0x03fc0000 && - stencil.align == 0x10000 && stencil.size == 0x010e0000 && - htile.align == 0x8000 && htile.size == 0x00030000, - "AGC 8x depth/stencil or fragment-independent HTile footprint regressed"); + Require("NativeMsaaState", "8x depth/stencil footprint", + TileGetDepthPitch(1920, 4, 3) == 1920 && + TileGetDepthSize( + 1920, 1080, 0, + Prospero::GpuEnumValue(Prospero::DepthFormat::kZ32F), + Prospero::GpuEnumValue(Prospero::StencilFormat::k8UInt), true, + &stencil, &htile, &depth, 3) && + depth.align == 0x10000 && depth.size == 0x03fc0000 && + stencil.align == 0x10000 && stencil.size == 0x010e0000 && + htile.align == 0x8000 && htile.size == 0x00030000, + "8x depth/stencil or fragment-independent HTile footprint regressed"); std::printf("[host] %-32s ok\n", "NativeMsaaState"); } @@ -13927,7 +14163,6 @@ int main(int argc, char **argv) { using namespace Libs::Graphics; EnsureConfigInitialized(); - TileInit(); if (argc == 2 && std::strcmp(argv[1], "--clip-control-only") == 0) { CheckClipControlDepthClipState(); return 0; @@ -13940,9 +14175,6 @@ int main(int argc, char **argv) { if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-death") == 0) { RunReverseRenderTargetDeathCase(); } - if (argc == 2 && std::strcmp(argv[1], "--standard64-rt-death") == 0) { - RunStandard64RenderTargetDeathCase(); - } if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-only") == 0) { CheckReverseRenderTargetFormatContract(); return 0; @@ -13965,9 +14197,6 @@ int main(int argc, char **argv) { } if (argc == 2 && std::strcmp(argv[1], "--layered-image-only") == 0) { CheckColorResolveLayers(); - CheckRenderTargetTileRoundTrip(); - CheckStorageTextureMipTailRoundTrip(); - CheckDepthTargetTileRoundTrip(); CheckGpuMetadataReuse(); return 0; } @@ -14015,8 +14244,6 @@ int main(int argc, char **argv) { return 0; } if (argc == 2 && std::strcmp(argv[1], "--depth-readback-only") == 0) { - CheckDepthTargetTileRoundTrip(); - CheckStencilTargetTileRoundTrip(); CheckDepthTargetFootprints(); return 0; } @@ -14050,11 +14277,8 @@ int main(int argc, char **argv) { CheckStorageImageSwizzleSpecializationId(); CheckColorResolveLayers(); CheckStandard64RenderTargetTileRoundTrip(); - CheckRenderTargetTileRoundTrip(); - CheckStorageTextureMipTailRoundTrip(); - CheckDepthTargetTileRoundTrip(); - CheckStencilTargetTileRoundTrip(); CheckStorageTextureVolumeUploadLayout(); + CheckStorageTextureVolumeMipRegions(); CheckStorageTextureGpuOwnedRebindState(); CheckStorageTextureSampledReuse(); CheckStorageTextureDepthAlias(); @@ -14082,6 +14306,7 @@ int main(int argc, char **argv) { CheckEmbeddedFetchLaneSpill(); CheckPs5GameExampleImageClearRuntimeShape(); VulkanHarness vulkan; + vulkan.CheckGpuTilerCpuParity(); vulkan.CheckQueryRegionImageClassification(); vulkan.CheckMutableStorageSrgbView(); vulkan.CheckMutableRenderTargetBgraStorageView();