Compare commits

...
Author SHA1 Message Date
nmzik 83152c8cad guest_gpu: remove memory-unmap submission deadlock + remove legacy agc buffering 2026-08-02 13:17:53 +02:00
nmzik 35a89d616a implement dynamic 2026-08-02 10:50:28 +02:00
21 changed files with 572 additions and 412 deletions
+5 -62
View File
@@ -32,11 +32,10 @@
namespace Libs::Graphics {
static thread_local CommandProcessor* g_current_processor = nullptr;
static thread_local Pm4Execution* g_current_execution = nullptr;
static thread_local uint32_t g_submission_pause_depth = 0;
static thread_local bool g_gpu_mutex_owned = false;
static thread_local bool g_gpu_thread = false;
static thread_local CommandProcessor* g_current_processor = nullptr;
static thread_local Pm4Execution* g_current_execution = nullptr;
static thread_local bool g_gpu_mutex_owned = false;
static thread_local bool g_gpu_thread = false;
class GpuMutexLock final {
public:
@@ -98,8 +97,6 @@ public:
bool trigger_agc_interrupt_on_done);
void SubmitFlipPreparation(uint64_t request_id);
void Done();
void PauseSubmissions();
void ResumeSubmissions();
void Shutdown();
[[nodiscard]] bool IsStopping();
void SendCommand(Common::UniqueFunction<void>&& command);
@@ -407,7 +404,7 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw
uint32_t write_control) {
const uint32_t dst_sel = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu);
const uint32_t cache_policy = (write_control >> 25u) & 0x3u;
const uint32_t increment = (write_control >> 16u) & 0x1u;
const uint32_t increment = (write_control >> 16u) & 0x1u;
const uint32_t write_confirm = (write_control >> 20u) & 0x1u;
switch (dst_sel) {
@@ -696,26 +693,6 @@ bool GpuState::Process(Submission& submission) {
return complete;
}
void GpuState::PauseSubmissions() {
if (g_gpu_mutex_owned) {
EXIT("GPU submissions are already paused by this thread\n");
}
g_gpu_mutex_owned = true;
m_submission_mutex.Lock();
if (!IsGpuThread()) {
WaitLocked();
}
m_renderer.GetCommandScheduler().DrainPriorityOperations();
}
void GpuState::ResumeSubmissions() {
if (!g_gpu_mutex_owned) {
EXIT("GPU submissions resumed without an active pause\n");
}
m_submission_mutex.Unlock();
g_gpu_mutex_owned = false;
}
Pm4ProcessResult CommandProcessor::Process(Pm4Execution& execution, uint32_t* buffer,
uint32_t size_dw) {
KYTY_PROFILER_BLOCK("CommandProcessor::Process");
@@ -1693,32 +1670,6 @@ int Gpu::GetFrameNum() const {
return m_state->GetFrameNum();
}
void Gpu::PauseSubmissions() {
m_state->PauseSubmissions();
}
void Gpu::ResumeSubmissions() {
m_state->ResumeSubmissions();
}
Gpu::SubmissionLock::SubmissionLock(Gpu& gpu): m_gpu(gpu) {
if (g_current_processor != nullptr || g_submission_pause_depth == UINT32_MAX) {
EXIT("cannot acquire GPU submission lock in the current state\n");
}
if (g_submission_pause_depth++ == 0) {
m_gpu.PauseSubmissions();
}
}
Gpu::SubmissionLock::~SubmissionLock() {
if (g_submission_pause_depth == 0) {
EXIT("GPU submission lock released without ownership\n");
}
if (--g_submission_pause_depth == 0) {
m_gpu.ResumeSubmissions();
}
}
bool Gpu::IsCommandProcessorThread() noexcept {
return g_current_processor != nullptr;
}
@@ -1727,12 +1678,4 @@ CommandProcessor* Gpu::CurrentCommandProcessor() noexcept {
return g_current_processor;
}
bool Gpu::SubmissionLockHeld() noexcept {
return g_submission_pause_depth != 0;
}
bool Gpu::MutexHeld() noexcept {
return g_gpu_mutex_owned;
}
} // namespace Libs::Graphics
-17
View File
@@ -35,25 +35,8 @@ public:
[[nodiscard]] static bool IsCommandProcessorThread() noexcept;
[[nodiscard]] static CommandProcessor* CurrentCommandProcessor() noexcept;
[[nodiscard]] static bool SubmissionLockHeld() noexcept;
[[nodiscard]] static bool MutexHeld() noexcept;
class SubmissionLock final {
public:
explicit SubmissionLock(Gpu& gpu);
~SubmissionLock();
KYTY_CLASS_NO_COPY(SubmissionLock);
private:
Gpu& m_gpu;
};
private:
friend class SubmissionLock;
void PauseSubmissions();
void ResumeSubmissions();
std::unique_ptr<GpuState> m_state;
};
} // namespace Libs::Graphics
+17 -5
View File
@@ -7,7 +7,8 @@
namespace Libs::Graphics {
GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
: m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex),
: m_scheduler(scheduler),
m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex),
m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache, m_resource_mutex) {}
GpuResourceManager::~GpuResourceManager() = default;
@@ -101,7 +102,22 @@ void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
}
void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
if (CommandScheduler::InDeferredOperation()) {
EXIT("unsupported memory unmap from an asynchronous GPU completion, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported memory unmap from a pre-owned resource transaction, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
const auto unmap = [this, vaddr, size] {
if (m_scheduler.Active()) {
const auto tick = m_scheduler.CurrentTick();
m_scheduler.FinishCurrent();
m_scheduler.WaitPriorityOperations(tick);
}
m_buffer_cache.UnmapMemory(vaddr, size);
m_texture_cache.UnmapMemory(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size);
@@ -109,13 +125,9 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
m_mapped_ranges.Subtract(vaddr, size);
};
if (m_gpu == nullptr) {
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("cannot synchronously unmap from a resource transaction\n");
}
unmap();
return;
}
Gpu::SubmissionLock submissions(*m_gpu);
m_gpu->SendCommandSync(unmap);
}
@@ -36,6 +36,7 @@ public:
private:
PageManager m_page_manager;
ResourceMutex m_resource_mutex;
CommandScheduler& m_scheduler;
BufferCache m_buffer_cache;
TextureCache m_texture_cache;
mutable std::shared_mutex m_mapped_ranges_mutex;
@@ -113,6 +113,13 @@ inline void ValidateStorageColorView(vk::Format image_format, vk::Format view_fo
[[nodiscard]] inline bool
IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resource) noexcept {
const bool supported_mip =
(resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None &&
resource.mip_levels == 1u) ||
(resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage &&
resource.mip_levels > 0u &&
resource.mip_levels <= ShaderRecompiler::IR::ImageResource::MaxMipLevels &&
!resource.read && !resource.atomic);
return (resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImage ||
resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint) &&
(resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1D ||
@@ -120,7 +127,7 @@ IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resou
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D ||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.written &&
supported_mip && resource.written &&
(!resource.atomic ||
(resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint &&
resource.read)) &&
@@ -94,10 +94,11 @@ vk::DescriptorBufferInfo BufferInfo(const BufferView& view) {
} // namespace
vk::DescriptorImageInfo DescriptorCache::MakeImageInfo(const TextureBinding& texture) {
EXIT_IF(!texture.image_id || texture.image_view == nullptr ||
texture.layout == vk::ImageLayout::eUndefined);
return {nullptr, texture.image_view, texture.layout};
vk::DescriptorImageInfo DescriptorCache::MakeImageInfo(const TextureBinding& texture,
uint32_t mip) {
const auto view = texture.mip_views.empty() ? texture.image_view : texture.mip_views.at(mip);
EXIT_IF(!texture.image_id || view == nullptr || texture.layout == vk::ImageLayout::eUndefined);
return {nullptr, view, texture.layout};
}
DescriptorCache::~DescriptorCache() {
@@ -154,7 +155,8 @@ void DescriptorCache::CreatePool() {
MaxSets * (ShaderRecompiler::IR::ShaderInfo::MaxBuffers +
ShaderRecompiler::IR::ShaderInfo::MaxAddresses + 3u)},
{vk::DescriptorType::eSampledImage, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages},
{vk::DescriptorType::eStorageImage, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages},
{vk::DescriptorType::eStorageImage, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages *
ShaderRecompiler::IR::ImageResource::MaxMipLevels},
{vk::DescriptorType::eSampler, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxSamplers},
};
vk::DescriptorPoolCreateInfo info {};
@@ -229,8 +231,10 @@ VulkanDescriptorSet& DescriptorCache::GetDescriptor(Stage
auto* set = Allocate(stage, program);
EXIT_NOT_IMPLEMENTED(set == nullptr);
const auto descriptor_count = program.info.buffers.size() + program.info.images.size() +
program.info.samplers.size() + program.info.addresses.size() + 3u;
uint32_t descriptor_count = 0;
for (const auto& binding: program.bindings.descriptors) {
descriptor_count += DescriptorCount(binding);
}
std::vector<vk::DescriptorBufferInfo> buffer_infos;
std::vector<vk::DescriptorImageInfo> image_infos;
std::vector<vk::WriteDescriptorSet> writes;
@@ -238,6 +242,7 @@ VulkanDescriptorSet& DescriptorCache::GetDescriptor(Stage
image_infos.reserve(descriptor_count);
writes.reserve(program.bindings.descriptors.size());
std::vector<uint32_t> image_mips(program.info.images.size());
for (const auto& binding: program.bindings.descriptors) {
vk::WriteDescriptorSet write {};
write.sType = vk::StructureType::eWriteDescriptorSet;
@@ -273,7 +278,11 @@ VulkanDescriptorSet& DescriptorCache::GetDescriptor(Stage
default: {
for (const auto resource: binding.resources) {
const auto& texture = data.images.at(resource);
image_infos.push_back(MakeImageInfo(texture));
const auto mip = program.info.images.at(resource).mip_mode ==
ShaderRecompiler::IR::ImageMipMode::DynamicStorage
? image_mips.at(resource)++
: 0u;
image_infos.push_back(MakeImageInfo(texture, mip));
}
break;
}
@@ -47,10 +47,11 @@ public:
enum class Stage { Unknown, Vertex, Pixel, Compute };
struct TextureBinding {
ImageId image_id;
vk::ImageView image_view = nullptr;
TextureCache::ImageDesc desc;
vk::ImageLayout layout = vk::ImageLayout::eUndefined;
ImageId image_id;
vk::ImageView image_view = nullptr;
TextureCache::ImageDesc desc;
vk::ImageLayout layout = vk::ImageLayout::eUndefined;
std::vector<vk::ImageView> mip_views;
};
struct NativeDescriptors {
@@ -94,7 +95,7 @@ private:
int next_free_pool = -1;
};
static vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture);
static vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture, uint32_t mip = 0);
void CreatePool();
VulkanDescriptorSet* Allocate(Stage stage, const ShaderRecompiler::IR::Program& program);
vk::DescriptorSetLayout
@@ -391,9 +391,16 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
const bool supported_swizzle =
IsValidImageSwizzle(swizzle) &&
(swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
const auto base_level = static_cast<uint32_t>(descriptor.BaseLevel());
const auto last_level = static_cast<uint32_t>(descriptor.LastLevel());
const auto mip_levels = last_level >= base_level ? last_level - base_level + 1u : 0u;
const bool dynamic_mip =
resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage;
const bool supported_mip_view = descriptor.BaseLevel() == 0 || is_1d || is_2d;
return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
supported_mip_view && descriptor.BaseLevel() == descriptor.LastLevel() &&
supported_mip_view && mip_levels != 0u &&
((dynamic_mip && mip_levels == resource.mip_levels) ||
(!dynamic_mip && descriptor.BaseLevel() == descriptor.LastLevel())) &&
descriptor.LastLevel() <= descriptor.MaxMip() && descriptor.MinLod() == 0 &&
supported_swizzle && descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth();
}
@@ -618,14 +625,15 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& reso
const bool multisampled = IsMultisampledTexture(type);
const auto levels = multisampled ? 1u : static_cast<uint32_t>(descriptor.MaxMip()) + 1u;
const auto tile = descriptor.TileMode();
const bool depth_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kDepth);
const bool depth_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kDepth);
const bool msaa_tile =
depth_tile || tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget);
const bool msaa_array = type == Prospero::ImageType::kColor2DMsaaArray;
if ((!multisampled && (base_level > last_level || last_level >= levels)) ||
(multisampled &&
(base_level != 0 || last_level == 0 || last_level > 3 ||
descriptor.MaxMip() != last_level || !msaa_tile || (descriptor.MsaaDepth() && !depth_tile) ||
descriptor.MaxMip() != last_level || !msaa_tile ||
(descriptor.MsaaDepth() && !depth_tile) ||
(!msaa_array && (descriptor.Depth() != 0 || descriptor.BaseArray5() != 0))))) {
EXIT("unsupported texture mip view: base=%u last=%u levels=%u max=%u type=%u tile=%u "
"kind=%u dimension=%u mip_mode=%u read=%d written=%d "
@@ -634,11 +642,15 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& reso
static_cast<uint32_t>(resource.kind), static_cast<uint32_t>(resource.dimension),
static_cast<uint32_t>(resource.mip_mode), resource.read, resource.written,
descriptor.fields[0], descriptor.fields[1], descriptor.fields[2], descriptor.fields[3],
descriptor.fields[4], descriptor.fields[5], descriptor.fields[6], descriptor.fields[7]);
descriptor.fields[4], descriptor.fields[5], descriptor.fields[6],
descriptor.fields[7]);
}
const auto samples = multisampled ? 1u << last_level : 1u;
const auto view_levels =
multisampled ? 1u : static_cast<uint32_t>(last_level - base_level) + 1u;
multisampled ||
(storage && resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage)
? 1u
: static_cast<uint32_t>(last_level - base_level) + 1u;
const auto depth = static_cast<uint32_t>(descriptor.Depth()) + 1u;
const auto format = descriptor.Format();
const bool sampled_numeric_class =
@@ -661,8 +673,8 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& reso
TileSizeAlign size {};
if (multisampled) {
const auto bytes = Prospero::NumBytesPerElement(format);
pitch = depth_tile ? TileGetDepthPitch(width, bytes, last_level)
: TileGetRenderTargetPitch(width, bytes, last_level);
pitch = depth_tile ? TileGetDepthPitch(width, bytes, last_level)
: TileGetRenderTargetPitch(width, bytes, last_level);
if (pitch == 0 || !TileGetRenderTargetSize(width, height, pitch, bytes, size, last_level) ||
size.size > UINT32_MAX / image_layers) {
EXIT("unsupported multisample texture layout\n");
@@ -679,8 +691,8 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& reso
ValidateStorageTexture(resource, descriptor, size.size);
}
const auto pixel_format = TextureGetFormat(format);
const auto storage_view_format =
const auto pixel_format = TextureGetFormat(format);
const auto storage_view_format =
storage && format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32SInt)
? vk::Format::eR32Uint
: SrgbStorageViewFormat(pixel_format);
@@ -893,7 +905,17 @@ void RenderExecutor::RebindImages(CommandBuffer& buffer,
}
auto& binding = images[i];
binding.image_view = texture_cache.FindTexture(binding.image_id, binding.desc);
auto& image = texture_cache.GetImage(binding.image_id);
auto& image = texture_cache.GetImage(binding.image_id);
binding.mip_views.clear();
if (program.info.images[i].mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage) {
binding.mip_views.reserve(program.info.images[i].mip_levels);
for (uint32_t mip = 0; mip < program.info.images[i].mip_levels; mip++) {
auto view = binding.desc.view_info;
view.base_level += mip;
view.level_count = 1;
binding.mip_views.push_back(mip == 0 ? binding.image_view : image.FindView(view));
}
}
const bool storage = binding.desc.type == TextureCache::BindingType::Storage;
image.usage.storage |= storage;
image.usage.texture |= !storage;
@@ -971,7 +993,11 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer,
auto& image = m_context.GetTextureCache().GetImage(descriptors.images[i].image_id);
auto& binding = descriptors.images[i];
const auto& view = binding.desc.view_info;
const ImageSubresourceRange range {view.base_level, view.level_count, view.base_layer,
const auto level_count =
program.info.images[i].mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage
? program.info.images[i].mip_levels
: view.level_count;
const ImageSubresourceRange range {view.base_level, level_count, view.base_layer,
view.layer_count};
const bool storage = binding.desc.type == TextureCache::BindingType::Storage;
if (image.info.data.Empty()) {
@@ -287,6 +287,10 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
LOGF("shaderStorageImageReadWithoutFormat is not supported\n");
skip_device = true;
}
if (features12.shaderStorageImageArrayNonUniformIndexing != VK_TRUE) {
LOGF("shaderStorageImageArrayNonUniformIndexing is not supported\n");
skip_device = true;
}
if (device_features2.features.shaderImageGatherExtended != VK_TRUE) {
LOGF("shaderImageGatherExtended is not supported\n");
@@ -514,6 +518,7 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
features12.pNext = &depth_clip_control;
features12.samplerMirrorClampToEdge = VK_TRUE;
features12.shaderStorageImageArrayNonUniformIndexing = VK_TRUE;
vk::PhysicalDeviceSubgroupSizeControlFeatures subgroup_size_control {};
subgroup_size_control.sType = vk::StructureType::ePhysicalDeviceSubgroupSizeControlFeatures;
@@ -237,8 +237,14 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
if (!ImageBinding(program.info.images[i], kind)) {
return Fail(error, "native shader plan has an invalid image class");
}
const auto bindings = program.info.images[i].NumBindings();
if (bindings == 0 || bindings > IR::ImageResource::MaxMipLevels) {
return Fail(error, "native shader plan has an invalid image descriptor count");
}
present[static_cast<size_t>(kind)] = true;
expected[static_cast<size_t>(kind)].push_back(i);
for (uint32_t binding = 0; binding < bindings; binding++) {
expected[static_cast<size_t>(kind)].push_back(i);
}
}
if (!program.info.samplers.empty()) {
Expect(Kind::Samplers, Dense(program.info.samplers.size()));
@@ -316,6 +322,11 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
inst.memory.image_dimension)) {
return Fail(error, "image instruction has an invalid dense resource");
}
if (inst.op == IR::Opcode::ImageStore &&
((program.info.images[inst.memory.resource].mip_mode ==
IR::ImageMipMode::DynamicStorage) != inst.memory.image_has_mip)) {
return Fail(error, "storage image mip mode does not match the instruction");
}
const bool address =
inst.op == IR::Opcode::SLoadDword || memory == IR::ResourceKind::Flat ||
memory == IR::ResourceKind::Global || memory == IR::ResourceKind::Scratch;
@@ -560,12 +560,21 @@ uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
uint32_t variable_id, uint32_t array_index,
IR::DescriptorBindingKind kind, uint32_t resource,
const char* variable_name) {
return DescriptorElementPointerId(state, result_ptr_type, variable_id,
ConstantU32(state, array_index), kind, resource,
variable_name);
}
uint32_t DescriptorElementPointerId(EmitterState& state, uint32_t result_ptr_type,
uint32_t variable_id, uint32_t array_index_id,
IR::DescriptorBindingKind kind, uint32_t resource,
const char* variable_name) {
if (variable_id == 0) {
ExitDescriptorBindingFailure(state, kind, resource, variable_name);
}
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)});
{OpAccessChain, result_ptr_type, pointer, variable_id, array_index_id});
return pointer;
}
@@ -33,13 +33,13 @@ uint32_t ConstantImageGatherHorizontalOffsets(EmitterState& state, ImageViewKind
}
uint32_t LoadStorageImageDescriptorAtIndex(EmitterState& state, uint32_t resource,
uint32_t array_index, bool uint_image,
uint32_t array_index_id, bool uint_image,
ImageViewKind view) {
const auto kind = StorageBindingKind(uint_image, view);
const auto& descriptors = state.storage_images[StorageImageIndex(uint_image, view)];
const auto pointer =
DescriptorElementPointer(state, descriptors.pointer_type, descriptors.variable, array_index,
kind, resource, "storage image descriptor array was not emitted");
const auto pointer = DescriptorElementPointerId(
state, descriptors.pointer_type, descriptors.variable, array_index_id, kind, resource,
"storage image descriptor array was not emitted");
const auto image = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, descriptors.image_type, image, pointer});
return image;
@@ -168,12 +168,38 @@ void EmitImageStore(EmitterState& state, const IR::Instruction& inst) {
const auto view = StorageImageViewKind(state, inst.memory, uint_image, inst.pc);
const auto binding =
ResourceForDescriptor(state, StorageBindingKind(uint_image, view), inst.memory.resource);
const auto image = LoadStorageImageDescriptorAtIndex(state, inst.memory.resource,
binding.array_index, uint_image, view);
const auto emit_write = [&](uint32_t descriptor_index, bool non_uniform) {
if (non_uniform) {
state.builder.AddAnnotation({OpDecorate, descriptor_index, DecorationNonUniform});
}
const auto image = LoadStorageImageDescriptorAtIndex(state, inst.memory.resource,
descriptor_index, uint_image, view);
if (non_uniform) {
state.builder.AddAnnotation({OpDecorate, image, DecorationNonUniform});
}
state.builder.AddFunction({OpImageWrite, image, EmitImageCoordU32(state, inst, view),
uint_image ? EmitImageStoreTexelU32(state, inst)
: EmitImageStoreTexelF32(state, inst)});
};
if (!inst.memory.image_has_mip) {
emit_write(ConstantU32(state, binding.array_index), false);
return;
}
const auto& resource = state.program.info.images[inst.memory.resource];
const auto mip = EmitImageMipLodU32(state, inst, inst.src[1], view);
const auto in_range = state.builder.AllocateId();
state.builder.AddFunction(
{OpImageWrite, image, EmitImageCoordU32(state, inst, view),
uint_image ? EmitImageStoreTexelU32(state, inst) : EmitImageStoreTexelF32(state, inst)});
{OpULessThan, state.bool_type, in_range, mip, ConstantU32(state, resource.mip_levels)});
EmitIfCondition(state, in_range, [&] {
auto descriptor_index = mip;
if (binding.array_index != 0) {
descriptor_index = state.builder.AllocateId();
state.builder.AddFunction({OpIAdd, state.uint_type, descriptor_index,
ConstantU32(state, binding.array_index), mip});
}
emit_write(descriptor_index, true);
});
}
void EmitImageSampleResult(EmitterState& state, const IR::Instruction& inst, uint32_t sample,
@@ -23,38 +23,40 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
enum : uint32_t {
ExecutionModelVertex = 0,
ExecutionModelFragment = 4,
ExecutionModelGLCompute = 5,
ExecutionModeOriginUpperLeft = 7,
ExecutionModeEarlyFragmentTests = 9,
ExecutionModeDepthReplacing = 12,
ExecutionModeLocalSize = 17,
ExecutionModeDerivativeGroupQuadsKHR = 5289,
AddressingModelLogical = 0,
MemoryModelGLSL450 = 1,
CapabilityShader = 1,
CapabilityImageGatherExtended = 25,
CapabilitySampled1D = 43,
CapabilityImage1D = 44,
CapabilityImageQuery = 50,
CapabilityStorageImageReadWithoutFormat = 55,
CapabilityStorageImageWriteWithoutFormat = 56,
CapabilityGroupNonUniform = 61,
CapabilityGroupNonUniformBallot = 64,
CapabilityGroupNonUniformShuffle = 65,
CapabilityComputeDerivativeGroupQuadsKHR = 5288,
StorageClassUniformConstant = 0,
StorageClassInput = 1,
StorageClassOutput = 3,
StorageClassWorkgroup = 4,
StorageClassFunction = 7,
StorageClassPushConstant = 9,
StorageClassImage = 11,
StorageClassStorageBuffer = 12,
FunctionControlNone = 0,
SelectionControlNone = 0,
LoopControlNone = 0,
ExecutionModelVertex = 0,
ExecutionModelFragment = 4,
ExecutionModelGLCompute = 5,
ExecutionModeOriginUpperLeft = 7,
ExecutionModeEarlyFragmentTests = 9,
ExecutionModeDepthReplacing = 12,
ExecutionModeLocalSize = 17,
ExecutionModeDerivativeGroupQuadsKHR = 5289,
AddressingModelLogical = 0,
MemoryModelGLSL450 = 1,
CapabilityShader = 1,
CapabilityImageGatherExtended = 25,
CapabilitySampled1D = 43,
CapabilityImage1D = 44,
CapabilityImageQuery = 50,
CapabilityStorageImageReadWithoutFormat = 55,
CapabilityStorageImageWriteWithoutFormat = 56,
CapabilityGroupNonUniform = 61,
CapabilityGroupNonUniformBallot = 64,
CapabilityGroupNonUniformShuffle = 65,
CapabilityShaderNonUniform = 5301,
CapabilityStorageImageArrayNonUniformIndexing = 5309,
CapabilityComputeDerivativeGroupQuadsKHR = 5288,
StorageClassUniformConstant = 0,
StorageClassInput = 1,
StorageClassOutput = 3,
StorageClassWorkgroup = 4,
StorageClassFunction = 7,
StorageClassPushConstant = 9,
StorageClassImage = 11,
StorageClassStorageBuffer = 12,
FunctionControlNone = 0,
SelectionControlNone = 0,
LoopControlNone = 0,
};
enum : uint32_t {
@@ -67,6 +69,7 @@ enum : uint32_t {
DecorationBinding = 33,
DecorationDescriptorSet = 34,
DecorationOffset = 35,
DecorationNonUniform = 5300,
};
enum : uint32_t {
@@ -679,6 +682,10 @@ uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
uint32_t variable_id, uint32_t array_index,
IR::DescriptorBindingKind kind, uint32_t resource,
const char* variable_name);
uint32_t DescriptorElementPointerId(EmitterState& state, uint32_t result_ptr_type,
uint32_t variable_id, uint32_t array_index_id,
IR::DescriptorBindingKind kind, uint32_t resource,
const char* variable_name);
ImageViewKind SampledImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
uint32_t use_pc);
@@ -484,6 +484,13 @@ void EmitHeaderAndTypes(EmitterState& state) {
if (state.needs_image_gather_extended) {
state.builder.AddCapability({CapabilityImageGatherExtended});
}
if (std::any_of(
state.program.info.images.begin(), state.program.info.images.end(),
[](const auto& image) { return image.mip_mode == IR::ImageMipMode::DynamicStorage; })) {
state.builder.AddCapability({CapabilityShaderNonUniform});
state.builder.AddCapability({CapabilityStorageImageArrayNonUniformIndexing});
state.builder.AddExtension("SPV_EXT_descriptor_indexing");
}
if (std::any_of(state.storage_images.begin(),
state.storage_images.begin() + StorageImageViewKindCount,
[](const auto& image) { return image.variable != 0; })) {
@@ -269,7 +269,10 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std
}
return false;
}
image_groups[static_cast<size_t>(group - ImageBindingKinds.begin())].push_back(i);
auto& resources = image_groups[static_cast<size_t>(group - ImageBindingKinds.begin())];
for (uint32_t binding = 0; binding < program.info.images[i].NumBindings(); binding++) {
resources.push_back(i);
}
}
for (uint32_t i = 0; i < image_groups.size(); i++) {
if (!image_groups[i].empty()) {
@@ -71,6 +71,17 @@ bool DescriptorIsCube(const DescriptorValue& descriptor) {
Prospero::ImageType::kCube;
}
bool DescriptorMipRange(const DescriptorValue& descriptor, uint32_t& count) {
const auto base_level = (descriptor.dwords[3] >> 12u) & 0xfu;
const auto last_level = (descriptor.dwords[3] >> 16u) & 0xfu;
const auto max_mip = (descriptor.dwords[5] >> 4u) & 0xfu;
if (base_level > last_level || last_level > max_mip) {
return false;
}
count = last_level - base_level + 1u;
return true;
}
bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResource& result) {
if (descriptor.dword_count != std::size(result.fields)) {
return false;
@@ -196,16 +207,31 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
}
continue;
}
if (image.mip_mode == ImageMipMode::DynamicStorage) {
uint32_t mip_levels = 0;
if (!DescriptorMipRange(descriptor, mip_levels) || mip_levels != image.mip_levels) {
if (error != nullptr) {
*error = fmt::format(
"image descriptor {} no longer matches specialized storage mip count", i);
}
return false;
}
} else if (image.mip_levels != 1u) {
if (error != nullptr) {
*error = fmt::format("image descriptor {} has invalid non-storage mip count", i);
}
return false;
}
const auto dimension = DescriptorDimension(descriptor, image.dimension);
if (dimension == Decoder::ImageDimension::Unknown || dimension != image.dimension ||
DescriptorIsCube(descriptor) != image.cube) {
if (error != nullptr) {
*error = fmt::format(
"image descriptor {} no longer matches specialized dimension: "
"{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x}",
i, descriptor.dwords[0], descriptor.dwords[1], descriptor.dwords[2],
descriptor.dwords[3], descriptor.dwords[4], descriptor.dwords[5],
descriptor.dwords[6], descriptor.dwords[7]);
*error =
fmt::format("image descriptor {} no longer matches specialized dimension: "
"{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x}",
i, descriptor.dwords[0], descriptor.dwords[1], descriptor.dwords[2],
descriptor.dwords[3], descriptor.dwords[4], descriptor.dwords[5],
descriptor.dwords[6], descriptor.dwords[7]);
}
return false;
}
@@ -224,10 +250,9 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
const bool raw_sint_storage =
storage && format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32SInt) &&
!image.read && !image.atomic;
const bool uint_descriptor =
Prospero::IsUintTextureFormat(format) || raw_sint_storage;
const auto uint_program = image.kind == ResourceKind::ImageUint ||
image.kind == ResourceKind::StorageImageUint;
const bool uint_descriptor = Prospero::IsUintTextureFormat(format) || raw_sint_storage;
const auto uint_program = image.kind == ResourceKind::ImageUint ||
image.kind == ResourceKind::StorageImageUint;
if (uint_descriptor != uint_program && !(image.atomic && uint_program)) {
if (error != nullptr) {
*error =
@@ -380,8 +405,9 @@ bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std
const auto& descriptor = snapshot.images[i];
auto& image = next.images[i];
if (NullImageDescriptor(descriptor)) {
image.dimension = Decoder::ImageDimension::Dim2D;
image.cube = false;
image.dimension = Decoder::ImageDimension::Dim2D;
image.cube = false;
image.mip_levels = 1;
switch (image.kind) {
case ResourceKind::ImageUint: image.kind = ResourceKind::Image; break;
case ResourceKind::StorageImageUint:
@@ -393,6 +419,16 @@ bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std
}
continue;
}
if (image.mip_mode == ImageMipMode::DynamicStorage) {
if (!DescriptorMipRange(descriptor, image.mip_levels)) {
if (error != nullptr) {
*error = fmt::format("image descriptor {} has invalid storage mip range", i);
}
return false;
}
} else {
image.mip_levels = 1;
}
const auto descriptor_dimension = DescriptorDimension(descriptor, image.dimension);
if (descriptor_dimension == Decoder::ImageDimension::Unknown) {
if (error != nullptr) {
@@ -598,11 +598,14 @@ enum class ImageMipMode { None, DynamicStorage };
constexpr uint32_t StorageImageIdentitySwizzle = 0x00000facu;
struct ImageResource {
static constexpr uint32_t MaxMipLevels = 16;
uint32_t source = 0;
uint32_t first_use_pc = 0;
ResourceKind kind = ResourceKind::None;
Decoder::ImageDimension dimension = Decoder::ImageDimension::Unknown;
ImageMipMode mip_mode = ImageMipMode::None;
uint32_t mip_levels = 1;
uint32_t storage_swizzle = StorageImageIdentitySwizzle;
bool read = false;
bool written = false;
@@ -610,6 +613,10 @@ struct ImageResource {
bool depth_compare = false;
bool cube = false;
[[nodiscard]] uint32_t NumBindings() const {
return mip_mode == ImageMipMode::DynamicStorage ? mip_levels : 1u;
}
bool operator==(const ImageResource& other) const = default;
};
+1
View File
@@ -1018,6 +1018,7 @@ static void ShaderAppendNativeSpecialization(std::vector<uint32_t>&
ids.push_back(static_cast<uint32_t>(image.kind));
ids.push_back(static_cast<uint32_t>(image.dimension));
ids.push_back(static_cast<uint32_t>(image.mip_mode));
ids.push_back(image.mip_levels);
ids.push_back(image.storage_swizzle);
}
ids.push_back(static_cast<uint32_t>(program.info.addresses.size()));
-199
View File
@@ -221,57 +221,6 @@ static RegisterDefaults* get_internal_register_defaults(uint32_t ver) {
return get_register_defaults(g_agc_internal_reg_defaults_by_version[index], &storage[index]);
}
struct PendingGraphicsSegment {
uint32_t* start = nullptr;
uint32_t* end = nullptr;
uint32_t* range_end = nullptr;
};
static std::mutex g_pending_graphics_segment_mutex;
static PendingGraphicsSegment g_pending_graphics_segment;
static void track_pending_graphics_segment_after_submit(uint32_t* dcb, uint32_t size_in_dwords) {
if (dcb == nullptr || size_in_dwords == 0) {
return;
}
auto* segment_start = dcb + size_in_dwords;
auto* range_end = segment_start + 0xfffffu;
std::lock_guard lock(g_pending_graphics_segment_mutex);
g_pending_graphics_segment.start = segment_start;
g_pending_graphics_segment.end = segment_start;
g_pending_graphics_segment.range_end = range_end;
}
static void track_pending_graphics_allocation(uint32_t* cmd, uint32_t size_dw) {
if (cmd == nullptr || size_dw == 0) {
return;
}
std::lock_guard lock(g_pending_graphics_segment_mutex);
auto* range_start = g_pending_graphics_segment.start;
auto* range_end = g_pending_graphics_segment.range_end;
if (range_start == nullptr || range_end == nullptr || cmd < range_start || cmd >= range_end) {
return;
}
auto* cmd_end = cmd + size_dw;
if (cmd > g_pending_graphics_segment.end) {
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1) < 64) {
LOGF("\t pending graphics segment: ignoring non-contiguous allocation cmd = "
"0x%016" PRIx64 ", tracked_end = 0x%016" PRIx64 "\n",
reinterpret_cast<uint64_t>(cmd),
reinterpret_cast<uint64_t>(g_pending_graphics_segment.end));
}
return;
}
if (cmd_end > g_pending_graphics_segment.end && cmd_end <= range_end) {
g_pending_graphics_segment.end = cmd_end;
}
}
struct CommandBuffer {
using Callback = KYTY_SYSV_ABI bool (*)(CommandBuffer*, uint32_t, void*);
@@ -370,7 +319,6 @@ struct CommandBuffer {
}
auto* ret_ptr = cursor_up;
cursor_up += size_dw;
track_pending_graphics_allocation(ret_ptr, size_dw);
return ret_ptr;
}
};
@@ -1918,7 +1866,6 @@ uint32_t* KYTY_SYSV_ABI GraphicsCbReleaseMem(CommandBuffer* buf, uint8_t action,
cmd[5] = static_cast<uint32_t>(packet_data & 0xffffffffu);
cmd[6] = static_cast<uint32_t>((packet_data >> 32u) & 0xffffffffu);
cmd[7] = interrupt_ctx_id & 0x07ffffffu;
return cmd;
}
@@ -3815,150 +3762,6 @@ static void submit_dcb(uint32_t* dcb, uint32_t size_in_dwords) {
EXIT_IF(g_renderer == nullptr);
g_renderer->GetGpu().Submit(dcb, size_in_dwords, nullptr, 0,
!dcb_has_queued_interrupt(dcb, size_in_dwords));
Gen5::track_pending_graphics_segment_after_submit(dcb, size_in_dwords);
}
static std::vector<uint64_t> collect_acb_wait_addresses(const uint32_t* acb,
uint32_t size_in_dwords) {
std::vector<uint64_t> addresses;
for (uint32_t offset = 0; offset < size_in_dwords;) {
auto cmd_id = acb[offset];
auto len = KYTY_PM4_LEN(cmd_id);
if (len == 0 || len > size_in_dwords - offset) {
return addresses;
}
auto op = (cmd_id >> 8u) & 0xffu;
if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd_id) == Pm4::R_WAIT_MEM_32 && len >= 7) {
auto address = static_cast<uint64_t>(acb[offset + 1]) |
(static_cast<uint64_t>(acb[offset + 2]) << 32u);
if (address != 0) {
addresses.push_back(address);
}
} else if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd_id) == Pm4::R_WAIT_MEM_64 && len >= 9) {
auto address = static_cast<uint64_t>(acb[offset + 1]) |
(static_cast<uint64_t>(acb[offset + 2]) << 32u);
if (address != 0) {
addresses.push_back(address);
}
}
offset += len;
}
return addresses;
}
static bool acb_waits_for_address(const std::vector<uint64_t>& wait_addresses,
uint64_t release_address) {
for (auto address: wait_addresses) {
if (address == release_address) {
return true;
}
}
return false;
}
static void flush_pending_graphics_segment_before_acb(const uint32_t* acb,
uint32_t acb_size_in_dwords) {
uint32_t* dcb = nullptr;
uint32_t size_in_dwords = 0;
auto wait_addresses = collect_acb_wait_addresses(acb, acb_size_in_dwords);
{
std::lock_guard lock(Gen5::g_pending_graphics_segment_mutex);
if (!wait_addresses.empty() && Gen5::g_pending_graphics_segment.start != nullptr) {
auto* scan = Gen5::g_pending_graphics_segment.start;
auto* matched_end = Gen5::g_pending_graphics_segment.start;
while (scan < Gen5::g_pending_graphics_segment.end) {
auto cmd_id = *scan;
if (cmd_id == 0x80000000u) {
scan++;
continue;
}
if ((cmd_id & 0xC0000000u) != 0xC0000000u) {
break;
}
auto len = KYTY_PM4_LEN(cmd_id);
if (len == 0 ||
len > static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end - scan)) {
break;
}
if (((cmd_id >> 8u) & 0xffu) == Pm4::IT_NOP &&
KYTY_PM4_R(cmd_id) == Pm4::R_RELEASE_MEM && len >= 7) {
auto release_addr =
static_cast<uint64_t>(scan[3]) | (static_cast<uint64_t>(scan[4]) << 32u);
if (acb_waits_for_address(wait_addresses, release_addr)) {
matched_end = scan + len;
}
}
scan += len;
}
if (matched_end > Gen5::g_pending_graphics_segment.start) {
Gen5::g_pending_graphics_segment.end = matched_end;
}
}
if (Gen5::g_pending_graphics_segment.start != nullptr &&
Gen5::g_pending_graphics_segment.end > Gen5::g_pending_graphics_segment.start) {
auto* scan = Gen5::g_pending_graphics_segment.start;
auto* valid_end = Gen5::g_pending_graphics_segment.start;
while (scan < Gen5::g_pending_graphics_segment.end) {
auto cmd_id = *scan;
if (cmd_id == 0x80000000u) {
scan++;
valid_end = scan;
continue;
}
if ((cmd_id & 0xC0000000u) != 0xC0000000u) {
break;
}
auto len = KYTY_PM4_LEN(cmd_id);
if (len == 0 ||
len > static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end - scan)) {
break;
}
scan += len;
valid_end = scan;
}
if (valid_end < Gen5::g_pending_graphics_segment.end) {
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1) < 64) {
LOGF("\t trimming pending graphics segment: addr = 0x%016" PRIx64
", old_dw = 0x%08" PRIx32 ", new_dw = 0x%08" PRIx32 "\n",
reinterpret_cast<uint64_t>(Gen5::g_pending_graphics_segment.start),
static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end -
Gen5::g_pending_graphics_segment.start),
static_cast<uint32_t>(valid_end - Gen5::g_pending_graphics_segment.start));
}
Gen5::g_pending_graphics_segment.end = valid_end;
}
}
if (Gen5::g_pending_graphics_segment.start == nullptr ||
Gen5::g_pending_graphics_segment.end <= Gen5::g_pending_graphics_segment.start) {
return;
}
dcb = Gen5::g_pending_graphics_segment.start;
size_in_dwords = static_cast<uint32_t>(Gen5::g_pending_graphics_segment.end -
Gen5::g_pending_graphics_segment.start);
}
LOGF("\t flushing pending graphics segment before ACB: addr = 0x%016" PRIx64
", dw_num = 0x%08" PRIx32 "\n",
reinterpret_cast<uint64_t>(dcb), size_in_dwords);
submit_dcb(dcb, size_in_dwords);
}
int KYTY_SYSV_ABI GraphicsDriverSubmitDcb(const Packet* packet) {
@@ -4074,8 +3877,6 @@ static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) {
LOGF("\t acb[%u] = 0x%08" PRIx32 "\n", i, acb[i]);
}
flush_pending_graphics_segment_before_acb(acb, size_in_dwords);
GraphicsDbgDumpDcb("a", size_in_dwords, acb);
const bool trigger_interrupt_on_done = !dcb_has_queued_interrupt(acb, size_in_dwords);
+27
View File
@@ -1468,6 +1468,32 @@ void TestNativeBindingLayoutOneDimensionalImages() {
"binding allocator did not preserve first-class 1D image groups");
}
void TestNativeBindingLayoutExpandsDynamicStorageMips() {
Program program;
program.stage = ShaderType::Compute;
program.blocks.resize(1);
auto dynamic = ImageUse(8, Opcode::ImageStore, ResourceKind::StorageImageUint,
Decoder::ImageDimension::Dim2D, 4);
dynamic.memory.image_has_mip = true;
program.blocks[0].instructions = {
dynamic, ImageUse(12, Opcode::ImageStore, ResourceKind::StorageImageUint,
Decoder::ImageDimension::Dim2D, 6)};
Prepare(program);
ShaderComputeInputInfo compute;
compute.thread_ids_num = 1;
std::string error;
Check(CollectShaderInfo(program, {.compute = &compute}, &error), error.c_str());
Check(program.info.images.size() == 2 &&
program.info.images[0].mip_mode == ImageMipMode::DynamicStorage,
"dynamic storage image was not tracked independently");
program.info.images[0].mip_levels = 3;
Check(AllocateBindings(program, {}, &error), error.c_str());
const auto* storage = FindBinding(program.bindings, DescriptorBindingKind::StorageUint2D);
Check(storage != nullptr && storage->resources == std::vector<uint32_t>({0, 0, 0, 1}),
"dynamic storage mip span overlapped the following logical image");
}
void TestNativeBindingLayoutSrtAndUserDataOverflow() {
Program srt;
srt.stage = ShaderType::Compute;
@@ -1936,6 +1962,7 @@ int main() {
RUN(TestTrackedProgramIsImmutable);
RUN(TestNativeBindingLayout);
RUN(TestNativeBindingLayoutOneDimensionalImages);
RUN(TestNativeBindingLayoutExpandsDynamicStorageMips);
RUN(TestNativeBindingLayoutSrtAndUserDataOverflow);
RUN(TestNativeBindingLayoutGds);
RUN(TestNativeBindingLayoutIsTransactional);
+287 -49
View File
@@ -58,6 +58,7 @@
#include <algorithm>
#include <array>
#include <bit>
#include <chrono>
#include <cinttypes>
#include <cmath>
#include <cstdint>
@@ -275,8 +276,9 @@ struct RenderExecutorTestAccess {
};
struct DescriptorCacheTestAccess {
static vk::DescriptorImageInfo MakeImageInfo(const DescriptorCache::TextureBinding& binding) {
return DescriptorCache::MakeImageInfo(binding);
static vk::DescriptorImageInfo MakeImageInfo(const DescriptorCache::TextureBinding& binding,
uint32_t mip = 0) {
return DescriptorCache::MakeImageInfo(binding, mip);
}
};
@@ -692,6 +694,8 @@ struct TestCase {
u32 storage_image_dwords_per_pixel = 4;
std::vector<u32> storage_image_r32ui;
std::vector<u32> expected_storage_image_r32ui;
std::vector<std::vector<u32>> storage_image_r32ui_mips;
std::vector<std::vector<u32>> expected_storage_image_r32ui_mips;
std::vector<std::string> required_spirv;
std::vector<std::string> forbidden_spirv;
ShaderComputeInputInfo compute_info {};
@@ -711,11 +715,6 @@ struct TestCase {
std::vector<u32> expected_gds;
};
struct SkippedCase {
const char* name = "";
const char* reason = "";
};
struct GraphicsCase {
const char* name = "";
std::vector<u32> fragment_code;
@@ -1061,6 +1060,7 @@ public:
vk::Image image = nullptr;
vk::DeviceMemory memory = nullptr;
vk::ImageView view = nullptr;
std::vector<vk::ImageView> mip_views;
vk::Format format = vk::Format::eUndefined;
vk::ImageLayout layout = vk::ImageLayout::eUndefined;
u32 width = 0;
@@ -1450,22 +1450,91 @@ public:
uint32_t ordered_suffix = 0;
std::jthread ordered([&] {
ordered_started.release();
Gpu::SubmissionLock submissions(gpu);
gpu.SendCommandSync([&] {
ordered_suffix = suffix;
ordered_finished = true;
});
gpu.Done();
ordered_suffix = suffix;
ordered_finished = true;
});
ordered_started.acquire();
gpu.SendCommandSync([&] {
Require("GpuCommandLane", "ordered barrier", !ordered_finished.load(),
"ordered host command overtook a queued submission");
Require("GpuCommandLane", "submit done barrier", !ordered_finished.load(),
"submit done returned before a queued submission");
label = 1;
});
ordered.join();
Require("GpuCommandLane", "ordered completion",
prefix == 11 && suffix == 22 && ordered_suffix == 22 && ordered_finished.load(),
"submission barrier did not drain prior PM4 work");
"submit done did not drain prior PM4 work");
auto& resources = context.GetGpuResources();
constexpr uint64_t empty_unmap_base = 0x0000000200400000ull;
constexpr uint64_t empty_unmap_size = 0x4000;
resources.MapMemory(empty_unmap_base, empty_unmap_size);
label = 0;
prefix = 0;
suffix = 0;
gpu.Submit(commands.data(), static_cast<uint32_t>(commands.size()), nullptr, 0);
std::binary_semaphore unmap_complete {0};
std::jthread unmap_thread([&] {
resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
unmap_complete.release();
});
const bool unmap_returned = unmap_complete.try_acquire_for(std::chrono::seconds(2));
gpu.SendCommandSync([&] { label = 1; });
if (!unmap_returned) {
unmap_complete.acquire();
}
unmap_thread.join();
gpu.Done();
Require("GpuCommandLane", "unmap queue progress",
unmap_returned && !resources.IsMapped(empty_unmap_base, empty_unmap_size) &&
prefix == 11 && suffix == 22,
"an unrelated unmap waited for a blocked PM4 submission");
auto& scheduler = context.GetCommandScheduler();
std::atomic<bool> normal_completed {false};
gpu.SendCommandSync(
[&] { scheduler.DeferOperation([&] { normal_completed = true; }); });
resources.MapMemory(empty_unmap_base, empty_unmap_size);
resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
Require("GpuCommandLane", "unmap native completion",
normal_completed.load() &&
!resources.IsMapped(empty_unmap_base, empty_unmap_size),
"unmap returned before an earlier native guest-memory callback");
std::binary_semaphore priority_entered {0};
std::binary_semaphore release_priority {0};
gpu.SendCommandSync([&] {
scheduler.DeferPriorityOperation([&] {
priority_entered.release();
release_priority.acquire();
});
scheduler.Flush();
});
priority_entered.acquire();
resources.MapMemory(empty_unmap_base, empty_unmap_size);
std::binary_semaphore priority_unmap_entered {0};
std::binary_semaphore priority_unmap_complete {0};
std::jthread priority_unmap_thread([&] {
gpu.SendCommandSync([&] {
priority_unmap_entered.release();
resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
});
priority_unmap_complete.release();
});
priority_unmap_entered.acquire();
const bool unmap_overtook_priority =
priority_unmap_complete.try_acquire_for(std::chrono::seconds(1));
release_priority.release();
if (!unmap_overtook_priority) {
priority_unmap_complete.acquire();
}
priority_unmap_thread.join();
Require("GpuCommandLane", "unmap priority ordering",
!unmap_overtook_priority &&
!resources.IsMapped(empty_unmap_base, empty_unmap_size),
"unmap returned before an earlier guest-memory callback");
constexpr uintptr_t fault_base = 0x0000000200500000ull;
constexpr uint64_t fault_size = 0x10000;
@@ -1483,7 +1552,6 @@ public:
Require("GpuCommandLane", "processor fault allocation",
fault_memory == reinterpret_cast<void*>(fault_base),
"fixed processor-fault allocation failed");
auto& resources = context.GetGpuResources();
resources.MapMemory(fault_base, fault_size);
constexpr uint64_t immediate_dst = fault_base + 0x1000;
@@ -1530,9 +1598,7 @@ public:
Require("GpuCommandLane", "DMA_DATA packet assembly", dma_cursor == dma_commands.size(),
"DMA_DATA GDS packet stream has the wrong size");
gpu.Submit(dma_commands.data(), static_cast<uint32_t>(dma_commands.size()), nullptr, 0);
{
Gpu::SubmissionLock submissions(gpu);
}
gpu.Done();
constexpr uint32_t clean_fill_value = 0xdecafbad;
gpu.SendCommandSyncWithProcessor([&](CommandProcessor&) {
auto& buffer_cache = resources.GetBufferCache();
@@ -5166,6 +5232,59 @@ public:
"non-array 1D specialization did not select the descriptor "
"base layer from its 1D-array backing");
auto dynamic_storage = storage;
const uint64_t dynamic_storage_address = base + 0xf0000;
const auto encoded_dynamic_address = dynamic_storage_address >> 8u;
dynamic_storage.fields[0] = static_cast<uint32_t>(encoded_dynamic_address);
dynamic_storage.fields[1] =
static_cast<uint32_t>(encoded_dynamic_address >> 32u) | (stencil_format << 20u);
dynamic_storage.fields[3] |= 1u << 16u;
dynamic_storage.fields[5] |= 1u << 4u;
ShaderRecompiler::IR::DescriptorValue dynamic_storage_descriptor {};
std::copy(std::begin(dynamic_storage.fields), std::end(dynamic_storage.fields),
dynamic_storage_descriptor.dwords.begin());
dynamic_storage_descriptor.dword_count = 8;
auto dynamic_program = storage_program;
dynamic_program.stage = ShaderType::Compute;
auto& dynamic_resource = dynamic_program.info.images[0];
dynamic_resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::DynamicStorage;
dynamic_resource.mip_levels = 2;
dynamic_program.bindings.descriptors.push_back(
{ShaderRecompiler::IR::DescriptorBindingKind::StorageUint2D, 0, {0, 0}});
dynamic_program.binding_layout_complete = true;
auto dynamic_snapshot = std::make_shared<ShaderRecompiler::IR::ResourceSnapshot>();
dynamic_snapshot->images.push_back(dynamic_storage_descriptor);
ShaderStageRuntime dynamic_runtime {
std::make_shared<const ShaderRecompiler::IR::Program>(std::move(dynamic_program)),
std::move(dynamic_snapshot)};
auto dynamic_bindings = executor.PrepareBindings(
scheduler.Current(), dynamic_runtime, vk::ShaderStageFlagBits::eCompute,
DescriptorCache::Stage::Compute);
executor.RebindImages(scheduler.Current(), dynamic_bindings);
auto& dynamic_binding = dynamic_bindings.resources.images[0];
Require(name, "dynamic mip singleton views",
dynamic_binding.desc.view_info.level_count == 1 &&
dynamic_binding.mip_views.size() == 2 &&
dynamic_binding.mip_views[0] == dynamic_binding.image_view &&
dynamic_binding.mip_views[0] != dynamic_binding.mip_views[1],
"production rebind did not acquire one distinct storage view per mip");
dynamic_binding.layout = vk::ImageLayout::eGeneral;
const auto mip0_info = DescriptorCacheTestAccess::MakeImageInfo(dynamic_binding, 0);
const auto mip1_info = DescriptorCacheTestAccess::MakeImageInfo(dynamic_binding, 1);
Require(name, "dynamic mip descriptor selection",
mip0_info.imageView == dynamic_binding.mip_views[0] &&
mip1_info.imageView == dynamic_binding.mip_views[1] &&
mip0_info.imageLayout == vk::ImageLayout::eGeneral &&
mip1_info.imageLayout == vk::ImageLayout::eGeneral,
"descriptor cache did not select the matching singleton mip view");
auto& descriptor_set = context.GetDescriptorCache().GetDescriptor(
DescriptorCache::Stage::Compute, *dynamic_runtime.program, dynamic_bindings.resources);
Require(name, "dynamic mip descriptor update", descriptor_set.set != nullptr,
"production descriptor update did not allocate the dynamic mip array");
context.GetDescriptorCache().Recycle(descriptor_set);
RenderExecutorTestAccess::ResetBindings(executor);
ShaderRecompiler::IR::ResourceSnapshot storage_snapshot {};
storage_snapshot.images.push_back(storage_descriptor);
ShaderStageRuntime storage_runtime {
@@ -6153,6 +6272,14 @@ public:
view_info.subresourceRange.layerCount = view_layers;
RequireVk(shader_name, "dispatch", m_device.createImageView(&view_info, nullptr, &ret.view),
"vkCreateImageView");
ret.mip_views.resize(ret.mip_levels);
view_info.subresourceRange.levelCount = 1;
for (u32 level = 0; level < ret.mip_levels; level++) {
view_info.subresourceRange.baseMipLevel = level;
RequireVk(shader_name, "dispatch",
m_device.createImageView(&view_info, nullptr, &ret.mip_views[level]),
"vkCreateImageView(mip)");
}
if (!initial_mips.empty()) {
size_t total_dwords = 0;
@@ -6186,6 +6313,10 @@ public:
if (image == nullptr) {
return;
}
for (auto view: image->mip_views) {
m_device.destroyImageView(view, nullptr);
}
image->mip_views.clear();
if (image->view != nullptr) {
m_device.destroyImageView(image->view, nullptr);
image->view = nullptr;
@@ -6200,10 +6331,12 @@ public:
}
}
std::vector<u32> ReadImage(const char* shader_name, Image* image) {
const auto dword_count = static_cast<size_t>(image->width) *
static_cast<size_t>(image->height) * image->layers *
image->dwords_per_pixel;
std::vector<u32> ReadImage(const char* shader_name, Image* image, u32 mip_level = 0) {
Require(shader_name, "readback", mip_level < image->mip_levels,
"image mip level is out of bounds");
const auto dword_count = ImageMipDwordCount(image->width, image->height,
image->dwords_per_pixel, mip_level,
image->layers);
auto staging = CreateHostBuffer(shader_name, dword_count * sizeof(u32),
vk::BufferUsageFlagBits::eTransferDst, {});
@@ -6216,11 +6349,11 @@ public:
vk::BufferImageCopy copy {};
copy.bufferOffset = 0;
copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
copy.imageSubresource.mipLevel = 0;
copy.imageSubresource.mipLevel = mip_level;
copy.imageSubresource.baseArrayLayer = 0;
copy.imageSubresource.layerCount = image->layers;
copy.imageExtent.width = image->width;
copy.imageExtent.height = image->height;
copy.imageExtent.width = MipExtent(image->width, mip_level);
copy.imageExtent.height = MipExtent(image->height, mip_level);
copy.imageExtent.depth = 1;
cmd.copyImageToBuffer(image->image, vk::ImageLayout::eTransferSrcOptimal, staging.buffer, 1,
&copy);
@@ -6579,13 +6712,32 @@ public:
storage_infos.resize(storage != nullptr ? storage->resources.size() : 0u);
storage_uint_infos.resize(storage_uint != nullptr ? storage_uint->resources.size()
: 0u);
for (auto& info: storage_infos) {
info.imageView = storage_image->view;
info.imageLayout = storage_image->layout;
const auto fill_storage_infos = [&](const auto* binding, const Image* image,
auto& infos) {
std::vector<u32> resource_mips(compiled.program.info.images.size());
for (u32 i = 0; i < infos.size(); i++) {
const u32 resource_index = binding->resources[i];
Require(test.name, "dispatch",
resource_index < compiled.program.info.images.size(),
"storage binding resource index is out of bounds");
const auto& resource = compiled.program.info.images[resource_index];
auto& info = infos[i];
if (resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage) {
const u32 mip = resource_mips[resource_index]++;
Require(test.name, "dispatch", mip < image->mip_views.size(),
"dynamic storage mip descriptor is out of bounds");
info.imageView = image->mip_views[mip];
} else {
info.imageView = image->view;
}
info.imageLayout = image->layout;
}
};
if (storage != nullptr) {
fill_storage_infos(storage, storage_image, storage_infos);
}
for (auto& info: storage_uint_infos) {
info.imageView = storage_image_uint->view;
info.imageLayout = storage_image_uint->layout;
if (storage_uint != nullptr) {
fill_storage_infos(storage_uint, storage_image_uint, storage_uint_infos);
}
if (storage != nullptr) {
vk::WriteDescriptorSet write {};
@@ -7860,6 +8012,9 @@ private:
Require("VulkanHarness", "dispatch",
available_features.shaderStorageImageReadWithoutFormat == true,
"shaderStorageImageReadWithoutFormat is not supported");
Require("VulkanHarness", "dispatch",
available_features12.shaderStorageImageArrayNonUniformIndexing == true,
"shaderStorageImageArrayNonUniformIndexing is not supported");
Require("VulkanHarness", "dispatch", available_features12.timelineSemaphore == true,
"timeline semaphores are not supported");
Require("VulkanHarness", "dispatch", available_features13.dynamicRendering == true,
@@ -7883,6 +8038,7 @@ private:
vk::PhysicalDeviceVulkan12Features device_features12 {};
device_features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
device_features12.timelineSemaphore = true;
device_features12.shaderStorageImageArrayNonUniformIndexing = true;
vk::PhysicalDeviceVulkan13Features device_features13 {};
device_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
device_features13.pNext = &device_features12;
@@ -8236,10 +8392,14 @@ void RunCase(VulkanHarness* vulkan, const TestCase& test) {
test.name, test.image_width, test.image_height, test.storage_image_format,
vk::ImageUsageFlagBits::eStorage, test.storage_image_rgba,
test.storage_image_dwords_per_pixel, vk::ImageLayout::eGeneral);
storage_image_uint =
vulkan->CreateImage2D(test.name, test.image_width, test.image_height,
vk::Format::eR32Uint, vk::ImageUsageFlagBits::eStorage,
test.storage_image_r32ui, 1, vk::ImageLayout::eGeneral);
auto storage_uint_mips = test.storage_image_r32ui_mips;
if (storage_uint_mips.empty() && !test.storage_image_r32ui.empty()) {
storage_uint_mips.push_back(test.storage_image_r32ui);
}
storage_image_uint = vulkan->CreateImageMips(
test.name, test.image_width, test.image_height, vk::Format::eR32Uint,
vk::ImageUsageFlagBits::eStorage, storage_uint_mips, 1, vk::ImageLayout::eGeneral,
vk::ImageType::e2D, vk::ImageViewType::e2D, 1);
}
if (needs_sampler) {
sampler = vulkan->CreateNearestSampler(test.name);
@@ -8266,6 +8426,12 @@ void RunCase(VulkanHarness* vulkan, const TestCase& test) {
CompareWords(test, "uint storage image readback", test.expected_storage_image_r32ui,
image_actual);
}
for (u32 mip = 0; mip < test.expected_storage_image_r32ui_mips.size(); mip++) {
auto image_actual = vulkan->ReadImage(test.name, &storage_image_uint, mip);
const auto& expected = test.expected_storage_image_r32ui_mips[mip];
image_actual.resize(expected.size());
CompareWords(test, "uint storage image mip readback", expected, image_actual);
}
if (sampler != nullptr) {
vulkan->Device().destroySampler(sampler, nullptr);
}
@@ -13643,10 +13809,65 @@ TestCase ImageGetResinfoDmaskMipLevels() {
return test;
}
SkippedCase ImageStoreMipWritesExplicitMip2D() {
return {"ImageStoreMipWritesExplicitMip2D",
"requires per-mip storage-image view descriptors; Vulkan "
"OpImageWrite cannot take Lod"};
TestCase ImageStoreMipWritesExplicitMip2D() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 1);
code.push_back(EncodeVop1(0x01, 22, Vgpr(0)));
AppendVMovLiteral(&code, 0, 0x12345678u);
code.push_back(EncodeMimg0(0x09, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
TestCase test;
test.name = "ImageStoreMipWritesExplicitMip2D";
test.code = code;
test.opcodes = {O::VMovB32, O::ImageStoreMip, O::SEndpgm};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.user_data[3] |= 1u << 16u;
test.user_data[5] |= 1u << 4u;
test.has_user_data = true;
test.storage_image_r32ui_mips = {std::vector<u32>(16, 0), std::vector<u32>(4, 0)};
test.expected_storage_image_r32ui_mips = test.storage_image_r32ui_mips;
test.expected_storage_image_r32ui_mips[0][5] = 0x12345678u;
test.expected_storage_image_r32ui_mips[1][3] = 0x12345678u;
test.required_spirv = {"OpCapability ShaderNonUniform",
"OpCapability StorageImageArrayNonUniformIndexing", "NonUniform",
"OpULessThan", "OpAccessChain", "OpImageWrite", "storage_uint_2d"};
test.compute_info.threads_num[0] = 2;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase ImageStoreMipOutOfRangeIsDiscarded() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 2);
AppendVMovLiteral(&code, 0, 0xdeadbeefu);
code.push_back(EncodeMimg0(0x09, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
TestCase test;
test.name = "ImageStoreMipOutOfRangeIsDiscarded";
test.code = code;
test.opcodes = {O::VMovB32, O::ImageStoreMip, O::SEndpgm};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.user_data[3] |= 1u << 16u;
test.user_data[5] |= 1u << 4u;
test.has_user_data = true;
test.storage_image_r32ui_mips = {std::vector<u32>(16, 0), std::vector<u32>(4, 0)};
test.expected_storage_image_r32ui_mips = test.storage_image_r32ui_mips;
test.required_spirv = {"OpULessThan", "OpImageWrite"};
return test;
}
TestCase ImageSampleAndGather() {
@@ -14598,6 +14819,8 @@ std::vector<TestCase> MakeCases() {
AddCase(ImageLoadA16UintCoordsOnGpu);
AddCase(ImageGetResinfoDmaskWidthHeight);
AddCase(ImageGetResinfoDmaskMipLevels);
AddCase(ImageStoreMipWritesExplicitMip2D);
AddCase(ImageStoreMipOutOfRangeIsDiscarded);
AddCase(ImageSampleAndGather);
AddCase(ImageSampleA16SamplerCoordsOnGpu);
AddCase(ImageSampleOpcodeAliasUsesNormalCoords);
@@ -14635,10 +14858,6 @@ std::vector<GraphicsCase> MakeGraphicsCases() {
};
}
std::vector<SkippedCase> MakeSkippedCases() {
return {ImageStoreMipWritesExplicitMip2D()};
}
void CheckPs5GameExampleImageClearRuntimeShape() {
const auto MakeCode = [] {
std::vector<u32> code;
@@ -14943,8 +15162,15 @@ void CheckRenderTargetFormatContract() {
resource.atomic = true;
} else if (std::strcmp(kind, "storage-compare") == 0) {
resource.depth_compare = true;
} else if (std::strcmp(kind, "storage-mip") == 0) {
} else if (std::strcmp(kind, "storage-mip-count") == 0) {
resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::DynamicStorage;
resource.mip_levels = 0;
} else if (std::strcmp(kind, "storage-mip-atomic") == 0) {
resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint;
resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::DynamicStorage;
resource.mip_levels = 2;
resource.read = true;
resource.atomic = true;
} else if (std::strcmp(kind, "storage-dimension") == 0) {
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Unknown;
} else {
@@ -15130,6 +15356,14 @@ void CheckSampledColorViews() {
Require("SampledColorViews", "atomic uint 2D storage resource",
IsSupportedStorageImageResource(storage_resource),
"atomic uint storage resource was rejected");
storage_resource.read = false;
storage_resource.atomic = false;
storage_resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::DynamicStorage;
storage_resource.mip_levels = 3;
Require("SampledColorViews", "dynamic uint 2D storage resource",
IsSupportedStorageImageResource(storage_resource) &&
storage_resource.NumBindings() == 3,
"write-only IMAGE_STORE_MIP resource was rejected");
char path[MAX_PATH] {};
Require("SampledColorViews", "host", GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
@@ -15138,7 +15372,8 @@ void CheckSampledColorViews() {
{"sampled-invalid-selector", "sampled-incompatible-format", "sampled-invalid-high",
"sampled-depth-format", "sampled-depth-swizzle", "storage-incompatible-format",
"storage-kind", "storage-no-write", "storage-nonuint-atomic", "storage-compare",
"storage-mip", "storage-dimension", "volume-mip-count", "volume-slice-range"}) {
"storage-mip-count", "storage-mip-atomic", "storage-dimension", "volume-mip-count",
"volume-slice-range"}) {
std::string command = std::string("\"") + path + "\" --image-view-death " + kind;
std::vector<char> mutable_command(command.begin(), command.end());
mutable_command.push_back('\0');
@@ -17256,6 +17491,12 @@ int main(int argc, char** argv) {
vulkan.CheckUnifiedImageViewCache();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--storage-mip-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ImageStoreMipWritesExplicitMip2D());
RunCase(&vulkan, ImageStoreMipOutOfRangeIsDiscarded());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--depth-readback-only") == 0) {
CheckDepthTargetFootprints();
return 0;
@@ -17384,9 +17625,6 @@ int main(int argc, char** argv) {
for (const auto& test: tests) {
RunCase(&vulkan, test);
}
for (const auto& test: MakeSkippedCases()) {
std::printf("[skip] %-32s %s\n", test.name, test.reason);
}
for (const auto& test: graphics_tests) {
RunGraphicsCase(&vulkan, test);
}