Embedded fetch shader: Fix overlapping buffer loads (#133)

Fix overlapping buffer loads. Fixes many games

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -88,15 +88,34 @@ TextureCache::~TextureCache() {
 
 bool TextureCache::SameBacking(const ImageInfo& cached, const ImageInfo& requested,
                                bool exact_format) {
-	const bool unit_extent =
-	    requested.extent.width == 1 && requested.extent.height == 1 && requested.extent.depth == 1;
-	return cached.data == requested.data && cached.extent == requested.extent &&
-	       cached.samples == requested.samples &&
-	       cached.bytes_per_block == requested.bytes_per_block &&
-	       (cached.type == requested.type || unit_extent) &&
-	       (exact_format
-	            ? cached.pixel_format == requested.pixel_format
-	            : ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format));
+	if (cached.data.address != requested.data.address) {
+		return false;
+	}
+	if (cached.data.size != requested.data.size) {
+		return false;
+	}
+	if (cached.extent != requested.extent) {
+		return false;
+	}
+	if (cached.samples != requested.samples) {
+		return false;
+	}
+	if (cached.bytes_per_block != requested.bytes_per_block) {
+		return false;
+	}
+	if (cached.tile_mode != requested.tile_mode) {
+		return false;
+	}
+	if (!ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format)) {
+		return false;
+	}
+	if (cached.type != requested.type && requested.extent != vk::Extent3D {1, 1, 1}) {
+		return false;
+	}
+	if (exact_format && cached.pixel_format != requested.pixel_format) {
+		return false;
+	}
+	return true;
 }
 
 TextureCache::BindingType TextureCache::UploadBinding(const Image& image) {
@@ -735,6 +754,12 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		    (requested.IsVolume() || cached.info.IsVolume())) {
 			return {ExpandImage(requested, cached_id)};
 		}
+		if (requested.tile_mode != cached.info.tile_mode) {
+			if (safe_to_delete) {
+				DeleteImages(std::array {cached_id}, cached_id);
+			}
+			return {merged_id};
+		}
 		if (requested.pixel_format != cached.info.pixel_format ||
 		    requested.data.size <= cached.info.data.size) {
 			const auto result_id = merged_id ? merged_id : cached_id;
@@ -747,12 +772,6 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		if (requested.type == cached.info.type && requested.resources > cached.info.resources) {
 			return {ExpandImage(requested, cached_id)};
 		}
-		if (requested.tile_mode != cached.info.tile_mode) {
-			if (safe_to_delete) {
-				DeleteImages(std::array {cached_id}, cached_id);
-			}
-			return {merged_id};
-		}
 		EXIT("TextureCache: unresolvable equal-address image overlap, address=0x%016" PRIx64
 		     " requested=%ux%u "
 		     "cached=%ux%u requested_size=0x%016" PRIx64 " cached_size=0x%016" PRIx64
@@ -1122,7 +1141,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 
 		for (const auto id: candidates) {
 			const auto owner = ResolveOwner(id);
-			if (owner == nullptr || owner->info.data != desc.info.data) {
+			if (owner == nullptr) {
 				continue;
 			}
 			if (SameBacking(owner->info, desc.info, exact_format)) {

src/graphics/shader/recompiler/decompiler/MemoryOps.cpp

@@ -421,13 +421,13 @@ bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	const uint32_t data0   = (word1 >> 8u) & 0xffu;
 	const uint32_t addr    = word1 & 0xffu;
 
-	inst.pc         = pc;
-	inst.word       = word0;
-	inst.word_count = 2;
-	inst.offset     = offset0 | (offset1 << 8u);
-	inst.gds        = ((word0 >> 17u) & 1u) != 0u;
-	inst.family     = Family::DS;
-	inst.opcode_id  = opcode;
+	inst.pc          = pc;
+	inst.word        = word0;
+	inst.word_count  = 2;
+	inst.offset      = offset0 | (offset1 << 8u);
+	inst.gds         = ((word0 >> 17u) & 1u) != 0u;
+	inst.family      = Family::DS;
+	inst.opcode_id   = opcode;
 	const auto* info = LookupMemoryOpcode(DS_OPS, static_cast<uint32_t>(std::size(DS_OPS)), opcode);
 	ApplyMemoryInfo(inst, info);
 	SetRawWords(inst, code, word_index, 2);
@@ -442,11 +442,6 @@ bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	     inst.opcode == Opcode::DsReadAddtidB32)) {
 		SetUnsupported(inst, Family::DS, opcode, "DS swizzle/addtid is available only for LDS");
 	}
-	if (inst.gds && (inst.opcode == Opcode::DsAppend || inst.opcode == Opcode::DsConsume) &&
-	    inst.offset != 0u) {
-		SetUnsupported(inst, Family::DS, opcode,
-		               "GDS append/consume requires a zero instruction offset");
-	}
 	if (inst.opcode == Opcode::DsWriteAddtidB32 && data1 != 0u) {
 		SetUnsupported(inst, Family::DS, opcode,
 		               "DS write addtid data1 operand is not implemented");

src/graphics/shader/recompiler/emitter/SpirvEmitter.cpp

@@ -1,7 +1,7 @@
 #include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
 
-#include "graphics/shader/recompiler/ir/SrtWalker.h"
 #include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/ir/SrtWalker.h"
 
 #include <algorithm>
 #include <array>
@@ -160,8 +160,7 @@ bool ValidateInstructionContract(const IR::Instruction& inst, std::string* error
 	                            inst.dst.kind != IR::OperandKind::Null)) ||
 	    ((inst.op == IR::Opcode::DsAppend || inst.op == IR::Opcode::DsConsume) &&
 	     (!ds_kind || !ds_resource || inst.src_count != 1 ||
-	      inst.dst.kind != IR::OperandKind::Register ||
-	      (kind == IR::ResourceKind::Gds && inst.memory.offset != 0))) ||
+	      inst.dst.kind != IR::OperandKind::Register)) ||
 	    ((inst.op == IR::Opcode::DsMinF32 || inst.op == IR::Opcode::DsMaxF32) &&
 	     (!ds_kind || !ds_resource || inst.src_count != 3 ||
 	      inst.dst.kind != IR::OperandKind::Null)) ||

src/graphics/shader/recompiler/emitter/spirvEmitterControlFlow.cpp

@@ -1025,15 +1025,47 @@ void EmitDispatcherSwitch(EmitterState& state, const IR::Program& program) {
 	EmitDispatcherExit(state);
 }
 
+size_t BufferLoadGroupSize(const IR::BasicBlock& block, size_t first_index) {
+	const auto& first = block.instructions[first_index];
+	if (first.op != IR::Opcode::BufferLoadDword || first.memory.component_index != 0u ||
+	    first.memory.component_count <= 1u) {
+		return 1u;
+	}
+
+	size_t count = 1u;
+	while (first_index + count < block.instructions.size() &&
+	       count < first.memory.component_count) {
+		const auto& next = block.instructions[first_index + count];
+		if (next.op != IR::Opcode::BufferLoadDword || next.pc != first.pc ||
+		    next.memory.component_index != count ||
+		    next.memory.component_count != first.memory.component_count) {
+			break;
+		}
+		count++;
+	}
+	return count;
+}
+
+void EmitBlockInstructions(EmitterState& state, const IR::BasicBlock& block) {
+	for (size_t i = 0; i < block.instructions.size();) {
+		const auto count = BufferLoadGroupSize(block, i);
+		if (count > 1u) {
+			EmitBufferLoadDwordGroup(state, block.instructions.data() + i,
+			                         static_cast<uint32_t>(count));
+		} else {
+			EmitInstruction(state, block.instructions[i]);
+		}
+		i += count;
+	}
+}
+
 void EmitDispatcherBlocks(EmitterState& state, const IR::Program& program) {
 	for (const auto& block: program.blocks) {
 		if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) {
 			continue;
 		}
 		state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)});
-		for (const auto& inst: block.instructions) {
-			EmitInstruction(state, inst);
-		}
+		EmitBlockInstructions(state, block);
 		EmitDispatcherTerminator(state, block.terminator);
 	}
 }
@@ -1083,9 +1115,7 @@ void EmitFunction(EmitterState& state, const IR::Program& program) {
 			continue;
 		}
 		state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)});
-		for (const auto& inst: block.instructions) {
-			EmitInstruction(state, inst);
-		}
+		EmitBlockInstructions(state, block);
 		EmitTerminator(state, block.terminator);
 	}
 

src/graphics/shader/recompiler/emitter/spirvEmitterInternal.h

@@ -3,11 +3,11 @@
 
 #include "common/common.h"
 #include "common/stringUtils.h"
-#include "graphics/shader/recompiler/ir/BindingLayout.h"
 #include "graphics/shader/recompiler/BufferFormat.h"
+#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
+#include "graphics/shader/recompiler/ir/BindingLayout.h"
 #include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
 
 #include <algorithm>
 #include <array>
@@ -312,117 +312,117 @@ struct EmitterState {
 	EmitterState(const IR::Program& program_, const IR::ResourceSnapshot& resources_)
 	    : program(program_), resources(resources_) {}
 
-	Builder                                builder;
-	const IR::Program&                     program;
-	const IR::ResourceSnapshot&            resources;
-	const ShaderVertexInputInfo*           vertex_input_info            = nullptr;
-	const ShaderPixelInputInfo*            pixel_input_info             = nullptr;
-	const ShaderComputeInputInfo*          compute_input_info           = nullptr;
-	ShaderType                             stage                        = ShaderType::Unknown;
-	uint32_t                               wave_size                    = 64;
-	bool                                   exact_subgroup_operations    = false;
-	bool                                   per_invocation_masks         = false;
-	uint32_t                               void_type                    = 0;
-	uint32_t                               bool_type                    = 0;
-	uint32_t                               uint_type                    = 0;
-	uint32_t                               uint_pair_type               = 0;
-	uint32_t                               int_pair_type                = 0;
-	uint32_t                               int_type                     = 0;
-	uint32_t                               float_type                   = 0;
-	uint32_t                               vec2_uint_type               = 0;
-	uint32_t                               vec3_uint_type               = 0;
-	uint32_t                               vec4_uint_type               = 0;
-	uint32_t                               vec2_int_type                = 0;
-	uint32_t                               vec3_int_type                = 0;
-	uint32_t                               vec4_int_type                = 0;
-	uint32_t                               vec2_float_type              = 0;
-	uint32_t                               vec3_float_type              = 0;
-	uint32_t                               vec4_float_type              = 0;
-	uint32_t                               ptr_func_uint                = 0;
-	uint32_t                               ptr_input_float              = 0;
-	uint32_t                               ptr_input_bool               = 0;
-	uint32_t                               ptr_input_int                = 0;
-	uint32_t                               ptr_input_uint               = 0;
-	uint32_t                               ptr_input_vec2_float         = 0;
-	uint32_t                               ptr_input_vec3_float         = 0;
-	uint32_t                               ptr_input_vec2_int           = 0;
-	uint32_t                               ptr_input_vec3_int           = 0;
-	uint32_t                               ptr_input_vec4_int           = 0;
-	uint32_t                               ptr_input_vec2_uint          = 0;
-	uint32_t                               ptr_input_vec3_uint          = 0;
-	uint32_t                               ptr_input_vec4_uint          = 0;
-	uint32_t                               ptr_input_vec4_float         = 0;
-	uint32_t                               sample_mask_array_type       = 0;
-	uint32_t                               ptr_output_int               = 0;
-	uint32_t                               ptr_output_sample_mask_array = 0;
-	uint32_t                               ptr_output_float             = 0;
-	uint32_t                               ptr_output_vec4_float        = 0;
-	uint32_t                               per_vertex_type              = 0;
-	uint32_t                               ptr_output_per_vertex        = 0;
-	uint32_t                               storage_runtime_array_type   = 0;
-	uint32_t                               storage_buffer_type          = 0;
-	uint32_t                               ptr_storage_buffer           = 0;
-	uint32_t                               ptr_storage_buffer_uint      = 0;
-	uint32_t                               storage_buffer_array_type    = 0;
-	uint32_t                               ptr_storage_buffer_array     = 0;
-	uint32_t                               storage_buffer_variable      = 0;
+	Builder                                          builder;
+	const IR::Program&                               program;
+	const IR::ResourceSnapshot&                      resources;
+	const ShaderVertexInputInfo*                     vertex_input_info  = nullptr;
+	const ShaderPixelInputInfo*                      pixel_input_info   = nullptr;
+	const ShaderComputeInputInfo*                    compute_input_info = nullptr;
+	ShaderType                                       stage              = ShaderType::Unknown;
+	uint32_t                                         wave_size          = 64;
+	bool                                             exact_subgroup_operations    = false;
+	bool                                             per_invocation_masks         = false;
+	uint32_t                                         void_type                    = 0;
+	uint32_t                                         bool_type                    = 0;
+	uint32_t                                         uint_type                    = 0;
+	uint32_t                                         uint_pair_type               = 0;
+	uint32_t                                         int_pair_type                = 0;
+	uint32_t                                         int_type                     = 0;
+	uint32_t                                         float_type                   = 0;
+	uint32_t                                         vec2_uint_type               = 0;
+	uint32_t                                         vec3_uint_type               = 0;
+	uint32_t                                         vec4_uint_type               = 0;
+	uint32_t                                         vec2_int_type                = 0;
+	uint32_t                                         vec3_int_type                = 0;
+	uint32_t                                         vec4_int_type                = 0;
+	uint32_t                                         vec2_float_type              = 0;
+	uint32_t                                         vec3_float_type              = 0;
+	uint32_t                                         vec4_float_type              = 0;
+	uint32_t                                         ptr_func_uint                = 0;
+	uint32_t                                         ptr_input_float              = 0;
+	uint32_t                                         ptr_input_bool               = 0;
+	uint32_t                                         ptr_input_int                = 0;
+	uint32_t                                         ptr_input_uint               = 0;
+	uint32_t                                         ptr_input_vec2_float         = 0;
+	uint32_t                                         ptr_input_vec3_float         = 0;
+	uint32_t                                         ptr_input_vec2_int           = 0;
+	uint32_t                                         ptr_input_vec3_int           = 0;
+	uint32_t                                         ptr_input_vec4_int           = 0;
+	uint32_t                                         ptr_input_vec2_uint          = 0;
+	uint32_t                                         ptr_input_vec3_uint          = 0;
+	uint32_t                                         ptr_input_vec4_uint          = 0;
+	uint32_t                                         ptr_input_vec4_float         = 0;
+	uint32_t                                         sample_mask_array_type       = 0;
+	uint32_t                                         ptr_output_int               = 0;
+	uint32_t                                         ptr_output_sample_mask_array = 0;
+	uint32_t                                         ptr_output_float             = 0;
+	uint32_t                                         ptr_output_vec4_float        = 0;
+	uint32_t                                         per_vertex_type              = 0;
+	uint32_t                                         ptr_output_per_vertex        = 0;
+	uint32_t                                         storage_runtime_array_type   = 0;
+	uint32_t                                         storage_buffer_type          = 0;
+	uint32_t                                         ptr_storage_buffer           = 0;
+	uint32_t                                         ptr_storage_buffer_uint      = 0;
+	uint32_t                                         storage_buffer_array_type    = 0;
+	uint32_t                                         ptr_storage_buffer_array     = 0;
+	uint32_t                                         storage_buffer_variable      = 0;
 	std::array<uint32_t, IR::ShaderInfo::MaxBuffers> storage_buffer_offsets {};
-	uint32_t                               address_memory_array_type    = 0;
-	uint32_t                               ptr_address_memory_array     = 0;
-	uint32_t                               address_memory_variable      = 0;
-	uint32_t                               gds_variable                 = 0;
-	uint32_t                               push_constant_array_type     = 0;
-	uint32_t                               push_constant_block_type     = 0;
-	uint32_t                               ptr_push_constant_block      = 0;
-	uint32_t                               ptr_push_constant_uint       = 0;
-	uint32_t                               push_constant_variable       = 0;
-	uint32_t                               vsharp_storage_variable      = 0;
-	uint32_t                               flattened_srt_variable       = 0;
-	uint32_t                               lds_array_type               = 0;
-	uint32_t                               ptr_workgroup_array          = 0;
-	uint32_t                               ptr_workgroup_uint           = 0;
-	uint32_t                               lds_variable                 = 0;
-	std::array<SampledImageDescriptors, 10> sampled_images;
-	std::array<StorageImageDescriptors, 10> storage_images;
-	uint32_t                               sampler_type                                  = 0;
-	uint32_t                               sampler_array_type                            = 0;
-	uint32_t                               ptr_uniform_sampler                           = 0;
-	uint32_t                               ptr_uniform_sampler_array                     = 0;
-	uint32_t                               sampler_variable                              = 0;
-	uint32_t                               ptr_image_uint                                = 0;
-	uint32_t                               func_type                                     = 0;
-	uint32_t                               main_func                                     = 0;
-	uint32_t                               entry_label                                   = 0;
-	uint32_t                               pixel_valid_mask_variable                     = 0;
-	bool                                   dispatcher_fallback                           = false;
-	uint32_t                               dispatch_pc_variable                          = 0;
-	uint32_t                               dispatch_header_label                         = 0;
-	uint32_t                               dispatch_select_label                         = 0;
-	uint32_t                               dispatch_default_label                        = 0;
-	uint32_t                               dispatch_after_switch_label                   = 0;
-	uint32_t                               dispatch_continue_label                       = 0;
-	uint32_t                               dispatch_merge_label                          = 0;
-	uint32_t                               glsl_std450                                   = 0;
-	uint32_t                               subgroup_local_invocation_id_variable         = 0;
-	uint32_t                               per_vertex_variable                           = 0;
-	uint32_t                               depth_variable                                = 0;
-	uint32_t                               sample_mask_variable                          = 0;
-	bool                                   needs_subgroup_ballot                         = false;
-	bool                                   needs_subgroup_shuffle                        = false;
-	bool                                   needs_subgroup_local_invocation_id            = false;
-	bool                                   needs_compute_derivatives                     = false;
-	bool                                   needs_image_gather_extended                   = false;
-	bool                                   needs_function_lds                            = false;
-	bool                                   needs_pixel_valid_mask                        = false;
-	std::vector<RegisterBinding>           registers;
-	std::vector<InputBinding>              inputs;
-	std::vector<OutputBinding>             outputs;
-	std::vector<uint32_t>                  interface_variables;
-	std::vector<bool>                      reachable_blocks;
-	std::map<uint32_t, uint32_t>           block_labels;
-	std::map<uint32_t, uint32_t>           constants;
-	std::map<uint32_t, uint32_t>           signed_constants;
-	std::map<uint32_t, uint32_t>           float_constants;
+	uint32_t                                         address_memory_array_type = 0;
+	uint32_t                                         ptr_address_memory_array  = 0;
+	uint32_t                                         address_memory_variable   = 0;
+	uint32_t                                         gds_variable              = 0;
+	uint32_t                                         push_constant_array_type  = 0;
+	uint32_t                                         push_constant_block_type  = 0;
+	uint32_t                                         ptr_push_constant_block   = 0;
+	uint32_t                                         ptr_push_constant_uint    = 0;
+	uint32_t                                         push_constant_variable    = 0;
+	uint32_t                                         vsharp_storage_variable   = 0;
+	uint32_t                                         flattened_srt_variable    = 0;
+	uint32_t                                         lds_array_type            = 0;
+	uint32_t                                         ptr_workgroup_array       = 0;
+	uint32_t                                         ptr_workgroup_uint        = 0;
+	uint32_t                                         lds_variable              = 0;
+	std::array<SampledImageDescriptors, 10>          sampled_images;
+	std::array<StorageImageDescriptors, 10>          storage_images;
+	uint32_t                                         sampler_type                          = 0;
+	uint32_t                                         sampler_array_type                    = 0;
+	uint32_t                                         ptr_uniform_sampler                   = 0;
+	uint32_t                                         ptr_uniform_sampler_array             = 0;
+	uint32_t                                         sampler_variable                      = 0;
+	uint32_t                                         ptr_image_uint                        = 0;
+	uint32_t                                         func_type                             = 0;
+	uint32_t                                         main_func                             = 0;
+	uint32_t                                         entry_label                           = 0;
+	uint32_t                                         pixel_valid_mask_variable             = 0;
+	bool                                             dispatcher_fallback                   = false;
+	uint32_t                                         dispatch_pc_variable                  = 0;
+	uint32_t                                         dispatch_header_label                 = 0;
+	uint32_t                                         dispatch_select_label                 = 0;
+	uint32_t                                         dispatch_default_label                = 0;
+	uint32_t                                         dispatch_after_switch_label           = 0;
+	uint32_t                                         dispatch_continue_label               = 0;
+	uint32_t                                         dispatch_merge_label                  = 0;
+	uint32_t                                         glsl_std450                           = 0;
+	uint32_t                                         subgroup_local_invocation_id_variable = 0;
+	uint32_t                                         per_vertex_variable                   = 0;
+	uint32_t                                         depth_variable                        = 0;
+	uint32_t                                         sample_mask_variable                  = 0;
+	bool                                             needs_subgroup_ballot                 = false;
+	bool                                             needs_subgroup_shuffle                = false;
+	bool                                             needs_subgroup_local_invocation_id    = false;
+	bool                                             needs_compute_derivatives             = false;
+	bool                                             needs_image_gather_extended           = false;
+	bool                                             needs_function_lds                    = false;
+	bool                                             needs_pixel_valid_mask                = false;
+	std::vector<RegisterBinding>                     registers;
+	std::vector<InputBinding>                        inputs;
+	std::vector<OutputBinding>                       outputs;
+	std::vector<uint32_t>                            interface_variables;
+	std::vector<bool>                                reachable_blocks;
+	std::map<uint32_t, uint32_t>                     block_labels;
+	std::map<uint32_t, uint32_t>                     constants;
+	std::map<uint32_t, uint32_t>                     signed_constants;
+	std::map<uint32_t, uint32_t>                     float_constants;
 };
 
 constexpr uint32_t PsInputOffsetMask = 0x0000001fu;
@@ -990,11 +990,6 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 uint32_t UnpackTBufferFormat(EmitterState& state, const IR::Instruction& inst,
                              const Format::BufferFormatInfo& info);
 
-bool EmitTypedTBufferLoad(EmitterState& state, const IR::Instruction& inst,
-                          const Format::BufferFormatInfo& info);
-
-bool EmitFormattedBufferLoad(EmitterState& state, const IR::Instruction& inst);
-
 uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format,
                                                  uint32_t               opcode_components);
 
@@ -1020,6 +1015,9 @@ void EmitBufferLoadSshort(EmitterState& state, const IR::Instruction& inst);
 
 void EmitBufferLoadDword(EmitterState& state, const IR::Instruction& inst);
 
+void EmitBufferLoadDwordGroup(EmitterState& state, const IR::Instruction* instructions,
+                              uint32_t count);
+
 void EmitBufferStoreDword(EmitterState& state, const IR::Instruction& inst);
 
 void EmitFlatLoadUbyte(EmitterState& state, const IR::Instruction& inst);

src/graphics/shader/recompiler/emitter/spirvEmitterMemory.cpp

@@ -34,16 +34,15 @@ uint32_t EmitDppWriteActiveBool(EmitterState& state, const IR::Operand& dst) {
 	    {OpShiftLeftLogical, state.uint_type, bank_bit, ConstantU32(state, 1), bank});
 	state.builder.AddFunction(
 	    {OpShiftLeftLogical, state.uint_type, row_bit, ConstantU32(state, 1), row});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, bank_hit,
-	                            ConstantU32(state, dst.dpp_bank_mask), bank_bit});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, bank_hit, ConstantU32(state, dst.dpp_bank_mask), bank_bit});
 	state.builder.AddFunction(
 	    {OpBitwiseAnd, state.uint_type, row_hit, ConstantU32(state, dst.dpp_row_mask), row_bit});
 	state.builder.AddFunction(
 	    {OpINotEqual, state.bool_type, bank_active, bank_hit, ConstantU32(state, 0)});
 	state.builder.AddFunction(
 	    {OpINotEqual, state.bool_type, row_active, row_hit, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpLogicalAnd, state.bool_type, dpp_active, bank_active, row_active});
+	state.builder.AddFunction({OpLogicalAnd, state.bool_type, dpp_active, bank_active, row_active});
 	uint32_t write_active = dpp_active;
 	if (!dst.dpp_bound_ctrl) {
 		const auto target  = EmitDppTargetLane(state, dst.dpp_ctrl);
@@ -102,7 +101,7 @@ void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value) {
 			const auto selected  = state.builder.AllocateId();
 			state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer});
 			state.builder.AddFunction({OpSelect, state.uint_type, selected,
-			                            EmitDppWriteActiveBool(state, dst), wave_value, old_value});
+			                           EmitDppWriteActiveBool(state, dst), wave_value, old_value});
 			state.builder.AddFunction({OpStore, pointer, selected});
 			return;
 		}
@@ -131,8 +130,7 @@ uint32_t EmitNotEqualZeroBool(EmitterState& state, uint32_t value) {
 uint32_t EmitSelectU32Value(EmitterState& state, uint32_t condition, uint32_t true_value,
                             uint32_t false_value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, ret, condition, true_value, false_value});
+	state.builder.AddFunction({OpSelect, state.uint_type, ret, condition, true_value, false_value});
 	return ret;
 }
 
@@ -212,12 +210,12 @@ bool IsStorageBufferMemoryKind(IR::ResourceKind kind) {
 
 void EmitStorageBufferOffsets(EmitterState& state) {
 	for (uint32_t i = 0; i < state.program.bindings.buffer_offset_count; i++) {
-		const auto word = EmitShaderDataDwordLoad(
-		    state, state.program.bindings.buffer_offset_dword + i / 4u);
-		const auto shift = ConstantU32(state, (i % 4u) * 8u + 2u);
+		const auto word =
+		    EmitShaderDataDwordLoad(state, state.program.bindings.buffer_offset_dword + i / 4u);
+		const auto shift                = ConstantU32(state, (i % 4u) * 8u + 2u);
 		state.storage_buffer_offsets[i] = EmitBinaryU32(
-		    state, OpBitwiseAnd,
-		    EmitBinaryU32(state, OpShiftRightLogical, word, shift), ConstantU32(state, 0x3fu));
+		    state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift),
+		    ConstantU32(state, 0x3fu));
 	}
 }
 
@@ -329,8 +327,7 @@ uint32_t EmitRelativeAddress(EmitterState& state, const IR::Instruction& inst, u
 
 uint32_t EmitFlatVirtualAddress(EmitterState& state, const IR::Instruction& inst,
                                 uint32_t first_src, uint32_t src_count) {
-	if (inst.memory.resource >= state.resources.addresses.size() ||
-	    src_count < 2) {
+	if (inst.memory.resource >= state.resources.addresses.size() || src_count < 2) {
 		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
 		                             inst.memory.resource, "flat address snapshot is missing");
 	}
@@ -429,20 +426,20 @@ uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInf
 		    ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
 		const auto pointer = state.builder.AllocateId();
 		state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
-		                            state.address_memory_variable,
-		                            ConstantU32(state, binding.array_index)});
+		                           state.address_memory_variable,
+		                           ConstantU32(state, binding.array_index)});
 		return pointer;
 	}
 	const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
 	const auto pointer = state.builder.AllocateId();
 	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
-	                            state.storage_buffer_variable,
-	                            ConstantU32(state, binding.array_index)});
+	                           state.storage_buffer_variable,
+	                           ConstantU32(state, binding.array_index)});
 	return pointer;
 }
 
 uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem,
-                                           uint32_t index, uint32_t use_pc) {
+                                          uint32_t index, uint32_t use_pc) {
 	index                = EmitStorageBufferIndex(state, mem, index, use_pc);
 	const auto object    = EmitStorageBufferObjectPointer(state, mem, use_pc);
 	const auto length    = state.builder.AllocateId();
@@ -453,7 +450,7 @@ uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryI
 }
 
 uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem,
-                                          uint32_t index, uint32_t use_pc) {
+                                         uint32_t index, uint32_t use_pc) {
 	index = EmitStorageBufferIndex(state, mem, index, use_pc);
 	if (IsFlatMemoryKind(mem.kind)) {
 		if (state.address_memory_variable == 0) {
@@ -589,9 +586,9 @@ uint32_t EmitMemoryLoadSubDwordValueU32(EmitterState& state, const IR::Instructi
 			const auto left       = state.builder.AllocateId();
 			const auto sign_shift = 32u - data_bits;
 			state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, left, masked,
-			                            ConstantU32(state, sign_shift)});
+			                           ConstantU32(state, sign_shift)});
 			state.builder.AddFunction({OpShiftRightArithmetic, state.uint_type, value, left,
-			                            ConstantU32(state, sign_shift)});
+			                           ConstantU32(state, sign_shift)});
 		}
 		return value;
 	};
@@ -659,16 +656,15 @@ void EmitAtomicUpdateU32(EmitterState& state, uint32_t pointer, IR::ResourceKind
 	state.builder.AddFunction({OpBranch, preheader});
 	state.builder.AddFunction({OpLabel, preheader});
 	state.builder.AddFunction({OpAtomicLoad, state.uint_type, initial, pointer,
-	                            ConstantU32(state, scope),
-	                            ConstantU32(state, MemorySemanticsNone)});
+	                           ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)});
 	state.builder.AddFunction({OpBranch, header});
 	state.builder.AddFunction({OpLabel, header});
 	state.builder.AddFunction(
 	    {OpPhi, state.uint_type, observed, initial, preheader, exchanged, continue_label});
 	const auto desired = desired_value(observed);
 	state.builder.AddFunction({OpAtomicCompareExchange, state.uint_type, exchanged, pointer,
-	                            ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone),
-	                            ConstantU32(state, MemorySemanticsNone), desired, observed});
+	                           ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone),
+	                           ConstantU32(state, MemorySemanticsNone), desired, observed});
 	const auto success = state.builder.AllocateId();
 	state.builder.AddFunction({OpIEqual, state.bool_type, success, exchanged, observed});
 	state.builder.AddFunction({OpLoopMerge, merge, continue_label, LoopControlNone});
@@ -793,8 +789,7 @@ uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
 uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
                                        uint32_t constant) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
+	state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
 	return ret;
 }
 
@@ -821,7 +816,7 @@ uint32_t EmitExtractFormatFieldU32(EmitterState& state, uint32_t raw_word, uint3
 		const auto signed_word = EmitTBufferBitcastU32ToI32(state, raw_word);
 		const auto extracted   = state.builder.AllocateId();
 		state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_word,
-		                            ConstantU32(state, offset), ConstantU32(state, bits)});
+		                           ConstantU32(state, offset), ConstantU32(state, bits)});
 		const auto ret = state.builder.AllocateId();
 		state.builder.AddFunction({OpBitcast, state.uint_type, ret, extracted});
 		return ret;
@@ -829,7 +824,7 @@ uint32_t EmitExtractFormatFieldU32(EmitterState& state, uint32_t raw_word, uint3
 
 	const auto extracted = state.builder.AllocateId();
 	state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, raw_word,
-	                            ConstantU32(state, offset), ConstantU32(state, bits)});
+	                           ConstantU32(state, offset), ConstantU32(state, bits)});
 	return extracted;
 }
 
@@ -940,7 +935,7 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 			state.builder.AddFunction(
 			    {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
 			state.builder.AddFunction({OpExtInst, state.float_type, clamped, state.glsl_std450,
-			                            GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
+			                           GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
 			return EmitTBufferBitcastF32ToU32(state, clamped);
 		}
 		case Format::ComponentType::Float:
@@ -961,18 +956,8 @@ uint32_t UnpackTBufferFormat(EmitterState& state, const IR::Instruction& inst,
 	return NormalizeFormatComponent(state, info, inst.memory.component_index, raw);
 }
 
-bool EmitTypedTBufferLoad(EmitterState& state, const IR::Instruction& inst,
-                          const Format::BufferFormatInfo& info) {
-	if (!Format::CanUseTypedBufferLoad(info.format)) {
-		return false;
-	}
-	const auto value = EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0,
-	                                               AddressSourceCount(inst, 0));
-	EmitStoreU32(state, inst.dst, value);
-	return true;
-}
-
-bool EmitFormattedBufferLoad(EmitterState& state, const IR::Instruction& inst) {
+bool EmitFormattedBufferLoadValueU32(EmitterState& state, const IR::Instruction& inst,
+                                     uint32_t& value) {
 	if (!IsFormattedBufferComponent(inst)) {
 		return false;
 	}
@@ -984,18 +969,29 @@ bool EmitFormattedBufferLoad(EmitterState& state, const IR::Instruction& inst) {
 
 	const auto info = Format::GetFormatInfo(format);
 	if (inst.memory.component_index >= info.component_count) {
-		EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
+		value = ConstantU32(state, 0);
 		return true;
 	}
 
-	if (EmitTypedTBufferLoad(state, inst, info)) {
+	if (Format::CanUseTypedBufferLoad(info.format)) {
+		value = EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0,
+		                                    AddressSourceCount(inst, 0));
 		return true;
 	}
 
-	EmitStoreU32(state, inst.dst, UnpackTBufferFormat(state, inst, info));
+	value = UnpackTBufferFormat(state, inst, info);
 	return true;
 }
 
+uint32_t EmitBufferLoadDwordValueU32(EmitterState& state, const IR::Instruction& inst) {
+	uint32_t value = 0;
+	if (EmitFormattedBufferLoadValueU32(state, inst, value)) {
+		return value;
+	}
+	return EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0,
+	                                   AddressSourceCount(inst, 0));
+}
+
 uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format,
                                                  uint32_t               opcode_components) {
 	switch (format) {
@@ -1117,8 +1113,8 @@ uint32_t EmitAtomicPointer(EmitterState& state, const IR::Instruction& inst) {
 			    StorageImageDescriptorPointer(state, inst.memory.resource, true, inst.pc, view);
 			const auto pointer = state.builder.AllocateId();
 			state.builder.AddFunction({OpImageTexelPointer, state.ptr_image_uint, pointer,
-			                            image_pointer, EmitImageCoordU32(state, inst, view),
-			                            ConstantU32(state, 0)});
+			                           image_pointer, EmitImageCoordU32(state, inst, view),
+			                           ConstantU32(state, 0)});
 			return pointer;
 		}
 		default: return 0;
@@ -1142,8 +1138,8 @@ void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t op
 			    EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc);
 			const auto result = state.builder.AllocateId();
 			state.builder.AddFunction({opcode, state.uint_type, result, pointer,
-			                            ConstantU32(state, ScopeDevice),
-			                            ConstantU32(state, MemorySemanticsNone), value});
+			                           ConstantU32(state, ScopeDevice),
+			                           ConstantU32(state, MemorySemanticsNone), value});
 			EmitDeviceAtomicMemoryBarrier(state);
 			return result;
 		});
@@ -1158,8 +1154,8 @@ void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t op
 			const auto pointer = EmitGdsElementPointer(state, index);
 			const auto result  = state.builder.AllocateId();
 			state.builder.AddFunction({opcode, state.uint_type, result, pointer,
-			                            ConstantU32(state, ScopeDevice),
-			                            ConstantU32(state, MemorySemanticsNone), value});
+			                           ConstantU32(state, ScopeDevice),
+			                           ConstantU32(state, MemorySemanticsNone), value});
 			EmitDeviceAtomicMemoryBarrier(state);
 			return result;
 		});
@@ -1177,7 +1173,7 @@ void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t op
 	const auto old   = state.builder.AllocateId();
 	const auto scope = inst.memory.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
 	state.builder.AddFunction({opcode, state.uint_type, old, pointer, ConstantU32(state, scope),
-	                            ConstantU32(state, MemorySemanticsNone), value});
+	                           ConstantU32(state, MemorySemanticsNone), value});
 	if (inst.memory.kind == IR::ResourceKind::StorageImageUint ||
 	    inst.memory.kind == IR::ResourceKind::Gds) {
 		EmitDeviceAtomicMemoryBarrier(state);
@@ -1199,8 +1195,8 @@ void EmitSLoadDword(EmitterState& state, const IR::Instruction& inst) {
 	    {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
 	const auto object = state.builder.AllocateId();
 	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, object,
-	                            state.address_memory_variable,
-	                            ConstantU32(state, binding.array_index)});
+	                           state.address_memory_variable,
+	                           ConstantU32(state, binding.array_index)});
 	const auto length    = state.builder.AllocateId();
 	const auto in_bounds = state.builder.AllocateId();
 	state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0});
@@ -1227,8 +1223,8 @@ void EmitLoadSrtDword(EmitterState& state, const IR::Instruction& inst) {
 	const auto pointer = state.builder.AllocateId();
 	const auto value   = state.builder.AllocateId();
 	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
-	                            state.flattened_srt_variable, ConstantU32(state, 0),
-	                            ConstantU32(state, inst.src[0].imm)});
+	                           state.flattened_srt_variable, ConstantU32(state, 0),
+	                           ConstantU32(state, inst.src[0].imm)});
 	state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
 	EmitStoreU32(state, inst.dst, value);
 }
@@ -1262,11 +1258,27 @@ void EmitBufferLoadSshort(EmitterState& state, const IR::Instruction& inst) {
 }
 
 void EmitBufferLoadDword(EmitterState& state, const IR::Instruction& inst) {
+	EmitGuardedByExec(
+	    state, [&]() { EmitStoreU32(state, inst.dst, EmitBufferLoadDwordValueU32(state, inst)); });
+}
+
+void EmitBufferLoadDwordGroup(EmitterState& state, const IR::Instruction* instructions,
+                              uint32_t count) {
+	if (instructions == nullptr || count == 0u) {
+		return;
+	}
+
 	EmitGuardedByExec(state, [&]() {
-		if (EmitFormattedBufferLoad(state, inst)) {
-			return;
+		// RDNA VMEM captures every VADDR component before making overlapping VDATA writes
+		// visible. Keep the split IR components instruction-atomic by deferring all stores.
+		std::vector<uint32_t> values;
+		values.reserve(count);
+		for (uint32_t i = 0; i < count; i++) {
+			values.push_back(EmitBufferLoadDwordValueU32(state, instructions[i]));
+		}
+		for (uint32_t i = 0; i < count; i++) {
+			EmitStoreU32(state, instructions[i].dst, values[i]);
 		}
-		EmitMemoryLoadU32(state, inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(inst, 0));
 	});
 }
 
@@ -1385,7 +1397,7 @@ DsCounterAddress EmitAppendConsumeAddress(EmitterState& state, const IR::Instruc
 	state.builder.AddFunction(
 	    {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
 	state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds,
-	                            ConstantU32(state, inst.memory.offset + 3u), size});
+	                           ConstantU32(state, inst.memory.offset + 3u), size});
 	return {index, size, in_bounds};
 }
 
@@ -1400,7 +1412,7 @@ uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) {
 uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) {
 	const auto pointer = state.builder.AllocateId();
 	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
-	                            state.gds_variable, ConstantU32(state, 0), index});
+	                           state.gds_variable, ConstantU32(state, 0), index});
 	return pointer;
 }
 
@@ -1431,7 +1443,7 @@ ExecMaskInfo EmitExecMaskInfo(EmitterState& state) {
 	if (state.per_invocation_masks) {
 		const auto ballot = state.builder.AllocateId();
 		state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
-		                            ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
+		                           ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
 		exec_lo = state.builder.AllocateId();
 		state.builder.AddFunction({OpCompositeExtract, state.uint_type, exec_lo, ballot, 0});
 		if (state.wave_size == 64u) {
@@ -1482,28 +1494,27 @@ void EmitDsAppendConsume(EmitterState& state, const IR::Instruction& inst, uint3
 	const auto do_atomic  = state.builder.AllocateId();
 	state.builder.AddFunction({OpIEqual, state.bool_type, first_lane, subid, exec.first_lane});
 	const auto first_active = EmitLogicalAndBool(state, first_lane, exec.any_active);
-	state.builder.AddFunction(
-	    {OpLogicalAnd, state.bool_type, do_atomic, first_active, in_bounds});
+	state.builder.AddFunction({OpLogicalAnd, state.bool_type, do_atomic, first_active, in_bounds});
 
 	const auto atomic_value = EmitValueOrZeroIfCondition(state, do_atomic, [&]() {
 		const auto pointer = gds ? EmitGdsElementPointer(state, address.index)
 		                         : EmitLdsElementPointer(state, address.index);
 		const auto result  = state.builder.AllocateId();
 		state.builder.AddFunction({atomic_opcode, state.uint_type, result, pointer,
-		                            ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
-		                            ConstantU32(state, MemorySemanticsNone), exec.active_count});
+		                           ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
+		                           ConstantU32(state, MemorySemanticsNone), exec.active_count});
 		if (gds) {
 			EmitDeviceAtomicMemoryBarrier(state);
 		} else {
 			const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
 			state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, ScopeWorkgroup),
-			                            ConstantU32(state, semantics)});
+			                           ConstantU32(state, semantics)});
 		}
 		return result;
 	});
 	const auto broadcast    = state.builder.AllocateId();
 	state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, broadcast,
-	                            ConstantU32(state, ScopeSubgroup), atomic_value, exec.first_lane});
+	                           ConstantU32(state, ScopeSubgroup), atomic_value, exec.first_lane});
 	const auto value = EmitSelectU32Value(state, exec.any_active, broadcast, ConstantU32(state, 0));
 	EmitStoreU32(state, inst.dst, value);
 }
@@ -1523,8 +1534,8 @@ void EmitDsFloatMinMaxF32(EmitterState& state, const IR::Instruction& inst, bool
 			const auto value_u32 = state.builder.AllocateId();
 			state.builder.AddFunction({OpBitcast, state.float_type, old_f32, old_u32});
 			state.builder.AddFunction({max_value ? OpFOrdGreaterThan : OpFOrdLessThan,
-			                            state.bool_type, store_src, max_value ? old_f32 : cmp_f32,
-			                            max_value ? cmp_f32 : old_f32});
+			                           state.bool_type, store_src, max_value ? old_f32 : cmp_f32,
+			                           max_value ? cmp_f32 : old_f32});
 			state.builder.AddFunction(
 			    {OpSelect, state.float_type, value_f32, store_src, data_f32, old_f32});
 			state.builder.AddFunction({OpBitcast, state.uint_type, value_u32, value_f32});
@@ -1575,14 +1586,13 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 	const auto xored  = state.builder.AllocateId();
 	const auto base   = state.builder.AllocateId();
 	const auto target = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
 	state.builder.AddFunction(
 	    {OpBitwiseAnd, state.uint_type, masked, lane, ConstantU32(state, control & 0x1fu)});
 	state.builder.AddFunction(
 	    {OpBitwiseOr, state.uint_type, ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
-	state.builder.AddFunction({OpBitwiseXor, state.uint_type, xored, ored,
-	                            ConstantU32(state, (control >> 10u) & 0x1fu)});
+	state.builder.AddFunction(
+	    {OpBitwiseXor, state.uint_type, xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
 	state.builder.AddFunction(
 	    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xffffffe0u)});
 	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, xored});
@@ -1596,7 +1606,7 @@ void EmitDsSwizzleB32(EmitterState& state, const IR::Instruction& inst) {
 	const auto target  = EmitDsSwizzleTargetLane(state, subid, control);
 	const auto value   = state.builder.AllocateId();
 	state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value,
-	                            ConstantU32(state, ScopeSubgroup), source, target});
+	                           ConstantU32(state, ScopeSubgroup), source, target});
 	const auto exec_active     = EmitLaneIndexActiveBool(state, target);
 	const auto subgroup_active = EmitSubgroupLaneActiveBool(state, target);
 	const auto source_active   = state.builder.AllocateId();

src/libs/libNet.cpp

@@ -654,6 +654,8 @@ namespace LibHttp2 {
 LIB_VERSION("Http2", 1, "Http2", 1, 1);
 
 constexpr int HTTP2_ERROR_INVALID_ID   = -2122641152; /* 0x817B1100 */
+constexpr int HTTP2_ERROR_BEFORE_SEND  = -2122641307; /* 0x817B1065 */
+constexpr int HTTP2_ERROR_TIMEOUT      = -2122641304; /* 0x817B1068 */
 constexpr int HTTP2_ERROR_NULL_POINTER = -2122640859; /* 0x817B1225 */
 
 struct Http2Options {
@@ -694,14 +696,14 @@ struct Http2Request {
 	std::string                                      url;
 	uint64_t                                         content_length = 0;
 	std::vector<std::pair<std::string, std::string>> headers;
-	bool                                             sent        = false;
-	int                                              status_code = 204;
-	std::string  response_headers                                = "HTTP/2 204 No Content\r\n\r\n";
-	std::string  response_body;
-	size_t       read_offset  = 0;
-	int          async_result = 0;
-	int          async_event  = 0;
-	Http2Options options;
+	int                                              send_result = HTTP2_ERROR_BEFORE_SEND;
+	int                                              status_code = 0;
+	std::string                                      response_headers;
+	std::string                                      response_body;
+	size_t                                           read_offset  = 0;
+	int                                              async_result = HTTP2_ERROR_BEFORE_SEND;
+	int                                              async_event  = 0;
+	Http2Options                                     options;
 };
 
 struct Http2AsyncResult {
@@ -1114,9 +1116,9 @@ static int KYTY_SYSV_ABI Http2SendRequest(int req_id, const void* post_data, siz
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	request->second.sent = true;
+	request->second.send_result = HTTP2_ERROR_TIMEOUT;
 
-	return 0;
+	return request->second.send_result;
 }
 
 static int KYTY_SYSV_ABI Http2SendRequestAsync(int req_id, const void* post_data, size_t size,
@@ -1136,8 +1138,8 @@ static int KYTY_SYSV_ABI Http2SendRequestAsync(int req_id, const void* post_data
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	request->second.sent         = true;
-	request->second.async_result = 0;
+	request->second.send_result  = HTTP2_ERROR_TIMEOUT;
+	request->second.async_result = request->second.send_result;
 	request->second.async_event  = 0;
 
 	return 0;
@@ -1159,11 +1161,10 @@ static int KYTY_SYSV_ABI Http2WaitAsync(int req_id, Http2AsyncResult* result, ui
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	request->second.sent = true;
-	*result              = {};
-	result->event_type   = request->second.async_event;
-	result->req_id       = req_id;
-	result->result       = request->second.async_result;
+	*result            = {};
+	result->event_type = request->second.async_event;
+	result->req_id     = req_id;
+	result->result     = request->second.async_result;
 
 	return 0;
 }
@@ -1178,13 +1179,19 @@ static int KYTY_SYSV_ABI Http2GetStatusCode(int req_id, int* status_code) {
 		return HTTP2_ERROR_NULL_POINTER;
 	}
 
+	*status_code = 0;
+
 	auto request = g_http2_requests.find(req_id);
 	if (request == g_http2_requests.end()) {
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	*status_code = request->second.status_code;
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		return send_result;
+	}
 
+	*status_code = request->second.status_code;
 	return 0;
 }
 
@@ -1200,14 +1207,21 @@ static int KYTY_SYSV_ABI Http2GetResponseContentLength(int req_id, int* result,
 		return HTTP2_ERROR_NULL_POINTER;
 	}
 
+	*result         = 0;
+	*content_length = 0;
+
 	auto request = g_http2_requests.find(req_id);
 	if (request == g_http2_requests.end()) {
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	*result         = 0; // SCE_HTTP2_CONTENTLEN_EXIST
-	*content_length = request->second.response_body.size();
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		*result = -1;
+		return send_result;
+	}
 
+	*content_length = request->second.response_body.size();
 	return 0;
 }
 
@@ -1223,14 +1237,21 @@ static int KYTY_SYSV_ABI Http2GetAllResponseHeaders(int req_id, char** header,
 		return HTTP2_ERROR_NULL_POINTER;
 	}
 
+	*header      = nullptr;
+	*header_size = 0;
+
 	auto request = g_http2_requests.find(req_id);
 	if (request == g_http2_requests.end()) {
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		return send_result;
+	}
+
 	*header      = const_cast<char*>(request->second.response_headers.c_str());
 	*header_size = request->second.response_headers.size();
-
 	return 0;
 }
 
@@ -1250,6 +1271,11 @@ static int KYTY_SYSV_ABI Http2ReadData(int req_id, void* data, size_t size) {
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		return send_result;
+	}
+
 	const auto& body = request->second.response_body;
 	const auto  remaining =
 	    request->second.read_offset < body.size() ? body.size() - request->second.read_offset : 0;
@@ -1280,6 +1306,13 @@ static int KYTY_SYSV_ABI Http2ReadDataAsync(int req_id, void* data, size_t size,
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		request->second.async_result = send_result;
+		request->second.async_event  = 1;
+		return 0;
+	}
+
 	const auto& body = request->second.response_body;
 	const auto  remaining =
 	    request->second.read_offset < body.size() ? body.size() - request->second.read_offset : 0;

tests/ShaderRecompilerComputeTests.cpp

@@ -4772,6 +4772,52 @@ public:
               "successive near-capacity image transfers replaced the shared "
               "download buffer");
 
+      constexpr uint64_t tile_alias_offset = 0x2000000;
+      constexpr uint64_t tile_alias_size = 0x400000;
+      constexpr uint32_t tile_alias_extent = 1024;
+      std::memset(memory + tile_alias_offset, 0,
+                  static_cast<size_t>(tile_alias_size));
+      auto render_target_alias = MakeLinearDesc(
+          base + tile_alias_offset, tile_alias_size,
+          vk::Format::eR8G8B8A8Unorm,
+          Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm),
+          Prospero::ImageType::kColor2D,
+          {tile_alias_extent, tile_alias_extent, 1}, 1, 4, 1);
+      render_target_alias.type = BindingType::Storage;
+      render_target_alias.info.tile_mode =
+          Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget);
+      render_target_alias.view_info.usage =
+          vk::ImageUsageFlagBits::eStorage;
+      const auto render_target_alias_image =
+          texture_cache.FindImage(render_target_alias);
+
+      auto standard_4kb_alias = render_target_alias;
+      standard_4kb_alias.type = BindingType::Texture;
+      standard_4kb_alias.info.tile_mode =
+          Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB);
+      standard_4kb_alias.view_info.usage =
+          vk::ImageUsageFlagBits::eSampled;
+      const auto standard_4kb_alias_image =
+          texture_cache.FindImage(standard_4kb_alias);
+      auto repeated_standard_4kb_alias = standard_4kb_alias;
+      const auto repeated_standard_4kb_alias_image =
+          texture_cache.FindImage(repeated_standard_4kb_alias);
+      Require(
+          name, "equal-size tile-mode alias",
+          render_target_alias_image && standard_4kb_alias_image &&
+              standard_4kb_alias_image != render_target_alias_image &&
+              repeated_standard_4kb_alias_image ==
+                  standard_4kb_alias_image &&
+              texture_cache.GetImage(render_target_alias_image)
+                      .info.tile_mode ==
+                  Prospero::GpuEnumValue(
+                      Prospero::TileMode::kRenderTarget) &&
+              texture_cache.GetImage(standard_4kb_alias_image)
+                      .info.tile_mode ==
+                  Prospero::GpuEnumValue(
+                      Prospero::TileMode::kStandard4KB),
+          "equal address/size lookup reused an incompatible tiled backing");
+
       for (auto &output : ms_observer_outputs) {
         DestroyBuffer(&output);
       }
@@ -12083,6 +12129,84 @@ TestCase BufferLoadVariants() {
            O::BufferLoadDwordx4, O::VMovB32, O::BufferStoreDword, O::SEndpgm}};
 }
 
+TestCase BufferLoadDwordx4SnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMubuf0(0x0eu, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadDwordx4SnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
+                  0,           0,           0,           0};
+  test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
+                   0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
+  test.opcodes = {O::VMovB32, O::BufferLoadDwordx4, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(16, 2);
+  test.has_user_data = true;
+  return test;
+}
+
+TestCase BufferLoadDwordx2SnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMubuf0(0x0du, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  for (u32 i = 0; i < 2; i++) {
+    AppendStoreVgpr(&code, 21 + i, 2 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadDwordx2SnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x11111111u, 0x22222222u, 0, 0};
+  test.expected = {0x11111111u, 0x22222222u, 0x11111111u, 0x22222222u};
+  test.opcodes = {O::VMovB32, O::BufferLoadDwordx2, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(8, 2);
+  test.has_user_data = true;
+  return test;
+}
+
+TestCase BufferLoadDwordx3SnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMubuf0(0x0fu, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  for (u32 i = 0; i < 3; i++) {
+    AppendStoreVgpr(&code, 21 + i, 3 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadDwordx3SnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0, 0, 0};
+  test.expected = {0x11111111u, 0x22222222u, 0x33333333u,
+                   0x11111111u, 0x22222222u, 0x33333333u};
+  test.opcodes = {O::VMovB32, O::BufferLoadDwordx3, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(12, 2);
+  test.has_user_data = true;
+  return test;
+}
+
 TestCase BufferStoreVariants() {
   using O = ShaderOpcode;
 
@@ -12151,6 +12275,69 @@ TestCase BufferFormatVariants() {
   return load;
 }
 
+TestCase BufferLoadFormatXyzwSnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMubuf0(0x03u, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadFormatXyzwSnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+                  0,           0,           0,           0};
+  test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+                   0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
+  test.opcodes = {O::VMovB32, O::BufferLoadFormatXyzw, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(
+      16, 2, false,
+      BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
+  test.has_user_data = true;
+  return test;
+}
+
+TestCase BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovLiteral(&code, 21, 0x11111111u);
+  AppendVMovLiteral(&code, 22, 0x22222222u);
+  AppendVMovLiteral(&code, 23, 0x33333333u);
+  AppendVMovLiteral(&code, 24, 0x44444444u);
+  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
+  code.push_back(EncodeMubuf0(0x03u, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  code.push_back(EncodeSMovB32(126, InlineU32(1)));
+  code.push_back(EncodeSMovB32(127, InlineU32(0)));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0xaaaaaaaa, 0xbbbbbbbb, 0xcccccccc, 0xdddddddd,
+                  0,          0,          0,          0};
+  test.expected = {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu,
+                   0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
+  test.opcodes = {O::VMovB32, O::SMovB64, O::BufferLoadFormatXyzw,
+                  O::SMovB32, O::BufferStoreDword, O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(
+      16, 2, false,
+      BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
+  test.has_user_data = true;
+  return test;
+}
+
 TestCase BufferFormatStoreVariants() {
   using O = ShaderOpcode;
 
@@ -12459,6 +12646,63 @@ TestCase TBufferLoadVariants() {
            O::BufferStoreDword, O::SEndpgm}};
 }
 
+TestCase TBufferLoadFormatXyzwSnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  constexpr auto format =
+      BufferFormat(Prospero::BufferFormat::k32_32_32_32Float);
+  code.push_back(
+      EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true));
+  code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "TBufferLoadFormatXyzwSnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+                  0,           0,           0,           0};
+  test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+                   0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
+  test.opcodes = {O::VMovB32, O::TBufferLoadFormatXyzw,
+                  O::BufferStoreDword, O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(16, 2);
+  test.has_user_data = true;
+  return test;
+}
+
+TestCase TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  constexpr auto format = BufferFormat(Prospero::BufferFormat::k8_8_8_8UInt);
+  code.push_back(
+      EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true));
+  code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x44332211u, 0, 0, 0, 0, 0, 0, 0};
+  test.expected = {0x44332211u, 0, 0, 0, 0x11u, 0x22u, 0x33u, 0x44u};
+  test.opcodes = {O::VMovB32, O::TBufferLoadFormatXyzw,
+                  O::BufferStoreDword, O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(4, 8);
+  test.has_user_data = true;
+  return test;
+}
+
 TestCase TBufferStoreFormatX8UintWritesOneByte() {
   using O = ShaderOpcode;
 
@@ -13593,23 +13837,36 @@ TestCase DsAppendUsesEncodedGdsSelector() {
   using O = ShaderOpcode;
 
   std::vector<u32> code;
-  AppendSMovLiteral(&code, 124, 0x00000001u);
+  AppendSMovLiteral(&code, 124, 0x00000008u);
   code.push_back(EncodeDs0(0x3e, 0, true));
   code.push_back(EncodeDs1(0, 0, 0));
   code.push_back(EncodeDs0(0x3d, 0, true));
   code.push_back(EncodeDs1(1, 0, 0));
+  code.push_back(EncodeDs0(0x3e, 4, true));
+  code.push_back(EncodeDs1(2, 0, 0));
+  code.push_back(EncodeDs0(0x3d, 4, true));
+  code.push_back(EncodeDs1(3, 0, 0));
+  AppendSMovLiteral(&code, 124, 0x00080008u);
+  code.push_back(EncodeDs0(0x3e, 4, true));
+  code.push_back(EncodeDs1(4, 0, 0));
+  code.push_back(EncodeDs0(0x3d, 4, true));
+  code.push_back(EncodeDs1(5, 0, 0));
   AppendStoreVgpr(&code, 0, 0);
   AppendStoreVgpr(&code, 1, 1);
+  AppendStoreVgpr(&code, 2, 2);
+  AppendStoreVgpr(&code, 3, 3);
+  AppendStoreVgpr(&code, 4, 4);
+  AppendStoreVgpr(&code, 5, 5);
   AppendEnd(&code);
 
   TestCase test{
       "DsAppendGdsSelector",
       code,
       {},
-      {10, 74},
+      {10, 74, 20, 84, 40, 104},
       {O::SMovB32, O::DsAppend, O::DsConsume, O::BufferStoreDword, O::SEndpgm}};
-  test.gds_initial = {10};
-  test.expected_gds = {10};
+  test.gds_initial = {10, 20, 30, 40};
+  test.expected_gds = {10, 20, 30, 40};
   return test;
 }
 
@@ -14780,8 +15037,13 @@ std::vector<TestCase> MakeCases() {
   AddCase(BufferStoreDwordAppliesHostOffset);
   AddCase(BufferOffsetsUsePackedLaneAndStorageFallback);
   AddCase(BufferLoadVariants);
+  AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress);
+  AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress);
+  AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress);
   AddCase(BufferStoreVariants);
   AddCase(BufferFormatVariants);
+  AddCase(BufferLoadFormatXyzwSnapshotsOverlappingAddress);
+  AddCase(BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress);
   AddCase(BufferFormatStoreVariants);
   AddCase(BufferStoreFormatXResource16UintWritesHalfword);
   AddCase(BufferLoadFormatXResource8UintZeroExtendsByte);
@@ -14795,6 +15057,8 @@ std::vector<TestCase> MakeCases() {
   AddCase(BufferStoreFormatXAddTidUsesLaneIndex);
   AddCase(BufferStoreFormatXDropsOutOfRangeRecord);
   AddCase(TBufferLoadVariants);
+  AddCase(TBufferLoadFormatXyzwSnapshotsOverlappingAddress);
+  AddCase(TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress);
   AddCase(TBufferLoadFormatX8UintZeroExtendsByte);
   AddCase(TBufferLoadFormatX8888UintExtractsFirstByte);
   AddCase(TBufferLoadFormatXIdxenUsesDescriptorStride);