shader: implement BUFFER_ATOMIC_FMIN, unhandled IR opcode now fails

This commit is contained in:
nmzik
2026-08-05 02:59:28 +02:00
parent 0a2f4814ac
commit df44ad3fac
14 changed files with 187 additions and 5 deletions
@@ -84,6 +84,7 @@ bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) {
case IR::Opcode::AtomicAndU32:
case IR::Opcode::AtomicOrU32:
case IR::Opcode::AtomicXorU32:
case IR::Opcode::AtomicFMinF32:
case IR::Opcode::FlatLoadUbyte:
case IR::Opcode::FlatLoadSbyte:
case IR::Opcode::FlatLoadUshort:
@@ -57,6 +57,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPS[] = {
{0x39u, Opcode::BufferAtomicAnd, 1, 32},
{0x3au, Opcode::BufferAtomicOr, 1, 32},
{0x3bu, Opcode::BufferAtomicXor, 1, 32},
{0x3fu, Opcode::BufferAtomicFMin, 1, 32},
};
constexpr MemoryOpcodeInfo MTBUF_OPS[] = {
@@ -835,6 +835,7 @@ std::string OpcodeToString(Opcode opcode) {
case Opcode::BufferAtomicAnd: return "buffer_atomic_and";
case Opcode::BufferAtomicOr: return "buffer_atomic_or";
case Opcode::BufferAtomicXor: return "buffer_atomic_xor";
case Opcode::BufferAtomicFMin: return "buffer_atomic_fmin";
case Opcode::FlatLoadUbyte: return "flat_load_ubyte";
case Opcode::FlatLoadSbyte: return "flat_load_sbyte";
case Opcode::FlatLoadUshort: return "flat_load_ushort";
@@ -1125,6 +1126,7 @@ std::string InstructionToString(const Instruction& inst) {
case Opcode::BufferAtomicAnd:
case Opcode::BufferAtomicOr:
case Opcode::BufferAtomicXor:
case Opcode::BufferAtomicFMin:
case Opcode::BufferLoadSbyte:
case Opcode::BufferLoadSshort:
case Opcode::FlatLoadUbyte:
@@ -434,6 +434,7 @@ enum class Opcode {
BufferAtomicAnd,
BufferAtomicOr,
BufferAtomicXor,
BufferAtomicFMin,
FlatLoadUbyte,
FlatLoadSbyte,
FlatLoadUshort,
@@ -131,7 +131,8 @@ bool IsAtomic(IR::Opcode op) {
case IR::Opcode::AtomicUMaxU32:
case IR::Opcode::AtomicAndU32:
case IR::Opcode::AtomicOrU32:
case IR::Opcode::AtomicXorU32: return true;
case IR::Opcode::AtomicXorU32:
case IR::Opcode::AtomicFMinF32: return true;
default: return false;
}
}
@@ -775,6 +775,9 @@ void EmitInstruction(EmitterState& state, const IR::Instruction& inst) {
case IR::Opcode::AtomicXorU32:
EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicXor); });
break;
case IR::Opcode::AtomicFMinF32:
EmitGuardedByExec(state, [&]() { EmitAtomicFMinF32(state, inst); });
break;
case IR::Opcode::FlatLoadUbyte: EmitFlatLoadUbyte(state, inst); break;
case IR::Opcode::FlatLoadSbyte: EmitFlatLoadSbyte(state, inst); break;
case IR::Opcode::FlatLoadUshort: EmitFlatLoadUshort(state, inst); break;
@@ -1011,6 +1011,8 @@ void EmitDeviceAtomicMemoryBarrier(EmitterState& state);
void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
void EmitAtomicFMinF32(EmitterState& state, const IR::Instruction& inst);
void EmitSLoadDword(EmitterState& state, const IR::Instruction& inst);
void EmitLoadSrtDword(EmitterState& state, const IR::Instruction& inst);
@@ -1573,19 +1575,22 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&
}
const auto then_label = state.builder.AllocateId();
const auto then_exit = state.builder.AllocateId();
const auto else_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId();
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
state.builder.AddFunction({OpBranchConditional, condition, then_label, else_label});
state.builder.AddFunction({OpLabel, then_label});
const auto then_value = fn();
state.builder.AddFunction({OpBranch, then_exit});
state.builder.AddFunction({OpLabel, then_exit});
state.builder.AddFunction({OpBranch, merge_label});
state.builder.AddFunction({OpLabel, else_label});
state.builder.AddFunction({OpBranch, merge_label});
state.builder.AddFunction({OpLabel, merge_label});
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpPhi, state.uint_type, value, then_value, then_label, ConstantU32(state, 0), else_label});
{OpPhi, state.uint_type, value, then_value, then_exit, ConstantU32(state, 0), else_label});
return value;
}
@@ -640,8 +640,8 @@ void EmitMemoryStoreU32(EmitterState& state, const IR::Instruction& inst, IR::Re
}
template <typename Fn>
void EmitAtomicUpdateU32(EmitterState& state, uint32_t pointer, IR::ResourceKind kind,
Fn&& desired_value) {
uint32_t EmitAtomicUpdateU32(EmitterState& state, uint32_t pointer, IR::ResourceKind kind,
Fn&& desired_value) {
const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
const auto memory = kind == IR::ResourceKind::Lds ? MemorySemanticsWorkgroupMemory
: MemorySemanticsUniformMemory;
@@ -675,6 +675,7 @@ void EmitAtomicUpdateU32(EmitterState& state, uint32_t pointer, IR::ResourceKind
const auto semantics = MemorySemanticsAcquireRelease | memory;
state.builder.AddFunction(
{OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)});
return observed;
}
void EmitMemoryStoreSubDwordU32(EmitterState& state, const IR::Instruction& inst,
@@ -1181,6 +1182,68 @@ void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t op
EmitStoreU32(state, inst.dst, old);
}
namespace {
uint32_t EmitF32BitsOrderedLessThan(EmitterState& state, uint32_t lhs_bits, uint32_t rhs_bits) {
struct ClassifiedBits {
uint32_t nan = 0;
uint32_t zero = 0;
uint32_t key = 0;
};
const auto Classify = [&](uint32_t bits) {
ClassifiedBits cls;
const auto abs_bits = EmitAndConstant(state, bits, 0x7fffffffu);
const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
const auto exponent_max =
EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0);
const auto negative = EmitCompareU32Constant(state, OpINotEqual,
EmitAndConstant(state, bits, 0x80000000u), 0);
const auto negative_key = state.builder.AllocateId();
const auto positive_key = state.builder.AllocateId();
// Map all non-NaN IEEE-754 encodings to monotonically increasing unsigned keys.
state.builder.AddFunction({OpNot, state.uint_type, negative_key, bits});
state.builder.AddFunction(
{OpBitwiseXor, state.uint_type, positive_key, bits, ConstantU32(state, 0x80000000u)});
cls.nan = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
cls.key = EmitSelectValueU32(state, negative, negative_key, positive_key);
return cls;
};
const auto lhs = Classify(lhs_bits);
const auto rhs = Classify(rhs_bits);
const auto any_nan = EmitLogicalOrBool(state, lhs.nan, rhs.nan);
// The key transform orders -0 below +0, while the floating comparison treats them as equal.
const auto both_zero = EmitLogicalAndBool(state, lhs.zero, rhs.zero);
const auto ordered_nonzero =
EmitLogicalNotBool(state, EmitLogicalOrBool(state, any_nan, both_zero));
const auto less = state.builder.AllocateId();
state.builder.AddFunction({OpULessThan, state.bool_type, less, lhs.key, rhs.key});
return EmitLogicalAndBool(state, ordered_nonzero, less);
}
} // namespace
void EmitAtomicFMinF32(EmitterState& state, const IR::Instruction& inst) {
const auto index =
EmitMemoryDwordIndex(state, inst, inst.memory, 1, AddressSourceCount(inst, 1));
const auto in_bounds = EmitStorageBufferElementInBounds(state, inst.memory, index, inst.pc);
const auto src_u32 = EmitValueLoad(state, inst.src[0]);
const auto old = EmitValueOrZeroIfCondition(state, in_bounds, [&]() {
const auto pointer = EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc);
return EmitAtomicUpdateU32(state, pointer, inst.memory.kind, [&](uint32_t old_u32) {
const auto replace_old = state.builder.AllocateId();
const auto less = EmitF32BitsOrderedLessThan(state, src_u32, old_u32);
state.builder.AddFunction(
{OpSelect, state.uint_type, replace_old, less, src_u32, old_u32});
return replace_old;
});
});
EmitStoreU32(state, inst.dst, old);
}
void EmitSLoadDword(EmitterState& state, const IR::Instruction& inst) {
if (state.address_memory_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
@@ -30,7 +30,8 @@ bool IsAtomic(Opcode op) {
case Opcode::AtomicUMaxU32:
case Opcode::AtomicAndU32:
case Opcode::AtomicOrU32:
case Opcode::AtomicXorU32: return true;
case Opcode::AtomicXorU32:
case Opcode::AtomicFMinF32: return true;
default: return false;
}
}
@@ -569,6 +569,8 @@ bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& blo
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicOrU32, error);
case Decoder::Opcode::BufferAtomicXor:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicXorU32, error);
case Decoder::Opcode::BufferAtomicFMin:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicFMinF32, error);
case Decoder::Opcode::FlatLoadUbyte:
case Decoder::Opcode::FlatLoadSbyte:
case Decoder::Opcode::FlatLoadUshort:
@@ -729,6 +731,7 @@ bool IsMemoryOpcode(Decoder::Opcode opcode) {
case Decoder::Opcode::BufferAtomicAnd:
case Decoder::Opcode::BufferAtomicOr:
case Decoder::Opcode::BufferAtomicXor:
case Decoder::Opcode::BufferAtomicFMin:
case Decoder::Opcode::FlatLoadUbyte:
case Decoder::Opcode::FlatLoadSbyte:
case Decoder::Opcode::FlatLoadUshort:
@@ -352,6 +352,7 @@ constexpr LowerMap LOWER_OPS[] = {
{Decoder::Opcode::BufferAtomicAnd, Opcode::AtomicAndU32},
{Decoder::Opcode::BufferAtomicOr, Opcode::AtomicOrU32},
{Decoder::Opcode::BufferAtomicXor, Opcode::AtomicXorU32},
{Decoder::Opcode::BufferAtomicFMin, Opcode::AtomicFMinF32},
{Decoder::Opcode::FlatLoadUbyte, Opcode::FlatLoadUbyte},
{Decoder::Opcode::FlatLoadSbyte, Opcode::FlatLoadSbyte},
{Decoder::Opcode::FlatLoadSshort, Opcode::FlatLoadSshort},
@@ -307,6 +307,7 @@ IR_OPCODE(AtomicUMaxU32, General)
IR_OPCODE(AtomicAndU32, General)
IR_OPCODE(AtomicOrU32, General)
IR_OPCODE(AtomicXorU32, General)
IR_OPCODE(AtomicFMinF32, General)
IR_OPCODE(FlatLoadUbyte, General)
IR_OPCODE(FlatLoadSbyte, General)
IR_OPCODE(FlatLoadUshort, General)
+91
View File
@@ -9050,6 +9050,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode, const std::set<ShaderOpcode>&
case Opcode::BufferAtomicAnd:
case Opcode::BufferAtomicOr:
case Opcode::BufferAtomicXor:
case Opcode::BufferAtomicFMin:
case Opcode::FlatLoadUbyte:
case Opcode::FlatLoadSbyte:
case Opcode::FlatLoadUshort:
@@ -13929,6 +13930,93 @@ TestCase BufferAtomicGlc0DoesNotReturnOldValue() {
{O::VMovB32, O::BufferAtomicAdd, O::BufferStoreDword, O::SEndpgm}};
}
TestCase BufferAtomicFMinExactRawGlcModes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x40000000u); // 2.0
code.push_back(0xe0fc0000u);
code.push_back(0x80010000u); // exact failing buffer_atomic_fmin, GLC=0
AppendStoreVgpr(&code, 0, 1);
AppendVMovLiteral(&code, 0, 0x3f800000u); // 1.0
code.push_back(0xe0fc4000u);
code.push_back(0x80010000u); // same instruction with GLC=1
AppendStoreVgpr(&code, 0, 2);
AppendEnd(&code);
TestCase test;
test.name = "BufferAtomicFMinExactRawGlcModes";
test.code = code;
test.initial = {0x40800000u, 0, 0}; // 4.0
test.expected = {0x3f800000u, 0x40000000u, 0x40000000u};
test.opcodes = {O::VMovB32, O::BufferAtomicFMin, O::BufferStoreDword, O::SEndpgm};
const auto descriptor =
MakeStructuredStorageBufferData(0, static_cast<u32>(test.initial.size() * sizeof(u32)));
std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4);
test.user_data[50] = 1u << 20u;
test.has_user_data = true;
return test;
}
TestCase BufferAtomicFMinSpecialValues() {
using O = ShaderOpcode;
const u32 values[] = {
0x40000000u, // 2.0
0x7f800000u, // +infinity
0xff800000u, // -infinity
0x3f800000u, // 1.0
0x7fc00000u, // quiet NaN
0x00000000u, // +0.0
0x80000000u, // -0.0
0x00000000u, // +0.0
0x80000001u, // smallest negative denorm
};
std::vector<u32> code;
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovU32(&code, 20, i * 4u);
AppendVMovLiteral(&code, i, values[i]);
AppendBufferStoreOpcode(&code, 0x3f, i, 20, true);
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, i, i + static_cast<u32>(std::size(values)));
}
AppendEnd(&code);
return {"BufferAtomicFMinSpecialValues",
code,
{0x40800000u, 0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u,
0x80000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0, 0, 0, 0, 0, 0, 0, 0, 0},
{0x40000000u, 0xbf800000u, 0xff800000u, 0x7fc00000u, 0x3f800000u,
0x80000000u, 0x00000000u, 0x00000000u, 0x80000001u, 0x40800000u,
0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u, 0x80000000u,
0x00000000u, 0x00000001u, 0x00000000u},
{O::VMovB32, O::BufferAtomicFMin, O::BufferStoreDword, O::SEndpgm}};
}
TestCase BufferAtomicFMinContendedWorkgroup() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // v_cvt_f32_u32 v1, thread_id.x
AppendVMovU32(&code, 20, 0);
AppendBufferStoreOpcode(&code, 0x3f, 1, 20);
AppendEnd(&code);
TestCase test;
test.name = "BufferAtomicFMinContendedWorkgroup";
test.code = code;
test.initial = {0x42c80000u}; // 100.0
test.expected = {0x00000000u}; // min(100.0, 0.0 .. 63.0)
test.opcodes = {O::VCvtF32U32, O::VMovB32, O::BufferAtomicFMin, O::SEndpgm};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
std::vector<u32> MakeRgbaImage(u32 width, u32 height, u32 value = 0) {
return std::vector<u32>(static_cast<size_t>(width) * height * 4u, value);
}
@@ -15279,6 +15367,9 @@ std::vector<TestCase> MakeCases() {
AddCase(DsSwizzleInvalidSourceLaneZero);
AddCase(BufferAtomicVariants);
AddCase(BufferAtomicGlc0DoesNotReturnOldValue);
AddCase(BufferAtomicFMinExactRawGlcModes);
AddCase(BufferAtomicFMinSpecialValues);
AddCase(BufferAtomicFMinContendedWorkgroup);
AddCase(ImageLoadVariants);
AddCase(ImageLoadR32UintUsesIntegerSampledImage);
AddCase(ImageLoad1DUsesScalarCoordinate);
+8
View File
@@ -4788,6 +4788,8 @@ void TestNewShaderRecompilerAtomicLowering() {
EncodeMubuf1(5, 0, 1), // buffer_atomic_or
EncodeMubuf0(0x3b, 40, true, true),
EncodeMubuf1(6, 0, 1), // buffer_atomic_xor
0xe0fc0000u,
0x80010000u, // exact buffer_atomic_fmin v0, s[4:7], 0 (GLC=0)
EncodeDs0(0x00),
EncodeDs1(0, 2, 1), // ds_add_u32
EncodeDs0(0x01),
@@ -4842,6 +4844,8 @@ void TestNewShaderRecompilerAtomicLowering() {
"new decoder did not decode buffer atomic signed max");
Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_xor"),
"new decoder did not decode buffer atomic xor");
Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_fmin"),
"new decoder did not decode buffer atomic float min");
Check(Common::ContainsStr(result.decoded_dump, "ds_add_u32"),
"new decoder did not decode DS atomic add");
Check(Common::ContainsStr(result.decoded_dump, "ds_sub_u32"),
@@ -4874,6 +4878,8 @@ void TestNewShaderRecompilerAtomicLowering() {
"buffer atomic or did not lower to IR");
Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 v6"),
"buffer atomic xor did not lower to IR");
Check(Common::ContainsStr(result.ir_dump, "AtomicFMinF32 null, v0"),
"buffer atomic float min did not lower to IR without a GLC return");
Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 null, v2"),
"DS no-return atomic add did not lower to IR");
Check(Common::ContainsStr(result.ir_dump, "AtomicSubU32 null, v10"),
@@ -4917,6 +4923,8 @@ void TestNewShaderRecompilerAtomicLowering() {
Check(SpirvContainsOpcode(result.spirv, 240), "SPIR-V binary does not contain OpAtomicAnd");
Check(SpirvContainsOpcode(result.spirv, 241), "SPIR-V binary does not contain OpAtomicOr");
Check(SpirvContainsOpcode(result.spirv, 242), "SPIR-V binary does not contain OpAtomicXor");
Check(SpirvContainsOpcode(result.spirv, 230),
"SPIR-V binary does not contain OpAtomicCompareExchange for float min");
Check(SpirvContainsOpcode(result.spirv, 225),
"buffer atomic SPIR-V binary does not contain OpMemoryBarrier");
CheckSpirvBinaryValidates(result.spirv);