H3cJP/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-08-19 06:52:36 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity
Files
df44ad3facfc431e1f88e45bc9fe979741eebb12
KytyPS5/src/graphics/shader/recompiler/emitter/spirvEmitterMemory.cpp
T

1686 lines
75 KiB
C++
Raw Blame History

#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
uint32_t EmitDppWriteActiveBool(EmitterState& state, const IR::Operand& dst) {
const auto exec_active = EmitExecActiveBool(state);
if (!dst.dpp) {
return exec_active;
}
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto bank_shift = state.builder.AllocateId();
const auto row_shift = state.builder.AllocateId();
const auto bank = state.builder.AllocateId();
const auto row = state.builder.AllocateId();
const auto bank_bit = state.builder.AllocateId();
const auto row_bit = state.builder.AllocateId();
const auto bank_hit = state.builder.AllocateId();
const auto row_hit = state.builder.AllocateId();
const auto bank_active = state.builder.AllocateId();
const auto row_active = state.builder.AllocateId();
const auto dpp_active = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, bank_shift, subid, ConstantU32(state, 2)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, bank, bank_shift, ConstantU32(state, 3)});
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, row_shift, subid, ConstantU32(state, 4)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, row, row_shift, ConstantU32(state, 3)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bank_bit, ConstantU32(state, 1), bank});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, row_bit, ConstantU32(state, 1), row});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, bank_hit, ConstantU32(state, dst.dpp_bank_mask), bank_bit});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, row_hit, ConstantU32(state, dst.dpp_row_mask), row_bit});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, bank_active, bank_hit, ConstantU32(state, 0)});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, row_active, row_hit, ConstantU32(state, 0)});
state.builder.AddFunction({OpLogicalAnd, state.bool_type, dpp_active, bank_active, row_active});
uint32_t write_active = dpp_active;
if (!dst.dpp_bound_ctrl) {
const auto target = EmitDppTargetLane(state, dst.dpp_ctrl);
const auto bounded = state.builder.AllocateId();
state.builder.AddFunction(
{OpLogicalAnd, state.bool_type, bounded, write_active, target.valid});
write_active = bounded;
}
state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, exec_active, write_active});
return ret;
}
uint32_t EmitSdwaDestinationMerge(EmitterState& state, uint32_t pointer, const IR::Operand& dst,
uint32_t value) {
if (dst.sdwa_sel == 6u) {
return value;
}
uint32_t offset = 0;
uint32_t width = 0;
if (!SdwaSelectorOffsetWidth(dst.sdwa_sel, offset, width) || width == 32u) {
return value;
}
const auto value_mask = width == 16u ? 0x0000ffffu : 0x000000ffu;
const auto field_mask = value_mask << offset;
const auto part = state.builder.AllocateId();
const auto shifted = offset != 0u ? state.builder.AllocateId() : part;
const auto old_value = state.builder.AllocateId();
const auto kept = state.builder.AllocateId();
const auto merged = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, part, value, ConstantU32(state, value_mask)});
if (offset != 0u) {
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, shifted, part, ConstantU32(state, offset)});
}
state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, kept, old_value, ConstantU32(state, ~field_mask)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, merged, kept, shifted});
return merged;
}
void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value) {
if (dst.kind != IR::OperandKind::Register) {
return;
}
const auto pointer = PointerForRegister(state, dst.reg);
if (pointer != 0) {
const auto store_value = EmitSdwaDestinationMerge(state, pointer, dst, value);
const auto wave_value =
IsInactiveWave32ExecHigh(state, dst.reg) ? ConstantU32(state, 0) : store_value;
if (dst.reg.file == IR::RegisterFile::Vector) {
const auto old_value = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer});
state.builder.AddFunction({OpSelect, state.uint_type, selected,
EmitDppWriteActiveBool(state, dst), wave_value, old_value});
state.builder.AddFunction({OpStore, pointer, selected});
return;
}
state.builder.AddFunction({OpStore, pointer, wave_value});
}
}
uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpIAdd, state.uint_type, ret, lhs, rhs});
return ret;
}
uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({opcode, state.uint_type, ret, lhs, rhs});
return ret;
}
uint32_t EmitNotEqualZeroBool(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpINotEqual, state.bool_type, ret, value, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitSelectU32Value(EmitterState& state, uint32_t condition, uint32_t true_value,
uint32_t false_value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, state.uint_type, ret, condition, true_value, false_value});
return ret;
}
uint32_t EmitByteAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src,
uint32_t src_count) {
uint32_t address = ConstantU32(state, inst.memory.offset);
for (uint32_t i = 0; i < src_count; i++) {
address = EmitAddU32(state, address, EmitValueLoad(state, inst.src[first_src + i]));
}
return address;
}
uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem,
uint32_t use_pc) {
(void)use_pc;
if (mem.resource >= state.program.info.buffers.size()) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
"buffer specialization is missing");
}
return state.program.info.buffers[mem.resource].packed_stride;
}
uint32_t StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem,
uint32_t use_pc) {
(void)use_pc;
if (mem.resource >= state.program.info.buffers.size()) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
"buffer specialization is missing");
}
return state.program.info.buffers[mem.resource].descriptor_format;
}
bool ShouldApplyBufferAddTid(const IR::Instruction& inst) {
return inst.memory.kind == IR::ResourceKind::Buffer && inst.memory.formatted &&
!inst.memory.typed && inst.op == IR::Opcode::BufferStoreDword;
}
uint32_t EmitBufferIndexWithAddTid(EmitterState& state, const IR::Instruction& inst, uint32_t index,
uint32_t packed) {
if (!ShouldApplyBufferAddTid(inst)) {
return index;
}
const auto add_tid_bit =
EmitBinaryU32(state, OpBitwiseAnd,
EmitBinaryU32(state, OpShiftRightLogical, packed, ConstantU32(state, 20)),
ConstantU32(state, 1));
const auto add_tid = EmitNotEqualZeroBool(state, add_tid_bit);
const auto lane =
EmitBinaryU32(state, OpBitwiseAnd, EmitLocalInvocationIndex(state), ConstantU32(state, 63));
const auto tid_index = EmitAddU32(state, index, lane);
return EmitSelectU32Value(state, add_tid, tid_index, index);
}
uint32_t EmitOptionalLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
if (lhs == 0) {
return rhs;
}
if (rhs == 0) {
return lhs;
}
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, lhs, rhs});
return ret;
}
bool IsStorageBufferMemoryKind(IR::ResourceKind kind) {
switch (kind) {
case IR::ResourceKind::ScalarBuffer:
case IR::ResourceKind::Buffer:
case IR::ResourceKind::Flat:
case IR::ResourceKind::Global:
case IR::ResourceKind::Scratch: return true;
default: return false;
}
}
void EmitStorageBufferOffsets(EmitterState& state) {
for (uint32_t i = 0; i < state.program.bindings.buffer_offset_count; i++) {
const auto word =
EmitShaderDataDwordLoad(state, state.program.bindings.buffer_offset_dword + i / 4u);
const auto shift = ConstantU32(state, (i % 4u) * 8u + 2u);
state.storage_buffer_offsets[i] = EmitBinaryU32(
state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift),
ConstantU32(state, 0x3fu));
}
}
uint32_t EmitBufferAddressFromParts(EmitterState& state, const IR::Instruction& inst,
uint32_t index, uint32_t offset, uint32_t soffset) {
const auto& mem = inst.memory;
const auto packed = ConstantU32(state, StorageBufferPackedStride(state, mem, inst.pc));
const auto address_index = EmitBufferIndexWithAddTid(state, inst, index, packed);
const auto stride = EmitBinaryU32(state, OpBitwiseAnd, packed, ConstantU32(state, 0x3fffu));
const auto swizzle_bit =
EmitBinaryU32(state, OpBitwiseAnd,
EmitBinaryU32(state, OpShiftRightLogical, packed, ConstantU32(state, 14)),
ConstantU32(state, 1));
const auto stride_nonzero = EmitNotEqualZeroBool(state, stride);
const auto swizzle_nonzero = EmitNotEqualZeroBool(state, swizzle_bit);
const auto use_swizzle = state.builder.AllocateId();
state.builder.AddFunction(
{OpLogicalAnd, state.bool_type, use_swizzle, swizzle_nonzero, stride_nonzero});
const auto index_stride_enum =
EmitBinaryU32(state, OpBitwiseAnd,
EmitBinaryU32(state, OpShiftRightLogical, packed, ConstantU32(state, 16)),
ConstantU32(state, 3));
const auto index_shift = EmitAddU32(state, index_stride_enum, ConstantU32(state, 3));
const auto index_stride =
EmitBinaryU32(state, OpShiftLeftLogical, ConstantU32(state, 8), index_stride_enum);
const auto index_stride_mask =
EmitBinaryU32(state, OpISub, index_stride, ConstantU32(state, 1));
const auto linear_index = EmitBinaryU32(state, OpIMul, address_index, stride);
const auto linear = EmitAddU32(state, linear_index, offset);
const auto index_msb = EmitBinaryU32(state, OpShiftRightLogical, address_index, index_shift);
const auto index_lsb = EmitBinaryU32(state, OpBitwiseAnd, address_index, index_stride_mask);
const auto offset_msb_bytes =
EmitBinaryU32(state, OpBitwiseAnd, offset, ConstantU32(state, ~3u));
const auto offset_lsb = EmitBinaryU32(state, OpBitwiseAnd, offset, ConstantU32(state, 3));
const auto index_part = EmitBinaryU32(state, OpIMul, index_msb, stride);
const auto msb_sum = EmitAddU32(state, index_part, offset_msb_bytes);
const auto msb = EmitBinaryU32(state, OpIMul, msb_sum, index_stride);
const auto lsb_index =
EmitBinaryU32(state, OpShiftLeftLogical, index_lsb, ConstantU32(state, 2));
const auto lsb = EmitAddU32(state, lsb_index, offset_lsb);
const auto swizzled = EmitAddU32(state, msb, lsb);
const auto buffer_offset = EmitSelectU32Value(state, use_swizzle, swizzled, linear);
return EmitAddU32(state, buffer_offset, soffset);
}
uint32_t EmitBufferByteAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src,
uint32_t src_count) {
const auto end = first_src + src_count;
auto cursor = first_src;
auto LoadAddressSource = [&]() {
if (cursor >= end) {
return ConstantU32(state, 0);
}
return EmitValueLoad(state, inst.src[cursor++]);
};
uint32_t index = ConstantU32(state, 0);
if (inst.memory.idxen) {
index = LoadAddressSource();
}
uint32_t offset = ConstantU32(state, inst.memory.offset);
if (inst.memory.offen) {
offset = EmitAddU32(state, offset, LoadAddressSource());
}
const auto soffset = LoadAddressSource();
return EmitBufferAddressFromParts(state, inst, index, offset, soffset);
}
bool IsFlatMemoryKind(IR::ResourceKind kind) {
return kind == IR::ResourceKind::Flat || kind == IR::ResourceKind::Global ||
kind == IR::ResourceKind::Scratch;
}
uint32_t EmitRelativeAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src,
uint32_t src_count, bool vector_only = true,
bool align_components = false) {
const auto delta = state.program.info.addresses[inst.memory.resource].specialized_base;
auto immediate = static_cast<int32_t>(inst.memory.offset);
if (align_components) {
immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
}
const auto initial = delta + static_cast<uint64_t>(static_cast<int64_t>(immediate));
uint32_t low = ConstantU32(state, static_cast<uint32_t>(initial));
uint32_t high = ConstantU32(state, static_cast<uint32_t>(initial >> 32u));
for (uint32_t i = 0; i < src_count; i++) {
const auto& src = inst.src[first_src + i];
if (!vector_only ||
(src.kind == IR::OperandKind::Register && src.reg.file == IR::RegisterFile::Vector)) {
auto value = EmitValueLoad(state, src);
if (align_components) {
value = EmitBinaryU32(state, OpBitwiseAnd, value, ConstantU32(state, ~3u));
}
const auto next = EmitAddU32(state, low, value);
const auto carry = state.builder.AllocateId();
state.builder.AddFunction({OpULessThan, state.bool_type, carry, next, low});
high = EmitAddU32(
state, high,
EmitSelectU32Value(state, carry, ConstantU32(state, 1), ConstantU32(state, 0)));
low = next;
}
}
const auto valid = state.builder.AllocateId();
state.builder.AddFunction({OpIEqual, state.bool_type, valid, high, ConstantU32(state, 0)});
return EmitSelectU32Value(state, valid, low, ConstantU32(state, UINT32_MAX));
}
uint32_t EmitFlatVirtualAddress(EmitterState& state, const IR::Instruction& inst,
uint32_t first_src, uint32_t src_count) {
if (inst.memory.resource >= state.resources.addresses.size() || src_count < 2) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
inst.memory.resource, "flat address snapshot is missing");
}
const auto low0 = EmitValueLoad(state, inst.src[first_src]);
const auto high0 = EmitValueLoad(state, inst.src[first_src + 1u]);
const auto immediate = ConstantU32(state, inst.memory.offset);
const auto low = EmitAddU32(state, low0, immediate);
const auto carry_bool = state.builder.AllocateId();
state.builder.AddFunction({OpULessThan, state.bool_type, carry_bool, low, low0});
const auto carry =
EmitSelectU32Value(state, carry_bool, ConstantU32(state, 1), ConstantU32(state, 0));
const auto immediate_high =
ConstantU32(state, (inst.memory.offset & 0x80000000u) != 0 ? UINT32_MAX : 0u);
const auto high = EmitAddU32(state, EmitAddU32(state, high0, immediate_high), carry);
const auto base = state.program.info.addresses[inst.memory.resource].specialized_base;
const auto base_low = static_cast<uint32_t>(base);
const auto base_high = static_cast<uint32_t>(base >> 32u);
const auto relative_low = EmitBinaryU32(state, OpISub, low, ConstantU32(state, base_low));
const auto borrow_bool = state.builder.AllocateId();
state.builder.AddFunction(
{OpULessThan, state.bool_type, borrow_bool, low, ConstantU32(state, base_low)});
const auto borrow =
EmitSelectU32Value(state, borrow_bool, ConstantU32(state, 1), ConstantU32(state, 0));
const auto relative_high = EmitBinaryU32(
state, OpISub, EmitBinaryU32(state, OpISub, high, ConstantU32(state, base_high)), borrow);
const auto valid = state.builder.AllocateId();
state.builder.AddFunction(
{OpIEqual, state.bool_type, valid, relative_high, ConstantU32(state, 0)});
return EmitSelectU32Value(state, valid, relative_low, ConstantU32(state, UINT32_MAX));
}
uint32_t EmitMemoryByteAddress(EmitterState& state, const IR::Instruction& inst,
const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count) {
if (mem.kind == IR::ResourceKind::Buffer) {
return EmitBufferByteAddress(state, inst, first_src, src_count);
}
if (mem.kind == IR::ResourceKind::Flat ||
((mem.kind == IR::ResourceKind::Global || mem.kind == IR::ResourceKind::Scratch) &&
state.program.info.addresses[mem.resource].source == IR::ScalarProvenance::Unknown)) {
return EmitFlatVirtualAddress(state, inst, first_src, src_count);
}
if (mem.kind == IR::ResourceKind::Global || mem.kind == IR::ResourceKind::Scratch) {
return EmitRelativeAddress(state, inst, first_src, src_count);
}
return EmitByteAddress(state, inst, first_src, src_count);
}
uint32_t EmitDwordIndex(EmitterState& state, const IR::Instruction& inst, uint32_t first_src,
uint32_t src_count) {
const auto address = EmitByteAddress(state, inst, first_src, src_count);
const auto index = state.builder.AllocateId();
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
return index;
}
uint32_t EmitMemoryDwordIndex(EmitterState& state, const IR::Instruction& inst,
const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count) {
const auto address = EmitMemoryByteAddress(state, inst, mem, first_src, src_count);
const auto index = state.builder.AllocateId();
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
return index;
}
DescriptorResourceBinding
StorageBufferBindingForMemory(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) {
(void)use_pc;
const auto binding =
ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
if (state.storage_buffer_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
"storage buffer descriptor array was not emitted");
}
return binding;
}
uint32_t EmitStorageBufferIndex(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
uint32_t use_pc) {
if (IsFlatMemoryKind(mem.kind)) {
return index;
}
const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
return EmitAddU32(state, index, state.storage_buffer_offsets[binding.array_index]);
}
uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem,
uint32_t use_pc) {
if (IsFlatMemoryKind(mem.kind)) {
if (state.address_memory_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
mem.resource, "address memory binding was not emitted");
}
const auto binding =
ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
state.address_memory_variable,
ConstantU32(state, binding.array_index)});
return pointer;
}
const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
state.storage_buffer_variable,
ConstantU32(state, binding.array_index)});
return pointer;
}
uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem,
uint32_t index, uint32_t use_pc) {
index = EmitStorageBufferIndex(state, mem, index, use_pc);
const auto object = EmitStorageBufferObjectPointer(state, mem, use_pc);
const auto length = state.builder.AllocateId();
const auto in_bounds = state.builder.AllocateId();
state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0});
state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length});
return in_bounds;
}
uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem,
uint32_t index, uint32_t use_pc) {
index = EmitStorageBufferIndex(state, mem, index, use_pc);
if (IsFlatMemoryKind(mem.kind)) {
if (state.address_memory_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
mem.resource, "address memory binding was not emitted");
}
const auto binding =
ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.address_memory_variable,
ConstantU32(state, binding.array_index), ConstantU32(state, 0), index});
return pointer;
}
const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.storage_buffer_variable,
ConstantU32(state, binding.array_index), ConstantU32(state, 0), index});
return pointer;
}
uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, state.ptr_workgroup_uint, pointer, state.lds_variable, index});
return pointer;
}
uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index) {
const auto in_bounds = state.builder.AllocateId();
const auto dwords = state.needs_function_lds ? 8192u : 1024u;
state.builder.AddFunction(
{OpULessThan, state.bool_type, in_bounds, index, ConstantU32(state, dwords)});
return in_bounds;
}
uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
uint32_t use_pc) {
if (mem.kind == IR::ResourceKind::Lds) {
return EmitLdsElementPointer(state, index);
}
if (mem.kind == IR::ResourceKind::Gds) {
return EmitGdsElementPointer(state, index);
}
return EmitStorageBufferElementPointer(state, mem, index, use_pc);
}
uint32_t EmitMemoryLoadDwordValueU32(EmitterState& state, const IR::Instruction& inst,
IR::ResourceKind kind, uint32_t first_src,
uint32_t src_count) {
auto mem = inst.memory;
mem.kind = kind;
const auto index = EmitMemoryDwordIndex(state, inst, mem, first_src, src_count);
auto LoadFromPointer = [&](uint32_t pointer) {
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return value;
};
if (mem.kind == IR::ResourceKind::Lds) {
return LoadFromPointer(EmitLdsElementPointer(state, index));
}
if (mem.kind == IR::ResourceKind::Gds) {
const auto in_bounds = EmitGdsElementInBounds(state, index);
return EmitValueOrZeroIfCondition(state, in_bounds, [&]() {
return LoadFromPointer(EmitGdsElementPointer(state, index));
});
}
if (IsStorageBufferMemoryKind(mem.kind)) {
const auto in_bounds = EmitStorageBufferElementInBounds(state, mem, index, inst.pc);
return EmitValueOrZeroIfCondition(state, in_bounds, [&]() {
return LoadFromPointer(EmitStorageBufferElementPointer(state, mem, index, inst.pc));
});
}
return ConstantU32(state, 0);
}
void EmitMemoryLoadU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind,
uint32_t first_src, uint32_t src_count) {
const auto value = EmitMemoryLoadDwordValueU32(state, inst, kind, first_src, src_count);
EmitStoreU32(state, inst.dst, value);
}
void EmitMemoryLoadDwordsU32(EmitterState& state, const IR::Instruction& inst,
IR::ResourceKind kind, uint32_t first_src, uint32_t src_count) {
const auto count = std::max(1u, inst.memory.data_dwords);
if (count == 1u) {
EmitMemoryLoadU32(state, inst, kind, first_src, src_count);
return;
}
std::vector<uint32_t> values;
values.reserve(count);
for (uint32_t i = 0; i < count; i++) {
auto load_inst = inst;
load_inst.memory.offset += i * 4u;
load_inst.memory.data_dwords = 1u;
load_inst.memory.component_index = i;
load_inst.memory.component_count = count;
values.push_back(EmitMemoryLoadDwordValueU32(state, load_inst, kind, first_src, src_count));
}
for (uint32_t i = 0; i < count; i++) {
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), values[i]);
}
}
uint32_t EmitMemoryLoadSubDwordValueU32(EmitterState& state, const IR::Instruction& inst,
IR::ResourceKind kind, uint32_t first_src,
uint32_t src_count, uint32_t data_bits, bool sign_extend) {
auto mem = inst.memory;
mem.kind = kind;
const auto address = EmitMemoryByteAddress(state, inst, mem, first_src, src_count);
const auto index = state.builder.AllocateId();
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
auto LoadFromPointer = [&](uint32_t pointer) {
const auto word = state.builder.AllocateId();
const auto byte_in_word = state.builder.AllocateId();
const auto shift = state.builder.AllocateId();
const auto shifted = state.builder.AllocateId();
const auto masked = state.builder.AllocateId();
const auto value = sign_extend ? state.builder.AllocateId() : masked;
const auto mask = data_bits == 8u ? 0xffu : 0xffffu;
state.builder.AddFunction({OpLoad, state.uint_type, word, pointer});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, byte_in_word, address, ConstantU32(state, 3)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, shift, byte_in_word, ConstantU32(state, 3)});
state.builder.AddFunction({OpShiftRightLogical, state.uint_type, shifted, word, shift});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, masked, shifted, ConstantU32(state, mask)});
if (sign_extend) {
const auto left = state.builder.AllocateId();
const auto sign_shift = 32u - data_bits;
state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, left, masked,
ConstantU32(state, sign_shift)});
state.builder.AddFunction({OpShiftRightArithmetic, state.uint_type, value, left,
ConstantU32(state, sign_shift)});
}
return value;
};
if (mem.kind == IR::ResourceKind::Lds) {
return LoadFromPointer(EmitLdsElementPointer(state, index));
}
if (mem.kind == IR::ResourceKind::Gds) {
const auto in_bounds = EmitGdsElementInBounds(state, index);
return EmitValueOrZeroIfCondition(state, in_bounds, [&]() {
return LoadFromPointer(EmitGdsElementPointer(state, index));
});
}
if (IsStorageBufferMemoryKind(mem.kind)) {
const auto in_bounds = EmitStorageBufferElementInBounds(state, mem, index, inst.pc);
return EmitValueOrZeroIfCondition(state, in_bounds, [&]() {
return LoadFromPointer(EmitStorageBufferElementPointer(state, mem, index, inst.pc));
});
}
return ConstantU32(state, 0);
}
void EmitMemoryLoadSubDwordU32(EmitterState& state, const IR::Instruction& inst,
IR::ResourceKind kind, uint32_t first_src, uint32_t src_count,
uint32_t data_bits, bool sign_extend) {
const auto value = EmitMemoryLoadSubDwordValueU32(state, inst, kind, first_src, src_count,
data_bits, sign_extend);
EmitStoreU32(state, inst.dst, value);
}
void EmitMemoryStoreU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind,
uint32_t first_src, uint32_t src_count) {
auto store_inst = inst;
store_inst.memory.kind = kind;
const auto value = EmitValueLoad(state, inst.src[0]);
uint32_t in_bounds = 0;
const auto index =
EmitMemoryDwordIndex(state, store_inst, store_inst.memory, first_src, src_count);
if (IsStorageBufferMemoryKind(store_inst.memory.kind)) {
in_bounds = EmitOptionalLogicalAndBool(
state, in_bounds,
EmitStorageBufferElementInBounds(state, store_inst.memory, index, inst.pc));
} else if (store_inst.memory.kind == IR::ResourceKind::Gds) {
in_bounds = EmitGdsElementInBounds(state, index);
}
EmitIfCondition(state, in_bounds, [&]() {
const auto pointer = EmitMemoryElementPointer(state, store_inst.memory, index, inst.pc);
state.builder.AddFunction({OpStore, pointer, value});
});
}
template <typename Fn>
uint32_t EmitAtomicUpdateU32(EmitterState& state, uint32_t pointer, IR::ResourceKind kind,
Fn&& desired_value) {
const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
const auto memory = kind == IR::ResourceKind::Lds ? MemorySemanticsWorkgroupMemory
: MemorySemanticsUniformMemory;
const auto preheader = state.builder.AllocateId();
const auto header = state.builder.AllocateId();
const auto continue_label = state.builder.AllocateId();
const auto merge = state.builder.AllocateId();
const auto initial = state.builder.AllocateId();
const auto observed = state.builder.AllocateId();
const auto exchanged = state.builder.AllocateId();
state.builder.AddFunction({OpBranch, preheader});
state.builder.AddFunction({OpLabel, preheader});
state.builder.AddFunction({OpAtomicLoad, state.uint_type, initial, pointer,
ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)});
state.builder.AddFunction({OpBranch, header});
state.builder.AddFunction({OpLabel, header});
state.builder.AddFunction(
{OpPhi, state.uint_type, observed, initial, preheader, exchanged, continue_label});
const auto desired = desired_value(observed);
state.builder.AddFunction({OpAtomicCompareExchange, state.uint_type, exchanged, pointer,
ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone),
ConstantU32(state, MemorySemanticsNone), desired, observed});
const auto success = state.builder.AllocateId();
state.builder.AddFunction({OpIEqual, state.bool_type, success, exchanged, observed});
state.builder.AddFunction({OpLoopMerge, merge, continue_label, LoopControlNone});
state.builder.AddFunction({OpBranchConditional, success, merge, continue_label});
state.builder.AddFunction({OpLabel, continue_label});
state.builder.AddFunction({OpBranch, header});
state.builder.AddFunction({OpLabel, merge});
const auto semantics = MemorySemanticsAcquireRelease | memory;
state.builder.AddFunction(
{OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)});
return observed;
}
void EmitMemoryStoreSubDwordU32(EmitterState& state, const IR::Instruction& inst,
IR::ResourceKind kind, uint32_t first_src, uint32_t src_count,
uint32_t data_bits) {
auto store_inst = inst;
store_inst.memory.kind = kind;
uint32_t in_bounds = 0;
const auto address =
EmitMemoryByteAddress(state, store_inst, store_inst.memory, first_src, src_count);
const auto index = state.builder.AllocateId();
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
if (IsStorageBufferMemoryKind(store_inst.memory.kind)) {
in_bounds = EmitOptionalLogicalAndBool(
state, in_bounds,
EmitStorageBufferElementInBounds(state, store_inst.memory, index, inst.pc));
} else if (store_inst.memory.kind == IR::ResourceKind::Gds) {
in_bounds = EmitGdsElementInBounds(state, index);
}
EmitIfCondition(state, in_bounds, [&]() {
const auto pointer = EmitMemoryElementPointer(state, store_inst.memory, index, inst.pc);
const auto value = EmitValueLoad(state, inst.src[0]);
const auto byte_in_word = state.builder.AllocateId();
const auto shift = state.builder.AllocateId();
const auto field_mask = state.builder.AllocateId();
const auto value_masked = state.builder.AllocateId();
const auto value_shifted = state.builder.AllocateId();
const auto mask = data_bits == 8u ? 0xffu : 0xffffu;
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, byte_in_word, address, ConstantU32(state, 3)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, shift, byte_in_word, ConstantU32(state, 3)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, field_mask, ConstantU32(state, mask), shift});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, value_masked, value, ConstantU32(state, mask)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, value_shifted, value_masked, shift});
auto Merge = [&](uint32_t word) {
const auto inverted = state.builder.AllocateId();
const auto cleared = state.builder.AllocateId();
const auto merged = state.builder.AllocateId();
state.builder.AddFunction({OpNot, state.uint_type, inverted, field_mask});
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, cleared, word, inverted});
state.builder.AddFunction(
{OpBitwiseOr, state.uint_type, merged, cleared, value_shifted});
return merged;
};
if (store_inst.memory.kind == IR::ResourceKind::Lds ||
store_inst.memory.kind == IR::ResourceKind::Gds) {
EmitAtomicUpdateU32(state, pointer, store_inst.memory.kind, Merge);
} else {
const auto word = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, word, pointer});
state.builder.AddFunction({OpStore, pointer, Merge(word)});
}
});
}
uint32_t AddressSourceCount(const IR::Instruction& inst, uint32_t first_src) {
return inst.src_count > first_src ? inst.src_count - first_src : 0u;
}
bool IsFormattedBufferComponent(const IR::Instruction& inst) {
return inst.memory.kind == IR::ResourceKind::Buffer && inst.memory.formatted &&
inst.memory.data_dwords == 1u;
}
Prospero::BufferFormat FormattedBufferFormat(const EmitterState& state,
const IR::Instruction& inst) {
return inst.memory.typed
? Format::DecodeTBufferFormat(inst.memory.data_format, inst.memory.number_format)
: static_cast<Prospero::BufferFormat>(
StorageBufferFormat(state, inst.memory, inst.pc));
}
IR::Instruction WithFormatComponentByteOffset(const IR::Instruction& inst,
Prospero::BufferFormat format) {
auto rebased = inst;
if (inst.memory.typed || inst.memory.component_index == 0u) {
return rebased;
}
const auto default_component_offset = inst.memory.component_index * 4u;
if (rebased.memory.offset >= default_component_offset) {
rebased.memory.offset =
rebased.memory.offset - default_component_offset +
Format::GetFormatComponentByteOffset(format, inst.memory.component_index);
}
return rebased;
}
uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.uint_type, ret, value});
return ret;
}
uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.float_type, ret, value});
return ret;
}
uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.int_type, ret, value});
return ret;
}
uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
uint32_t constant) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
return ret;
}
uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
uint32_t false_value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.float_type, ret, condition, true_value, false_value});
return ret;
}
bool IsSignedFormatComponent(Format::ComponentType type) {
return type == Format::ComponentType::Sint || type == Format::ComponentType::Snorm ||
type == Format::ComponentType::Sscaled;
}
uint32_t EmitExtractFormatFieldU32(EmitterState& state, uint32_t raw_word, uint32_t offset,
uint32_t bits, bool sign_extend) {
if (bits == 32u && offset == 0u) {
return raw_word;
}
if (sign_extend) {
const auto signed_word = EmitTBufferBitcastU32ToI32(state, raw_word);
const auto extracted = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_word,
ConstantU32(state, offset), ConstantU32(state, bits)});
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.uint_type, ret, extracted});
return ret;
}
const auto extracted = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, raw_word,
ConstantU32(state, offset), ConstantU32(state, bits)});
return extracted;
}
uint32_t EmitHalfToF32Bits(EmitterState& state, uint32_t raw) {
const auto unpacked = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, raw});
state.builder.AddFunction({OpCompositeExtract, state.float_type, value, unpacked, 0});
return EmitTBufferBitcastF32ToU32(state, value);
}
uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits) {
const auto mantissa_bits = bits == 11u ? 6u : 5u;
const auto mantissa_mask = (1u << mantissa_bits) - 1u;
const auto mantissa =
EmitBinaryU32(state, OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask));
const auto exponent = EmitBinaryU32(
state, OpBitwiseAnd,
EmitBinaryU32(state, OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)),
ConstantU32(state, 0x1fu));
const auto exponent_32 = EmitAddU32(state, exponent, ConstantU32(state, 127u - 15u));
const auto exponent_bits =
EmitBinaryU32(state, OpShiftLeftLogical, exponent_32, ConstantU32(state, 23));
const auto mantissa_bits_32 =
EmitBinaryU32(state, OpShiftLeftLogical, mantissa, ConstantU32(state, 23u - mantissa_bits));
const auto normal_bits = EmitBinaryU32(state, OpBitwiseOr, exponent_bits, mantissa_bits_32);
const auto normal = EmitTBufferBitcastU32ToF32(state, normal_bits);
const auto special_bits =
EmitBinaryU32(state, OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32);
const auto special = EmitTBufferBitcastU32ToF32(state, special_bits);
const auto mantissa_f32 = state.builder.AllocateId();
const auto subnormal = state.builder.AllocateId();
state.builder.AddFunction({OpConvertUToF, state.float_type, mantissa_f32, mantissa});
state.builder.AddFunction(
{OpFMul, state.float_type, subnormal, mantissa_f32,
ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast<int>(mantissa_bits)))});
const auto zero_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 0);
const auto special_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 31);
const auto finite = EmitTBufferSelectF32(state, zero_exp, subnormal, normal);
const auto result = EmitTBufferSelectF32(state, special_exp, special, finite);
return EmitTBufferBitcastF32ToU32(state, result);
}
uint32_t EmitFormatRawComponent(EmitterState& state, const IR::Instruction& inst,
const Format::BufferFormatInfo& info) {
const auto component = inst.memory.component_index;
if (component >= info.component_count) {
return ConstantU32(state, 0);
}
auto load_inst = WithFormatComponentByteOffset(inst, info.format);
const auto bits = info.component_bits[component];
const auto sign_extend = IsSignedFormatComponent(info.type);
if (info.packed_bitfield) {
const auto raw_word = EmitMemoryLoadDwordValueU32(
state, load_inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(load_inst, 0));
return EmitExtractFormatFieldU32(state, raw_word, info.component_bit_offset[component],
bits, sign_extend);
}
if (bits == 32u) {
return EmitMemoryLoadDwordValueU32(state, load_inst, IR::ResourceKind::Buffer, 0,
AddressSourceCount(load_inst, 0));
}
return EmitMemoryLoadSubDwordValueU32(state, load_inst, IR::ResourceKind::Buffer, 0,
AddressSourceCount(load_inst, 0), bits, sign_extend);
}
uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info,
uint32_t component, uint32_t raw) {
const auto bits = info.component_bits[component];
switch (info.type) {
case Format::ComponentType::Uint:
case Format::ComponentType::Sint: return raw;
case Format::ComponentType::Uscaled: {
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw});
return EmitTBufferBitcastF32ToU32(state, value);
}
case Format::ComponentType::Sscaled: {
const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw});
return EmitTBufferBitcastF32ToU32(state, value);
}
case Format::ComponentType::Unorm: {
const auto value = state.builder.AllocateId();
const auto normalized = state.builder.AllocateId();
const auto max_value = static_cast<float>((1u << bits) - 1u);
state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw});
state.builder.AddFunction(
{OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
return EmitTBufferBitcastF32ToU32(state, normalized);
}
case Format::ComponentType::Snorm: {
const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
const auto value = state.builder.AllocateId();
const auto normalized = state.builder.AllocateId();
const auto clamped = state.builder.AllocateId();
const auto max_value = static_cast<float>((1u << (bits - 1u)) - 1u);
state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw});
state.builder.AddFunction(
{OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
state.builder.AddFunction({OpExtInst, state.float_type, clamped, state.glsl_std450,
GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
return EmitTBufferBitcastF32ToU32(state, clamped);
}
case Format::ComponentType::Float:
if (bits == 32u) {
return raw;
}
if (bits == 16u) {
return EmitHalfToF32Bits(state, raw);
}
return EmitUFloatToF32Bits(state, raw, bits);
default: return raw;
}
}
uint32_t UnpackTBufferFormat(EmitterState& state, const IR::Instruction& inst,
const Format::BufferFormatInfo& info) {
const auto raw = EmitFormatRawComponent(state, inst, info);
return NormalizeFormatComponent(state, info, inst.memory.component_index, raw);
}
bool EmitFormattedBufferLoadValueU32(EmitterState& state, const IR::Instruction& inst,
uint32_t& value) {
if (!IsFormattedBufferComponent(inst)) {
return false;
}
const auto format = FormattedBufferFormat(state, inst);
if (!Format::IsKnownFormat(format)) {
return false;
}
const auto info = Format::GetFormatInfo(format);
if (inst.memory.component_index >= info.component_count) {
value = ConstantU32(state, 0);
return true;
}
if (Format::CanUseTypedBufferLoad(info.format)) {
value = EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0,
AddressSourceCount(inst, 0));
return true;
}
value = UnpackTBufferFormat(state, inst, info);
return true;
}
uint32_t EmitBufferLoadDwordValueU32(EmitterState& state, const IR::Instruction& inst) {
uint32_t value = 0;
if (EmitFormattedBufferLoadValueU32(state, inst, value)) {
return value;
}
return EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0,
AddressSourceCount(inst, 0));
}
uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format,
uint32_t opcode_components) {
switch (format) {
case Prospero::BufferFormat::k32UInt:
case Prospero::BufferFormat::k32SInt:
case Prospero::BufferFormat::k32Float: return opcode_components >= 1u ? 1u : 0u;
case Prospero::BufferFormat::k32_32UInt:
case Prospero::BufferFormat::k32_32SInt:
case Prospero::BufferFormat::k32_32Float: return opcode_components >= 2u ? 2u : 0u;
case Prospero::BufferFormat::k32_32_32UInt:
case Prospero::BufferFormat::k32_32_32SInt:
case Prospero::BufferFormat::k32_32_32Float: return opcode_components >= 3u ? 3u : 0u;
case Prospero::BufferFormat::k32_32_32_32UInt:
case Prospero::BufferFormat::k32_32_32_32SInt:
case Prospero::BufferFormat::k32_32_32_32Float: return opcode_components >= 4u ? 4u : 0u;
default: return 0u;
}
}
bool EmitBufferIntegerFormatStore(EmitterState& state, const IR::Instruction& inst) {
if (!IsFormattedBufferComponent(inst)) {
return false;
}
const auto format = FormattedBufferFormat(state, inst);
if (!inst.memory.typed) {
const auto dword_components =
FormattedBufferDwordStoreComponentCount(format, inst.memory.component_count);
if (dword_components != 0u) {
if (inst.memory.component_index >= dword_components) {
return true;
}
EmitMemoryStoreU32(state, inst, IR::ResourceKind::Buffer, 1,
AddressSourceCount(inst, 1));
return true;
}
switch (format) {
case Prospero::BufferFormat::k8UInt:
case Prospero::BufferFormat::k8SInt:
if (inst.memory.component_index != 0u) {
return true;
}
EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1,
AddressSourceCount(inst, 1), 8);
return true;
case Prospero::BufferFormat::k16UInt:
case Prospero::BufferFormat::k16SInt:
if (inst.memory.component_index != 0u) {
return true;
}
EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1,
AddressSourceCount(inst, 1), 16);
return true;
case Prospero::BufferFormat::k8_8UInt:
case Prospero::BufferFormat::k8_8SInt: {
if (inst.memory.component_index >= 2u) {
return true;
}
const auto rebased = WithFormatComponentByteOffset(inst, format);
EmitMemoryStoreSubDwordU32(state, rebased, IR::ResourceKind::Buffer, 1,
AddressSourceCount(rebased, 1), 8);
return true;
}
case Prospero::BufferFormat::k16_16UInt:
case Prospero::BufferFormat::k16_16SInt: {
if (inst.memory.component_index >= 2u) {
return true;
}
const auto rebased = WithFormatComponentByteOffset(inst, format);
EmitMemoryStoreSubDwordU32(state, rebased, IR::ResourceKind::Buffer, 1,
AddressSourceCount(rebased, 1), 16);
return true;
}
default: return false;
}
}
switch (format) {
case Prospero::BufferFormat::k8UInt:
case Prospero::BufferFormat::k8SInt:
EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1,
AddressSourceCount(inst, 1), 8);
return true;
case Prospero::BufferFormat::k16UInt:
case Prospero::BufferFormat::k16SInt:
EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1,
AddressSourceCount(inst, 1), 16);
return true;
case Prospero::BufferFormat::k8_8UInt:
case Prospero::BufferFormat::k8_8SInt:
EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1,
AddressSourceCount(inst, 1), 8);
return true;
case Prospero::BufferFormat::k16_16UInt:
case Prospero::BufferFormat::k16_16SInt:
EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1,
AddressSourceCount(inst, 1), 16);
return true;
default: break;
}
return false;
}
uint32_t EmitAtomicPointer(EmitterState& state, const IR::Instruction& inst) {
switch (inst.memory.kind) {
case IR::ResourceKind::Lds:
return EmitLdsElementPointer(state, EmitDwordIndex(state, inst, 1, 1));
case IR::ResourceKind::Gds:
return EmitGdsElementPointer(state, EmitDwordIndex(state, inst, 1, 1));
case IR::ResourceKind::Buffer: {
const auto index =
EmitMemoryDwordIndex(state, inst, inst.memory, 1, AddressSourceCount(inst, 1));
return EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc);
}
case IR::ResourceKind::StorageImageUint: {
const auto view = StorageImageViewKind(state, inst.memory, true, inst.pc);
const auto image_pointer =
StorageImageDescriptorPointer(state, inst.memory.resource, true, inst.pc, view);
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpImageTexelPointer, state.ptr_image_uint, pointer,
image_pointer, EmitImageCoordU32(state, inst, view),
ConstantU32(state, 0)});
return pointer;
}
default: return 0;
}
}
void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsUniformMemory;
state.builder.AddFunction(
{OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)});
}
void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) {
if (IsStorageBufferMemoryKind(inst.memory.kind)) {
const auto index =
EmitMemoryDwordIndex(state, inst, inst.memory, 1, AddressSourceCount(inst, 1));
const auto in_bounds = EmitStorageBufferElementInBounds(state, inst.memory, index, inst.pc);
const auto value = EmitValueLoad(state, inst.src[0]);
const auto old = EmitValueOrZeroIfCondition(state, in_bounds, [&]() {
const auto pointer =
EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc);
const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, state.uint_type, result, pointer,
ConstantU32(state, ScopeDevice),
ConstantU32(state, MemorySemanticsNone), value});
EmitDeviceAtomicMemoryBarrier(state);
return result;
});
EmitStoreU32(state, inst.dst, old);
return;
}
if (inst.memory.kind == IR::ResourceKind::Gds) {
const auto index = EmitDwordIndex(state, inst, 1, 1);
const auto in_bounds = EmitGdsElementInBounds(state, index);
const auto value = EmitValueLoad(state, inst.src[0]);
const auto old = EmitValueOrZeroIfCondition(state, in_bounds, [&]() {
const auto pointer = EmitGdsElementPointer(state, index);
const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, state.uint_type, result, pointer,
ConstantU32(state, ScopeDevice),
ConstantU32(state, MemorySemanticsNone), value});
EmitDeviceAtomicMemoryBarrier(state);
return result;
});
EmitStoreU32(state, inst.dst, old);
return;
}
const auto pointer = EmitAtomicPointer(state, inst);
if (pointer == 0) {
EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
return;
}
const auto value = EmitValueLoad(state, inst.src[0]);
const auto old = state.builder.AllocateId();
const auto scope = inst.memory.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
state.builder.AddFunction({opcode, state.uint_type, old, pointer, ConstantU32(state, scope),
ConstantU32(state, MemorySemanticsNone), value});
if (inst.memory.kind == IR::ResourceKind::StorageImageUint ||
inst.memory.kind == IR::ResourceKind::Gds) {
EmitDeviceAtomicMemoryBarrier(state);
}
EmitStoreU32(state, inst.dst, old);
}
namespace {
uint32_t EmitF32BitsOrderedLessThan(EmitterState& state, uint32_t lhs_bits, uint32_t rhs_bits) {
struct ClassifiedBits {
uint32_t nan = 0;
uint32_t zero = 0;
uint32_t key = 0;
};
const auto Classify = [&](uint32_t bits) {
ClassifiedBits cls;
const auto abs_bits = EmitAndConstant(state, bits, 0x7fffffffu);
const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
const auto exponent_max =
EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0);
const auto negative = EmitCompareU32Constant(state, OpINotEqual,
EmitAndConstant(state, bits, 0x80000000u), 0);
const auto negative_key = state.builder.AllocateId();
const auto positive_key = state.builder.AllocateId();
// Map all non-NaN IEEE-754 encodings to monotonically increasing unsigned keys.
state.builder.AddFunction({OpNot, state.uint_type, negative_key, bits});
state.builder.AddFunction(
{OpBitwiseXor, state.uint_type, positive_key, bits, ConstantU32(state, 0x80000000u)});
cls.nan = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
cls.key = EmitSelectValueU32(state, negative, negative_key, positive_key);
return cls;
};
const auto lhs = Classify(lhs_bits);
const auto rhs = Classify(rhs_bits);
const auto any_nan = EmitLogicalOrBool(state, lhs.nan, rhs.nan);
// The key transform orders -0 below +0, while the floating comparison treats them as equal.
const auto both_zero = EmitLogicalAndBool(state, lhs.zero, rhs.zero);
const auto ordered_nonzero =
EmitLogicalNotBool(state, EmitLogicalOrBool(state, any_nan, both_zero));
const auto less = state.builder.AllocateId();
state.builder.AddFunction({OpULessThan, state.bool_type, less, lhs.key, rhs.key});
return EmitLogicalAndBool(state, ordered_nonzero, less);
}
} // namespace
void EmitAtomicFMinF32(EmitterState& state, const IR::Instruction& inst) {
const auto index =
EmitMemoryDwordIndex(state, inst, inst.memory, 1, AddressSourceCount(inst, 1));
const auto in_bounds = EmitStorageBufferElementInBounds(state, inst.memory, index, inst.pc);
const auto src_u32 = EmitValueLoad(state, inst.src[0]);
const auto old = EmitValueOrZeroIfCondition(state, in_bounds, [&]() {
const auto pointer = EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc);
return EmitAtomicUpdateU32(state, pointer, inst.memory.kind, [&](uint32_t old_u32) {
const auto replace_old = state.builder.AllocateId();
const auto less = EmitF32BitsOrderedLessThan(state, src_u32, old_u32);
state.builder.AddFunction(
{OpSelect, state.uint_type, replace_old, less, src_u32, old_u32});
return replace_old;
});
});
EmitStoreU32(state, inst.dst, old);
}
void EmitSLoadDword(EmitterState& state, const IR::Instruction& inst) {
if (state.address_memory_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
inst.memory.resource,
"scalar memory descriptor array was not emitted");
}
const auto binding = ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory,
inst.memory.resource);
const auto address = EmitRelativeAddress(state, inst, 0, 1, false, true);
const auto index = state.builder.AllocateId();
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
const auto object = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, object,
state.address_memory_variable,
ConstantU32(state, binding.array_index)});
const auto length = state.builder.AllocateId();
const auto in_bounds = state.builder.AllocateId();
state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0});
state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length});
const auto value = EmitValueOrZeroIfCondition(state, in_bounds, [&]() {
const auto pointer = state.builder.AllocateId();
const auto loaded = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.address_memory_variable,
ConstantU32(state, binding.array_index), ConstantU32(state, 0), index});
state.builder.AddFunction({OpLoad, state.uint_type, loaded, pointer});
return loaded;
});
EmitStoreU32(state, inst.dst, value);
}
void EmitLoadSrtDword(EmitterState& state, const IR::Instruction& inst) {
if (state.flattened_srt_variable == 0 || inst.src_count != 1 ||
inst.src[0].kind != IR::OperandKind::ImmediateU32) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::FlattenedSrt,
inst.src_count != 0 ? inst.src[0].imm : 0,
"invalid flattened SRT load");
}
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
state.flattened_srt_variable, ConstantU32(state, 0),
ConstantU32(state, inst.src[0].imm)});
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
EmitStoreU32(state, inst.dst, value);
}
void EmitBufferLoadUbyte(EmitterState& state, const IR::Instruction& inst) {
EmitGuardedByExec(state, [&]() {
EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0,
AddressSourceCount(inst, 0), 8, false);
});
}
void EmitBufferLoadSbyte(EmitterState& state, const IR::Instruction& inst) {
EmitGuardedByExec(state, [&]() {
EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0,
AddressSourceCount(inst, 0), 8, true);
});
}
void EmitBufferLoadUshort(EmitterState& state, const IR::Instruction& inst) {
EmitGuardedByExec(state, [&]() {
EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0,
AddressSourceCount(inst, 0), 16, false);
});
}
void EmitBufferLoadSshort(EmitterState& state, const IR::Instruction& inst) {
EmitGuardedByExec(state, [&]() {
EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0,
AddressSourceCount(inst, 0), 16, true);
});
}
void EmitBufferLoadDword(EmitterState& state, const IR::Instruction& inst) {
EmitGuardedByExec(
state, [&]() { EmitStoreU32(state, inst.dst, EmitBufferLoadDwordValueU32(state, inst)); });
}
void EmitBufferLoadDwordGroup(EmitterState& state, const IR::Instruction* instructions,
uint32_t count) {
if (instructions == nullptr || count == 0u) {
return;
}
EmitGuardedByExec(state, [&]() {
// RDNA VMEM captures every VADDR component before making overlapping VDATA writes
// visible. Keep the split IR components instruction-atomic by deferring all stores.
std::vector<uint32_t> values;
values.reserve(count);
for (uint32_t i = 0; i < count; i++) {
values.push_back(EmitBufferLoadDwordValueU32(state, instructions[i]));
}
for (uint32_t i = 0; i < count; i++) {
EmitStoreU32(state, instructions[i].dst, values[i]);
}
});
}
void EmitBufferStoreDword(EmitterState& state, const IR::Instruction& inst) {
if (EmitBufferIntegerFormatStore(state, inst)) {
return;
}
EmitMemoryStoreU32(state, inst, IR::ResourceKind::Buffer, 1, AddressSourceCount(inst, 1));
}
void EmitFlatLoadUbyte(EmitterState& state, const IR::Instruction& inst) {
EmitGuardedByExec(state, [&]() {
EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 8, false);
});
}
void EmitFlatLoadSbyte(EmitterState& state, const IR::Instruction& inst) {
EmitGuardedByExec(state, [&]() {
EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 8, true);
});
}
void EmitFlatLoadUshort(EmitterState& state, const IR::Instruction& inst) {
EmitGuardedByExec(state, [&]() {
EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 16, false);
});
}
void EmitFlatLoadSshort(EmitterState& state, const IR::Instruction& inst) {
EmitGuardedByExec(state, [&]() {
EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 16, true);
});
}
void EmitFlatLoadDword(EmitterState& state, const IR::Instruction& inst) {
EmitGuardedByExec(state, [&]() {
EmitMemoryLoadDwordsU32(state, inst, inst.memory.kind, 0, inst.src_count);
});
}
uint32_t EmitDsAddtidDwordIndex(EmitterState& state, const IR::Instruction& inst,
uint32_t m0_src_index) {
if (state.stage != ShaderType::Compute) {
const auto m0 = m0_src_index < inst.src_count ? EmitValueLoad(state, inst.src[m0_src_index])
: ConstantU32(state, 0);
const auto m0_lo = state.builder.AllocateId();
const auto base = state.builder.AllocateId();
const auto index = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, m0_lo, m0, ConstantU32(state, 0x0000ffffu)});
state.builder.AddFunction(
{OpIAdd, state.uint_type, base, m0_lo, ConstantU32(state, inst.memory.offset)});
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, index, base, ConstantU32(state, 2)});
return index;
}
const auto local_index = EmitLocalInvocationIndex(state);
const auto lane = state.builder.AllocateId();
const auto lane_bytes = state.builder.AllocateId();
const auto m0 = m0_src_index < inst.src_count ? EmitValueLoad(state, inst.src[m0_src_index])
: ConstantU32(state, 0);
const auto m0_lo = state.builder.AllocateId();
const auto base = state.builder.AllocateId();
const auto address = state.builder.AllocateId();
const auto index = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, local_index, ConstantU32(state, 63)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, lane_bytes, lane, ConstantU32(state, 2)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, m0_lo, m0, ConstantU32(state, 0x0000ffffu)});
state.builder.AddFunction(
{OpIAdd, state.uint_type, base, m0_lo, ConstantU32(state, inst.memory.offset)});
state.builder.AddFunction({OpIAdd, state.uint_type, address, base, lane_bytes});
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
return index;
}
void EmitDsWriteAddtidB32(EmitterState& state, const IR::Instruction& inst) {
const auto value = EmitValueLoad(state, inst.src[0]);
const auto index = EmitDsAddtidDwordIndex(state, inst, 1);
const auto pointer = EmitLdsElementPointer(state, index);
state.builder.AddFunction({OpStore, pointer, value});
}
void EmitDsReadAddtidB32(EmitterState& state, const IR::Instruction& inst) {
const auto index = EmitDsAddtidDwordIndex(state, inst, 0);
const auto pointer = EmitLdsElementPointer(state, index);
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
EmitStoreU32(state, inst.dst, value);
}
struct DsCounterAddress {
uint32_t index = 0;
uint32_t size = 0;
uint32_t in_lds_bounds = 0;
};
DsCounterAddress EmitAppendConsumeAddress(EmitterState& state, const IR::Instruction& inst) {
const auto m0 = inst.src_count > 0 ? EmitValueLoad(state, inst.src[0]) : ConstantU32(state, 0);
const auto base = state.builder.AllocateId();
const auto size = state.builder.AllocateId();
const auto address = state.builder.AllocateId();
const auto index = state.builder.AllocateId();
const auto in_bounds = state.builder.AllocateId();
// RDNA2 puts the DS append/consume byte base in M0[31:16].
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, base, m0, ConstantU32(state, 16)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, size, m0, ConstantU32(state, 0xffffu)});
state.builder.AddFunction(
{OpIAdd, state.uint_type, address, base, ConstantU32(state, inst.memory.offset)});
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds,
ConstantU32(state, inst.memory.offset + 3u), size});
return {index, size, in_bounds};
}
uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) {
const auto length = state.builder.AllocateId();
const auto in_bounds = state.builder.AllocateId();
state.builder.AddFunction({OpArrayLength, state.uint_type, length, state.gds_variable, 0});
state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length});
return in_bounds;
}
uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
state.gds_variable, ConstantU32(state, 0), index});
return pointer;
}
uint32_t EmitBitCountU32Value(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitCount, state.uint_type, ret, value});
return ret;
}
uint32_t EmitFindLsbU32Value(EmitterState& state, uint32_t value) {
const auto i32 = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpExtInst, state.int_type, i32, state.glsl_std450, GlslFindILsb, value});
state.builder.AddFunction({OpBitcast, state.uint_type, ret, i32});
return ret;
}
struct ExecMaskInfo {
uint32_t active_count = 0;
uint32_t first_lane = 0;
uint32_t any_active = 0;
};
ExecMaskInfo EmitExecMaskInfo(EmitterState& state) {
uint32_t exec_lo = 0;
uint32_t exec_hi = 0;
if (state.per_invocation_masks) {
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
exec_lo = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, exec_lo, ballot, 0});
if (state.wave_size == 64u) {
exec_hi = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, exec_hi, ballot, 1});
} else {
exec_hi = ConstantU32(state, 0);
}
} else {
exec_lo = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
exec_hi = WaveMaskHighLoad(state, IR::RegisterFile::Exec);
}
const auto lo_count = EmitBitCountU32Value(state, exec_lo);
const auto hi_count = EmitBitCountU32Value(state, exec_hi);
const auto active_count = state.builder.AllocateId();
state.builder.AddFunction({OpIAdd, state.uint_type, active_count, lo_count, hi_count});
const auto lo_nonzero = EmitNotEqualZeroBool(state, exec_lo);
const auto hi_nonzero = EmitNotEqualZeroBool(state, exec_hi);
const auto any_active = EmitLogicalOrBool(state, lo_nonzero, hi_nonzero);
const auto hi_lsb = EmitFindLsbU32Value(state, exec_hi);
const auto hi_lane = state.builder.AllocateId();
state.builder.AddFunction({OpIAdd, state.uint_type, hi_lane, hi_lsb, ConstantU32(state, 32)});
const auto lo_lsb = EmitFindLsbU32Value(state, exec_lo);
const auto selected = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, state.uint_type, selected, lo_nonzero, lo_lsb, hi_lane});
const auto first_lane = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.uint_type, first_lane, any_active, selected, ConstantU32(state, 0)});
return {active_count, first_lane, any_active};
}
void EmitDsAppendConsume(EmitterState& state, const IR::Instruction& inst, uint32_t atomic_opcode) {
const auto gds = inst.memory.kind == IR::ResourceKind::Gds;
if (gds && state.gds_variable == 0) {
EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
return;
}
const auto address = EmitAppendConsumeAddress(state, inst);
const auto storage_bounds = gds ? EmitGdsElementInBounds(state, address.index)
: EmitLdsElementInBounds(state, address.index);
const auto m0_bounds = gds ? EmitNotEqualZeroBool(state, address.size) : address.in_lds_bounds;
const auto in_bounds = EmitLogicalAndBool(state, m0_bounds, storage_bounds);
const auto exec = EmitExecMaskInfo(state);
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto first_lane = state.builder.AllocateId();
const auto do_atomic = state.builder.AllocateId();
state.builder.AddFunction({OpIEqual, state.bool_type, first_lane, subid, exec.first_lane});
const auto first_active = EmitLogicalAndBool(state, first_lane, exec.any_active);
state.builder.AddFunction({OpLogicalAnd, state.bool_type, do_atomic, first_active, in_bounds});
const auto atomic_value = EmitValueOrZeroIfCondition(state, do_atomic, [&]() {
const auto pointer = gds ? EmitGdsElementPointer(state, address.index)
: EmitLdsElementPointer(state, address.index);
const auto result = state.builder.AllocateId();
state.builder.AddFunction({atomic_opcode, state.uint_type, result, pointer,
ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
ConstantU32(state, MemorySemanticsNone), exec.active_count});
if (gds) {
EmitDeviceAtomicMemoryBarrier(state);
} else {
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, ScopeWorkgroup),
ConstantU32(state, semantics)});
}
return result;
});
const auto broadcast = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, broadcast,
ConstantU32(state, ScopeSubgroup), atomic_value, exec.first_lane});
const auto value = EmitSelectU32Value(state, exec.any_active, broadcast, ConstantU32(state, 0));
EmitStoreU32(state, inst.dst, value);
}
void EmitDsFloatMinMaxF32(EmitterState& state, const IR::Instruction& inst, bool max_value) {
const auto index = EmitDwordIndex(state, inst, 1, 1);
const auto in_bounds =
inst.memory.kind == IR::ResourceKind::Gds ? EmitGdsElementInBounds(state, index) : 0u;
EmitIfCondition(state, in_bounds, [&]() {
const auto pointer = EmitMemoryElementPointer(state, inst.memory, index, inst.pc);
const auto data_f32 = EmitFloatLoad(state, inst.src[0]);
const auto cmp_f32 = EmitFloatLoad(state, inst.src[2]);
EmitAtomicUpdateU32(state, pointer, inst.memory.kind, [&](uint32_t old_u32) {
const auto old_f32 = state.builder.AllocateId();
const auto store_src = state.builder.AllocateId();
const auto value_f32 = state.builder.AllocateId();
const auto value_u32 = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.float_type, old_f32, old_u32});
state.builder.AddFunction({max_value ? OpFOrdGreaterThan : OpFOrdLessThan,
state.bool_type, store_src, max_value ? old_f32 : cmp_f32,
max_value ? cmp_f32 : old_f32});
state.builder.AddFunction(
{OpSelect, state.float_type, value_f32, store_src, data_f32, old_f32});
state.builder.AddFunction({OpBitcast, state.uint_type, value_u32, value_f32});
return value_u32;
});
});
}
uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
if ((control & 0xc000u) == 0xc000u) {
const uint32_t mask = control & 0x1fu;
const uint32_t rotate = (control >> 5u) & 0x1fu;
const uint32_t rotate_delta = (control & 0x400u) != 0u ? ((32u - rotate) & 0x1fu) : rotate;
const auto lane = EmitAndConstant(state, subid, 31);
const auto rotated_sum = EmitAddU32(state, lane, ConstantU32(state, rotate_delta));
const auto rotated = EmitAndConstant(state, rotated_sum, 31);
const auto kept = EmitAndConstant(state, lane, mask);
const auto moved = EmitAndConstant(state, rotated, (~mask) & 31u);
const auto combined = EmitOrU32(state, kept, moved);
const auto base = EmitAndConstant(state, subid, 0xffffffe0u);
return EmitOrU32(state, base, combined);
}
if ((control & 0x8000u) != 0) {
const auto lane2 = state.builder.AllocateId();
const auto shift = state.builder.AllocateId();
const auto perm0 = state.builder.AllocateId();
const auto perm = state.builder.AllocateId();
const auto base = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane2, subid, ConstantU32(state, 3)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, shift, lane2, ConstantU32(state, 1)});
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, perm0, ConstantU32(state, control), shift});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, perm, perm0, ConstantU32(state, 3)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xfffffffcu)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, perm});
return target;
}
const auto lane = state.builder.AllocateId();
const auto masked = state.builder.AllocateId();
const auto ored = state.builder.AllocateId();
const auto xored = state.builder.AllocateId();
const auto base = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, masked, lane, ConstantU32(state, control & 0x1fu)});
state.builder.AddFunction(
{OpBitwiseOr, state.uint_type, ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
state.builder.AddFunction(
{OpBitwiseXor, state.uint_type, xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xffffffe0u)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, xored});
return target;
}
void EmitDsSwizzleB32(EmitterState& state, const IR::Instruction& inst) {
const auto source = EmitValueLoad(state, inst.src[0]);
const auto control = inst.src_count > 1u ? inst.src[1].imm & 0xffffu : 0u;
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto target = EmitDsSwizzleTargetLane(state, subid, control);
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value,
ConstantU32(state, ScopeSubgroup), source, target});
const auto exec_active = EmitLaneIndexActiveBool(state, target);
const auto subgroup_active = EmitSubgroupLaneActiveBool(state, target);
const auto source_active = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
// RDNA2 DS_SWIZZLE returns zero when the selected source thread is invalid.
state.builder.AddFunction(
{OpLogicalAnd, state.bool_type, source_active, exec_active, subgroup_active});
state.builder.AddFunction(
{OpSelect, state.uint_type, selected, source_active, value, ConstantU32(state, 0)});
EmitStoreU32(state, inst.dst, selected);
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
Reference in New Issue View Git Blame Copy Permalink