summaryrefslogtreecommitdiff
path: root/source/slang/hlsl.meta.slang
diff options
context:
space:
mode:
authorYong He <yonghe@outlook.com>2024-10-17 20:14:22 -0700
committerGitHub <noreply@github.com>2024-10-17 20:14:22 -0700
commita618b8c5e249b0f20e6c0c95f9da1b5cbfdbf08b (patch)
treed583c373d574a265fefe7f288a96c4b382e259b8 /source/slang/hlsl.meta.slang
parent11e1ecafa09396a3559fe245d729b40ce4f25d52 (diff)
Cleanup atomic intrinsics. (#5324)
* Cleanup atomic intrinsics. * Fix. * Fix glsl. * Remove hacky intrinsic expansion logic for glsl image atomics. * Fix all tests. * Fix. * Add `InterlockedAddF16Emulated`. * Fix glsl intrinsic. * Fix.
Diffstat (limited to 'source/slang/hlsl.meta.slang')
-rw-r--r--source/slang/hlsl.meta.slang2417
1 files changed, 366 insertions, 2051 deletions
diff --git a/source/slang/hlsl.meta.slang b/source/slang/hlsl.meta.slang
index 191fa3195..1c01c2f6b 100644
--- a/source/slang/hlsl.meta.slang
+++ b/source/slang/hlsl.meta.slang
@@ -3923,475 +3923,36 @@ ${{{{
}
}}}}
-// AtomicAdd
-// Make the GLSL atomicAdd available.
-// We have separate int/float implementations, as the float version requires some specific extensions
-// https://www.khronos.org/registry/OpenGL/extensions/NV/NV_shader_atomic_float.txt
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_float)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_float1)]
-float __atomicAdd(__ref float value, float amount)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicAdd($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpExtension "SPV_EXT_shader_atomic_float_add";
- OpCapability AtomicFloat32AddEXT;
- result:$$float = OpAtomicFAddEXT &value Device None $amount
- };
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_NV_shader_atomic_fp16_vector)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_halfvec)]
-half2 __atomicAdd(__ref half2 value, half2 amount)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicAdd($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpExtension "SPV_EXT_shader_atomic_float_add";
- OpCapability AtomicFloat32AddEXT;
- result:$$half2 = OpAtomicFAddEXT &value Device None $amount
- };
- }
-}
-
-// Helper for hlsl, using NVAPI
-[__requiresNVAPI]
-[require(hlsl, atomic_hlsl_nvapi)]
-uint2 __atomicAdd(RWByteAddressBuffer buf, uint offset, uint2)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "NvInterlockedAddUint64($0, $1, $2)";
- }
-}
-
-// atomic add for hlsl using SM6.6
-[require(hlsl, atomic_hlsl_sm_6_6)]
-void __atomicAdd(RWByteAddressBuffer buf, uint offset, int64_t value, out int64_t originalValue)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "$0.InterlockedAdd64($1, $2, $3)";
- }
-}
-
-[require(hlsl, atomic_hlsl_sm_6_6)]
-void __atomicAdd(RWByteAddressBuffer buf, uint offset, uint64_t value, out uint64_t originalValue)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "$0.InterlockedAdd64($1, $2, $3)";
- }
-}
-
-// Int versions require glsl 4.30
-// https://www.khronos.org/registry/OpenGL-Refpages/gl4/html/atomicAdd.xhtml
-
-__glsl_version(430)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl)]
-int __atomicAdd(__ref int value, int amount)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicAdd($0, $1)";
- case spirv:
- return spirv_asm
- {
- result:$$int = OpAtomicIAdd &value Device None $amount;
- };
- }
-}
-
-__glsl_version(430)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl)]
-uint __atomicAdd(__ref uint value, uint amount)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicAdd($0, $1)";
- case spirv:
- return spirv_asm
- {
- result:$$uint = OpAtomicIAdd &value Device None $amount;
- };
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_int64)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_int64)]
-int64_t __atomicAdd(__ref int64_t value, int64_t amount)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicAdd($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpCapability Int64Atomics;
- result:$$int64_t = OpAtomicIAdd &value Device None $amount
- };
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_int64)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_int64)]
-uint64_t __atomicAdd(__ref uint64_t value, uint64_t amount)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicAdd($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpCapability Int64Atomics;
- result:$$uint64_t = OpAtomicIAdd &value Device None $amount
- };
- }
-}
-
-// Cas - Compare and swap
-
-// Helper for HLSL, using NVAPI
-
-[__requiresNVAPI]
-[require(hlsl, atomic_hlsl_nvapi)]
-uint2 __cas(RWByteAddressBuffer buf, uint offset, uint2 compareValue, uint2 value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "NvInterlockedCompareExchangeUint64($0, $1, $2, $3)";
- }
-}
-
-// CAS using SM6.6
-[require(hlsl, atomic_hlsl_sm_6_6)]
-void __cas(RWByteAddressBuffer buf, uint offset, in int64_t compare_value, in int64_t value, out int64_t original_value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "$0.InterlockedCompareExchange64($1, $2, $3, $4)";
- }
-}
-
-[require(hlsl, atomic_hlsl_sm_6_6)]
-void __cas(RWByteAddressBuffer buf, uint offset, in uint64_t compare_value, in uint64_t value, out uint64_t original_value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "$0.InterlockedCompareExchange64($1, $2, $3, $4)";
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_int64)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_int64)]
-int64_t __cas(__ref int64_t ioValue, int64_t compareValue, int64_t newValue)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicCompSwap($0, $1, $2)";
- case spirv:
- return spirv_asm
- {
- OpCapability Int64Atomics;
- result:$$int64_t = OpAtomicCompareExchange &ioValue Device None None $newValue $compareValue
- };
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_int64)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_int64)]
-uint64_t __cas(__ref uint64_t ioValue, uint64_t compareValue, uint64_t newValue)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicCompSwap($0, $1, $2)";
- case spirv:
- return spirv_asm
- {
- OpCapability Int64Atomics;
- result:$$uint64_t = OpAtomicCompareExchange &ioValue Device None None $newValue $compareValue
- };
- }
-}
-
-// Max
-
-[__requiresNVAPI]
-[require(hlsl, atomic_hlsl_nvapi)]
-uint2 __atomicMax(RWByteAddressBuffer buf, uint offset, uint2 value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "NvInterlockedMaxUint64($0, $1, $2)";
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_int64)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_int64)]
-uint64_t __atomicMax(__ref uint64_t ioValue, uint64_t value)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicMax($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpCapability Int64Atomics;
- result:$$uint64_t = OpAtomicUMax &ioValue Device None $value
- };
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_float2)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_float2)]
-float __atomicMax(__ref float ioValue, float value)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicMax($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpExtension "SPV_EXT_shader_atomic_float_min_max";
- OpCapability AtomicFloat32MinMaxEXT;
- result:$$float = OpAtomicFMaxEXT &ioValue Device None $value
- };
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_float2)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_float2)]
-half __atomicMax(__ref half ioValue, half value)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicMax($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpExtension "SPV_EXT_shader_atomic_float_min_max";
- OpCapability AtomicFloat16MinMaxEXT;
- result:$$half = OpAtomicFMaxEXT &ioValue Device None $value
- };
- }
-}
-
-// Min
-
-[__requiresNVAPI]
-[require(hlsl, atomic_hlsl_nvapi)]
-uint2 __atomicMin(RWByteAddressBuffer buf, uint offset, uint2 value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "NvInterlockedMinUint64($0, $1, $2)";
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_int64)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_int64)]
-uint64_t __atomicMin(__ref uint64_t ioValue, uint64_t value)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicMin($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpCapability Int64Atomics;
- result:$$uint64_t = OpAtomicUMin &ioValue Device None $value
- };
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_float2)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_float2)]
-float __atomicMin(__ref float ioValue, float value)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicMin($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpExtension "SPV_EXT_shader_atomic_float_min_max";
- OpCapability AtomicFloat32MinMaxEXT;
- result:$$float = OpAtomicFMinEXT &ioValue Device None $value
- };
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_float2)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_float2)]
-half __atomicMin(__ref half ioValue, half value)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicMin($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpExtension "SPV_EXT_shader_atomic_float_min_max";
- OpCapability AtomicFloat16MinMaxEXT;
- result:$$half = OpAtomicFMinEXT &ioValue Device None $value
- };
- }
-}
-
-// And
-
-[__requiresNVAPI]
-[require(hlsl, atomic_hlsl_nvapi)]
-uint2 __atomicAnd(RWByteAddressBuffer buf, uint offset, uint2 value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "NvInterlockedAndUint64($0, $1, $2)";
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_int64)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_int64)]
-uint64_t __atomicAnd(__ref uint64_t ioValue, uint64_t value)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicAnd($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpCapability Int64Atomics;
- result:$$uint64_t = OpAtomicAnd &ioValue Device None $value
- };
- }
-}
-
-// Or
-
-[__requiresNVAPI]
-[require(hlsl, atomic_hlsl_nvapi)]
-uint2 __atomicOr(RWByteAddressBuffer buf, uint offset, uint2 value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "NvInterlockedOrUint64($0, $1, $2)";
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_int64)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_int64)]
-uint64_t __atomicOr(__ref uint64_t ioValue, uint64_t value)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicOr($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpCapability Int64Atomics;
- result:$$uint64_t = OpAtomicOr &ioValue Device None $value
- };
- }
-}
-
-// Xor
-
-[__requiresNVAPI]
-[require(hlsl, atomic_hlsl_nvapi)]
-uint2 __atomicXor(RWByteAddressBuffer buf, uint offset, uint2 value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "NvInterlockedXorUint64($0, $1, $2)";
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_int64)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_int64)]
-uint64_t __atomicXor(__ref uint64_t ioValue, uint64_t value)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicXor($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpCapability Int64Atomics;
- result:$$uint64_t = OpAtomicXor &ioValue Device None $value
- };
- }
-}
-
-// Exchange
-
-[__requiresNVAPI]
-[require(hlsl, atomic_hlsl_nvapi)]
-uint2 __atomicExchange(RWByteAddressBuffer buf, uint offset, uint2 value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "NvInterlockedExchangeUint64($0, $1, $2)";
- }
-}
-
-__glsl_version(430)
-__glsl_extension(GL_EXT_shader_atomic_int64)
-[ForceInline]
-[require(glsl_spirv, atomic_glsl_int64)]
-uint64_t __atomicExchange(__ref uint64_t ioValue, uint64_t value)
-{
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicExchange($0, $1)";
- case spirv:
- return spirv_asm
- {
- OpCapability Int64Atomics;
- result:$$uint64_t = OpAtomicExchange &ioValue Device None $value
- };
- }
-}
+// Atomic intrinsic insts.
+
+__intrinsic_op($(kIROp_AtomicExchange))
+T __atomic_exchange<T>(__ref T val, T newValue, MemoryOrder order = MemoryOrder.Relaxed);
+__intrinsic_op($(kIROp_AtomicCompareExchange))
+T __atomic_compare_exchange<T>(
+ __ref T val,
+ T compareValue,
+ T newValue,
+ MemoryOrder successOrder = MemoryOrder.Relaxed,
+ MemoryOrder failOrder = MemoryOrder.Relaxed);
+__intrinsic_op($(kIROp_AtomicAdd))
+T __atomic_add<T>(__ref T val, T value, MemoryOrder order = MemoryOrder.Relaxed);
+__intrinsic_op($(kIROp_AtomicSub))
+T __atomic_sub<T>(__ref T val, T value, MemoryOrder order = MemoryOrder.Relaxed);
+__intrinsic_op($(kIROp_AtomicMax))
+T __atomic_max<T>(__ref T val, T value, MemoryOrder order = MemoryOrder.Relaxed);
+__intrinsic_op($(kIROp_AtomicMin))
+T __atomic_min<T>(__ref T val, T value, MemoryOrder order = MemoryOrder.Relaxed);
+__intrinsic_op($(kIROp_AtomicAnd))
+T __atomic_and<T>(__ref T val, T value, MemoryOrder order = MemoryOrder.Relaxed);
+__intrinsic_op($(kIROp_AtomicOr))
+T __atomic_or<T>(__ref T val, T value, MemoryOrder order = MemoryOrder.Relaxed);
+__intrinsic_op($(kIROp_AtomicXor))
+T __atomic_xor<T>(__ref T val, T value, MemoryOrder order = MemoryOrder.Relaxed);
+__intrinsic_op($(kIROp_AtomicInc))
+T __atomic_increment<T>(__ref T val, MemoryOrder order = MemoryOrder.Relaxed);
+__intrinsic_op($(kIROp_AtomicDec))
+T __atomic_decrement<T>(__ref T val, MemoryOrder order = MemoryOrder.Relaxed);
// Conversion between uint64_t and uint2
@@ -4802,6 +4363,20 @@ struct $(item.name)
}
${{{{
+ struct BufferAtomicOps
+ {
+ const char* name;
+ const char* internalName;
+ };
+ const BufferAtomicOps bufferAtomicOps[] = {
+ {"Max", "max"},
+ {"Min", "min"},
+ {"Add", "add"},
+ {"And", "and"},
+ {"Or", "or"},
+ {"Xor", "xor"},
+ {"Exchange", "exchange"}
+ };
if (item.op == kIROp_HLSLRWByteAddressBufferType)
{
}}}}
@@ -4822,6 +4397,13 @@ ${{{{
// F32 Add
+ /// Perform a 32-bit floating point atomic add operation at `byteAddress`.
+ /// @param byteAddress The address at which to perform the atomic add operation.
+ /// @param valueToAdd The value to add to the value at `byteAddress`.
+ /// @param originalValue The original value at `byteAddress` before the add operation.
+ /// @remarks For SPIR-V, this function maps to `OpAtomicFAdd`. For HLSL, this function translates to an NVAPI call
+ /// due to lack of native HLSL intrinsic for floating point atomic add. For CUDA, this function
+ /// maps to `atomicAdd`.
__cuda_sm_version(2.0)
[__requiresNVAPI]
[ForceInline]
@@ -4832,35 +4414,45 @@ ${{{{
{
case hlsl: __intrinsic_asm "($3 = NvInterlockedAddFp32($0, $1, $2))";
case cuda: __intrinsic_asm "(*$3 = atomicAdd($0._getPtrAt<float>($1), $2))";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<float>(this);
- __metalInterlocked_add(__getMetalAtomicRef(buf[byteAddress / 4]), valueToAdd, originalValue);
- return;
- }
- case glsl:
- case spirv:
+ default:
{
let buf = __getEquivalentStructuredBuffer<float>(this);
- originalValue = __atomicAdd(buf[byteAddress / 4], valueToAdd);
+ originalValue = __atomic_add(buf[byteAddress / 4], valueToAdd);
return;
}
}
}
// FP16x2
+
+ /// @internal
+ /// Maps to the `NvInterlockedAddFp16x2` NVAPI function.
+ ///
[__requiresNVAPI]
[ForceInline]
- [require(hlsl, atomic_hlsl_nvapi)]
+ [require(cuda_hlsl_spirv)]
uint _NvInterlockedAddFp16x2(uint byteAddress, uint fp16x2Value)
{
__target_switch
{
case hlsl:
__intrinsic_asm "NvInterlockedAddFp16x2($0, $1, $2)";
+ default:
+ let buf = __getEquivalentStructuredBuffer<half2>(this);
+ return bit_cast<uint>(__atomic_add(buf[byteAddress / 4], bit_cast<half2>(fp16x2Value)));
}
}
+
+ /// Perform a 16-bit floating point atomic add operation at `byteAddress`.
+ /// @param byteAddress The address at which to perform the atomic add operation.
+ /// @param valueToAdd The value to add to the value at `byteAddress`.
+ /// @param originalValue The original value at `byteAddress` before the add operation.
+ /// @remarks For SPIR-V, this function maps to `OpAtomicFAdd` and requires `SPV_EXT_shader_atomic_float16_add` extension.
+ ///
+ /// For HLSL, this function translates to an NVAPI call
+ /// due to lack of native HLSL intrinsic for floating point atomic add. For CUDA, this function
+ /// maps to `atomicAdd`.
[__requiresNVAPI]
[ForceInline]
void InterlockedAddF16(uint byteAddress, half value, out half originalValue)
@@ -4880,17 +4472,55 @@ ${{{{
originalValue = asfloat16((uint16_t)(_NvInterlockedAddFp16x2(byteAddress, packedInput) >> 16));
}
return;
- case glsl:
- case spirv:
+ default:
+ {
+ let buf = __getEquivalentStructuredBuffer<half>(this);
+ originalValue = __atomic_add(buf[byteAddress/2], value);
+ return;
+ }
+ }
+ }
+
+ /// Perform a 16-bit floating point atomic add operation at `byteAddress` through emulation using `half2` atomics.
+ /// @param byteAddress The address at which to perform the atomic add operation.
+ /// @param valueToAdd The value to add to the value at `byteAddress`.
+ /// @param originalValue The original value at `byteAddress` before the add operation.
+ /// @remarks For SPIR-V, this function maps to `OpAtomicFAdd` on a `half2` vector with the correct part set to `value`
+ /// and the remaining part set to 0. This requires the `AtomicFloat16VectorNV` capability introduced by the `SPV_NV_shader_atomic_fp16_vector`
+ /// extension.
+ ///
+ /// For HLSL, this function translates to an equivalent NVAPI call
+ /// due to lack of native HLSL intrinsic for floating point atomic add. For CUDA, this function
+ /// maps to `atomicAdd`.
+ [__requiresNVAPI]
+ [ForceInline]
+ void InterlockedAddF16Emulated(uint byteAddress, half value, out half originalValue)
+ {
+ __target_switch
+ {
+ case hlsl:
+ if ((byteAddress & 2) == 0)
+ {
+ uint packedInput = asuint16(value);
+ originalValue = asfloat16((uint16_t)_NvInterlockedAddFp16x2(byteAddress, packedInput));
+ }
+ else
+ {
+ byteAddress = byteAddress & ~3;
+ uint packedInput = ((uint)asuint16(value)) << 16;
+ originalValue = asfloat16((uint16_t)(_NvInterlockedAddFp16x2(byteAddress, packedInput) >> 16));
+ }
+ return;
+ default:
{
let buf = __getEquivalentStructuredBuffer<half2>(this);
if ((byteAddress & 2) == 0)
{
- originalValue = __atomicAdd(buf[byteAddress/4], half2(value, half(0.0))).x;
+ originalValue = __atomic_add(buf[byteAddress/4], half2(value, half(0.0))).x;
}
else
{
- originalValue = __atomicAdd(buf[byteAddress/4], half2(half(0.0), value)).y;
+ originalValue = __atomic_add(buf[byteAddress/4], half2(half(0.0), value)).y;
}
return;
}
@@ -4908,484 +4538,207 @@ ${{{{
__target_switch
{
case hlsl: __intrinsic_asm "(NvInterlockedAddFp32($0, $1, $2))";
- case cuda: __intrinsic_asm "atomicAdd($0._getPtrAt<float>($1), $2)";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<float>(this);
- __metalInterlocked_add(__getMetalAtomicRef(buf[byteAddress / 4]), valueToAdd);
- return;
- }
- case glsl:
- case spirv:
+ default:
{
let buf = __getEquivalentStructuredBuffer<float>(this);
- __atomicAdd(buf[byteAddress / 4], valueToAdd);
+ __atomic_add(buf[byteAddress / 4], valueToAdd);
return;
}
}
}
// Int64 Add
+
+ /// Perform a 64-bit integer atomic add operation at `byteAddress`.
+ /// @param byteAddress The address at which to perform the atomic add operation.
+ /// @param valueToAdd The value to add to the value at `byteAddress`.
+ /// @param originalValue The original value at `byteAddress` before the add operation.
+ /// @remarks For SPIR-V, this function maps to `OpAtomicAdd`. For HLSL, this function
+ /// translates to `InterlockedAdd64` and requires shader model 6.6.
+ /// For CUDA, this function maps to `atomicAdd`.
[ForceInline]
- __cuda_sm_version(6.0)
- [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_nvapi_cuda6_int64)]
+ [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
void InterlockedAddI64(uint byteAddress, int64_t valueToAdd, out int64_t originalValue)
{
- __target_switch
- {
- case cuda: __intrinsic_asm "(*$3 = atomicAdd($0._getPtrAt<uint64_t>($1), $2))";
- case hlsl:
- originalValue = __asuint64(__atomicAdd(this, byteAddress, __asuint2(valueToAdd)));
- case glsl:
- case spirv:
- {
- let buf = __getEquivalentStructuredBuffer<int64_t>(this);
- originalValue = __atomicAdd(buf[byteAddress / 8], valueToAdd);
- }
- }
+ InterlockedAdd64(byteAddress, valueToAdd, originalValue);
}
// Without returning original value
- __cuda_sm_version(6.0)
- [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_nvapi_cuda6_int64)]
+ [ForceInline]
+ [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
void InterlockedAddI64(uint byteAddress, int64_t valueToAdd)
{
- __target_switch
- {
- case cuda: __intrinsic_asm "atomicAdd($0._getPtrAt<uint64_t>($1), $2)";
- case hlsl:
- __atomicAdd(this, byteAddress, __asuint2(valueToAdd));
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<int64_t>(this);
- __atomicAdd(buf[byteAddress / 8], valueToAdd);
- }
+ InterlockedAdd64(byteAddress, valueToAdd);
}
// Cas uint64_t
- [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_nvapi_cuda9_int64)]
+ /// Perform a 64-bit integer atomic compare-and-exchange operation at `byteAddress`.
+ /// @param byteAddress The address at which to perform the atomic compare-and-exchange operation.
+ /// @param compareValue The value to compare to the value at `byteAddress`.
+ /// @param value The value to store at `byteAddress` if the comparison is successful.
+ /// @param originalValue The original value at `byteAddress` before the add operation.
+ /// @remarks For SPIR-V, this function maps to `OpAtomicCompareExchange`. For HLSL, this function
+ /// translates to `InterlockedCompareExchange64` and requires shader model 6.6.
+ /// For CUDA, this function maps to `atomicCAS`.
+ [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
void InterlockedCompareExchangeU64(uint byteAddress, uint64_t compareValue, uint64_t value, out uint64_t outOriginalValue)
{
__target_switch
{
case cuda: __intrinsic_asm "(*$4 = atomicCAS($0._getPtrAt<uint64_t>($1), $2, $3))";
case hlsl:
- outOriginalValue = __asuint64(__cas(this, byteAddress, __asuint2(compareValue), __asuint2(value)));
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint64_t>(this);
- outOriginalValue = __cas(buf[byteAddress / 8], compareValue, value);
- }
- }
-
- // Max
-
- __cuda_sm_version(5.0)
- [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_nvapi_cuda5_int64)]
- uint64_t InterlockedMaxU64(uint byteAddress, uint64_t value)
- {
- __target_switch
- {
- case cuda: __intrinsic_asm "atomicMax($0._getPtrAt<uint64_t>($1), $2)";
- case hlsl:
- return __asuint64(__atomicMax(this, byteAddress, __asuint2(value)));
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint64_t>(this);
- return __atomicMax(buf[byteAddress / 8], value);
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedMax64(uint byteAddress, int64_t value)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedMax64";
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedMax64(uint byteAddress, int64_t value, out int64_t outOriginalValue)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedMax64";
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedMax64(uint byteAddress, uint64_t value)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedMax64";
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedMax64(uint byteAddress, uint64_t value, out uint64_t outOriginalValue)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedMax64";
- }
- }
-
- // Min
-
- __cuda_sm_version(5.0)
- [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_nvapi_cuda5_int64)]
- uint64_t InterlockedMinU64(uint byteAddress, uint64_t value)
- {
- __target_switch
- {
- case cuda: __intrinsic_asm "atomicMin($0._getPtrAt<uint64_t>($1), $2)";
- case hlsl:
- return __asuint64(__atomicMin(this, byteAddress, __asuint2(value)));
- case glsl:
- case spirv:
+ __intrinsic_asm ".InterlockedCompareExchange64";
+ default:
let buf = __getEquivalentStructuredBuffer<uint64_t>(this);
- return __atomicMin(buf[byteAddress / 8], value);
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedMin64(uint byteAddress, int64_t value)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedMin64";
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedMin64(uint byteAddress, int64_t value, out int64_t outOriginalValue)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedMin64";
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedMin64(uint byteAddress, uint64_t value)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedMin64";
+ outOriginalValue = __atomic_compare_exchange(buf[byteAddress / 8], compareValue, value);
}
}
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedMin64(uint byteAddress, uint64_t value, out uint64_t outOriginalValue)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedMin64";
- }
- }
-
- // And
-
- __cuda_sm_version(5.0)
- [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_nvapi_cuda5_int64)]
- uint64_t InterlockedAndU64(uint byteAddress, uint64_t value)
- {
- __target_switch
- {
- case cuda: __intrinsic_asm "atomicAnd($0._getPtrAt<uint64_t>($1), $2)";
- case hlsl:
- return __asuint64(__atomicAnd(this, byteAddress, __asuint2(value)));
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint64_t>(this);
- return __atomicAnd(buf[byteAddress / 8], value);
- }
- }
+ // SM6.6 6 64bit atomics.
+ // InterlockedMax64, InterlockedMin64, InterlockedAdd64, InterlockedAnd64, InterlockedOr64, InterlockedXor64, InterlockedExchange64
+${{{{
+ for (auto op : bufferAtomicOps) {
+}}}}
[ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedAnd64(uint byteAddress, uint64_t value)
+ [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
+ uint64_t Interlocked$(op.name)U64(uint byteAddress, uint64_t value)
{
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedAnd64";
- }
+ uint64_t originalValue;
+ Interlocked$(op.name)64(byteAddress, value, originalValue);
+ return originalValue;
}
[ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedAnd64(uint byteAddress, uint64_t value, out uint64_t outOriginalValue)
+ [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
+ void Interlocked$(op.name)64(uint byteAddress, int64_t value)
{
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedAnd64";
- }
- }
-
- // Or
-
- __cuda_sm_version(5.0)
- [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_nvapi_cuda5_int64)]
- uint64_t InterlockedOrU64(uint byteAddress, uint64_t value)
- {
- __target_switch
- {
- case cuda: __intrinsic_asm "atomicOr($0._getPtrAt<uint64_t>($1), $2)";
- case hlsl:
- return __asuint64(__atomicOr(this, byteAddress, __asuint2(value)));
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint64_t>(this);
- return __atomicOr(buf[byteAddress / 8], value);
- }
+ int64_t oldValue;
+ Interlocked$(op.name)64(byteAddress, value, oldValue);
}
+ /// Perform a 64-bit integer atomic $(op.internalName) operation at `byteAddress`.
+ /// @param byteAddress The address at which to perform the atomic $(op.internalName) operation.
+ /// @param value The operand for the $(op.internalName) operation.
+ /// @param originalValue The original value at `byteAddress` before the $(op.internalName) operation.
[ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedOr64(uint byteAddress, uint64_t value)
+ [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
+ void Interlocked$(op.name)64<T:__BuiltinInt64Type>(uint byteAddress, T value, out T outOriginalValue)
{
__target_switch
{
- case hlsl: __intrinsic_asm ".InterlockedOr64";
+ case hlsl: __intrinsic_asm ".Interlocked$(op.name)64";
+ default:
+ let buf = __getEquivalentStructuredBuffer<T>(this);
+ outOriginalValue = __atomic_$(op.internalName)(buf[byteAddress / 8], value);
+ return;
}
}
+${{{{
+} // for (each bufferOps)
+}}}}
+ /// Perform a 64-bit integer atomic compare-and-exchange operation at `byteAddress`.
+ /// @param byteAddress The address at which to perform the atomic compare-and-exchange operation.
+ /// @param compareValue The value to compare to the value at `byteAddress`.
+ /// @param value The value to store at `byteAddress` if the comparison is successful.
+ /// @param outOriginalValue The original value at `byteAddress` before the add operation.
+ /// @remarks For SPIR-V, this function maps to `OpAtomicCompareExchange`. For HLSL, this function
+ /// translates to `InterlockedCompareExchange64` and requires shader model 6.6.
+ /// For CUDA, this function maps to `atomicCAS`.
[ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedOr64(uint byteAddress, uint64_t value, out uint64_t outOriginalValue)
+ [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
+ void InterlockedCompareExchange64<T:__BuiltinInt64Type>(uint byteAddress, T compareValue, T value, out T outOriginalValue)
{
__target_switch
{
- case hlsl: __intrinsic_asm ".InterlockedOr64";
- }
- }
-
- // Xor
-
- __cuda_sm_version(5.0)
- [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_nvapi_cuda5_int64)]
- uint64_t InterlockedXorU64(uint byteAddress, uint64_t value)
- {
- __target_switch
- {
- case cuda: __intrinsic_asm "atomicXor($0._getPtrAt<uint64_t>($1), $2)";
case hlsl:
- return __asuint64(__atomicXor(this, byteAddress, __asuint2(value)));
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint64_t>(this);
- return __atomicXor(buf[byteAddress / 8], value);
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedXor64(uint byteAddress, uint64_t value)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedXor64";
+ __intrinsic_asm ".InterlockedCompareExchange64";
+ default:
+ let buf = __getEquivalentStructuredBuffer<T>(this);
+ outOriginalValue = __atomic_compare_exchange(buf[byteAddress / 8], compareValue, value);
+ return;
}
}
[ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedXor64(uint byteAddress, uint64_t value, out uint64_t outOriginalValue)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedXor64";
- }
- }
-
- // Exchange
-
- [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_nvapi_cuda9_int64)]
- uint64_t InterlockedExchangeU64(uint byteAddress, uint64_t value)
+ [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
+ void InterlockedCompareExchangeFloatBitwise(uint byteAddress, float compareValue, float value, out float outOriginalValue)
{
__target_switch
{
- case cuda: __intrinsic_asm "atomicExch($0._getPtrAt<uint64_t>($1), $2)";
- case hlsl:
- return __asuint64(__atomicExchange(this, byteAddress, __asuint2(value)));
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint64_t>(this);
- return __atomicExchange(buf[byteAddress / 8], value);
+ case hlsl: __intrinsic_asm ".InterlockedCompareExchangeFloatBitwise";
+ default:
+ let buf = __getEquivalentStructuredBuffer<float>(this);
+ outOriginalValue = __atomic_compare_exchange(buf[byteAddress / 4], compareValue, value);
+ return;
}
}
+ /// Perform a floating-point atomic bitwise exchange operation at `byteAddress`.
+ /// @param byteAddress The address at which to perform the atomic exchange operation.
+ /// @param value The value to store at `byteAddress`.
+ /// @param [out] outOriginalValue The original value at `byteAddress` before the exchange operation.
+ /// @remarks For SPIR-V, this function maps to `OpAtomicExchange`. For HLSL, this function
+ /// translates to `InterlockedExchangeFloat` and requires shader model 6.6.
+ /// For CUDA, this function maps to `atomicExch`.
[ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
+ [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
void InterlockedExchangeFloat(uint byteAddress, float value, out float outOriginalValue)
{
__target_switch
{
case hlsl: __intrinsic_asm ".InterlockedExchangeFloat";
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedExchange64(uint byteAddress, int64_t value, out int64_t outOriginalValue)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedExchange64";
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedExchange64(uint byteAddress, uint64_t value, out uint64_t outOriginalValue)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedExchange64";
- }
- }
-
- // SM6.6 6 64bit atomics.
- [ForceInline]
- [require(glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
- void InterlockedAdd64(uint byteAddress, int64_t valueToAdd)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedAdd64";
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<int64_t>(this);
- __atomicAdd(buf[byteAddress / 8], valueToAdd);
- }
- }
-
- [ForceInline]
- [require(glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
- void InterlockedAdd64(uint byteAddress, int64_t valueToAdd, out int64_t outOriginalValue)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedAdd64";
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<int64_t>(this);
- outOriginalValue = __atomicAdd(buf[byteAddress / 8], valueToAdd);
- return;
- }
- }
-
- [ForceInline]
- [require(glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
- void InterlockedAdd64(uint byteAddress, uint64_t valueToAdd)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedAdd64";
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint64_t>(this);
- __atomicAdd(buf[byteAddress / 8], valueToAdd);
- }
- }
-
- [ForceInline]
- [require(glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
- void InterlockedAdd64(uint byteAddress, uint64_t valueToAdd, out uint64_t outOriginalValue)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedAdd64";
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint64_t>(this);
- outOriginalValue = __atomicAdd(buf[byteAddress / 8], valueToAdd);
+ default:
+ let buf = __getEquivalentStructuredBuffer<float>(this);
+ outOriginalValue = __atomic_exchange(buf[byteAddress / 4], value);
return;
}
}
+ /// Perform a 64-bit integer atomic compare-and-store operation at `byteAddress`.
+ /// @param byteAddress The address at which to perform the atomic store operation.
+ /// @param compareValue The value to compare to the value at `byteAddress`.
+ /// @param value The value to store at `byteAddress` if the the value at address is equal to `compareValue`.
+ /// @param [out] outOriginalValue The original value at `byteAddress` before the store operation.
+ /// @remarks For SPIR-V, this function maps to `OpAtomicCompareExchange`. For HLSL, this function
+ /// translates to `InterlockedCompareStore64` and requires shader model 6.6.
+ /// For CUDA, this function maps to `atomicCAS`.
[ForceInline]
- [require(glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
- void InterlockedCompareExchange64(uint byteAddress, int64_t compareValue, int64_t value, out int64_t outOriginalValue)
- {
- __target_switch
- {
- case hlsl:
- __cas(this, byteAddress, compareValue, value, outOriginalValue);
- return;
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<int64_t>(this);
- outOriginalValue = __cas(buf[byteAddress / 8], compareValue, value);
- return;
- }
- }
-
[ForceInline]
- [require(glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
- void InterlockedCompareExchange64(uint byteAddress, uint64_t compareValue, uint64_t value, out uint64_t outOriginalValue)
+ [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
+ void InterlockedCompareStore64<T:__BuiltinInt64Type>(uint byteAddress, T compareValue, T value)
{
__target_switch
{
- case hlsl:
- __cas(this, byteAddress, compareValue, value, outOriginalValue);
- return;
- case glsl:
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint64_t>(this);
- outOriginalValue = __cas(buf[byteAddress / 8], compareValue, value);
+ case hlsl: __intrinsic_asm ".InterlockedCompareStore64";
+ default:
+ let buf = __getEquivalentStructuredBuffer<T>(this);
+ __atomic_compare_exchange(buf[byteAddress / 4], compareValue, value);
return;
}
}
-
+
+ /// Perform a floating-point atomic bitwise compare-and-store operation at `byteAddress`.
+ /// @param byteAddress The address at which to perform the atomic compare-and-exchange operation.
+ /// @param compareValue The value to perform bitwise comparison to the value at `byteAddress`.
+ /// @param value The value to store at `byteAddress` if the comparison is successful.
+ /// @param [out] outOriginalValue The original value at `byteAddress` before the compare-and-exchange operation.
+ /// @remarks For SPIR-V, this function maps to `OpAtomicCompareExchange`. For HLSL, this function
+ /// translates to `InterlockedCompareStoreFloatBitwise` and requires shader model 6.6.
+ /// For CUDA, this function maps to `atomicCAS`.
[ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
+ [require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_cuda9_int64)]
void InterlockedCompareStoreFloatBitwise(uint byteAddress, float compareValue, float value)
{
__target_switch
{
case hlsl: __intrinsic_asm ".InterlockedCompareStoreFloatBitwise";
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedCompareExchangeFloatBitwise(uint byteAddress, float compareValue, float value, out float outOriginalValue)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedCompareExchangeFloatBitwise";
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedCompareStore64(uint byteAddress, int64_t compareValue, int64_t value)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedCompareStore64";
- }
- }
-
- [ForceInline]
- [require(hlsl, atomic_hlsl_sm_6_6)]
- void InterlockedCompareStore64(uint byteAddress, uint64_t compareValue, uint64_t value)
- {
- __target_switch
- {
- case hlsl: __intrinsic_asm ".InterlockedCompareStore64";
+ default:
+ let buf = __getEquivalentStructuredBuffer<float>(this);
+ __atomic_compare_exchange(buf[byteAddress / 4], compareValue, value);
+ return;
}
}
@@ -5393,103 +4746,62 @@ ${{{{
} // endif (type == RWByteAddressBuffer)
}}}}
- // Added operations:
- [ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedAdd(
- UINT dest,
- UINT value,
- out UINT original_value)
- {
- __target_switch
- {
- case glsl: __intrinsic_asm "($3 = atomicAdd($0._data[$1/4], $2))";
- case cuda: __intrinsic_asm "(*$3 = atomicAdd($0._getPtrAt<uint32_t>($1), $2))";
- case hlsl: __intrinsic_asm ".InterlockedAdd";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_add(__getMetalAtomicRef(buf[dest / 4]), value, original_value);
- return;
- }
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedAdd(buf[dest / 4], value, original_value);
- }
- }
-
- [ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedAdd(
- UINT dest,
- UINT value)
- {
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicAdd($0._data[$1/4], $2)";
- case cuda: __intrinsic_asm "atomicAdd($0._getPtrAt<uint32_t>($1), $2)";
- case hlsl: __intrinsic_asm ".InterlockedAdd";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_add(__getMetalAtomicRef(buf[dest / 4]), value);
- return;
- }
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedAdd(buf[dest / 4], value);
- }
- }
+ // 32-bit atomic operations:
+ // InterlockedMax, InterlockedMin, InterlockedAdd, InterlockedAnd, InterlockedOr, InterlockedXor, InterlockedExchange
+${{{{
+ for (auto op : bufferAtomicOps) {
+}}}}
+ /// Perform an atomic $(op.internalName) operation at the specified byte
+ /// location of the byte address buffer.
+ /// @param dest The byte address at which to perform the atomic $(op.internalName) operation.
+ /// @param value The operand of the atomic operation.
+ /// @param original_value The original value at `dest` before the $(op.internalName) operation.
[ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedAnd(
+ [require(cuda_glsl_hlsl_metal_spirv, atomic_glsl_hlsl_cuda_metal, byteaddressbuffer_rw)]
+ void Interlocked$(op.name)(
UINT dest,
UINT value,
out UINT original_value)
{
__target_switch
{
- case glsl: __intrinsic_asm "$3 = atomicAnd($0._data[$1/4], $2)";
- case cuda: __intrinsic_asm "(*$3 = atomicAnd($0._getPtrAt<uint32_t>($1), $2))";
- case hlsl: __intrinsic_asm ".InterlockedAnd";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_and(__getMetalAtomicRef(buf[dest / 4]), value, original_value);
- return;
- }
- case spirv:
+ case hlsl: __intrinsic_asm ".Interlocked$(op.name)";
+ default:
let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedAnd(buf[dest / 4], value, original_value);
+ ::Interlocked$(op.name)(buf[dest / 4], value, original_value);
}
}
[ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedAnd(
+ [require(cuda_glsl_hlsl_metal_spirv, atomic_glsl_hlsl_cuda_metal, byteaddressbuffer_rw)]
+ void Interlocked$(op.name)(
UINT dest,
UINT value)
{
__target_switch
{
- case glsl: __intrinsic_asm "atomicAnd($0._data[$1/4], $2)";
- case cuda: __intrinsic_asm "atomicAnd($0._getPtrAt<uint32_t>($1), $2)";
- case hlsl: __intrinsic_asm ".InterlockedAnd";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_and(__getMetalAtomicRef(buf[dest / 4]), value);
- return;
- }
- case spirv:
+ case hlsl: __intrinsic_asm ".Interlocked$(op.name)";
+ default:
let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedAnd(buf[dest / 4], value);
+ ::Interlocked$(op.name)(buf[dest / 4], value);
}
}
+${{{{
+} // for (buffer atomic ops)
+}}}}
+ /// Perform a 32-bit integer atomic compare-and-exchange operation at
+ /// the specified byte address within the `RWByteAddressBuffer`.
+ /// @param dest The address at which to perform the atomic compare-and-exchange operation.
+ /// @param compare_value The value to perform bitwise comparison to the value at `byteAddress`.
+ /// @param value The value to store at `byteAddress` if the comparison is successful.
+ /// @param original_value The original value at `byteAddress` before the compare-and-exchange operation.
+ /// @remarks For SPIR-V, this function maps to `OpAtomicCompareExchange`. For HLSL, this function
+ /// translates to `InterlockedCompareExchange`.
+ /// For CUDA, this function maps to `atomicCAS`.
[ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
+ [require(cuda_glsl_hlsl_metal_spirv, atomic_glsl_hlsl_cuda_metal, byteaddressbuffer_rw)]
void InterlockedCompareExchange(
UINT dest,
UINT compare_value,
@@ -5498,23 +4810,23 @@ ${{{{
{
__target_switch
{
- case glsl: __intrinsic_asm "($4 = atomicCompSwap($0._data[$1/4], $2, $3))";
- case cuda: __intrinsic_asm "(*$4 = atomicCAS($0._getPtrAt<uint32_t>($1), $2, $3))";
case hlsl: __intrinsic_asm ".InterlockedCompareExchange";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_compare_exchange(__getMetalAtomicRef(buf[dest / 4]), compare_value, value, original_value);
- return;
- }
- case spirv:
+ default:
let buf = __getEquivalentStructuredBuffer<uint>(this);
::InterlockedCompareExchange(buf[dest / 4], compare_value, value, original_value);
}
}
+ /// Perform a 32-bit integer atomic compare-and-store operation at
+ /// the specified byte address within the `RWByteAddressBuffer`.
+ /// @param dest The address at which to perform the atomic add operation.
+ /// @param compare_value The value to perform comparison to the value at `byteAddress`.
+ /// @param value The value to store at `byteAddress` if the comparison is successful.
+ /// @remarks For SPIR-V, this function maps to `OpAtomicCompareExchange`. For HLSL, this function
+ /// translates to `InterlockedCompareStore`.
+ /// For CUDA, this function maps to `atomicCAS`.
[ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
+ [require(cuda_glsl_hlsl_metal_spirv, atomic_glsl_hlsl_cuda_metal, byteaddressbuffer_rw)]
void InterlockedCompareStore(
UINT dest,
UINT compare_value,
@@ -5522,232 +4834,13 @@ ${{{{
{
__target_switch
{
- case glsl: __intrinsic_asm "atomicCompSwap($0._data[$1/4], $2, $3)";
- case cuda: __intrinsic_asm "atomicCAS($0._getPtrAt<uint32_t>($1), $2, $3)";
case hlsl: __intrinsic_asm ".InterlockedCompareStore";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_compare_exchange(__getMetalAtomicRef(buf[dest / 4]), compare_value, value);
- return;
- }
- case spirv:
+ default:
let buf = __getEquivalentStructuredBuffer<uint>(this);
::InterlockedCompareStore(buf[dest / 4], compare_value, value);
}
}
- [ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedExchange(
- UINT dest,
- UINT value,
- out UINT original_value)
- {
- __target_switch
- {
- case glsl: __intrinsic_asm "($3 = atomicExchange($0._data[$1/4], $2))";
- case cuda: __intrinsic_asm "(*$3 = atomicExch($0._getPtrAt<uint32_t>($1), $2))";
- case hlsl: __intrinsic_asm ".InterlockedExchange";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_exchange(__getMetalAtomicRef(buf[dest / 4]), value, original_value);
- return;
- }
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedExchange(buf[dest / 4], value, original_value);
- }
- }
-
- [ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedMax(
- UINT dest,
- UINT value,
- out UINT original_value)
- {
- __target_switch
- {
- case glsl: __intrinsic_asm "($3 = atomicMax($0._data[$1/4], $2))";
- case cuda: __intrinsic_asm "(*$3 = atomicMax($0._getPtrAt<uint32_t>($1), $2))";
- case hlsl: __intrinsic_asm ".InterlockedMax";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_max(__getMetalAtomicRef(buf[dest / 4]), value, original_value);
- return;
- }
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedMax(buf[dest / 4], value, original_value);
- }
- }
-
- [ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedMax(
- UINT dest,
- UINT value)
- {
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicMax($0._data[$1/4], $2)";
- case cuda: __intrinsic_asm "atomicMax($0._getPtrAt<uint32_t>($1), $2)";
- case hlsl: __intrinsic_asm ".InterlockedMax";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_max(__getMetalAtomicRef(buf[dest / 4]), value);
- return;
- }
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedMax(buf[dest / 4], value);
- }
- }
-
- [ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedMin(
- UINT dest,
- UINT value,
- out UINT original_value)
- {
- __target_switch
- {
- case glsl: __intrinsic_asm "($3 = atomicMin($0._data[$1/4], $2))";
- case cuda: __intrinsic_asm "(*$3 = atomicMin($0._getPtrAt<uint32_t>($1), $2))";
- case hlsl: __intrinsic_asm ".InterlockedMin";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_min(__getMetalAtomicRef(buf[dest / 4]), value, original_value);
- return;
- }
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedMin(buf[dest / 4], value, original_value);
- }
- }
-
- [ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedMin(
- UINT dest,
- UINT value)
- {
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicMin($0._data[$1/4], $2)";
- case cuda: __intrinsic_asm "atomicMin($0._getPtrAt<uint32_t>($1), $2)";
- case hlsl: __intrinsic_asm ".InterlockedMin";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_min(__getMetalAtomicRef(buf[dest / 4]), value);
- return;
- }
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedMin(buf[dest / 4], value);
- }
- }
-
- [ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedOr(
- UINT dest,
- UINT value,
- out UINT original_value)
- {
- __target_switch
- {
- case glsl: __intrinsic_asm "($3 = atomicOr($0._data[$1/4], $2))";
- case cuda: __intrinsic_asm "(*$3 = atomicOr($0._getPtrAt<uint32_t>($1), $2))";
- case hlsl: __intrinsic_asm ".InterlockedOr";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_or(__getMetalAtomicRef(buf[dest / 4]), value, original_value);
- return;
- }
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedOr(buf[dest / 4], value, original_value);
- }
- }
-
- [ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedOr(
- UINT dest,
- UINT value)
- {
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicOr($0._data[$1/4], $2)";
- case cuda: __intrinsic_asm "atomicOr($0._getPtrAt<uint32_t>($1), $2)";
- case hlsl: __intrinsic_asm ".InterlockedOr";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_or(__getMetalAtomicRef(buf[dest / 4]), value);
- return;
- }
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedOr(buf[dest / 4], value);
- }
- }
-
- [ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedXor(
- UINT dest,
- UINT value,
- out UINT original_value)
- {
- __target_switch
- {
- case glsl: __intrinsic_asm "($3 = atomicXor($0._data[$1/4], $2))";
- case cuda: __intrinsic_asm "(*$3 = atomicXor($0._getPtrAt<uint32_t>($1), $2))";
- case hlsl: __intrinsic_asm ".InterlockedXor";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_xor(__getMetalAtomicRef(buf[dest / 4]), value, original_value);
- return;
- }
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedXor(buf[dest / 4], value, original_value);
- }
- }
-
- [ForceInline]
- [require(cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
- void InterlockedXor(
- UINT dest,
- UINT value)
- {
- __target_switch
- {
- case glsl: __intrinsic_asm "atomicXor($0._data[$1/4], $2)";
- case cuda: __intrinsic_asm "atomicXor($0._getPtrAt<uint32_t>($1), $2)";
- case hlsl: __intrinsic_asm ".InterlockedXor";
- case metal:
- {
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- __metalInterlocked_xor(__getMetalAtomicRef(buf[dest / 4]), value);
- return;
- }
- case spirv:
- let buf = __getEquivalentStructuredBuffer<uint>(this);
- ::InterlockedXor(buf[dest / 4], value);
- }
- }
[ForceInline]
[require(cpp_cuda_glsl_hlsl_metal_spirv, byteaddressbuffer_rw)]
@@ -9699,26 +8792,6 @@ void GroupMemoryBarrierWithGroupSync()
// Atomics
-__generic<T>
-__intrinsic_op($(kIROp_MetalAtomicCast))
-[require(metal)]
-T* __getMetalAtomicRef(__ref T x);
-
-// Checks if input is a ImageSubscript
-__generic<T>
-__intrinsic_op($(kIROp_IsTextureAccess))
-bool __isTextureAccess(__ref T x);
-
-// Checks if input is a texture of T type scalar
-__generic<T>
-__intrinsic_op($(kIROp_IsTextureScalarAccess))
-bool __isTextureScalarAccess(__ref T x);
-
-// Checks if input is a texture array
-__generic<T>
-__intrinsic_op($(kIROp_IsTextureArrayAccess))
-bool __isTextureArrayAccess(__ref T x);
-
// Accepts an ImageSubscript
// Gets Texture used with ImageSubscript.
__generic<TextureAccess>
@@ -9738,414 +8811,6 @@ __intrinsic_op($(kIROp_ExtractArrayCoordFromTextureAccess))
uint __extractArrayCoordFromTextureAccess(__ref TextureAccess x);
${{{{
-for (bool isArray : {false, true})
-{
- StringBuilder coordBuilder;
- StringBuilder coordFetchBuilder;
-
- StringBuilder threeParamsASMBuilder;
- StringBuilder threeParamsOutputParamASMBuilder;
-
- StringBuilder fourParamsASMBuilder;
-
- coordBuilder << "Coord coord";
- coordFetchBuilder << "coord";
-
- threeParamsASMBuilder << "$1, $2";
-
- fourParamsASMBuilder << "$1, $2, $3";
- if(isArray)
- {
- coordBuilder << ", uint arrayCoord";
- coordFetchBuilder << ", arrayCoord";
- threeParamsASMBuilder << ", $3";
- fourParamsASMBuilder << ", $4";
- threeParamsOutputParamASMBuilder << "$4";
- }
- else
- {
- threeParamsOutputParamASMBuilder << "$3";
- }
- auto coordString = coordBuilder.toString();
- auto coordFetchString = coordFetchBuilder.toString();
-
- auto threeParamsASMString = threeParamsASMBuilder.toString();
- auto threeParamsOutputParamASMString = threeParamsOutputParamASMBuilder.toString();
-
- auto fourParamsASMString = fourParamsASMBuilder.toString();
-}}}}
-
-${{{{
- for (const char* atomicOperation : {"add", "and", "max", "min", "or", "sub", "xor"})
- {
-}}}}
- __generic<TextureType, T, Coord>
- [ForceInline]
- [require(metal)]
- vector<T, 4> __metalImageInterlocked_$(atomicOperation)(TextureType tex, $(coordString), vector<T, 4> value)
- {
- static_assert(T is int || T is uint, "__metalImageInterlocked only allows 'int'/'uint' textures");
- static_assert(Coord is uint || Coord is vector<uint,2> || Coord is vector<uint,3> || Coord is vector<uint,4>,
- "__metalImageInterlocked implementation only allows 'uint' coordinates");
- __intrinsic_asm "$0.atomic_fetch_$(atomicOperation)($(threeParamsASMString))";
- }
-
- __generic<TextureType, T, Coord>
- [ForceInline]
- [require(metal)]
- void __metalImageInterlocked_$(atomicOperation)(TextureType tex, $(coordString), vector<T, 4> value, out T original_value)
- {
- static_assert(T is int || T is uint, "__metalImageInterlocked only allows 'int'/'uint' textures");
- static_assert(Coord is uint || Coord is vector<uint,2> || Coord is vector<uint,3> || Coord is vector<uint,4>,
- "__metalImageInterlocked implementation only allows 'uint' coordinates");
- original_value = __metalImageInterlocked_$(atomicOperation)(tex, $(coordFetchString), value)[0];
- }
-${{{{
- } // atomicOperation
-}}}}
-
- __generic<TextureType, T, Coord>
- [ForceInline]
- [require(metal)]
- vector<T, 4> __metalImageInterlocked_exchange(TextureType tex, $(coordString), vector<T, 4> value)
- {
- static_assert(T is int || T is uint, "__metalImageInterlocked only allows 'int'/'uint' textures");
- static_assert(Coord is uint || Coord is vector<uint,2> || Coord is vector<uint,3> || Coord is vector<uint,4>,
- "__metalImageInterlocked implementation only allows 'uint' coordinates");
- __intrinsic_asm "($0.atomic_exchange($(threeParamsASMString)))";
- }
- __generic<TextureType, T, Coord>
- [ForceInline]
- [require(metal)]
- void __metalImageInterlocked_exchange(TextureType tex, $(coordString), vector<T, 4> value, out T original_value)
- {
- static_assert(T is int || T is uint, "Metal atomic texture operations only allow 'int'/'uint' textures");
- static_assert(Coord is uint || Coord is vector<uint,2> || Coord is vector<uint,3> || Coord is vector<uint,4>,
- "__metalImageInterlocked implementation only allows 'uint' coordinates");
- original_value = __metalImageInterlocked_exchange(tex, $(coordFetchString), value)[0];
- }
-
- __generic<TextureType, T, Coord>
- [ForceInline]
- [require(metal)]
- void __metalImageInterlocked_compare_exchange(TextureType tex, $(coordString), __ref vector<T, 4> compare_value, vector<T, 4> value)
- {
- static_assert(T is int || T is uint, "__metalImageInterlocked only allows 'int'/'uint' textures");
- static_assert(Coord is uint || Coord is vector<uint,2> || Coord is vector<uint,3> || Coord is vector<uint,4>,
- "__metalImageInterlocked implementation only allows 'uint' coordinates");
- __intrinsic_asm "($0.atomic_compare_exchange_weak($(fourParamsASMString)))";
- }
- __generic<TextureType, T, Coord>
- [ForceInline]
- [require(metal)]
- void __metalImageInterlocked_compare_exchange(TextureType tex, $(coordString), vector<T, 4> compare_value, vector<T, 4> value, out T original_value)
- {
- static_assert(T is int || T is uint, "__metalImageInterlocked only allows 'int'/'uint' textures");
- static_assert(Coord is uint || Coord is vector<uint,2> || Coord is vector<uint,3> || Coord is vector<uint,4>,
- "__metalImageInterlocked implementation only allows 'uint' coordinates");
- __metalImageInterlocked_compare_exchange(tex, $(coordFetchString), compare_value, value);
- original_value = compare_value[0];
- }
-
-${{{{
-} // isArray
-}}}}
-
-${{{{
-
-// Generated functions:
-
-// atomicAdd, InterlockedAdd, atomic_fetch_add_explicit, OpAtomicIAdd, OpAtomicFAddEXT
-// __cudaInterlocked_add, __glslInterlocked_add, __hlslInterlocked_add, __metalInterlocked_add, __spirvInterlocked_add
-
-// atomicAnd, InterlockedAnd, atomic_fetch_and_explicit, OpAtomicAnd
-// __cudaInterlocked_and, __glslInterlocked_and, __hlslInterlocked_and, __metalInterlocked_and, __spirvInterlocked_and
-
-// atomicMax, InterlockedMax, atomic_fetch_max_explicit, OpAtomicUMax, OpAtomicSMax, OpAtomicFMaxEXT
-// __cudaInterlocked_max, __glslInterlocked_max, __hlslInterlocked_max, __metalInterlocked_max, __spirvInterlocked_max
-
-// atomicMin, InterlockedMin, atomic_fetch_min_explicit, OpAtomicUMin, OpAtomicSMin, OpAtomicFMinEXT
-// __cudaInterlocked_min, __glslInterlocked_min, __hlslInterlocked_min, __metalInterlocked_min, __spirvInterlocked_min
-
-// atomicOr, InterlockedOr, atomic_fetch_or_explicit, OpAtomicOr
-// __cudaInterlocked_or, __glslInterlocked_or, __hlslInterlocked_or, __metalInterlocked_or, __spirvInterlocked_or
-
-// atomicXor, InterlockedXor, atomic_fetch_xor_explicit, OpAtomicXor
-// __cudaInterlocked_xor, __glslInterlocked_xor, __hlslInterlocked_xor, __metalInterlocked_xor, __spirvInterlocked_xor
-
-// atomicExchange, atomicExch, InterlockedExchange, atomic_exchange_explicit, OpAtomicExchange
-// __cudaInterlocked_exchange, __glslInterlocked_exchange, __hlslInterlocked_exchange, __metalInterlocked_exchange, __spirvInterlocked_exchange
-
-struct InternalAtomicOperationInfo
-{
- const char* slangSuffix;
- const char* cudaSuffix;
- const char* glslSuffix;
- const char* hlslSuffix;
- const char* metalSuffix;
- const char* spirvFloatSuffix;
- const char* spirvUIntSuffix;
- const char* spirvIntSuffix;
-
- const char* assertExpr;
-};
-
-InternalAtomicOperationInfo internalAtomicOperationInfo[7] = {
- { "add", "Add", "Add", "Add", "fetch_add", "FAddEXT", "IAdd", "IAdd", "true" },
- { "and", "And", "And", "And", "fetch_and", "And", "And", "And", "!__isFloat<T>()" },
- { "max", "Max", "Max", "Max", "fetch_max", "FMaxEXT", "UMax", "SMax", "true" },
- { "min", "Min", "Min", "Min", "fetch_min", "FMinEXT", "UMin", "SMin", "true" },
- { "or", "Or", "Or", "Or", "fetch_or", "Or", "Or", "Or", "!__isFloat<T>()" },
- { "xor", "Xor", "Xor", "Xor", "fetch_xor", "Xor", "Xor", "Xor", "!__isFloat<T>()" },
- { "exchange", "Exch", "Exchange", "Exchange", "exchange", "Exchange", "Exchange", "Exchange", "true" },
-};
-
-for (InternalAtomicOperationInfo atomicOp : internalAtomicOperationInfo)
-{
-}}}}
- __generic<AtomicType, T>
- [ForceInline]
- [require(metal)]
- void __metalInterlocked_$(atomicOp.slangSuffix)(AtomicType dest, T value)
- {
- static_assert($(atomicOp.assertExpr), "Unable to use float with Atomic$(atomicOp.slangSuffix)");
- __intrinsic_asm "atomic_$(atomicOp.metalSuffix)_explicit($0, $1, memory_order_relaxed)";
- }
-
- __generic<AtomicType, T>
- [ForceInline]
- [require(metal)]
- void __metalInterlocked_$(atomicOp.slangSuffix)(AtomicType dest, T value, out T original_value)
- {
- static_assert($(atomicOp.assertExpr), "Unable to use float with Atomic$(atomicOp.slangSuffix)");
- __intrinsic_asm "((*($2)) = (atomic_$(atomicOp.metalSuffix)_explicit($0, $1, memory_order_relaxed)))";
- }
-
- __generic<T>
- [ForceInline]
- [require(cuda)]
- void __cudaInterlocked_$(atomicOp.slangSuffix)(__ref T dest, T value)
- {
- static_assert($(atomicOp.assertExpr), "Unable to use float with Atomic$(atomicOp.slangSuffix)");
- __intrinsic_asm "atomic$(atomicOp.cudaSuffix)((int*)$0, $1)";
- }
-
- __generic<T>
- [ForceInline]
- [require(cuda)]
- void __cudaInterlocked_$(atomicOp.slangSuffix)(__ref T dest, T value, out T original_value)
- {
- static_assert($(atomicOp.assertExpr), "Unable to use float with Atomic$(atomicOp.slangSuffix)");
- __intrinsic_asm "(*$2 = atomic$(atomicOp.cudaSuffix)((int*)$0, $1))";
- }
-
- __generic<T>
- [ForceInline]
- [require(glsl)]
- void __glslInterlocked_$(atomicOp.slangSuffix)(__ref T dest, T value)
- {
- static_assert($(atomicOp.assertExpr), "Unable to use float with Atomic$(atomicOp.slangSuffix)");
- __intrinsic_asm "$atomic$(atomicOp.glslSuffix)($A, $1)";
- }
-
- __generic<T>
- [ForceInline]
- [require(glsl)]
- void __glslInterlocked_$(atomicOp.slangSuffix)(__ref T dest, T value, out T original_value)
- {
- static_assert($(atomicOp.assertExpr), "Unable to use float with Atomic$(atomicOp.slangSuffix)");
- __intrinsic_asm "($2 = $atomic$(atomicOp.glslSuffix)($A, $1))";
- }
-
- __generic<T>
- [ForceInline]
- [require(hlsl)]
- void __hlslInterlocked_$(atomicOp.slangSuffix)(__ref T dest, T value)
- {
- static_assert($(atomicOp.assertExpr), "Unable to use float with Atomic$(atomicOp.slangSuffix)");
- __intrinsic_asm "Interlocked$(atomicOp.hlslSuffix)";
- }
-
- __generic<T>
- [ForceInline]
- [require(hlsl)]
- void __hlslInterlocked_$(atomicOp.slangSuffix)(__ref T dest, T value, out T original_value)
- {
- static_assert($(atomicOp.assertExpr), "Unable to use float with Atomic$(atomicOp.slangSuffix)");
- __intrinsic_asm "Interlocked$(atomicOp.hlslSuffix)";
- }
-
- __generic<T>
- [ForceInline]
- [require(spirv)]
- void __spirvInterlocked_$(atomicOp.slangSuffix)(__ref T dest, T value)
- {
- static_assert($(atomicOp.assertExpr), "Unable to use float with Atomic$(atomicOp.slangSuffix)");
- if (__isFloat<T>())
- {
- spirv_asm
- {
- result:$$T = OpAtomic$(atomicOp.spirvFloatSuffix) &dest Device None $value
- };
- }
- else if (__isUnsignedInt<T>())
- {
- spirv_asm
- {
- result:$$T = OpAtomic$(atomicOp.spirvUIntSuffix) &dest Device None $value
- };
- }
- else if (__isInt<T>())
- {
- spirv_asm
- {
- result:$$T = OpAtomic$(atomicOp.spirvIntSuffix) &dest Device None $value
- };
- }
- }
-
- __generic<T>
- [ForceInline]
- [require(spirv)]
- void __spirvInterlocked_$(atomicOp.slangSuffix)(__ref T dest, T value, out T original_value)
- {
- static_assert($(atomicOp.assertExpr), "Unable to use float with Atomic$(atomicOp.slangSuffix)");
- if (__isFloat<T>())
- {
- spirv_asm
- {
- %original:$$T = OpAtomic$(atomicOp.spirvFloatSuffix) &dest Device None $value;
- OpStore &original_value %original
- };
- }
- else if (__isUnsignedInt<T>())
- {
- spirv_asm
- {
- %original:$$T = OpAtomic$(atomicOp.spirvUIntSuffix) &dest Device None $value;
- OpStore &original_value %original
- };
- }
- else if (__isInt<T>())
- {
- spirv_asm
- {
- %original:$$T = OpAtomic$(atomicOp.spirvIntSuffix) &dest Device None $value;
- OpStore &original_value %original
- };
- }
- }
-
-${{{{
-} // fetchAndModify
-}}}}
-
-__generic<AtomicType, T>
-[ForceInline]
-[require(metal)]
-void __metalInterlocked_compare_exchange(AtomicType dest, __ref T compare_value, T value)
-{
- __intrinsic_asm "atomic_compare_exchange_weak_explicit($0, $1, $2, memory_order_relaxed, memory_order_relaxed)";
-}
-
-__generic<AtomicType, T>
-[ForceInline]
-[require(metal)]
-void __metalInterlocked_compare_exchange(AtomicType dest, T compare_value, T value, out T original_value)
-{
- __metalInterlocked_compare_exchange(dest, compare_value, value);
- original_value = compare_value;
-}
-
-__generic<T>
-__glsl_version(430)
-[ForceInline]
-[require(cuda)]
-void __cudaInterlocked_compare_exchange(__ref T dest, __ref T compare_value, T value)
-{
- __intrinsic_asm "atomicCAS($0, $1, $2)";
-}
-
-__generic<T>
-[ForceInline]
-[require(cuda)]
-void __cudaInterlocked_compare_exchange(__ref T dest, T compare_value, T value, out T original_value)
-{
- __intrinsic_asm "*$3 = atomicCAS($0, $1, $2)";
-}
-
-__generic<T>
-[ForceInline]
-[require(glsl)]
-void __glslInterlocked_compare_exchange(__ref T dest, __ref T compare_value, T value)
-{
- __intrinsic_asm "$atomicCompSwap($A, $1, $2)";
-}
-
-__generic<T>
-[ForceInline]
-[require(glsl)]
-void __glslInterlocked_compare_exchange(__ref T dest, T compare_value, T value, out T original_value)
-{
- __intrinsic_asm "($3 = $atomicCompSwap($A, $1, $2))";
-}
-
-__generic<T>
-[ForceInline]
-[require(hlsl)]
-void __hlslInterlocked_compare_exchange(__ref T dest, __ref T compare_value, T value)
-{
- __intrinsic_asm "InterlockedCompareExchange";
-}
-
-__generic<T>
-[ForceInline]
-[require(hlsl)]
-void __hlslInterlocked_compare_exchange(__ref T dest, T compare_value, T value, out T original_value)
-{
- __intrinsic_asm "InterlockedCompareExchange";
-}
-
-__generic<T>
-[ForceInline]
-[require(spirv)]
-void __spirvInterlocked_compare_exchange(__ref T dest, __ref T compare_value, T value)
-{
- spirv_asm
- {
- %result:$$T = OpAtomicCompareExchange &dest Device None None $value $compare_value;
- };
-}
-
-__generic<T>
-[ForceInline]
-[require(spirv)]
-void __spirvInterlocked_compare_exchange(__ref T dest, T compare_value, T value, out T original_value)
-{
- spirv_asm
- {
- %original:$$T = OpAtomicCompareExchange &dest Device None None $value $compare_value;
- OpStore &original_value %original
- };
-}
-
-__generic<T>
-[ForceInline]
-[require(hlsl)]
-void __hlslInterlocked_compare_exchange_float_bitwise(__ref T dest, T compare_value, T value)
-{
- __intrinsic_asm "InterlockedCompareExchangeFloatBitwise";
-}
-
-__generic<T>
-[ForceInline]
-[require(hlsl)]
-void __hlslInterlocked_compare_exchange_float_bitwise(__ref T dest, T compare_value, T value, out T original_value)
-{
- __intrinsic_asm "InterlockedCompareExchangeFloatBitwise";
-}
-
-${{{{
// Generates code for:
// InterlockedAdd, InterlockedAnd, InterlockedOr, InterlockedXor,
// InterlockedMax, InterlockedMin, InterlockedExchange
@@ -10153,516 +8818,166 @@ struct SlangAtomicOperationInfo
{
const char* slangCallSuffix;
const char* internalCallSuffix;
+ const char* interface;
};
SlangAtomicOperationInfo slangAtomicOperationInfo[7] = {
- { "Add", "add" },
- { "And", "and" },
- { "Or", "or" },
- { "Xor", "xor" },
- { "Max", "max" },
- { "Min", "min" },
- { "Exchange", "exchange" },
+ { "Add", "add", "IArithmeticAtomicable" },
+ { "And", "and", "IArithmeticAtomicable" },
+ { "Or", "or", "IArithmeticAtomicable" },
+ { "Xor", "xor", "IArithmeticAtomicable" },
+ { "Max", "max", "IArithmeticAtomicable" },
+ { "Min", "min", "IArithmeticAtomicable" },
+ { "Exchange", "exchange", "IAtomicable" },
};
for (SlangAtomicOperationInfo atomicOp : slangAtomicOperationInfo)
{
- for(const char* T : {"int", "uint"})
- {
}}}}
+/// Perform an atomic $(atomicOp.internalCallSuffix) operation on `dest`.
+/// @param T The type of the value to perform the atomic operation on.
+/// @param dest The value to perform the atomic operation on.
+/// @param value The operand to the atomic operation.
+/// @param original_value The value of `dest` before the operation.
+/// @remarks When targeting HLSL, it is invalid to call this function with `T` being a floating-point type, since
+/// HLSL does not allow atomic operations on floating point types. For `InterlockedAdd`, consider using
+/// `RWByteAddressBuffer.InterlockedAddF32` or `RWByteAddressBuffer.InterlockedAddF16` instead when NVAPI is available.
+/// On SPIR-V (Vulkan), all integer and floating point types are supported.
+/// On Metal and WGSL, all floating-point types are not supported.
+/// @category atomic Atomic functions
[ForceInline]
__glsl_version(430)
[require(cuda_glsl_hlsl_metal_spirv, atomic_glsl_hlsl_cuda_metal)]
-void Interlocked$(atomicOp.slangCallSuffix)(__ref $(T) dest, $(T) value)
+void Interlocked$(atomicOp.slangCallSuffix)<T:$(atomicOp.interface)>(__ref T dest, T value)
{
- static_assert(__isTextureScalarAccess(dest) || !__isTextureAccess(dest), "Atomic must be applied to scalar texture or non-texture");
- __target_switch
- {
- case hlsl: __hlslInterlocked_$(atomicOp.internalCallSuffix)(dest, value);
- case cuda: __cudaInterlocked_$(atomicOp.internalCallSuffix)(dest, value);
- case glsl: __glslInterlocked_$(atomicOp.internalCallSuffix)(dest, value);
- case spirv: __spirvInterlocked_$(atomicOp.internalCallSuffix)(dest, value);
- case metal:
- if (__isTextureAccess(dest))
- {
- if(__isTextureArrayAccess(dest))
- {
- __metalImageInterlocked_$(atomicOp.internalCallSuffix)(__extractTextureFromTextureAccess(dest),
- __extractCoordFromTextureAccess(dest), __extractArrayCoordFromTextureAccess(dest), vector<$(T), 4>(value));
- }
- else
- {
- __metalImageInterlocked_$(atomicOp.internalCallSuffix)(__extractTextureFromTextureAccess(dest),
- __extractCoordFromTextureAccess(dest), vector<$(T), 4>(value));
- }
- }
- else
- {
- __metalInterlocked_$(atomicOp.internalCallSuffix)(__getMetalAtomicRef(dest), value);
- }
- return;
- }
+ __atomic_$(atomicOp.internalCallSuffix)(dest, value);
}
[ForceInline]
__glsl_version(430)
[require(cuda_glsl_hlsl_metal_spirv, atomic_glsl_hlsl_cuda_metal)]
-void Interlocked$(atomicOp.slangCallSuffix)(__ref $(T) dest, $(T) value, out $(T) original_value)
+void Interlocked$(atomicOp.slangCallSuffix)<T:$(atomicOp.interface)>(__ref T dest, T value, out T original_value)
{
- static_assert(__isTextureScalarAccess(dest) || !__isTextureAccess(dest), "Atomic must be applied to a scalar texture or non-texture");
- __target_switch
- {
- case hlsl: __hlslInterlocked_$(atomicOp.internalCallSuffix)(dest, value, original_value);
- case cuda: __cudaInterlocked_$(atomicOp.internalCallSuffix)(dest, value, original_value);
- case glsl: __glslInterlocked_$(atomicOp.internalCallSuffix)(dest, value, original_value);
- case spirv: __spirvInterlocked_$(atomicOp.internalCallSuffix)(dest, value, original_value);
- case metal:
- if (__isTextureAccess(dest))
- if(__isTextureArrayAccess(dest))
- {
- __metalImageInterlocked_$(atomicOp.internalCallSuffix)(__extractTextureFromTextureAccess(dest),
- __extractCoordFromTextureAccess(dest), __extractArrayCoordFromTextureAccess(dest), vector<$(T),4>(value), original_value);
- }
- else
- {
- __metalImageInterlocked_$(atomicOp.internalCallSuffix)(__extractTextureFromTextureAccess(dest),
- __extractCoordFromTextureAccess(dest), vector<$(T),4>(value), original_value);
- }
- else
- __metalInterlocked_$(atomicOp.internalCallSuffix)(__getMetalAtomicRef(dest), value, original_value);
- return;
- }
+ original_value = __atomic_$(atomicOp.internalCallSuffix)(dest, value);
}
-${{{{
- } // for(const char* T : {"int64_t", "uint64_t"})
-}}}}
-
[ForceInline]
+__glsl_version(430)
+[require(cuda_glsl_hlsl_metal_spirv, atomic_glsl_hlsl_cuda_metal)]
void Interlocked$(atomicOp.slangCallSuffix)(__ref uint dest, int value)
{
- Interlocked$(atomicOp.slangCallSuffix)(dest, (uint)value);
+ __atomic_$(atomicOp.internalCallSuffix)(dest, (uint)value);
}
${{{{
} // for (SlangAtomicOperationInfo atomicOp : slangAtomicOperationInfo)
}}}}
-${{{{
-for(const char* T : {"int64_t", "uint64_t"})
-{
-}}}}
-/// @category atomic Atomic functions
-[ForceInline]
-[require(cuda_glsl_hlsl_spirv, atomic_glsl_hlsl_cuda_metal)]
-void InterlockedAdd(__ref $(T) dest, $(T) value)
-{
- __target_switch
- {
- case hlsl: __hlslInterlocked_add(dest, value);
- case cuda: __cudaInterlocked_add(dest, value);
- case glsl:
- __requireGLSLExtension("GL_EXT_shader_atomic_int64");
- __glslInterlocked_add(dest, value);
- case spirv:
- spirv_asm
- {
- OpCapability Int64Atomics;
- result:$$$(T) = OpAtomicIAdd &dest Device None $value;
- };
- }
-}
-
-[ForceInline]
-void InterlockedAdd(__ref $(T) dest, $(T) value, out $(T) original_value)
-{
- __target_switch
- {
- case hlsl: __hlslInterlocked_add(dest, value, original_value);
- case cuda: __cudaInterlocked_add(dest, value, original_value);
- case glsl:
- __requireGLSLExtension("GL_EXT_shader_atomic_int64");
- __glslInterlocked_add(dest, value, original_value);
- case spirv:
- spirv_asm
- {
- OpCapability Int64Atomics;
- %origin:$$$(T) = OpAtomicIAdd &dest Device None $value;
- OpStore &original_value %origin
- };
- }
-}
-
-/// @category atomic
-[ForceInline]
-void InterlockedAnd(__ref $(T) dest, $(T) value)
-{
- __target_switch
- {
- case hlsl: __hlslInterlocked_and(dest, value);
- }
-}
-
-[ForceInline]
-void InterlockedAnd(__ref $(T) dest, $(T) value, out $(T) original_value)
-{
- __target_switch
- {
- case hlsl: __hlslInterlocked_and(dest, value, original_value);
- }
-}
-
-/// @category atomic
-[ForceInline]
-void InterlockedCompareExchange(__ref $(T) dest, $(T) compare_value, $(T) value)
-{
- __target_switch
- {
- case hlsl: __hlslInterlocked_compare_exchange(dest, compare_value, value);
- }
-}
-
-[ForceInline]
-void InterlockedCompareExchange(__ref $(T) dest, $(T) compare_value, $(T) value, out $(T) original_value)
-{
- __target_switch
- {
- case hlsl: __hlslInterlocked_compare_exchange(dest, compare_value, value, original_value);
- }
-}
-
-[ForceInline]
-void InterlockedCompareStore(__ref $(T) dest, $(T) compare_value, $(T) value);
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "InterlockedCompareStore";
- }
-}
-
-/// @category atomic
-[ForceInline]
-void InterlockedExchange(__ref $(T) dest, $(T) value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "InterlockedExchange";
- }
-}
-
-[ForceInline]
-void InterlockedExchange(__ref $(T) dest, $(T) value, out $(T) original_value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "InterlockedExchange";
- }
-}
-
-/// @category atomic
-[ForceInline]
-void InterlockedMax(__ref $(T) dest, $(T) value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "InterlockedMax";
- }
-}
-
-[ForceInline]
-void InterlockedMax(__ref $(T) dest, $(T) value, out $(T) original_value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "InterlockedMax";
- }
-}
-
-/// @category atomic
-[ForceInline]
-void InterlockedMin(__ref $(T) dest, $(T) value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "InterlockedMin";
- }
-}
-
-[ForceInline]
-void InterlockedMin(__ref $(T) dest, $(T) value, out $(T) original_value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "InterlockedMin";
- }
-}
-
-/// @category atomic
-[ForceInline]
-void InterlockedOr(__ref $(T) dest, $(T) value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "InterlockedOr";
- }
-}
-
-[ForceInline]
-void InterlockedOr(__ref $(T) dest, $(T) value, out $(T) original_value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "InterlockedOr";
- }
-}
-
-/// @category atomic
-[ForceInline]
-void InterlockedXor(__ref $(T) dest, $(T) value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "InterlockedXor";
- }
-}
-
-[ForceInline]
-void InterlockedXor(__ref $(T) dest, $(T) value, out $(T) original_value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "InterlockedXor";
- }
-}
-
-${{{{
-} // for(const char* T : {"int64_t", "uint64_t"})
-}}}}
-
+/// Perform an atomic compare and exchange operation on `dest`.
+/// @param T The type of the value to perform the atomic operation on.
+/// @param dest The value to perform the atomic operation on.
+/// @param compare_value The value to compare `dest` with.
+/// @param value The value to store into `dest` if the compare result is equal.
+/// @param original_value The value of `dest` before the operation.
+/// @remarks When targeting HLSL, a call to this function with `T` being `float` will translate to a call to
+/// `InterlockedCompareExchangeFloatBitwise`, which means the comparison is done as a bitwise comparison.
+///
+/// On SPIR-V (Vulkan), this function maps to `OpAtomicCompareExchange`.
+///
+/// On Metal and WGSL, all floating-point types are not supported.
+///
+/// On CUDA, this function maps to `atomicCAS`.
/// @category atomic
[ForceInline]
-__glsl_version(430)
[require(cuda_glsl_hlsl_metal_spirv, atomic_glsl_hlsl_cuda_metal)]
-void InterlockedCompareExchange(__ref int dest, int compare_value, int value, out int original_value)
+void InterlockedCompareExchange<T:IAtomicable>(__ref T dest, T compare_value, T value, out T original_value)
{
- static_assert(__isTextureScalarAccess(dest) || !__isTextureAccess(dest), "Atomic must be applied to scalar texture or non-texture");
- __target_switch
- {
- case hlsl: __hlslInterlocked_compare_exchange(dest, compare_value, value, original_value);
- case glsl: __glslInterlocked_compare_exchange(dest, compare_value, value, original_value);
- case cuda: __cudaInterlocked_compare_exchange(dest, compare_value, value, original_value);
- case spirv: __spirvInterlocked_compare_exchange(dest, compare_value, value, original_value);
- case metal:
- if (__isTextureAccess(dest))
- {
- vector<int, 4> vec_compare_value = vector<int, 4>(compare_value);
- if(__isTextureArrayAccess(dest))
- {
- __metalImageInterlocked_compare_exchange(__extractTextureFromTextureAccess(dest),
- __extractCoordFromTextureAccess(dest), __extractArrayCoordFromTextureAccess(dest), vec_compare_value, vector<int, 4>(value), original_value);
- }
- else
- {
- __metalImageInterlocked_compare_exchange(__extractTextureFromTextureAccess(dest),
- __extractCoordFromTextureAccess(dest), vec_compare_value, vector<int, 4>(value), original_value);
- }
- }
- else
- {
- __metalInterlocked_compare_exchange(__getMetalAtomicRef(dest), compare_value, value, original_value);
- }
- return;
- }
-}
-
-[ForceInline]
-__glsl_version(430)
-[require(cuda_glsl_hlsl_metal_spirv, atomic_glsl_hlsl_cuda_metal)]
-void InterlockedCompareExchange(__ref uint dest, uint compare_value, uint value, out uint original_value)
-{
- static_assert(__isTextureScalarAccess(dest) || !__isTextureAccess(dest), "Atomic must be applied to scalar texture or non-texture");
- __target_switch
- {
- case hlsl: __hlslInterlocked_compare_exchange(dest, compare_value, value, original_value);
- case cuda: __cudaInterlocked_compare_exchange(dest, compare_value, value, original_value);
- case glsl: __glslInterlocked_compare_exchange(dest, compare_value, value, original_value);
- case spirv: __spirvInterlocked_compare_exchange(dest, compare_value, value, original_value);
- case metal:
- if (__isTextureAccess(dest))
- {
- vector<uint, 4> vec_compare_value = vector<uint, 4>(compare_value);
- if(__isTextureArrayAccess(dest))
- {
- __metalImageInterlocked_compare_exchange(__extractTextureFromTextureAccess(dest),
- __extractCoordFromTextureAccess(dest), __extractArrayCoordFromTextureAccess(dest), vec_compare_value, vector<uint, 4>(value), original_value);
- }
- else
- {
- __metalImageInterlocked_compare_exchange(__extractTextureFromTextureAccess(dest),
- __extractCoordFromTextureAccess(dest), vec_compare_value, vector<uint, 4>(value), original_value);
- }
- }
- else
- {
- __metalInterlocked_compare_exchange(__getMetalAtomicRef(dest), compare_value, value, original_value);
- }
- return;
- }
+ original_value = __atomic_compare_exchange(dest, compare_value, value);
}
+/// Perform an atomic compare and exchange operation on `dest`.
+/// @param T The type of the value to perform the atomic operation on.
+/// @param dest The value to perform the atomic operation on.
+/// @param compare_value The value to compare `dest` with.
+/// @param value The value to store into `dest` if the compare result is equal.
+/// @param original_value The value of `dest` before the operation.
+/// @remarks When targeting HLSL, a call to this function will translate to a call to
+/// `InterlockedCompareExchangeFloatBitwise`, which means the comparison is done as a bitwise comparison.
+///
+/// On SPIR-V (Vulkan), this function maps to `OpAtomicCompareExchange`.
+///
+/// On Metal and WGSL, this function is not available.
+///
+/// On CUDA, this function maps to `atomicCAS`.
/// @category atomic
[ForceInline]
void InterlockedCompareExchangeFloatBitwise(__ref float dest, float compare_value, float value)
{
- static_assert(__isTextureScalarAccess(dest) || !__isTextureAccess(dest), "Atomic must be applied to scalar texture or non-texture");
- __target_switch
- {
- case hlsl: __hlslInterlocked_compare_exchange_float_bitwise(dest, compare_value, value);
- case metal:
- static_assert(!__isTextureAccess(dest), "float atomic texture operations are disallowed with Metal target's");
- __metalInterlocked_compare_exchange(__getMetalAtomicRef(dest), compare_value, value);
- return;
- }
+ __atomic_compare_exchange(dest, compare_value, value);
}
[ForceInline]
void InterlockedCompareExchangeFloatBitwise(__ref float dest, float compare_value, float value, out float original_value)
{
- static_assert(__isTextureScalarAccess(dest) || !__isTextureAccess(dest), "Atomic must be applied to scalar texture or non-texture");
- __target_switch
- {
- case hlsl: __hlslInterlocked_compare_exchange_float_bitwise(dest, compare_value, value, original_value);
- case metal:
- static_assert(!__isTextureAccess(dest), "float atomic texture operations are disallowed with Metal target's");
- __metalInterlocked_compare_exchange(__getMetalAtomicRef(dest), compare_value, value, original_value);
- return;
- }
+ original_value = __atomic_compare_exchange(dest, compare_value, value);
}
+/// Perform an atomic compare and store operation on `dest`.
+/// @param T The type of the value to perform the atomic operation on.
+/// @param dest The value to perform the atomic operation on.
+/// @param compare_value The value to compare `dest` with.
+/// @param value The value to store into `dest` if the compare result is equal.
+/// @remarks When targeting HLSL, a call to this function with `T` being `float` will translate to a call to
+/// `InterlockedCompareStoreFloatBitwise`, which means the comparison is done as a bitwise comparison.
+///
+/// On SPIR-V (Vulkan), this function maps to `OpAtomicCompareExchange`.
+///
+/// On Metal and WGSL, this function is not available.
+///
+/// On CUDA, this function maps to `atomicCAS`.
/// @category atomic
[ForceInline]
__glsl_version(430)
[require(cuda_glsl_hlsl_metal_spirv, atomic_glsl_hlsl_cuda_metal)]
-void InterlockedCompareStore(__ref int dest, int compare_value, int value)
+void InterlockedCompareStore<T:IAtomicable>(__ref T dest, T compare_value, T value)
{
__target_switch
{
case hlsl: __intrinsic_asm "InterlockedCompareStore";
- case glsl: __intrinsic_asm "$atomicCompSwap($A, $1, $2)";
- case cuda: __intrinsic_asm "atomicCAS($0, $1, $2)";
- case spirv:
- {
- spirv_asm
- {
- result:$$int = OpAtomicCompareExchange &dest Device None None $value $compare_value;
- };
- return;
- }
- case metal:
- {
- if (__isTextureAccess(dest))
- {
- vector<int, 4> vec_compare_value = vector<int, 4>(compare_value);
- if(__isTextureArrayAccess(dest))
- {
- __metalImageInterlocked_compare_exchange(__extractTextureFromTextureAccess(dest),
- __extractCoordFromTextureAccess(dest), __extractArrayCoordFromTextureAccess(dest), vec_compare_value, vector<int, 4>(value));
- }
- else
- {
- __metalImageInterlocked_compare_exchange(__extractTextureFromTextureAccess(dest),
- __extractCoordFromTextureAccess(dest), vec_compare_value, vector<int, 4>(value));
- }
- }
- else
- {
- __metalInterlocked_compare_exchange(__getMetalAtomicRef(dest), compare_value, value);
- }
- return;
- }
- }
-}
-
-[ForceInline]
-__glsl_version(430)
-[require(cuda_glsl_hlsl_metal_spirv, atomic_glsl_hlsl_cuda_metal)]
-void InterlockedCompareStore(__ref uint dest, uint compare_value, uint value)
-{
- __target_switch
- {
- case hlsl: __intrinsic_asm "InterlockedCompareStore";
- case glsl: __intrinsic_asm "$atomicCompSwap($A, $1, $2)";
- case cuda: __intrinsic_asm "atomicCAS((int*)$0, $1, $2)";
- case spirv:
- spirv_asm
- {
- result:$$uint = OpAtomicCompareExchange &dest Device None None $value $compare_value;
- };
- case metal:
- if (__isTextureAccess(dest))
- {
- vector<uint, 4> vec_compare_value = vector<uint, 4>(compare_value);
- if(__isTextureArrayAccess(dest))
- {
- __metalImageInterlocked_compare_exchange(__extractTextureFromTextureAccess(dest),
- __extractCoordFromTextureAccess(dest), __extractArrayCoordFromTextureAccess(dest), vec_compare_value, vector<uint, 4>(value));
- }
- else
- {
- __metalImageInterlocked_compare_exchange(__extractTextureFromTextureAccess(dest),
- __extractCoordFromTextureAccess(dest), vec_compare_value, vector<uint, 4>(value));
- }
- }
- else
- {
- __metalInterlocked_compare_exchange(__getMetalAtomicRef(dest), compare_value, value);
- }
+ default:
+ __atomic_compare_exchange(dest, compare_value, value);
return;
}
}
+/// Perform an atomic compare and store operation on `dest`.
+/// @param T The type of the value to perform the atomic operation on.
+/// @param dest The value to perform the atomic operation on.
+/// @param compare_value The value to compare `dest` with.
+/// @param value The value to store into `dest` if the compare result is equal.
+/// @remarks When targeting HLSL, a call to this function will translate to a call to
+/// `InterlockedCompareStoreFloatBitwise`, which means the comparison is done as a bitwise comparison.
+///
+/// On SPIR-V (Vulkan), this function maps to `OpAtomicCompareExchange`.
+///
+/// On Metal and WGSL, this function is not available.
+///
+/// On CUDA, this function maps to `atomicCAS`.
/// @category atomic
[ForceInline]
-void InterlockedCompareStoreFloatBitwise(__ref float dest, float compare_value, float value)
+void InterlockedCompareStoreFloatBitwise<T:IAtomicable>(__ref T dest, T compare_value, T value)
{
__target_switch
{
case hlsl: __intrinsic_asm "InterlockedCompareStoreFloatBitwise";
- }
-}
-
-/// @category atomic
-[ForceInline]
-void InterlockedExchange(__ref float dest, float value)
-{
- static_assert(__isTextureScalarAccess(dest) || !__isTextureAccess(dest), "Atomic must be applied to scalar texture or non-texture");
- __target_switch
- {
- case hlsl: __hlslInterlocked_exchange(dest, value);
- case metal:
- static_assert(!__isTextureAccess(dest), "'float' atomic texture operations are disallowed with Metal target's");
- __metalInterlocked_exchange(__getMetalAtomicRef(dest), value);
- return;
- }
-}
-
-[ForceInline]
-void InterlockedExchange(__ref float dest, float value, out float original_value)
-{
- static_assert(__isTextureScalarAccess(dest) || !__isTextureAccess(dest), "Atomic must be applied to scalar texture or non-texture");
- __target_switch
- {
- case hlsl: __hlslInterlocked_exchange(dest, value, original_value);
- case metal:
- static_assert(!__isTextureAccess(dest), "'float' atomic texture operations are disallowed with Metal target's");
- __metalInterlocked_exchange(__getMetalAtomicRef(dest), value, original_value);
+ default:
+ __atomic_compare_exchange(dest, compare_value, value);
return;
}
}
-
/// Test if a floating-point value finite.
/// @category math
__generic<T : __BuiltinFloatingPointType>
@@ -21245,13 +19560,13 @@ extension _Texture<float, Shape, 0, 0, 0, $(kStdlibResourceAccessReadWrite), 0,
{
__target_switch
{
- case spirv:
- originalValue = __atomicAdd(this[coord], value);
+ default:
+ originalValue = __atomic_add(this[coord], value);
return;
- case glsl:
- __intrinsic_asm "$3 = imageAtomicAdd($0, $1, $2)";
case hlsl:
__intrinsic_asm "$3 = NvInterlockedAddFp32($0, $1, $2)";
+ case glsl:
+ __intrinsic_asm "$3 = imageAtomicAdd($0, $1, $2)";
}
}