diff options
Diffstat (limited to 'source/slang/slang-emit-spirv.cpp')
| -rw-r--r-- | source/slang/slang-emit-spirv.cpp | 262 |
1 files changed, 144 insertions, 118 deletions
diff --git a/source/slang/slang-emit-spirv.cpp b/source/slang/slang-emit-spirv.cpp index 3a8a913ec..8bcd1429f 100644 --- a/source/slang/slang-emit-spirv.cpp +++ b/source/slang/slang-emit-spirv.cpp @@ -4569,33 +4569,37 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex break; case kIROp_AtomicLoad: { - IRBuilder builder{inst}; - if (isAtomicableAddressSpace(inst->getOperand(0)->getDataType())) + IRAtomicLoad* atomicLoad = as<IRAtomicLoad>(inst); + auto ptr = atomicLoad->getPtr(); + IRBuilder builder{atomicLoad}; + if (isAtomicableAddressSpace(ptr->getDataType())) { if (m_memoryModel == SpvMemoryModelVulkan) requireSPIRVCapability(SpvCapabilityVulkanMemoryModelDeviceScope); const auto memoryScope = emitIntConstant(IRIntegerValue{SpvScopeDevice}, builder.getUIntType()); - const auto memorySemantics = - emitMemorySemanticMask(inst->getOperand(1), inst->getOperand(0)); + const auto memorySemantics = emitMemorySemanticMask(inst->getOperand(1), ptr); result = emitOpAtomicLoad( parent, inst, inst->getFullType(), - inst->getOperand(0), + ptr, memoryScope, memorySemantics); ensureAtomicCapability(inst, SpvOpAtomicLoad); } else { - result = emitLoadMaybeCoherent(parent, inst); + result = emitLoad(parent, inst, ptr); } } break; case kIROp_AtomicStore: { + IRAtomicStore* atomicStore = as<IRAtomicStore>(inst); + auto ptr = atomicStore->getPtr(); + auto val = atomicStore->getVal(); IRBuilder builder{inst}; if (isAtomicableAddressSpace(inst->getOperand(0)->getDataType())) { @@ -4604,48 +4608,44 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex const auto memoryScope = emitIntConstant(IRIntegerValue{SpvScopeDevice}, builder.getUIntType()); - const auto memorySemantics = - emitMemorySemanticMask(inst->getOperand(2), inst->getOperand(0)); - result = emitOpAtomicStore( - parent, - inst, - inst->getOperand(0), - memoryScope, - memorySemantics, - inst->getOperand(1)); + const auto memorySemantics = emitMemorySemanticMask(inst->getOperand(2), ptr); + result = + emitOpAtomicStore(parent, inst, ptr, memoryScope, memorySemantics, val); ensureAtomicCapability(inst, SpvOpAtomicStore); } else { - result = emitStoreMaybeCoherent(parent, inst); + result = emitStore(parent, inst, ptr, val); } } break; case kIROp_AtomicExchange: { + IRAtomicExchange* atomicExchange = as<IRAtomicExchange>(inst); + auto ptr = atomicExchange->getPtr(); + auto val = atomicExchange->getOperand(1); IRBuilder builder{inst}; - if (isAtomicableAddressSpace(inst->getOperand(0)->getDataType())) + if (isAtomicableAddressSpace(ptr->getDataType())) { if (m_memoryModel == SpvMemoryModelVulkan) requireSPIRVCapability(SpvCapabilityVulkanMemoryModelDeviceScope); const auto memoryScope = emitIntConstant(IRIntegerValue{SpvScopeDevice}, builder.getUIntType()); - const auto memorySemantics = - emitMemorySemanticMask(inst->getOperand(2), inst->getOperand(0)); + const auto memorySemantics = emitMemorySemanticMask(inst->getOperand(2), ptr); result = emitOpAtomicExchange( parent, inst, inst->getFullType(), - inst->getOperand(0), + ptr, memoryScope, memorySemantics, - inst->getOperand(1)); + val); ensureAtomicCapability(inst, SpvOpAtomicExchange); } else { - result = emitStoreMaybeCoherent(parent, inst); + result = emitStore(parent, inst, ptr, val); } } break; @@ -7082,6 +7082,8 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex SpvInst* emitGetOffsetPtr(SpvInstParent* parent, IRInst* inst) { + requireVariableBufferCapabilityIfNeeded(inst->getDataType()); + return emitOpPtrAccessChain( parent, inst, @@ -7174,54 +7176,100 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex } } - SpvInst* emitLoad(SpvInstParent* parent, IRLoad* inst) + enum class MemoryAccessType { - requireVariableBufferCapabilityIfNeeded(inst->getDataType()); + Load, + Store + }; + + template<MemoryAccessType memoryAccessType> + void getMemoryAccessOperandsOfLoadStore( + IRInst* inst, + IRInst* ptr, + int& memoryAccessMaskOut, + int& alignmentOut, + MemoryScope& memoryScopeOut) + { + IRAlignedAttr* alignedAttr = nullptr; + IRMemoryScopeAttr* memoryScopeAttr = nullptr; - auto ptrType = as<IRPtrTypeBase>(inst->getPtr()->getDataType()); - if (ptrType && addressSpaceToStorageClass(ptrType->getAddressSpace()) == - SpvStorageClassPhysicalStorageBuffer) + for (auto attr : inst->getAllAttrs()) { - IRSizeAndAlignment sizeAndAlignment; - if (auto alignedAttr = inst->findAttr<IRAlignedAttr>()) + if (auto foundAlignedAttr = as<IRAlignedAttr>(attr)) + alignedAttr = foundAlignedAttr; + else if (auto foundMemoryScopeAttr = as<IRMemoryScopeAttr>(attr)) + memoryScopeAttr = foundMemoryScopeAttr; + } + + // Determine coherence + { + bool isCoherent = false; + if (memoryScopeAttr) { - sizeAndAlignment.alignment = (int)getIntVal(alignedAttr->getAlignment()); + memoryScopeOut = (MemoryScope)getIntVal(memoryScopeAttr->getMemoryScope()); + if (m_memoryModel != SpvMemoryModelVulkan) + SLANG_ASSERT_FAILURE( + "Explicit coherent operations require vulkan-memory-model, " + "specify the capability 'vk_mem_model'"); + isCoherent = true; } else { - getNaturalSizeAndAlignment( - m_targetProgram->getOptionSet(), - ptrType->getValueType(), - &sizeAndAlignment); + if (NeedToUseCoherentLoadOrStore(ptr)) + { + memoryScopeOut = MemoryScope::Device; + isCoherent = true; + } + } + if (isCoherent) + { + + memoryAccessMaskOut |= SpvMemoryAccessNonPrivatePointerMask; + if constexpr (memoryAccessType == MemoryAccessType::Load) + memoryAccessMaskOut |= SpvMemoryAccessMakePointerVisibleMask; + else + memoryAccessMaskOut |= SpvMemoryAccessMakePointerAvailableMask; + if (memoryScopeOut == MemoryScope::Device) + requireSPIRVCapability(SpvCapabilityVulkanMemoryModelDeviceScope); } - return emitOpLoadAligned( - parent, - inst, - inst->getDataType(), - inst->getPtr(), - SpvLiteralInteger::from32(sizeAndAlignment.alignment)); } - else + + // Determine alignment { - return emitLoadMaybeCoherent(parent, inst); + auto ptrType = as<IRPtrTypeBase>(ptr->getDataType()); + if (ptrType && addressSpaceToStorageClass(ptrType->getAddressSpace()) == + SpvStorageClassPhysicalStorageBuffer) + { + IRSizeAndAlignment sizeAndAlignment; + if (alignedAttr) + sizeAndAlignment.alignment = (int)getIntVal(alignedAttr->getAlignment()); + else + getNaturalSizeAndAlignment( + m_targetProgram->getOptionSet(), + ptrType->getValueType(), + &sizeAndAlignment); + + alignmentOut = sizeAndAlignment.alignment; + if (alignmentOut != -1) + memoryAccessMaskOut |= SpvMemoryAccessAlignedMask; + } } } - SpvInst* emitLoadMaybeCoherent(SpvInstParent* parent, IRInst* inst) + SpvInst* emitLoad(SpvInstParent* parent, IRInst* inst, IRInst* ptr) { - IRBuilder builder{inst}; - builder.setInsertBefore(inst); - - SpvInst* deviceScope = nullptr; - IRInst* pointer = inst->getOperand(0); - - bool coherentPointer = NeedToUseCoherentLoadOrStore(pointer); - if (coherentPointer) - { - requireSPIRVCapability(SpvCapabilityVulkanMemoryModelDeviceScope); - deviceScope = emitIntConstant(IRIntegerValue{SpvScopeDevice}, builder.getUIntType()); - } + requireVariableBufferCapabilityIfNeeded(inst->getDataType()); + IRBuilder builder(inst); + int memoryAccessMask = 0; + int alignment = -1; + MemoryScope memoryScope{}; + getMemoryAccessOperandsOfLoadStore<MemoryAccessType::Load>( + inst, + ptr, + memoryAccessMask, + alignment, + memoryScope); return emitInstCustomOperandFunc( parent, inst, @@ -7230,85 +7278,61 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex { emitOperand(inst->getFullType()); emitOperand(kResultID); - emitOperand(pointer); - - if (coherentPointer) + emitOperand(ptr); + if (memoryAccessMask) { - emitOperand( - SpvMemoryAccessMakePointerVisibleMask | - SpvMemoryAccessNonPrivatePointerMask); - - emitOperand(deviceScope); + emitOperand(SpvLiteralInteger::from32(memoryAccessMask)); + if (memoryAccessMask & SpvMemoryAccessAlignedMask) + emitOperand(SpvLiteralInteger::from32((uint32_t)alignment)); + if (memoryAccessMask & SpvMemoryAccessMakePointerVisibleMask) + emitOperand( + emitIntConstant((IRIntegerValue)memoryScope, builder.getIntType())); } }); } - SpvInst* emitStore(SpvInstParent* parent, IRStore* inst) + SpvInst* emitLoad(SpvInstParent* parent, IRLoad* inst) { - auto ptrType = as<IRPtrTypeBase>(inst->getPtr()->getDataType()); - if (ptrType && addressSpaceToStorageClass(ptrType->getAddressSpace()) == - SpvStorageClassPhysicalStorageBuffer) - { - IRSizeAndAlignment sizeAndAlignment; - if (auto alignedAttr = inst->findAttr<IRAlignedAttr>()) - { - sizeAndAlignment.alignment = (int)getIntVal(alignedAttr->getAlignment()); - } - else - { - getNaturalSizeAndAlignment( - m_targetProgram->getOptionSet(), - ptrType->getValueType(), - &sizeAndAlignment); - } - return emitOpStoreAligned( - parent, - inst, - inst->getPtr(), - inst->getVal(), - SpvLiteralInteger::from32(sizeAndAlignment.alignment)); - } - else - { - return emitStoreMaybeCoherent(parent, inst); - } + return emitLoad(parent, inst, inst->getPtr()); } - SpvInst* emitStoreMaybeCoherent(SpvInstParent* parent, IRInst* inst) + SpvInst* emitStore(SpvInstParent* parent, IRInst* inst, IRInst* ptr, IRInst* val) { - IRBuilder builder{inst}; - builder.setInsertBefore(inst); - - SpvInst* deviceScope = nullptr; - IRInst* pointer = inst->getOperand(0); - IRInst* object = inst->getOperand(1); - - bool coherentPointer = NeedToUseCoherentLoadOrStore(pointer); - if (coherentPointer) - { - requireSPIRVCapability(SpvCapabilityVulkanMemoryModelDeviceScope); - deviceScope = emitIntConstant(IRIntegerValue{SpvScopeDevice}, builder.getUIntType()); - } + requireVariableBufferCapabilityIfNeeded(inst->getDataType()); + IRBuilder builder(inst); + int memoryAccessMask = 0; + int alignment = -1; + MemoryScope memoryScope{}; + getMemoryAccessOperandsOfLoadStore<MemoryAccessType::Store>( + inst, + ptr, + memoryAccessMask, + alignment, + memoryScope); return emitInstCustomOperandFunc( parent, inst, SpvOpStore, [&]() { - emitOperand(pointer); - emitOperand(object); - - if (coherentPointer) + emitOperand(ptr); + emitOperand(val); + if (memoryAccessMask) { - emitOperand( - SpvMemoryAccessMakePointerAvailableMask | - SpvMemoryAccessNonPrivatePointerMask); - - emitOperand(deviceScope); + emitOperand(SpvLiteralInteger::from32(memoryAccessMask)); + if (memoryAccessMask & SpvMemoryAccessAlignedMask) + emitOperand(SpvLiteralInteger::from32((uint32_t)alignment)); + if (memoryAccessMask & SpvMemoryAccessMakePointerAvailableMask) + emitOperand( + emitIntConstant((IRIntegerValue)memoryScope, builder.getIntType())); } }); } + SpvInst* emitStore(SpvInstParent* parent, IRStore* inst) + { + return emitStore(parent, inst, inst->getPtr(), inst->getVal()); + } SpvInst* emitSwizzledStore(SpvInstParent* parent, IRSwizzledStore* inst) { @@ -8613,6 +8637,8 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex SpvInst* emitDebugValue(SpvInstParent* parent, IRDebugValue* debugValue) { + auto debugVar = debugValue->getDebugVar(); + auto debugValueVal = debugValue->getValue(); // We are asked to update the value for a debug variable. // A debug variable is already emited as a OpDebugVariable + // OpVariable + OpDebugDeclare. We only need to store the new value @@ -8628,7 +8654,7 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex // variable. If it doesn't, we can't emit a store. // List<IRInst*> irAccessChain; - auto rootVar = getRootAddr(debugValue->getDebugVar(), irAccessChain); + auto rootVar = getRootAddr(debugVar, irAccessChain); SpvInst* spvDebugVar = nullptr; if (!m_mapIRInstToSpvInst.tryGetValue(rootVar, spvDebugVar)) return nullptr; @@ -8644,7 +8670,7 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex // be fully static. We will skip emitting the debug inst if the access chain // isn't static. // - auto type = unwrapAttributedType(debugValue->getDebugVar()->getDataType()); + auto type = unwrapAttributedType(debugVar->getDataType()); List<SpvInst*> accessChain; bool isConstAccessChain = translateIRAccessChain(builder, type, irAccessChain, accessChain); @@ -8657,7 +8683,7 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex m_voidType, getNonSemanticDebugInfoExtInst(), rootVar, - debugValue->getValue(), + debugValueVal, getDwarfExpr(), accessChain); } @@ -8669,7 +8695,7 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex // The ordinary case is the debug variable has a backing ordinary variable. // We can simply emit a store into the backing variable for the DebugValue operation. // - return emitStoreMaybeCoherent(parent, debugValue); + return emitStore(parent, debugValue, debugVar, debugValueVal); } IRInst* getName(IRInst* inst) |