hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch tips, but the experience is degraded.

yum-mirror/glslang

git clone https://git.yummers.dev/yum-mirror/glslang

Guang-035Fix Uniform storage class for buffer_type uniform descriptors43c384bf2026-10-14

1//
2// Copyright (C) 2014-2016 LunarG, Inc.
3// Copyright (C) 2015-2020 Google, Inc.
4// Copyright (C) 2017, 2022-2025 Arm Limited.
5// Modifications Copyright (C) 2020 Advanced Micro Devices, Inc. All rights reserved.
6//
7// All rights reserved.
8//
9// Redistribution and use in source and binary forms, with or without
10// modification, are permitted provided that the following conditions
11// are met:
12//
13//    Redistributions of source code must retain the above copyright
14//    notice, this list of conditions and the following disclaimer.
15//
16//    Redistributions in binary form must reproduce the above
17//    copyright notice, this list of conditions and the following
18//    disclaimer in the documentation and/or other materials provided
19//    with the distribution.
20//
21//    Neither the name of 3Dlabs Inc. Ltd. nor the names of its
22//    contributors may be used to endorse or promote products derived
23//    from this software without specific prior written permission.
24//
25// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
28// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
29// COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
30// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
31// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
32// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
33// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
35// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36// POSSIBILITY OF SUCH DAMAGE.
37
38//
39// Visit the nodes in the glslang intermediate tree representation to
40// translate them to SPIR-V.
41//
42
43#include "spirv.hpp11"
44#include "GlslangToSpv.h"
45#include "SpvBuilder.h"
46#include "SpvTools.h"
47#include "spvUtil.h"
48
49namespace spv {
50    #include "GLSL.std.450.h"
51    #include "GLSL.ext.KHR.h"
52    #include "GLSL.ext.EXT.h"
53    #include "GLSL.ext.AMD.h"
54    #include "GLSL.ext.NV.h"
55    #include "GLSL.ext.ARM.h"
56    #include "GLSL.ext.QCOM.h"
57    #include "NonSemanticDebugPrintf.h"
58}
59
60// Glslang includes
61#include "../glslang/MachineIndependent/localintermediate.h"
62#include "../glslang/MachineIndependent/SymbolTable.h"
63#include "../glslang/Include/Common.h"
64
65// Build-time generated includes
66#include "glslang/build_info.h"
67
68#include <fstream>
69#include <iomanip>
70#include <list>
71#include <map>
72#include <optional>
73#include <stack>
74#include <string>
75#include <vector>
76
77namespace {
78
79namespace {
80class SpecConstantOpModeGuard {
81public:
82    SpecConstantOpModeGuard(spv::Builder* builder)
83        : builder_(builder) {
84        previous_flag_ = builder->isInSpecConstCodeGenMode();
85    }
86    ~SpecConstantOpModeGuard() {
87        previous_flag_ ? builder_->setToSpecConstCodeGenMode()
88                       : builder_->setToNormalCodeGenMode();
89    }
90    void turnOnSpecConstantOpMode() {
91        builder_->setToSpecConstCodeGenMode();
92    }
93
94private:
95    spv::Builder* builder_;
96    bool previous_flag_;
97};
98
99struct OpDecorations {
100    public:
101        OpDecorations(spv::Decoration precision, spv::Decoration noContraction, spv::Decoration nonUniform) :
102            precision(precision)
103            ,
104            noContraction(noContraction),
105            nonUniform(nonUniform)
106        { }
107
108    spv::Decoration precision;
109
110        void addNoContraction(spv::Builder& builder, spv::Id t) { builder.addDecoration(t, noContraction); }
111        void addNonUniform(spv::Builder& builder, spv::Id t)  { builder.addDecoration(t, nonUniform); }
112    protected:
113    spv::Decoration noContraction;
114    spv::Decoration nonUniform;
115};
116
117void addDerivativeGroupExecutionMode(spv::Builder& builder, const glslang::TIntermediate& intermediate,
118                                     spv::Function* shaderEntry)
119{
120    if (intermediate.getLayoutDerivativeModeNone() == glslang::LayoutDerivativeGroupQuads) {
121        if (intermediate.getLayoutDerivativeExtension() == glslang::EdgKHR) {
122            builder.addCapability(spv::Capability::ComputeDerivativeGroupQuadsKHR);
123            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::DerivativeGroupQuadsKHR);
124            builder.addExtension(spv::E_SPV_KHR_compute_shader_derivatives);
125        } else {
126            builder.addCapability(spv::Capability::ComputeDerivativeGroupQuadsNV);
127            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::DerivativeGroupQuadsNV);
128            builder.addExtension(spv::E_SPV_NV_compute_shader_derivatives);
129        }
130    } else if (intermediate.getLayoutDerivativeModeNone() == glslang::LayoutDerivativeGroupLinear) {
131        if (intermediate.getLayoutDerivativeExtension() == glslang::EdgKHR) {
132            builder.addCapability(spv::Capability::ComputeDerivativeGroupLinearKHR);
133            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::DerivativeGroupLinearKHR);
134            builder.addExtension(spv::E_SPV_KHR_compute_shader_derivatives);
135        } else {
136            builder.addCapability(spv::Capability::ComputeDerivativeGroupLinearNV);
137            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::DerivativeGroupLinearNV);
138            builder.addExtension(spv::E_SPV_NV_compute_shader_derivatives);
139        }
140    }
141}
142
143} // namespace
144
145class TGlslangToSpvTraverser;
146
147class DescHeapLayoutEmitter {
148public:
149    explicit DescHeapLayoutEmitter(TGlslangToSpvTraverser& traverser);
150
151    bool isDescriptorPayload(const glslang::TType& type) const;
152    bool containsDescriptorPayload(const glslang::TType& type) const;
153
154    spv::Id getOrCreateValueSize(const glslang::TType& type);
155    spv::Id getOrCreateValueAlignment(const glslang::TType& type);
156
157    // Runtime arrays return spv::NoResult for total array size.
158    spv::Id getOrCreateArraySize(const glslang::TType& arrayType);
159    spv::Id getOrCreateArrayAlignment(const glslang::TType& arrayType);
160    std::vector<spv::Id> getOrCreateArrayStrides(const glslang::TType& arrayType);
161
162    // Returns OffsetIdEXT operands relative to the struct start.
163    std::vector<spv::Id> getOrCreateStructMemberOffsets(const glslang::TType& structType);
164
165private:
166    // compute* does not write TType* caches; it is safe for temporary array element types.
167    spv::Id computeValueSize(const glslang::TType& type);
168    spv::Id computeValueAlignment(const glslang::TType& type);
169    spv::Id getOrCreateStructSize(const glslang::TType& structType);
170    spv::Id getOrCreateStructAlignment(const glslang::TType& structType);
171
172    spv::Id makeUint(unsigned value);
173    spv::Id makeSpecOp(spv::Op op, const std::vector<spv::Id>& operands);
174    spv::Id getMemberSize(const glslang::TType& memberType);
175    spv::Id getMemberAlignment(const glslang::TType& memberType);
176
177    void getInnermostArrayElementType(const glslang::TType& arrayType, glslang::TType& elementType) const;
178
179    TGlslangToSpvTraverser& traverser;
180
181    // TType* caches are only for stable AST/member/symbol types.
182    std::unordered_map<const glslang::TType*, spv::Id> valueSizeCache;
183    std::unordered_map<const glslang::TType*, spv::Id> valueAlignmentCache;
184    std::unordered_map<const glslang::TType*, spv::Id> arraySizeCache;
185    std::unordered_map<const glslang::TType*, spv::Id> arrayAlignmentCache;
186    std::unordered_map<const glslang::TType*, std::vector<spv::Id>> arrayStridesCache;
187    std::unordered_map<const glslang::TTypeList*, spv::Id> structSizeCache;
188    std::unordered_map<const glslang::TTypeList*, spv::Id> structAlignmentCache;
189    std::unordered_map<const glslang::TTypeList*, std::vector<spv::Id>> structMemberOffsetsCache;
190};
191
192//
193// The main holder of information for translating glslang to SPIR-V.
194//
195// Derives from the AST walking base class.
196//
197class TGlslangToSpvTraverser : public glslang::TIntermTraverser {
198public:
199    TGlslangToSpvTraverser(unsigned int spvVersion, const glslang::TIntermediate*, spv::SpvBuildLogger* logger,
200        glslang::SpvOptions& options);
201    virtual ~TGlslangToSpvTraverser() { }
202
203    bool visitAggregate(glslang::TVisit, glslang::TIntermAggregate*) override;
204    bool visitBinary(glslang::TVisit, glslang::TIntermBinary*) override;
205    void visitConstantUnion(glslang::TIntermConstantUnion*) override;
206    bool visitSelection(glslang::TVisit, glslang::TIntermSelection*) override;
207    bool visitSwitch(glslang::TVisit, glslang::TIntermSwitch*) override;
208    void visitSymbol(glslang::TIntermSymbol* symbol) override;
209    bool visitUnary(glslang::TVisit, glslang::TIntermUnary*) override;
210    bool visitLoop(glslang::TVisit, glslang::TIntermLoop*) override;
211    bool visitBranch(glslang::TVisit visit, glslang::TIntermBranch*) override;
212    bool visitVariableDecl(glslang::TVisit, glslang::TIntermVariableDecl*) override;
213
214    void finishSpv(bool compileOnly);
215    void dumpSpv(std::vector<unsigned int>& out);
216
217protected:
218    friend class DescHeapLayoutEmitter;
219
220    TGlslangToSpvTraverser(TGlslangToSpvTraverser&);
221    TGlslangToSpvTraverser& operator=(TGlslangToSpvTraverser&);
222
223    spv::Decoration TranslateInterpolationDecoration(const glslang::TQualifier& qualifier);
224    spv::Decoration TranslateAuxiliaryStorageDecoration(const glslang::TQualifier& qualifier);
225    spv::Decoration TranslateNonUniformDecoration(const glslang::TQualifier& qualifier);
226    spv::Decoration TranslateNonUniformDecoration(const spv::Builder::AccessChain::CoherentFlags& coherentFlags);
227    spv::Builder::AccessChain::CoherentFlags TranslateCoherent(const glslang::TType& type);
228    spv::MemoryAccessMask TranslateMemoryAccess(const spv::Builder::AccessChain::CoherentFlags &coherentFlags);
229    spv::ImageOperandsMask TranslateImageOperands(const spv::Builder::AccessChain::CoherentFlags &coherentFlags);
230    spv::Scope TranslateMemoryScope(const spv::Builder::AccessChain::CoherentFlags &coherentFlags);
231    spv::BuiltIn TranslateBuiltInDecoration(glslang::TBuiltInVariable, bool memberDeclaration);
232    spv::ImageFormat TranslateImageFormat(const glslang::TType& type);
233    spv::SelectionControlMask TranslateSelectionControl(const glslang::TIntermSelection&) const;
234    spv::SelectionControlMask TranslateSwitchControl(const glslang::TIntermSwitch&) const;
235    spv::LoopControlMask TranslateLoopControl(const glslang::TIntermLoop&, std::vector<unsigned int>& operands) const;
236    spv::FunctionControlMask TranslateFunctionControl(unsigned int) const;
237    spv::StorageClass TranslateStorageClass(const glslang::TType&);
238    void TranslateLiterals(const glslang::TVector<const glslang::TIntermConstantUnion*>&, std::vector<unsigned>&) const;
239    void addIndirectionIndexCapabilities(const glslang::TType& baseType, const glslang::TType& indexType);
240    spv::Id createSpvVariable(const glslang::TIntermSymbol*, spv::Id forcedType);
241    spv::Id getSampledType(const glslang::TSampler&);
242    spv::Id getInvertedSwizzleType(const glslang::TIntermTyped&);
243    spv::Id createInvertedSwizzle(spv::Decoration precision, const glslang::TIntermTyped&, spv::Id parentResult);
244    void convertSwizzle(const glslang::TIntermAggregate&, std::vector<unsigned>& swizzle);
245    spv::Id convertGlslangToSpvType(const glslang::TType& type, bool forwardReferenceOnly = false);
246    spv::Id convertGlslangToSpvType(const glslang::TType& type, glslang::TLayoutPacking, const glslang::TQualifier&,
247        bool lastBufferBlockMember, bool forwardReferenceOnly = false);
248    void applySpirvDecorate(const glslang::TType& type, spv::Id id, std::optional<int> member);
249    bool filterMember(const glslang::TType& member);
250    spv::Id convertGlslangStructToSpvType(const glslang::TType&, const glslang::TTypeList* glslangStruct,
251                                          glslang::TLayoutPacking, const glslang::TQualifier&);
252    spv::LinkageType convertGlslangLinkageToSpv(glslang::TLinkType glslangLinkType);
253    bool isDescHeapDescriptorType(const glslang::TType& type) const;
254    bool containsDescHeapDescriptorType(const glslang::TType& type) const;
255    spv::Id makeDescHeapImageArrayWrapperType(const glslang::TIntermSymbol& symbol, spv::Id arrayType);
256
257    void decorateStructType(const glslang::TType&, const glslang::TTypeList* glslangStruct, glslang::TLayoutPacking,
258                            const glslang::TQualifier&, spv::Id, const std::vector<spv::Id>& spvMembers);
259    spv::Id makeArraySizeId(const glslang::TArraySizes&, int dim, bool allowZero = false, bool boolType = false);
260    spv::Id makeHeapOffsetId(const glslang::TType& type);
261    spv::Id accessChainLoad(const glslang::TType& type);
262    void    accessChainStore(const glslang::TType& type, spv::Id rvalue);
263    void multiTypeStore(const glslang::TType&, spv::Id rValue);
264    spv::Id convertLoadedBoolInUniformToUint(const glslang::TType& type, spv::Id nominalTypeId, spv::Id loadedId);
265    glslang::TLayoutPacking getExplicitLayout(const glslang::TType& type) const;
266    int getArrayStride(const glslang::TType& arrayType, glslang::TLayoutPacking, glslang::TLayoutMatrix);
267    int getMatrixStride(const glslang::TType& matrixType, glslang::TLayoutPacking, glslang::TLayoutMatrix);
268    void updateMemberOffset(const glslang::TType& structType, const glslang::TType& memberType, int& currentOffset,
269                            int& nextOffset, glslang::TLayoutPacking, glslang::TLayoutMatrix);
270    void declareUseOfStructMember(const glslang::TTypeList& members, int glslangMember);
271
272    bool isShaderEntryPoint(const glslang::TIntermAggregate* node);
273    bool writableParam(glslang::TStorageQualifier) const;
274    bool originalParam(glslang::TStorageQualifier, const glslang::TType&, bool implicitThisParam);
275    void makeFunctions(const glslang::TIntermSequence&);
276    void makeGlobalInitializers(const glslang::TIntermSequence&);
277    void collectRayTracingLinkerObjects();
278    void visitFunctions(const glslang::TIntermSequence&);
279    void translateArguments(const glslang::TIntermAggregate& node, std::vector<spv::Id>& arguments,
280        spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags);
281    void translateArguments(glslang::TIntermUnary& node, std::vector<spv::Id>& arguments);
282    spv::Id createImageTextureFunctionCall(glslang::TIntermOperator* node);
283    spv::Id handleUserFunctionCall(const glslang::TIntermAggregate*);
284
285    spv::Id createBinaryOperation(glslang::TOperator op, OpDecorations&, spv::Id typeId, spv::Id left, spv::Id right,
286                                  glslang::TBasicType typeProxy, bool reduceComparison = true);
287    spv::Id createBinaryMatrixOperation(spv::Op, OpDecorations&, spv::Id typeId, spv::Id left, spv::Id right);
288    spv::Id createUnaryOperation(glslang::TOperator op, OpDecorations&, spv::Id typeId, spv::Id operand,
289                                 glslang::TBasicType typeProxy,
290                                 const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags,
291                                 const glslang::TType &opType);
292    spv::Id createUnaryMatrixOperation(spv::Op op, OpDecorations&, spv::Id typeId, spv::Id operand,
293                                       glslang::TBasicType typeProxy);
294    spv::Id createConversion(glslang::TOperator op, OpDecorations&, spv::Id destTypeId, spv::Id operand,
295                             glslang::TBasicType resultBasicType, glslang::TBasicType operandBasicType,
296                             bool sameCoopMatUse = true);
297    spv::Id createCoopMatConversion(spv::Id destType, spv::Id operand,
298                                    glslang::TBasicType resultBasicType, glslang::TBasicType operandBasicType,
299                                    bool sameUse, bool transpose);
300
301    spv::Id createIntWidthConversion(spv::Id operand, int vectorSize, spv::Id destType,
302                                     glslang::TBasicType resultBasicType, glslang::TBasicType operandBasicType);
303    bool specConstUConvertNeedsWorkaround();
304    spv::Id createSpecConstUConvert(spv::Id typeId, spv::Id operand, int vectorSize, int operandWidth,
305                                    int resultWidth);
306    spv::Id makeSmearedConstant(spv::Id constant, int vectorSize);
307    spv::Id createAtomicOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId,
308        std::vector<spv::Id>& operands, glslang::TBasicType typeProxy,
309        const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags,
310        const glslang::TType &opType);
311    spv::Id createInvocationsOperation(glslang::TOperator op, spv::Id typeId, std::vector<spv::Id>& operands,
312        glslang::TBasicType typeProxy);
313    spv::Id CreateInvocationsVectorOperation(spv::Op op, spv::GroupOperation groupOperation,
314        spv::Id typeId, std::vector<spv::Id>& operands);
315    spv::Id createSubgroupOperation(glslang::TOperator op, spv::Id typeId, std::vector<spv::Id>& operands,
316        glslang::TBasicType typeProxy);
317    spv::Id createMiscOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId,
318        std::vector<spv::Id>& operands, glslang::TBasicType typeProxy);
319    spv::Id createNoArgOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId);
320    spv::Id getSymbolId(const glslang::TIntermSymbol* node);
321    void addMeshNVDecoration(spv::Id id, int member, const glslang::TQualifier & qualifier);
322    bool hasQCOMImageProceessingDecoration(spv::Id id, spv::Decoration decor);
323    void addImageProcessingQCOMDecoration(spv::Id id, spv::Decoration decor);
324    void addImageProcessing2QCOMDecoration(spv::Id id, bool isForGather);
325    spv::Id createSpvConstant(const glslang::TIntermTyped&);
326    spv::Id createSpvConstantFromConstUnionArray(const glslang::TType& type, const glslang::TConstUnionArray&,
327        int& nextConst, bool specConstant);
328    bool isTrivialLeaf(const glslang::TIntermTyped* node);
329    bool isTrivial(const glslang::TIntermTyped* node);
330    spv::Id createShortCircuit(glslang::TOperator, glslang::TIntermTyped& left, glslang::TIntermTyped& right);
331    spv::Id getExtBuiltins(const char* name);
332    std::pair<spv::Id, spv::Id> getForcedType(glslang::TBuiltInVariable builtIn, const glslang::TType&);
333    spv::Id translateForcedType(spv::Id object, const glslang::TType& type);
334    spv::Id createCompositeConstruct(spv::Id typeId, std::vector<spv::Id> constituents);
335    void recordDescHeapAccessChainInfo(glslang::TIntermBinary* node);
336    void createAbortEXT(const glslang::TIntermSequence &glslangOperands);
337    void enableCoopMatConversions();
338
339    glslang::SpvOptions& options;
340    spv::Function* shaderEntry;
341    spv::Function* currentFunction;
342    spv::Instruction* entryPoint;
343    int sequenceDepth;
344
345    // GL_EXT_opacity_micromap_ray_query_mode: the spec constant emitted for gl_EnableOpacityMicromapEXT,
346    // if it was materialized during traversal; spv::NoResult otherwise. finishSpv() references it (or
347    // synthesizes one) for the OpacityMicromapIdKHR execution mode.
348    spv::Id opacityMicromapEnableConstId;
349
350    spv::SpvBuildLogger* logger;
351
352    // There is a 1:1 mapping between a spv builder and a module; this is thread safe
353    spv::Builder builder;
354    bool inEntryPoint;
355    bool entryPointTerminated;
356    bool linkageOnly;                  // true when visiting the set of objects in the AST present only for
357                                       // establishing interface, whether or not they were statically used
358    std::set<spv::Id> iOSet;           // all input/output variables from either static use or declaration of interface
359    const glslang::TIntermediate* glslangIntermediate;
360    bool nanMinMaxClamp;               // true if use NMin/NMax/NClamp instead of FMin/FMax/FClamp
361    spv::Id stdBuiltins;
362    spv::Id nonSemanticDebugPrintf;
363    std::unordered_map<std::string, spv::Id> extBuiltinMap;
364
365    std::unordered_map<long long, spv::Id> symbolValues;
366    std::unordered_map<uint32_t, spv::Id> builtInVariableIds;
367    std::unordered_set<long long> rValueParameters;  // set of formal function parameters passed as rValues,
368                                               // rather than a pointer
369    std::unordered_map<std::string, spv::Function*> functionMap;
370    std::unordered_map<const glslang::TTypeList*, spv::Id> structMap[glslang::ElpCount][glslang::ElmCount];
371    // for mapping glslang block indices to spv indices (e.g., due to hidden members),
372    // keyed by the block's member list so distinct block types never share a slot:
373    std::unordered_map<const glslang::TTypeList*, std::vector<int>> memberRemapper;
374    std::stack<bool> breakForLoop;  // false means break for switch
375    std::unordered_map<std::string, const glslang::TIntermSymbol*> counterOriginator;
376    // Map pointee types for EbtReference to their forward pointers
377    std::map<const glslang::TType *, spv::Id> forwardPointers;
378    // Type forcing, for when SPIR-V wants a different type than the AST,
379    // requiring local translation to and from SPIR-V type on every access.
380    // Maps <builtin-variable-id -> AST-required-type-id>
381    std::unordered_map<spv::Id, spv::Id> forceType;
382    // Used by Task shader while generating opearnds for OpEmitMeshTasksEXT
383    spv::Id taskPayloadID;
384    // Used later for generating OpTraceKHR/OpExecuteCallableKHR/OpHitObjectRecordHit*/OpHitObjectGetShaderBindingTableData
385    std::unordered_map<unsigned int, glslang::TIntermSymbol *> locationToSymbol[4];
386    std::unordered_map<spv::Id, std::vector<spv::Decoration> > idToQCOMDecorations;
387
388    // Descriptor heap root types used as OpUntypedAccessChainKHR base type operands.
389    std::unordered_map<long long, spv::Id> heapDescHeapBaseType;
390    // Descriptor heap block payload types. Opaque descriptor types are already deduplicated by SpvBuilder.
391    std::unordered_map<const glslang::TTypeList*, spv::Id> heapDescHeapBlockDescriptorTypes;
392
393    DescHeapLayoutEmitter descHeapLayout;
394
395};
396
397//
398// Helper functions for translating glslang representations to SPIR-V enumerants.
399//
400
401// Translate glslang profile to SPIR-V source language.
402spv::SourceLanguage TranslateSourceLanguage(glslang::EShSource source, EProfile profile)
403{
404    switch (source) {
405    case glslang::EShSourceGlsl:
406        switch (profile) {
407        case ENoProfile:
408        case ECoreProfile:
409        case ECompatibilityProfile:
410            return spv::SourceLanguage::GLSL;
411        case EEsProfile:
412            return spv::SourceLanguage::ESSL;
413        default:
414            return spv::SourceLanguage::Unknown;
415        }
416    case glslang::EShSourceHlsl:
417        return spv::SourceLanguage::HLSL;
418    default:
419        return spv::SourceLanguage::Unknown;
420    }
421}
422
423// Translate glslang language (stage) to SPIR-V execution model.
424spv::ExecutionModel TranslateExecutionModel(EShLanguage stage, bool isMeshShaderEXT = false)
425{
426    switch (stage) {
427    case EShLangVertex:           return spv::ExecutionModel::Vertex;
428    case EShLangFragment:         return spv::ExecutionModel::Fragment;
429    case EShLangCompute:          return spv::ExecutionModel::GLCompute;
430    case EShLangTessControl:      return spv::ExecutionModel::TessellationControl;
431    case EShLangTessEvaluation:   return spv::ExecutionModel::TessellationEvaluation;
432    case EShLangGeometry:         return spv::ExecutionModel::Geometry;
433    case EShLangRayGen:           return spv::ExecutionModel::RayGenerationKHR;
434    case EShLangIntersect:        return spv::ExecutionModel::IntersectionKHR;
435    case EShLangAnyHit:           return spv::ExecutionModel::AnyHitKHR;
436    case EShLangClosestHit:       return spv::ExecutionModel::ClosestHitKHR;
437    case EShLangMiss:             return spv::ExecutionModel::MissKHR;
438    case EShLangCallable:         return spv::ExecutionModel::CallableKHR;
439    case EShLangTask:             return (isMeshShaderEXT)? spv::ExecutionModel::TaskEXT : spv::ExecutionModel::TaskNV;
440    case EShLangMesh:             return (isMeshShaderEXT)? spv::ExecutionModel::MeshEXT : spv::ExecutionModel::MeshNV;
441    default:
442        assert(0);
443        return spv::ExecutionModel::Fragment;
444    }
445}
446
447// Translate glslang sampler type to SPIR-V dimensionality.
448spv::Dim TranslateDimensionality(const glslang::TSampler& sampler)
449{
450    switch (sampler.dim) {
451    case glslang::Esd1D:      return spv::Dim::Dim1D;
452    case glslang::Esd2D:      return spv::Dim::Dim2D;
453    case glslang::Esd3D:      return spv::Dim::Dim3D;
454    case glslang::EsdCube:    return spv::Dim::Cube;
455    case glslang::EsdRect:    return spv::Dim::Rect;
456    case glslang::EsdBuffer:  return spv::Dim::Buffer;
457    case glslang::EsdSubpass: return spv::Dim::SubpassData;
458    case glslang::EsdAttachmentEXT: return spv::Dim::TileImageDataEXT;
459    default:
460        assert(0);
461        return spv::Dim::Dim2D;
462    }
463}
464
465// Translate glslang precision to SPIR-V precision decorations.
466spv::Decoration TranslatePrecisionDecoration(glslang::TPrecisionQualifier glslangPrecision)
467{
468    switch (glslangPrecision) {
469    case glslang::EpqLow:    return spv::Decoration::RelaxedPrecision;
470    case glslang::EpqMedium: return spv::Decoration::RelaxedPrecision;
471    default:
472        return spv::NoPrecision;
473    }
474}
475
476// Translate glslang type to SPIR-V precision decorations.
477spv::Decoration TranslatePrecisionDecoration(const glslang::TType& type)
478{
479    return TranslatePrecisionDecoration(type.getQualifier().precision);
480}
481
482// Translate glslang type to SPIR-V block decorations.
483spv::Decoration TranslateBlockDecoration(const glslang::TStorageQualifier storage, bool useStorageBuffer)
484{
485    switch (storage) {
486    case glslang::EvqUniform:      return spv::Decoration::Block;
487    case glslang::EvqBuffer:       return useStorageBuffer ? spv::Decoration::Block : spv::Decoration::BufferBlock;
488    case glslang::EvqVaryingIn:    return spv::Decoration::Block;
489    case glslang::EvqVaryingOut:   return spv::Decoration::Block;
490    case glslang::EvqShared:       return spv::Decoration::Block;
491    case glslang::EvqPayload:      return spv::Decoration::Block;
492    case glslang::EvqPayloadIn:    return spv::Decoration::Block;
493    case glslang::EvqHitAttr:      return spv::Decoration::Block;
494    case glslang::EvqCallableData:   return spv::Decoration::Block;
495    case glslang::EvqCallableDataIn: return spv::Decoration::Block;
496    case glslang::EvqHitObjectAttrNV: return spv::Decoration::Block;
497    case glslang::EvqHitObjectAttrEXT: return spv::Decoration::Block;
498    case glslang::EvqResourceHeap:  return spv::Decoration::Block;
499    case glslang::EvqSamplerHeap:   return spv::Decoration::Block;
500    default:
501        assert(0);
502        break;
503    }
504
505    return spv::Decoration::Max;
506}
507
508// Translate glslang type to SPIR-V memory decorations.
509void TranslateMemoryDecoration(const glslang::TQualifier& qualifier, std::vector<spv::Decoration>& memory,
510    bool useVulkanMemoryModel)
511{
512    if (!useVulkanMemoryModel) {
513        if (qualifier.isVolatile()) {
514            memory.push_back(spv::Decoration::Volatile);
515            memory.push_back(spv::Decoration::Coherent);
516        } else if (qualifier.isCoherent()) {
517            memory.push_back(spv::Decoration::Coherent);
518        }
519    }
520    if (qualifier.isRestrict())
521        memory.push_back(spv::Decoration::Restrict);
522    if (qualifier.isReadOnly())
523        memory.push_back(spv::Decoration::NonWritable);
524    if (qualifier.isWriteOnly())
525        memory.push_back(spv::Decoration::NonReadable);
526}
527
528// Translate glslang type to SPIR-V layout decorations.
529spv::Decoration TranslateLayoutDecoration(const glslang::TType& type, glslang::TLayoutMatrix matrixLayout)
530{
531    if (type.isMatrix()) {
532        switch (matrixLayout) {
533        case glslang::ElmRowMajor:
534            return spv::Decoration::RowMajor;
535        case glslang::ElmColumnMajor:
536            return spv::Decoration::ColMajor;
537        default:
538            // opaque layouts don't need a majorness
539            return spv::Decoration::Max;
540        }
541    } else {
542        switch (type.getBasicType()) {
543        default:
544            return spv::Decoration::Max;
545            break;
546        case glslang::EbtBlock:
547            switch (type.getQualifier().storage) {
548            case glslang::EvqShared:
549            case glslang::EvqUniform:
550            case glslang::EvqBuffer:
551                switch (type.getQualifier().layoutPacking) {
552                case glslang::ElpShared:  return spv::Decoration::GLSLShared;
553                case glslang::ElpPacked:  return spv::Decoration::GLSLPacked;
554                default:
555                    return spv::Decoration::Max;
556                }
557            case glslang::EvqVaryingIn:
558            case glslang::EvqVaryingOut:
559                if (type.getQualifier().isTaskMemory()) {
560                    switch (type.getQualifier().layoutPacking) {
561                    case glslang::ElpShared:  return spv::Decoration::GLSLShared;
562                    case glslang::ElpPacked:  return spv::Decoration::GLSLPacked;
563                    default: break;
564                    }
565                } else {
566                    assert(type.getQualifier().layoutPacking == glslang::ElpNone);
567                }
568                return spv::Decoration::Max;
569            case glslang::EvqPayload:
570            case glslang::EvqPayloadIn:
571            case glslang::EvqHitAttr:
572            case glslang::EvqCallableData:
573            case glslang::EvqCallableDataIn:
574            case glslang::EvqHitObjectAttrNV:
575            case glslang::EvqHitObjectAttrEXT:
576            case glslang::EvqResourceHeap:
577            case glslang::EvqSamplerHeap:
578                return spv::Decoration::Max;
579            default:
580                assert(0);
581                return spv::Decoration::Max;
582            }
583        }
584    }
585}
586
587// Translate glslang type to SPIR-V interpolation decorations.
588// Returns spv::Decoration::Max when no decoration
589// should be applied.
590spv::Decoration TGlslangToSpvTraverser::TranslateInterpolationDecoration(const glslang::TQualifier& qualifier)
591{
592    if (qualifier.smooth)
593        // Smooth decoration doesn't exist in SPIR-V 1.0
594        return spv::Decoration::Max;
595    else if (qualifier.isNonPerspective())
596        return spv::Decoration::NoPerspective;
597    else if (qualifier.flat)
598        return spv::Decoration::Flat;
599    else if (qualifier.isExplicitInterpolation()) {
600        builder.addExtension(spv::E_SPV_AMD_shader_explicit_vertex_parameter);
601        return spv::Decoration::ExplicitInterpAMD;
602    }
603    else
604        return spv::Decoration::Max;
605}
606
607// Translate glslang type to SPIR-V auxiliary storage decorations.
608// Returns spv::Decoration::Max when no decoration
609// should be applied.
610spv::Decoration TGlslangToSpvTraverser::TranslateAuxiliaryStorageDecoration(const glslang::TQualifier& qualifier)
611{
612    if (qualifier.centroid)
613        return spv::Decoration::Centroid;
614    else if (qualifier.patch)
615        return spv::Decoration::Patch;
616    else if (qualifier.sample) {
617        builder.addCapability(spv::Capability::SampleRateShading);
618        return spv::Decoration::Sample;
619    }
620
621    return spv::Decoration::Max;
622}
623
624// If glslang type is invariant, return SPIR-V invariant decoration.
625spv::Decoration TranslateInvariantDecoration(const glslang::TQualifier& qualifier)
626{
627    if (qualifier.invariant)
628        return spv::Decoration::Invariant;
629    else
630        return spv::Decoration::Max;
631}
632
633// If glslang type is noContraction, return SPIR-V NoContraction decoration.
634spv::Decoration TranslateNoContractionDecoration(const glslang::TQualifier& qualifier)
635{
636    if (qualifier.isNoContraction())
637        return spv::Decoration::NoContraction;
638    else
639        return spv::Decoration::Max;
640}
641
642// If glslang type is nonUniform, return SPIR-V NonUniform decoration.
643spv::Decoration TGlslangToSpvTraverser::TranslateNonUniformDecoration(const glslang::TQualifier& qualifier)
644{
645    if (qualifier.isNonUniform()) {
646        builder.addIncorporatedExtension("SPV_EXT_descriptor_indexing", spv::Spv_1_5);
647        builder.addCapability(spv::Capability::ShaderNonUniformEXT);
648
649        auto& extensions = glslangIntermediate->getRequestedExtensions();
650        if (extensions.find("GL_EXT_descriptor_heap") != extensions.end()) {
651            builder.addExtension("SPV_EXT_descriptor_heap");
652            builder.addCapability(spv::Capability::DescriptorHeapEXT);
653        }
654        else {
655            return spv::Decoration::NonUniformEXT;
656        }
657    }
658
659    return spv::Decoration::Max;
660}
661
662// If lvalue flags contains nonUniform, return SPIR-V NonUniform decoration.
663spv::Decoration TGlslangToSpvTraverser::TranslateNonUniformDecoration(
664    const spv::Builder::AccessChain::CoherentFlags& coherentFlags)
665{
666    if (coherentFlags.isNonUniform()) {
667        builder.addIncorporatedExtension("SPV_EXT_descriptor_indexing", spv::Spv_1_5);
668        builder.addCapability(spv::Capability::ShaderNonUniformEXT);
669
670        auto& extensions = glslangIntermediate->getRequestedExtensions();
671        if (extensions.find("GL_EXT_descriptor_heap") != extensions.end()) {
672            builder.addExtension("SPV_EXT_descriptor_heap");
673            builder.addCapability(spv::Capability::DescriptorHeapEXT);
674        }
675        else {
676            return spv::Decoration::NonUniformEXT;
677        }
678    }
679
680    return spv::Decoration::Max;
681}
682
683spv::MemoryAccessMask TGlslangToSpvTraverser::TranslateMemoryAccess(
684    const spv::Builder::AccessChain::CoherentFlags &coherentFlags)
685{
686    spv::MemoryAccessMask mask = spv::MemoryAccessMask::MaskNone;
687
688    if (!glslangIntermediate->usingVulkanMemoryModel() || coherentFlags.isImage)
689        return mask;
690
691    if (coherentFlags.isVolatile() || coherentFlags.anyCoherent()) {
692        mask = mask | spv::MemoryAccessMask::MakePointerAvailableKHR |
693                      spv::MemoryAccessMask::MakePointerVisibleKHR;
694    }
695
696    if (coherentFlags.nonprivate) {
697        mask = mask | spv::MemoryAccessMask::NonPrivatePointerKHR;
698    }
699    if (coherentFlags.volatil) {
700        mask = mask | spv::MemoryAccessMask::Volatile;
701    }
702    if (coherentFlags.nontemporal) {
703        mask = mask | spv::MemoryAccessMask::Nontemporal;
704    }
705    if (mask != spv::MemoryAccessMask::MaskNone) {
706        builder.addCapability(spv::Capability::VulkanMemoryModelKHR);
707    }
708
709    return mask;
710}
711
712spv::ImageOperandsMask TGlslangToSpvTraverser::TranslateImageOperands(
713    const spv::Builder::AccessChain::CoherentFlags &coherentFlags)
714{
715    spv::ImageOperandsMask mask = spv::ImageOperandsMask::MaskNone;
716
717    if (!glslangIntermediate->usingVulkanMemoryModel())
718        return mask;
719
720    if (coherentFlags.volatil ||
721        coherentFlags.anyCoherent()) {
722        mask = mask | spv::ImageOperandsMask::MakeTexelAvailableKHR |
723                      spv::ImageOperandsMask::MakeTexelVisibleKHR;
724    }
725    if (coherentFlags.nonprivate) {
726        mask = mask | spv::ImageOperandsMask::NonPrivateTexelKHR;
727    }
728    if (coherentFlags.volatil) {
729        mask = mask | spv::ImageOperandsMask::VolatileTexelKHR;
730    }
731    if (coherentFlags.nontemporal && builder.getSpvVersion() >= spv::Spv_1_6) {
732        mask = mask | spv::ImageOperandsMask::Nontemporal;
733    }
734    if (mask != spv::ImageOperandsMask::MaskNone) {
735        builder.addCapability(spv::Capability::VulkanMemoryModelKHR);
736    }
737
738    return mask;
739}
740
741spv::Builder::AccessChain::CoherentFlags TGlslangToSpvTraverser::TranslateCoherent(const glslang::TType& type)
742{
743    spv::Builder::AccessChain::CoherentFlags flags = {};
744    flags.coherent = type.getQualifier().coherent;
745    flags.devicecoherent = type.getQualifier().devicecoherent;
746    flags.queuefamilycoherent = type.getQualifier().queuefamilycoherent;
747    // shared variables are implicitly workgroupcoherent in GLSL.
748    flags.workgroupcoherent = type.getQualifier().workgroupcoherent ||
749                              type.getQualifier().storage == glslang::EvqShared;
750    flags.subgroupcoherent = type.getQualifier().subgroupcoherent;
751    flags.shadercallcoherent = type.getQualifier().shadercallcoherent;
752    flags.volatil = type.getQualifier().volatil;
753    flags.nontemporal = type.getQualifier().nontemporal;
754    // *coherent variables are implicitly nonprivate in GLSL
755    flags.nonprivate = type.getQualifier().nonprivate ||
756                       flags.anyCoherent() ||
757                       flags.volatil;
758    flags.isImage = type.getBasicType() == glslang::EbtSampler;
759    flags.nonUniform = type.getQualifier().nonUniform;
760    return flags;
761}
762
763spv::Scope TGlslangToSpvTraverser::TranslateMemoryScope(
764    const spv::Builder::AccessChain::CoherentFlags &coherentFlags)
765{
766    spv::Scope scope = spv::Scope::Max;
767
768    if (coherentFlags.volatil || coherentFlags.coherent) {
769        // coherent defaults to Device scope in the old model, QueueFamilyKHR scope in the new model
770        scope = glslangIntermediate->usingVulkanMemoryModel() ? spv::Scope::QueueFamilyKHR : spv::Scope::Device;
771    } else if (coherentFlags.devicecoherent) {
772        scope = spv::Scope::Device;
773    } else if (coherentFlags.queuefamilycoherent) {
774        scope = spv::Scope::QueueFamilyKHR;
775    } else if (coherentFlags.workgroupcoherent) {
776        scope = spv::Scope::Workgroup;
777    } else if (coherentFlags.subgroupcoherent) {
778        scope = spv::Scope::Subgroup;
779    } else if (coherentFlags.shadercallcoherent) {
780        scope = spv::Scope::ShaderCallKHR;
781    }
782    if (glslangIntermediate->usingVulkanMemoryModel() && scope == spv::Scope::Device) {
783        builder.addCapability(spv::Capability::VulkanMemoryModelDeviceScopeKHR);
784    }
785
786    return scope;
787}
788
789// Translate a glslang built-in variable to a SPIR-V built in decoration.  Also generate
790// associated capabilities when required.  For some built-in variables, a capability
791// is generated only when using the variable in an executable instruction, but not when
792// just declaring a struct member variable with it.  This is true for PointSize,
793// ClipDistance, and CullDistance.
794spv::BuiltIn TGlslangToSpvTraverser::TranslateBuiltInDecoration(glslang::TBuiltInVariable builtIn,
795    bool memberDeclaration)
796{
797    switch (builtIn) {
798    case glslang::EbvPointSize:
799        // Defer adding the capability until the built-in is actually used.
800        if (! memberDeclaration) {
801            switch (glslangIntermediate->getStage()) {
802            case EShLangGeometry:
803                builder.addCapability(spv::Capability::GeometryPointSize);
804                break;
805            case EShLangTessControl:
806            case EShLangTessEvaluation:
807                builder.addCapability(spv::Capability::TessellationPointSize);
808                break;
809            default:
810                break;
811            }
812        }
813        return spv::BuiltIn::PointSize;
814
815    case glslang::EbvPosition:             return spv::BuiltIn::Position;
816    case glslang::EbvVertexId:             return spv::BuiltIn::VertexId;
817    case glslang::EbvInstanceId:           return spv::BuiltIn::InstanceId;
818    case glslang::EbvVertexIndex:          return spv::BuiltIn::VertexIndex;
819    case glslang::EbvInstanceIndex:        return spv::BuiltIn::InstanceIndex;
820
821    case glslang::EbvFragCoord:            return spv::BuiltIn::FragCoord;
822    case glslang::EbvPointCoord:           return spv::BuiltIn::PointCoord;
823    case glslang::EbvFace:                 return spv::BuiltIn::FrontFacing;
824    case glslang::EbvFragDepth:            return spv::BuiltIn::FragDepth;
825
826    case glslang::EbvNumWorkGroups:        return spv::BuiltIn::NumWorkgroups;
827    case glslang::EbvWorkGroupSize:        return spv::BuiltIn::WorkgroupSize;
828    case glslang::EbvWorkGroupId:          return spv::BuiltIn::WorkgroupId;
829    case glslang::EbvLocalInvocationId:    return spv::BuiltIn::LocalInvocationId;
830    case glslang::EbvLocalInvocationIndex: return spv::BuiltIn::LocalInvocationIndex;
831    case glslang::EbvGlobalInvocationId:   return spv::BuiltIn::GlobalInvocationId;
832
833    // These *Distance capabilities logically belong here, but if the member is declared and
834    // then never used, consumers of SPIR-V prefer the capability not be declared.
835    // They are now generated when used, rather than here when declared.
836    // Potentially, the specification should be more clear what the minimum
837    // use needed is to trigger the capability.
838    //
839    case glslang::EbvClipDistance:
840        if (!memberDeclaration)
841            builder.addCapability(spv::Capability::ClipDistance);
842        return spv::BuiltIn::ClipDistance;
843
844    case glslang::EbvCullDistance:
845        if (!memberDeclaration)
846            builder.addCapability(spv::Capability::CullDistance);
847        return spv::BuiltIn::CullDistance;
848
849    case glslang::EbvViewportIndex:
850        if (glslangIntermediate->getStage() == EShLangGeometry ||
851            glslangIntermediate->getStage() == EShLangFragment) {
852            builder.addCapability(spv::Capability::MultiViewport);
853        }
854        if (glslangIntermediate->getStage() == EShLangVertex ||
855            glslangIntermediate->getStage() == EShLangTessControl ||
856            glslangIntermediate->getStage() == EShLangTessEvaluation) {
857
858            if (builder.getSpvVersion() < spv::Spv_1_5) {
859                builder.addIncorporatedExtension(spv::E_SPV_EXT_shader_viewport_index_layer, spv::Spv_1_5);
860                builder.addCapability(spv::Capability::ShaderViewportIndexLayerEXT);
861            }
862            else
863                builder.addCapability(spv::Capability::ShaderViewportIndex);
864        }
865        return spv::BuiltIn::ViewportIndex;
866
867    case glslang::EbvSampleId:
868        builder.addCapability(spv::Capability::SampleRateShading);
869        return spv::BuiltIn::SampleId;
870
871    case glslang::EbvSamplePosition:
872        builder.addCapability(spv::Capability::SampleRateShading);
873        return spv::BuiltIn::SamplePosition;
874
875    case glslang::EbvSampleMask:
876        return spv::BuiltIn::SampleMask;
877
878    case glslang::EbvLayer:
879        if (glslangIntermediate->getStage() == EShLangMesh) {
880            return spv::BuiltIn::Layer;
881        }
882        if (glslangIntermediate->getStage() == EShLangGeometry ||
883            glslangIntermediate->getStage() == EShLangFragment) {
884            builder.addCapability(spv::Capability::Geometry);
885        }
886        if (glslangIntermediate->getStage() == EShLangVertex ||
887            glslangIntermediate->getStage() == EShLangTessControl ||
888            glslangIntermediate->getStage() == EShLangTessEvaluation) {
889
890            if (builder.getSpvVersion() < spv::Spv_1_5) {
891                builder.addIncorporatedExtension(spv::E_SPV_EXT_shader_viewport_index_layer, spv::Spv_1_5);
892                builder.addCapability(spv::Capability::ShaderViewportIndexLayerEXT);
893            } else
894                builder.addCapability(spv::Capability::ShaderLayer);
895        }
896        return spv::BuiltIn::Layer;
897
898    case glslang::EbvBaseVertex:
899        builder.addIncorporatedExtension(spv::E_SPV_KHR_shader_draw_parameters, spv::Spv_1_3);
900        builder.addCapability(spv::Capability::DrawParameters);
901        return spv::BuiltIn::BaseVertex;
902
903    case glslang::EbvBaseInstance:
904        builder.addIncorporatedExtension(spv::E_SPV_KHR_shader_draw_parameters, spv::Spv_1_3);
905        builder.addCapability(spv::Capability::DrawParameters);
906        return spv::BuiltIn::BaseInstance;
907
908    case glslang::EbvDrawId:
909        builder.addIncorporatedExtension(spv::E_SPV_KHR_shader_draw_parameters, spv::Spv_1_3);
910        builder.addCapability(spv::Capability::DrawParameters);
911        return spv::BuiltIn::DrawIndex;
912
913    case glslang::EbvPrimitiveId:
914        if (glslangIntermediate->getStage() == EShLangFragment)
915            builder.addCapability(spv::Capability::Geometry);
916        return spv::BuiltIn::PrimitiveId;
917
918    case glslang::EbvFragStencilRef:
919        builder.addExtension(spv::E_SPV_EXT_shader_stencil_export);
920        builder.addCapability(spv::Capability::StencilExportEXT);
921        return spv::BuiltIn::FragStencilRefEXT;
922
923    case glslang::EbvShadingRateKHR:
924        builder.addExtension(spv::E_SPV_KHR_fragment_shading_rate);
925        builder.addCapability(spv::Capability::FragmentShadingRateKHR);
926        return spv::BuiltIn::ShadingRateKHR;
927
928    case glslang::EbvPrimitiveShadingRateKHR:
929        builder.addExtension(spv::E_SPV_KHR_fragment_shading_rate);
930        builder.addCapability(spv::Capability::FragmentShadingRateKHR);
931        return spv::BuiltIn::PrimitiveShadingRateKHR;
932
933    case glslang::EbvInvocationId:         return spv::BuiltIn::InvocationId;
934    case glslang::EbvTessLevelInner:       return spv::BuiltIn::TessLevelInner;
935    case glslang::EbvTessLevelOuter:       return spv::BuiltIn::TessLevelOuter;
936    case glslang::EbvTessCoord:            return spv::BuiltIn::TessCoord;
937    case glslang::EbvPatchVertices:        return spv::BuiltIn::PatchVertices;
938    case glslang::EbvHelperInvocation:     return spv::BuiltIn::HelperInvocation;
939
940    case glslang::EbvSubGroupSize:
941        builder.addExtension(spv::E_SPV_KHR_shader_ballot);
942        builder.addCapability(spv::Capability::SubgroupBallotKHR);
943        return spv::BuiltIn::SubgroupSize;
944
945    case glslang::EbvSubGroupInvocation:
946        builder.addExtension(spv::E_SPV_KHR_shader_ballot);
947        builder.addCapability(spv::Capability::SubgroupBallotKHR);
948        return spv::BuiltIn::SubgroupLocalInvocationId;
949
950    case glslang::EbvSubGroupEqMask:
951        builder.addExtension(spv::E_SPV_KHR_shader_ballot);
952        builder.addCapability(spv::Capability::SubgroupBallotKHR);
953        return spv::BuiltIn::SubgroupEqMask;
954
955    case glslang::EbvSubGroupGeMask:
956        builder.addExtension(spv::E_SPV_KHR_shader_ballot);
957        builder.addCapability(spv::Capability::SubgroupBallotKHR);
958        return spv::BuiltIn::SubgroupGeMask;
959
960    case glslang::EbvSubGroupGtMask:
961        builder.addExtension(spv::E_SPV_KHR_shader_ballot);
962        builder.addCapability(spv::Capability::SubgroupBallotKHR);
963        return spv::BuiltIn::SubgroupGtMask;
964
965    case glslang::EbvSubGroupLeMask:
966        builder.addExtension(spv::E_SPV_KHR_shader_ballot);
967        builder.addCapability(spv::Capability::SubgroupBallotKHR);
968        return spv::BuiltIn::SubgroupLeMask;
969
970    case glslang::EbvSubGroupLtMask:
971        builder.addExtension(spv::E_SPV_KHR_shader_ballot);
972        builder.addCapability(spv::Capability::SubgroupBallotKHR);
973        return spv::BuiltIn::SubgroupLtMask;
974
975    case glslang::EbvNumSubgroups:
976        builder.addCapability(spv::Capability::GroupNonUniform);
977        return spv::BuiltIn::NumSubgroups;
978
979    case glslang::EbvSubgroupID:
980        builder.addCapability(spv::Capability::GroupNonUniform);
981        return spv::BuiltIn::SubgroupId;
982
983    case glslang::EbvSubgroupSize2:
984        builder.addCapability(spv::Capability::GroupNonUniform);
985        return spv::BuiltIn::SubgroupSize;
986
987    case glslang::EbvSubgroupInvocation2:
988        builder.addCapability(spv::Capability::GroupNonUniform);
989        return spv::BuiltIn::SubgroupLocalInvocationId;
990
991    case glslang::EbvSubgroupEqMask2:
992        builder.addCapability(spv::Capability::GroupNonUniform);
993        builder.addCapability(spv::Capability::GroupNonUniformBallot);
994        return spv::BuiltIn::SubgroupEqMask;
995
996    case glslang::EbvSubgroupGeMask2:
997        builder.addCapability(spv::Capability::GroupNonUniform);
998        builder.addCapability(spv::Capability::GroupNonUniformBallot);
999        return spv::BuiltIn::SubgroupGeMask;
1000
1001    case glslang::EbvSubgroupGtMask2:
1002        builder.addCapability(spv::Capability::GroupNonUniform);
1003        builder.addCapability(spv::Capability::GroupNonUniformBallot);
1004        return spv::BuiltIn::SubgroupGtMask;
1005
1006    case glslang::EbvSubgroupLeMask2:
1007        builder.addCapability(spv::Capability::GroupNonUniform);
1008        builder.addCapability(spv::Capability::GroupNonUniformBallot);
1009        return spv::BuiltIn::SubgroupLeMask;
1010
1011    case glslang::EbvSubgroupLtMask2:
1012        builder.addCapability(spv::Capability::GroupNonUniform);
1013        builder.addCapability(spv::Capability::GroupNonUniformBallot);
1014        return spv::BuiltIn::SubgroupLtMask;
1015
1016    case glslang::EbvBaryCoordNoPersp:
1017        builder.addExtension(spv::E_SPV_AMD_shader_explicit_vertex_parameter);
1018        return spv::BuiltIn::BaryCoordNoPerspAMD;
1019
1020    case glslang::EbvBaryCoordNoPerspCentroid:
1021        builder.addExtension(spv::E_SPV_AMD_shader_explicit_vertex_parameter);
1022        return spv::BuiltIn::BaryCoordNoPerspCentroidAMD;
1023
1024    case glslang::EbvBaryCoordNoPerspSample:
1025        builder.addExtension(spv::E_SPV_AMD_shader_explicit_vertex_parameter);
1026        return spv::BuiltIn::BaryCoordNoPerspSampleAMD;
1027
1028    case glslang::EbvBaryCoordSmooth:
1029        builder.addExtension(spv::E_SPV_AMD_shader_explicit_vertex_parameter);
1030        return spv::BuiltIn::BaryCoordSmoothAMD;
1031
1032    case glslang::EbvBaryCoordSmoothCentroid:
1033        builder.addExtension(spv::E_SPV_AMD_shader_explicit_vertex_parameter);
1034        return spv::BuiltIn::BaryCoordSmoothCentroidAMD;
1035
1036    case glslang::EbvBaryCoordSmoothSample:
1037        builder.addExtension(spv::E_SPV_AMD_shader_explicit_vertex_parameter);
1038        return spv::BuiltIn::BaryCoordSmoothSampleAMD;
1039
1040    case glslang::EbvBaryCoordPullModel:
1041        builder.addExtension(spv::E_SPV_AMD_shader_explicit_vertex_parameter);
1042        return spv::BuiltIn::BaryCoordPullModelAMD;
1043
1044    case glslang::EbvDeviceIndex:
1045        builder.addIncorporatedExtension(spv::E_SPV_KHR_device_group, spv::Spv_1_3);
1046        builder.addCapability(spv::Capability::DeviceGroup);
1047        return spv::BuiltIn::DeviceIndex;
1048
1049    case glslang::EbvViewIndex:
1050        builder.addIncorporatedExtension(spv::E_SPV_KHR_multiview, spv::Spv_1_3);
1051        builder.addCapability(spv::Capability::MultiView);
1052        return spv::BuiltIn::ViewIndex;
1053
1054    case glslang::EbvFragSizeEXT:
1055        builder.addExtension(spv::E_SPV_EXT_fragment_invocation_density);
1056        builder.addCapability(spv::Capability::FragmentDensityEXT);
1057        return spv::BuiltIn::FragSizeEXT;
1058
1059    case glslang::EbvFragInvocationCountEXT:
1060        builder.addExtension(spv::E_SPV_EXT_fragment_invocation_density);
1061        builder.addCapability(spv::Capability::FragmentDensityEXT);
1062        return spv::BuiltIn::FragInvocationCountEXT;
1063
1064    case glslang::EbvViewportMaskNV:
1065        if (!memberDeclaration) {
1066            builder.addExtension(spv::E_SPV_NV_viewport_array2);
1067            builder.addCapability(spv::Capability::ShaderViewportMaskNV);
1068        }
1069        return spv::BuiltIn::ViewportMaskNV;
1070    case glslang::EbvSecondaryPositionNV:
1071        if (!memberDeclaration) {
1072            builder.addExtension(spv::E_SPV_NV_stereo_view_rendering);
1073            builder.addCapability(spv::Capability::ShaderStereoViewNV);
1074        }
1075        return spv::BuiltIn::SecondaryPositionNV;
1076    case glslang::EbvSecondaryViewportMaskNV:
1077        if (!memberDeclaration) {
1078            builder.addExtension(spv::E_SPV_NV_stereo_view_rendering);
1079            builder.addCapability(spv::Capability::ShaderStereoViewNV);
1080        }
1081        return spv::BuiltIn::SecondaryViewportMaskNV;
1082    case glslang::EbvPositionPerViewNV:
1083        if (!memberDeclaration) {
1084            builder.addExtension(spv::E_SPV_NVX_multiview_per_view_attributes);
1085            builder.addCapability(spv::Capability::PerViewAttributesNV);
1086        }
1087        return spv::BuiltIn::PositionPerViewNV;
1088    case glslang::EbvViewportMaskPerViewNV:
1089        if (!memberDeclaration) {
1090            builder.addExtension(spv::E_SPV_NVX_multiview_per_view_attributes);
1091            builder.addCapability(spv::Capability::PerViewAttributesNV);
1092        }
1093        return spv::BuiltIn::ViewportMaskPerViewNV;
1094    case glslang::EbvFragFullyCoveredNV:
1095        builder.addExtension(spv::E_SPV_EXT_fragment_fully_covered);
1096        builder.addCapability(spv::Capability::FragmentFullyCoveredEXT);
1097        return spv::BuiltIn::FullyCoveredEXT;
1098    case glslang::EbvFragmentSizeNV:
1099        builder.addExtension(spv::E_SPV_NV_shading_rate);
1100        builder.addCapability(spv::Capability::ShadingRateNV);
1101        return spv::BuiltIn::FragmentSizeNV;
1102    case glslang::EbvInvocationsPerPixelNV:
1103        builder.addExtension(spv::E_SPV_NV_shading_rate);
1104        builder.addCapability(spv::Capability::ShadingRateNV);
1105        return spv::BuiltIn::InvocationsPerPixelNV;
1106
1107    // ray tracing
1108    case glslang::EbvLaunchId:
1109        return spv::BuiltIn::LaunchIdKHR;
1110    case glslang::EbvLaunchSize:
1111        return spv::BuiltIn::LaunchSizeKHR;
1112    case glslang::EbvWorldRayOrigin:
1113        return spv::BuiltIn::WorldRayOriginKHR;
1114    case glslang::EbvWorldRayDirection:
1115        return spv::BuiltIn::WorldRayDirectionKHR;
1116    case glslang::EbvObjectRayOrigin:
1117        return spv::BuiltIn::ObjectRayOriginKHR;
1118    case glslang::EbvObjectRayDirection:
1119        return spv::BuiltIn::ObjectRayDirectionKHR;
1120    case glslang::EbvRayTmin:
1121        return spv::BuiltIn::RayTminKHR;
1122    case glslang::EbvRayTmax:
1123        return spv::BuiltIn::RayTmaxKHR;
1124    case glslang::EbvCullMask:
1125        return spv::BuiltIn::CullMaskKHR;
1126    case glslang::EbvPositionFetch:
1127        return spv::BuiltIn::HitTriangleVertexPositionsKHR;
1128    case glslang::EbvInstanceCustomIndex:
1129        return spv::BuiltIn::InstanceCustomIndexKHR;
1130    case glslang::EbvHitKind:
1131        return spv::BuiltIn::HitKindKHR;
1132    case glslang::EbvObjectToWorld:
1133    case glslang::EbvObjectToWorld3x4:
1134        return spv::BuiltIn::ObjectToWorldKHR;
1135    case glslang::EbvWorldToObject:
1136    case glslang::EbvWorldToObject3x4:
1137        return spv::BuiltIn::WorldToObjectKHR;
1138    case glslang::EbvIncomingRayFlags:
1139        return spv::BuiltIn::IncomingRayFlagsKHR;
1140    case glslang::EbvGeometryIndex:
1141        return spv::BuiltIn::RayGeometryIndexKHR;
1142    case glslang::EbvCurrentRayTimeNV:
1143        builder.addExtension(spv::E_SPV_NV_ray_tracing_motion_blur);
1144        builder.addCapability(spv::Capability::RayTracingMotionBlurNV);
1145        return spv::BuiltIn::CurrentRayTimeNV;
1146    case glslang::EbvMicroTrianglePositionNV:
1147        builder.addCapability(spv::Capability::RayTracingDisplacementMicromapNV);
1148        builder.addExtension("SPV_NV_displacement_micromap");
1149        return spv::BuiltIn::HitMicroTriangleVertexPositionsNV;
1150    case glslang::EbvMicroTriangleBaryNV:
1151        builder.addCapability(spv::Capability::RayTracingDisplacementMicromapNV);
1152        builder.addExtension("SPV_NV_displacement_micromap");
1153        return spv::BuiltIn::HitMicroTriangleVertexBarycentricsNV;
1154    case glslang::EbvHitKindFrontFacingMicroTriangleNV:
1155        builder.addCapability(spv::Capability::RayTracingDisplacementMicromapNV);
1156        builder.addExtension("SPV_NV_displacement_micromap");
1157        return spv::BuiltIn::HitKindFrontFacingMicroTriangleNV;
1158    case glslang::EbvHitKindBackFacingMicroTriangleNV:
1159        builder.addCapability(spv::Capability::RayTracingDisplacementMicromapNV);
1160        builder.addExtension("SPV_NV_displacement_micromap");
1161        return spv::BuiltIn::HitKindBackFacingMicroTriangleNV;
1162    case glslang::EbvClusterIDNV:
1163        builder.addCapability(spv::Capability::RayTracingClusterAccelerationStructureNV);
1164        builder.addExtension("SPV_NV_cluster_acceleration_structure");
1165        return spv::BuiltIn::ClusterIDNV;
1166    case glslang::EbvHitIsSphereNV:
1167        builder.addCapability(spv::Capability::RayTracingSpheresGeometryNV);
1168        builder.addExtension("SPV_NV_linear_swept_spheres");
1169        return spv::BuiltIn::HitIsSphereNV;
1170    case glslang::EbvHitIsLSSNV:
1171        builder.addCapability(spv::Capability::RayTracingLinearSweptSpheresGeometryNV);
1172        builder.addExtension("SPV_NV_linear_swept_spheres");
1173        return spv::BuiltIn::HitIsLSSNV;
1174    case glslang::EbvHitSpherePositionNV:
1175        builder.addCapability(spv::Capability::RayTracingSpheresGeometryNV);
1176        builder.addExtension("SPV_NV_linear_swept_spheres");
1177        return spv::BuiltIn::HitSpherePositionNV;
1178    case glslang::EbvHitSphereRadiusNV:
1179        builder.addCapability(spv::Capability::RayTracingSpheresGeometryNV);
1180        builder.addExtension("SPV_NV_linear_swept_spheres");
1181        return spv::BuiltIn::HitSphereRadiusNV;
1182    case glslang::EbvHitLSSPositionsNV:
1183        builder.addCapability(spv::Capability::RayTracingLinearSweptSpheresGeometryNV);
1184        builder.addExtension("SPV_NV_linear_swept_spheres");
1185        return spv::BuiltIn::HitLSSPositionsNV;
1186    case glslang::EbvHitLSSRadiiNV:
1187        builder.addCapability(spv::Capability::RayTracingLinearSweptSpheresGeometryNV);
1188        builder.addExtension("SPV_NV_linear_swept_spheres");
1189        return spv::BuiltIn::HitLSSRadiiNV;
1190
1191    // barycentrics
1192    case glslang::EbvBaryCoordNV:
1193        builder.addExtension(spv::E_SPV_NV_fragment_shader_barycentric);
1194        builder.addCapability(spv::Capability::FragmentBarycentricNV);
1195        return spv::BuiltIn::BaryCoordNV;
1196    case glslang::EbvBaryCoordNoPerspNV:
1197        builder.addExtension(spv::E_SPV_NV_fragment_shader_barycentric);
1198        builder.addCapability(spv::Capability::FragmentBarycentricNV);
1199        return spv::BuiltIn::BaryCoordNoPerspNV;
1200
1201    case glslang::EbvBaryCoordEXT:
1202        builder.addExtension(spv::E_SPV_KHR_fragment_shader_barycentric);
1203        builder.addCapability(spv::Capability::FragmentBarycentricKHR);
1204        return spv::BuiltIn::BaryCoordKHR;
1205    case glslang::EbvBaryCoordNoPerspEXT:
1206        builder.addExtension(spv::E_SPV_KHR_fragment_shader_barycentric);
1207        builder.addCapability(spv::Capability::FragmentBarycentricKHR);
1208        return spv::BuiltIn::BaryCoordNoPerspKHR;
1209
1210    // mesh shaders
1211    case glslang::EbvTaskCountNV:
1212        return spv::BuiltIn::TaskCountNV;
1213    case glslang::EbvPrimitiveCountNV:
1214        return spv::BuiltIn::PrimitiveCountNV;
1215    case glslang::EbvPrimitiveIndicesNV:
1216        return spv::BuiltIn::PrimitiveIndicesNV;
1217    case glslang::EbvClipDistancePerViewNV:
1218        return spv::BuiltIn::ClipDistancePerViewNV;
1219    case glslang::EbvCullDistancePerViewNV:
1220        return spv::BuiltIn::CullDistancePerViewNV;
1221    case glslang::EbvLayerPerViewNV:
1222        return spv::BuiltIn::LayerPerViewNV;
1223    case glslang::EbvMeshViewCountNV:
1224        return spv::BuiltIn::MeshViewCountNV;
1225    case glslang::EbvMeshViewIndicesNV:
1226        return spv::BuiltIn::MeshViewIndicesNV;
1227
1228    // SPV_EXT_mesh_shader
1229    case glslang::EbvPrimitivePointIndicesEXT:
1230        return spv::BuiltIn::PrimitivePointIndicesEXT;
1231    case glslang::EbvPrimitiveLineIndicesEXT:
1232        return spv::BuiltIn::PrimitiveLineIndicesEXT;
1233    case glslang::EbvPrimitiveTriangleIndicesEXT:
1234        return spv::BuiltIn::PrimitiveTriangleIndicesEXT;
1235    case glslang::EbvCullPrimitiveEXT:
1236        return spv::BuiltIn::CullPrimitiveEXT;
1237
1238    // sm builtins
1239    case glslang::EbvWarpsPerSM:
1240        builder.addExtension(spv::E_SPV_NV_shader_sm_builtins);
1241        builder.addCapability(spv::Capability::ShaderSMBuiltinsNV);
1242        return spv::BuiltIn::WarpsPerSMNV;
1243    case glslang::EbvSMCount:
1244        builder.addExtension(spv::E_SPV_NV_shader_sm_builtins);
1245        builder.addCapability(spv::Capability::ShaderSMBuiltinsNV);
1246        return spv::BuiltIn::SMCountNV;
1247    case glslang::EbvWarpID:
1248        builder.addExtension(spv::E_SPV_NV_shader_sm_builtins);
1249        builder.addCapability(spv::Capability::ShaderSMBuiltinsNV);
1250        return spv::BuiltIn::WarpIDNV;
1251    case glslang::EbvSMID:
1252        builder.addExtension(spv::E_SPV_NV_shader_sm_builtins);
1253        builder.addCapability(spv::Capability::ShaderSMBuiltinsNV);
1254        return spv::BuiltIn::SMIDNV;
1255
1256   // ARM builtins
1257    case glslang::EbvCoreCountARM:
1258        builder.addExtension(spv::E_SPV_ARM_core_builtins);
1259        builder.addCapability(spv::Capability::CoreBuiltinsARM);
1260        return spv::BuiltIn::CoreCountARM;
1261    case glslang::EbvCoreIDARM:
1262        builder.addExtension(spv::E_SPV_ARM_core_builtins);
1263        builder.addCapability(spv::Capability::CoreBuiltinsARM);
1264        return spv::BuiltIn::CoreIDARM;
1265    case glslang::EbvCoreMaxIDARM:
1266        builder.addExtension(spv::E_SPV_ARM_core_builtins);
1267        builder.addCapability(spv::Capability::CoreBuiltinsARM);
1268        return spv::BuiltIn::CoreMaxIDARM;
1269    case glslang::EbvWarpIDARM:
1270        builder.addExtension(spv::E_SPV_ARM_core_builtins);
1271        builder.addCapability(spv::Capability::CoreBuiltinsARM);
1272        return spv::BuiltIn::WarpIDARM;
1273    case glslang::EbvWarpMaxIDARM:
1274        builder.addExtension(spv::E_SPV_ARM_core_builtins);
1275        builder.addCapability(spv::Capability::CoreBuiltinsARM);
1276        return spv::BuiltIn::WarpMaxIDARM;
1277
1278    // QCOM builtins
1279    case glslang::EbvTileOffsetQCOM:
1280        builder.addExtension(spv::E_SPV_QCOM_tile_shading);
1281        return spv::BuiltIn::TileOffsetQCOM;
1282    case glslang::EbvTileDimensionQCOM:
1283        builder.addExtension(spv::E_SPV_QCOM_tile_shading);
1284        return spv::BuiltIn::TileDimensionQCOM;
1285    case glslang::EbvTileApronSizeQCOM:
1286        builder.addExtension(spv::E_SPV_QCOM_tile_shading);
1287        return spv::BuiltIn::TileApronSizeQCOM;
1288
1289    // SPV_EXT_descriptor_heap
1290    case glslang::EbvSamplerHeapEXT:
1291        builder.addExtension(spv::E_SPV_EXT_descriptor_heap);
1292        builder.addExtension(spv::E_SPV_KHR_untyped_pointers);
1293        builder.addCapability(spv::Capability::DescriptorHeapEXT);
1294        builder.addCapability(spv::Capability::UntypedPointersKHR);
1295        // Add SamplerHeapEXT Symbol for spv level.
1296        if (builtInVariableIds.find(uint32_t(spv::BuiltIn::SamplerHeapEXT)) == builtInVariableIds.end()) {
1297            spv::Id samplerHeapEXT =
1298                builder.createUntypedVariable(spv::NoPrecision, spv::StorageClass::UniformConstant, "sampler_heap");
1299            if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4)
1300                entryPoint->addIdOperand(samplerHeapEXT);
1301            builder.addDecoration(samplerHeapEXT, spv::Decoration::BuiltIn, (int)spv::BuiltIn::SamplerHeapEXT);
1302            builtInVariableIds.insert({uint32_t(spv::BuiltIn::SamplerHeapEXT), samplerHeapEXT});
1303        }
1304        return spv::BuiltIn::SamplerHeapEXT;
1305    case glslang::EbvResourceHeapEXT:
1306        builder.addExtension(spv::E_SPV_EXT_descriptor_heap);
1307        builder.addExtension(spv::E_SPV_KHR_untyped_pointers);
1308        builder.addCapability(spv::Capability::DescriptorHeapEXT);
1309        builder.addCapability(spv::Capability::UntypedPointersKHR);
1310        // Add ResourceHeapEXT Symbol for spv level.
1311        if (builtInVariableIds.find(uint32_t(spv::BuiltIn::ResourceHeapEXT)) == builtInVariableIds.end()) {
1312            spv::Id resourceHeapEXT =
1313                builder.createUntypedVariable(spv::NoPrecision, spv::StorageClass::UniformConstant, "resource_heap");
1314            if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4)
1315                entryPoint->addIdOperand(resourceHeapEXT);
1316            builder.addDecoration(resourceHeapEXT, spv::Decoration::BuiltIn, (int)spv::BuiltIn::ResourceHeapEXT);
1317            builtInVariableIds.insert({uint32_t(spv::BuiltIn::ResourceHeapEXT), resourceHeapEXT});
1318        }
1319        return spv::BuiltIn::ResourceHeapEXT;
1320
1321    default:
1322        return spv::BuiltIn::Max;
1323    }
1324}
1325
1326// Translate glslang image layout format to SPIR-V image format.
1327spv::ImageFormat TGlslangToSpvTraverser::TranslateImageFormat(const glslang::TType& type)
1328{
1329    assert(type.getBasicType() == glslang::EbtSampler);
1330
1331    // Check for capabilities
1332    switch (type.getQualifier().getFormat()) {
1333    case glslang::ElfRg32f:
1334    case glslang::ElfRg16f:
1335    case glslang::ElfR11fG11fB10f:
1336    case glslang::ElfR16f:
1337    case glslang::ElfRgba16:
1338    case glslang::ElfRgb10A2:
1339    case glslang::ElfRg16:
1340    case glslang::ElfRg8:
1341    case glslang::ElfR16:
1342    case glslang::ElfR8:
1343    case glslang::ElfRgba16Snorm:
1344    case glslang::ElfRg16Snorm:
1345    case glslang::ElfRg8Snorm:
1346    case glslang::ElfR16Snorm:
1347    case glslang::ElfR8Snorm:
1348
1349    case glslang::ElfRg32i:
1350    case glslang::ElfRg16i:
1351    case glslang::ElfRg8i:
1352    case glslang::ElfR16i:
1353    case glslang::ElfR8i:
1354
1355    case glslang::ElfRgb10a2ui:
1356    case glslang::ElfRg32ui:
1357    case glslang::ElfRg16ui:
1358    case glslang::ElfRg8ui:
1359    case glslang::ElfR16ui:
1360    case glslang::ElfR8ui:
1361        builder.addCapability(spv::Capability::StorageImageExtendedFormats);
1362        break;
1363
1364    case glslang::ElfR64ui:
1365    case glslang::ElfR64i:
1366        builder.addExtension(spv::E_SPV_EXT_shader_image_int64);
1367        builder.addCapability(spv::Capability::Int64ImageEXT);
1368        break;
1369    default:
1370        break;
1371    }
1372
1373    // do the translation
1374    switch (type.getQualifier().getFormat()) {
1375    case glslang::ElfNone:          return spv::ImageFormat::Unknown;
1376    case glslang::ElfRgba32f:       return spv::ImageFormat::Rgba32f;
1377    case glslang::ElfRgba16f:       return spv::ImageFormat::Rgba16f;
1378    case glslang::ElfR32f:          return spv::ImageFormat::R32f;
1379    case glslang::ElfRgba8:         return spv::ImageFormat::Rgba8;
1380    case glslang::ElfRgba8Snorm:    return spv::ImageFormat::Rgba8Snorm;
1381    case glslang::ElfRg32f:         return spv::ImageFormat::Rg32f;
1382    case glslang::ElfRg16f:         return spv::ImageFormat::Rg16f;
1383    case glslang::ElfR11fG11fB10f:  return spv::ImageFormat::R11fG11fB10f;
1384    case glslang::ElfR16f:          return spv::ImageFormat::R16f;
1385    case glslang::ElfRgba16:        return spv::ImageFormat::Rgba16;
1386    case glslang::ElfRgb10A2:       return spv::ImageFormat::Rgb10A2;
1387    case glslang::ElfRg16:          return spv::ImageFormat::Rg16;
1388    case glslang::ElfRg8:           return spv::ImageFormat::Rg8;
1389    case glslang::ElfR16:           return spv::ImageFormat::R16;
1390    case glslang::ElfR8:            return spv::ImageFormat::R8;
1391    case glslang::ElfRgba16Snorm:   return spv::ImageFormat::Rgba16Snorm;
1392    case glslang::ElfRg16Snorm:     return spv::ImageFormat::Rg16Snorm;
1393    case glslang::ElfRg8Snorm:      return spv::ImageFormat::Rg8Snorm;
1394    case glslang::ElfR16Snorm:      return spv::ImageFormat::R16Snorm;
1395    case glslang::ElfR8Snorm:       return spv::ImageFormat::R8Snorm;
1396    case glslang::ElfRgba32i:       return spv::ImageFormat::Rgba32i;
1397    case glslang::ElfRgba16i:       return spv::ImageFormat::Rgba16i;
1398    case glslang::ElfRgba8i:        return spv::ImageFormat::Rgba8i;
1399    case glslang::ElfR32i:          return spv::ImageFormat::R32i;
1400    case glslang::ElfRg32i:         return spv::ImageFormat::Rg32i;
1401    case glslang::ElfRg16i:         return spv::ImageFormat::Rg16i;
1402    case glslang::ElfRg8i:          return spv::ImageFormat::Rg8i;
1403    case glslang::ElfR16i:          return spv::ImageFormat::R16i;
1404    case glslang::ElfR8i:           return spv::ImageFormat::R8i;
1405    case glslang::ElfRgba32ui:      return spv::ImageFormat::Rgba32ui;
1406    case glslang::ElfRgba16ui:      return spv::ImageFormat::Rgba16ui;
1407    case glslang::ElfRgba8ui:       return spv::ImageFormat::Rgba8ui;
1408    case glslang::ElfR32ui:         return spv::ImageFormat::R32ui;
1409    case glslang::ElfRg32ui:        return spv::ImageFormat::Rg32ui;
1410    case glslang::ElfRg16ui:        return spv::ImageFormat::Rg16ui;
1411    case glslang::ElfRgb10a2ui:     return spv::ImageFormat::Rgb10a2ui;
1412    case glslang::ElfRg8ui:         return spv::ImageFormat::Rg8ui;
1413    case glslang::ElfR16ui:         return spv::ImageFormat::R16ui;
1414    case glslang::ElfR8ui:          return spv::ImageFormat::R8ui;
1415    case glslang::ElfR64ui:         return spv::ImageFormat::R64ui;
1416    case glslang::ElfR64i:          return spv::ImageFormat::R64i;
1417    default:                        return spv::ImageFormat::Max;
1418    }
1419}
1420
1421spv::SelectionControlMask TGlslangToSpvTraverser::TranslateSelectionControl(
1422    const glslang::TIntermSelection& selectionNode) const
1423{
1424    if (selectionNode.getFlatten())
1425        return spv::SelectionControlMask::Flatten;
1426    if (selectionNode.getDontFlatten())
1427        return spv::SelectionControlMask::DontFlatten;
1428    return spv::SelectionControlMask::MaskNone;
1429}
1430
1431spv::SelectionControlMask TGlslangToSpvTraverser::TranslateSwitchControl(const glslang::TIntermSwitch& switchNode)
1432    const
1433{
1434    if (switchNode.getFlatten())
1435        return spv::SelectionControlMask::Flatten;
1436    if (switchNode.getDontFlatten())
1437        return spv::SelectionControlMask::DontFlatten;
1438    return spv::SelectionControlMask::MaskNone;
1439}
1440
1441spv::FunctionControlMask TGlslangToSpvTraverser::TranslateFunctionControl(unsigned int functionControl) const
1442{
1443    spv::FunctionControlMask control = spv::FunctionControlMask::MaskNone;
1444
1445    if (functionControl & glslang::EfcInline)
1446        control = control | spv::FunctionControlMask::Inline;
1447    if (functionControl & glslang::EfcDontInline)
1448        control = control | spv::FunctionControlMask::DontInline;
1449
1450    return control;
1451}
1452
1453// return a non-0 dependency if the dependency argument must be set
1454spv::LoopControlMask TGlslangToSpvTraverser::TranslateLoopControl(const glslang::TIntermLoop& loopNode,
1455    std::vector<unsigned int>& operands) const
1456{
1457    spv::LoopControlMask control = spv::LoopControlMask::MaskNone;
1458
1459    if (loopNode.getDontUnroll())
1460        control = control | spv::LoopControlMask::DontUnroll;
1461    if (loopNode.getUnroll())
1462        control = control | spv::LoopControlMask::Unroll;
1463    if (unsigned(loopNode.getLoopDependency()) == glslang::TIntermLoop::dependencyInfinite)
1464        control = control | spv::LoopControlMask::DependencyInfinite;
1465    else if (loopNode.getLoopDependency() > 0) {
1466        control = control | spv::LoopControlMask::DependencyLength;
1467        operands.push_back((unsigned int)loopNode.getLoopDependency());
1468    }
1469    if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) {
1470        if (loopNode.getMinIterations() > 0) {
1471            control = control | spv::LoopControlMask::MinIterations;
1472            operands.push_back(loopNode.getMinIterations());
1473        }
1474        if (loopNode.getMaxIterations() < glslang::TIntermLoop::iterationsInfinite) {
1475            control = control | spv::LoopControlMask::MaxIterations;
1476            operands.push_back(loopNode.getMaxIterations());
1477        }
1478        if (loopNode.getIterationMultiple() > 1) {
1479            control = control | spv::LoopControlMask::IterationMultiple;
1480            operands.push_back(loopNode.getIterationMultiple());
1481        }
1482        if (loopNode.getPeelCount() > 0) {
1483            control = control | spv::LoopControlMask::PeelCount;
1484            operands.push_back(loopNode.getPeelCount());
1485        }
1486        if (loopNode.getPartialCount() > 0) {
1487            control = control | spv::LoopControlMask::PartialCount;
1488            operands.push_back(loopNode.getPartialCount());
1489        }
1490    }
1491    if (loopNode.getMultipleWaitQueuesQCOM() != loopNode.noMultipleQaitQueues) {
1492        control = control | spv::LoopControlMask::MultipleWaitQueuesQCOM;
1493        operands.push_back(loopNode.getMultipleWaitQueuesQCOM());
1494    }
1495
1496    return control;
1497}
1498
1499// Translate glslang type to SPIR-V storage class.
1500spv::StorageClass TGlslangToSpvTraverser::TranslateStorageClass(const glslang::TType& type)
1501{
1502    if (type.getBasicType() == glslang::EbtRayQuery || type.getBasicType() == glslang::EbtHitObjectNV
1503        || type.getBasicType() == glslang::EbtHitObjectEXT)
1504        return spv::StorageClass::Private;
1505    if (type.getQualifier().isSpirvByReference()) {
1506        if (type.getQualifier().isParamInput() || type.getQualifier().isParamOutput())
1507            return spv::StorageClass::Function;
1508    }
1509    if (type.getQualifier().isPipeInput())
1510        return spv::StorageClass::Input;
1511    if (type.getQualifier().isPipeOutput())
1512        return spv::StorageClass::Output;
1513    if (type.getQualifier().storage == glslang::EvqTileImageEXT || type.isAttachmentEXT()) {
1514        builder.addExtension(spv::E_SPV_EXT_shader_tile_image);
1515        builder.addCapability(spv::Capability::TileImageColorReadAccessEXT);
1516        return spv::StorageClass::TileImageEXT;
1517    }
1518
1519    if (type.getQualifier().isTileAttachmentQCOM()) {
1520        builder.addExtension(spv::E_SPV_QCOM_tile_shading);
1521        builder.addCapability(spv::Capability::TileShadingQCOM);
1522        return spv::StorageClass::TileAttachmentQCOM;
1523    }
1524
1525    if (glslangIntermediate->getSource() != glslang::EShSourceHlsl ||
1526            type.getQualifier().storage == glslang::EvqUniform) {
1527        if (type.isAtomic())
1528            return spv::StorageClass::AtomicCounter;
1529        if (type.containsOpaque() && !glslangIntermediate->getBindlessMode()) {
1530            if (type.getQualifier().storage == glslang::EvqResourceHeap ||
1531                type.getQualifier().storage == glslang::EvqSamplerHeap)
1532                return spv::StorageClass::Uniform;
1533            else
1534                return spv::StorageClass::UniformConstant;
1535        }
1536    }
1537
1538    if (type.getQualifier().isUniformOrBuffer() &&
1539        type.getQualifier().isShaderRecord()) {
1540        return spv::StorageClass::ShaderRecordBufferKHR;
1541    }
1542
1543    if (glslangIntermediate->usingStorageBuffer() && type.getQualifier().storage == glslang::EvqBuffer) {
1544        builder.addIncorporatedExtension(spv::E_SPV_KHR_storage_buffer_storage_class, spv::Spv_1_3);
1545        return spv::StorageClass::StorageBuffer;
1546    }
1547
1548    if (type.getQualifier().isUniformOrBuffer()) {
1549        if (type.getQualifier().isPushConstant())
1550            return spv::StorageClass::PushConstant;
1551        if (type.getBasicType() == glslang::EbtBlock ||
1552            type.getQualifier().layoutDescriptorHeap ||
1553            type.getQualifier().isBufferType())
1554            return spv::StorageClass::Uniform;
1555        return spv::StorageClass::UniformConstant;
1556    }
1557
1558    if (type.getQualifier().storage == glslang::EvqShared && type.getBasicType() == glslang::EbtBlock) {
1559        builder.addExtension(spv::E_SPV_KHR_workgroup_memory_explicit_layout);
1560        builder.addCapability(spv::Capability::WorkgroupMemoryExplicitLayoutKHR);
1561        return spv::StorageClass::Workgroup;
1562    }
1563
1564    switch (type.getQualifier().storage) {
1565    case glslang::EvqGlobal:        return spv::StorageClass::Private;
1566    case glslang::EvqConstReadOnly: return spv::StorageClass::Function;
1567    case glslang::EvqTemporary:     return spv::StorageClass::Function;
1568    case glslang::EvqShared:           return spv::StorageClass::Workgroup;
1569    case glslang::EvqPayload:        return spv::StorageClass::RayPayloadKHR;
1570    case glslang::EvqPayloadIn:      return spv::StorageClass::IncomingRayPayloadKHR;
1571    case glslang::EvqHitAttr:        return spv::StorageClass::HitAttributeKHR;
1572    case glslang::EvqCallableData:   return spv::StorageClass::CallableDataKHR;
1573    case glslang::EvqCallableDataIn: return spv::StorageClass::IncomingCallableDataKHR;
1574    case glslang::EvqtaskPayloadSharedEXT : return spv::StorageClass::TaskPayloadWorkgroupEXT;
1575    case glslang::EvqHitObjectAttrNV: return spv::StorageClass::HitObjectAttributeNV;
1576    case glslang::EvqHitObjectAttrEXT: return spv::StorageClass::HitObjectAttributeEXT;
1577    case glslang::EvqSpirvStorageClass: return static_cast<spv::StorageClass>(type.getQualifier().spirvStorageClass);
1578    default:
1579        assert(0);
1580        break;
1581    }
1582
1583    return spv::StorageClass::Function;
1584}
1585
1586// Translate glslang constants to SPIR-V literals
1587void TGlslangToSpvTraverser::TranslateLiterals(const glslang::TVector<const glslang::TIntermConstantUnion*>& constants,
1588                                               std::vector<unsigned>& literals) const
1589{
1590    for (auto constant : constants) {
1591        if (constant->getBasicType() == glslang::EbtFloat) {
1592            float floatValue = static_cast<float>(constant->getConstArray()[0].getDConst());
1593            unsigned literal;
1594            static_assert(sizeof(literal) == sizeof(floatValue), "sizeof(unsigned) != sizeof(float)");
1595            memcpy(&literal, &floatValue, sizeof(literal));
1596            literals.push_back(literal);
1597        } else if (constant->getBasicType() == glslang::EbtInt) {
1598            unsigned literal = constant->getConstArray()[0].getIConst();
1599            literals.push_back(literal);
1600        } else if (constant->getBasicType() == glslang::EbtUint) {
1601            unsigned literal = constant->getConstArray()[0].getUConst();
1602            literals.push_back(literal);
1603        } else if (constant->getBasicType() == glslang::EbtBool) {
1604            unsigned literal = constant->getConstArray()[0].getBConst();
1605            literals.push_back(literal);
1606        } else if (constant->getBasicType() == glslang::EbtString) {
1607            auto str = constant->getConstArray()[0].getSConst()->c_str();
1608            unsigned literal = 0;
1609            char* literalPtr = reinterpret_cast<char*>(&literal);
1610            unsigned charCount = 0;
1611            char ch = 0;
1612            do {
1613                ch = *(str++);
1614                *(literalPtr++) = ch;
1615                ++charCount;
1616                if (charCount == 4) {
1617                    literals.push_back(literal);
1618                    literalPtr = reinterpret_cast<char*>(&literal);
1619                    charCount = 0;
1620                }
1621            } while (ch != 0);
1622
1623            // Partial literal is padded with 0
1624            if (charCount > 0) {
1625                for (; charCount < 4; ++charCount)
1626                    *(literalPtr++) = 0;
1627                literals.push_back(literal);
1628            }
1629        } else
1630            assert(0); // Unexpected type
1631    }
1632}
1633
1634// Add capabilities pertaining to how an array is indexed.
1635void TGlslangToSpvTraverser::addIndirectionIndexCapabilities(const glslang::TType& baseType,
1636                                                             const glslang::TType& indexType)
1637{
1638    if (indexType.getQualifier().isNonUniform()) {
1639
1640        // deal with an asserted non-uniform index
1641        // SPV_EXT_descriptor_indexing already added in TranslateNonUniformDecoration
1642        if (baseType.getBasicType() == glslang::EbtSampler) {
1643            if (baseType.getQualifier().hasAttachment())
1644                builder.addCapability(spv::Capability::InputAttachmentArrayNonUniformIndexingEXT);
1645            else if (baseType.isImage() && baseType.getSampler().isBuffer())
1646                builder.addCapability(spv::Capability::StorageTexelBufferArrayNonUniformIndexingEXT);
1647            else if (baseType.isTexture() && baseType.getSampler().isBuffer())
1648                builder.addCapability(spv::Capability::UniformTexelBufferArrayNonUniformIndexingEXT);
1649            else if (baseType.isImage())
1650                builder.addCapability(spv::Capability::StorageImageArrayNonUniformIndexingEXT);
1651            else if (baseType.isTexture())
1652                builder.addCapability(spv::Capability::SampledImageArrayNonUniformIndexingEXT);
1653        } else if (baseType.getBasicType() == glslang::EbtBlock) {
1654            if (baseType.getQualifier().storage == glslang::EvqBuffer)
1655                builder.addCapability(spv::Capability::StorageBufferArrayNonUniformIndexingEXT);
1656            else if (baseType.getQualifier().storage == glslang::EvqUniform)
1657                builder.addCapability(spv::Capability::UniformBufferArrayNonUniformIndexingEXT);
1658        }
1659    } else {
1660        // assume a dynamically uniform index
1661        if (baseType.getBasicType() == glslang::EbtSampler) {
1662            if (baseType.getQualifier().hasAttachment()) {
1663                builder.addIncorporatedExtension("SPV_EXT_descriptor_indexing", spv::Spv_1_5);
1664                builder.addCapability(spv::Capability::InputAttachmentArrayDynamicIndexingEXT);
1665            } else if (baseType.isImage() && baseType.getSampler().isBuffer()) {
1666                builder.addIncorporatedExtension("SPV_EXT_descriptor_indexing", spv::Spv_1_5);
1667                builder.addCapability(spv::Capability::StorageTexelBufferArrayDynamicIndexingEXT);
1668            } else if (baseType.isTexture() && baseType.getSampler().isBuffer()) {
1669                builder.addIncorporatedExtension("SPV_EXT_descriptor_indexing", spv::Spv_1_5);
1670                builder.addCapability(spv::Capability::UniformTexelBufferArrayDynamicIndexingEXT);
1671            }
1672        }
1673    }
1674}
1675
1676// Return whether or not the given type is something that should be tied to a
1677// descriptor set.
1678bool IsDescriptorResource(const glslang::TType& type)
1679{
1680    // uniform and buffer blocks are included, unless it is a push_constant
1681    if (type.getBasicType() == glslang::EbtBlock)
1682        return type.getQualifier().isUniformOrBuffer() &&
1683        ! type.getQualifier().isShaderRecord() &&
1684        ! type.getQualifier().isPushConstant();
1685
1686    // non block...
1687    // basically samplerXXX/subpass/sampler/texture are all included
1688    // if they are the global-scope-class, not the function parameter
1689    // (or local, if they ever exist) class.
1690    if (type.getBasicType() == glslang::EbtSampler ||
1691        type.getBasicType() == glslang::EbtAccStruct)
1692        return type.getQualifier().isUniformOrBuffer();
1693
1694    // Tensors are tied to a descriptor.
1695    if (type.isTensorARM())
1696        return true;
1697
1698    // None of the above.
1699    return false;
1700}
1701
1702void InheritQualifiers(glslang::TQualifier& child, const glslang::TQualifier& parent)
1703{
1704    if (child.layoutMatrix == glslang::ElmNone)
1705        child.layoutMatrix = parent.layoutMatrix;
1706
1707    if (parent.invariant)
1708        child.invariant = true;
1709    if (parent.flat)
1710        child.flat = true;
1711    if (parent.centroid)
1712        child.centroid = true;
1713    if (parent.nopersp)
1714        child.nopersp = true;
1715    if (parent.explicitInterp)
1716        child.explicitInterp = true;
1717    if (parent.perPrimitiveNV)
1718        child.perPrimitiveNV = true;
1719    if (parent.perViewNV)
1720        child.perViewNV = true;
1721    if (parent.perTaskNV)
1722        child.perTaskNV = true;
1723    if (parent.storage == glslang::EvqtaskPayloadSharedEXT)
1724        child.storage = glslang::EvqtaskPayloadSharedEXT;
1725    if (parent.patch)
1726        child.patch = true;
1727    if (parent.sample)
1728        child.sample = true;
1729    if (parent.coherent)
1730        child.coherent = true;
1731    if (parent.devicecoherent)
1732        child.devicecoherent = true;
1733    if (parent.queuefamilycoherent)
1734        child.queuefamilycoherent = true;
1735    if (parent.workgroupcoherent)
1736        child.workgroupcoherent = true;
1737    if (parent.subgroupcoherent)
1738        child.subgroupcoherent = true;
1739    if (parent.shadercallcoherent)
1740        child.shadercallcoherent = true;
1741    if (parent.nonprivate)
1742        child.nonprivate = true;
1743    if (parent.volatil)
1744        child.volatil = true;
1745    if (parent.nontemporal)
1746        child.nontemporal = true;
1747    if (parent.restrict)
1748        child.restrict = true;
1749    if (parent.readonly)
1750        child.readonly = true;
1751    if (parent.writeonly)
1752        child.writeonly = true;
1753    if (parent.nonUniform)
1754        child.nonUniform = true;
1755}
1756
1757bool HasNonLayoutQualifiers(const glslang::TType& type, const glslang::TQualifier& qualifier)
1758{
1759    // This should list qualifiers that simultaneous satisfy:
1760    // - struct members might inherit from a struct declaration
1761    //     (note that non-block structs don't explicitly inherit,
1762    //      only implicitly, meaning no decoration involved)
1763    // - affect decorations on the struct members
1764    //     (note smooth does not, and expecting something like volatile
1765    //      to effect the whole object)
1766    // - are not part of the offset/st430/etc or row/column-major layout
1767    return qualifier.invariant || (qualifier.hasLocation() && type.getBasicType() == glslang::EbtBlock);
1768}
1769
1770//
1771// Implement the TGlslangToSpvTraverser class.
1772//
1773
1774TGlslangToSpvTraverser::TGlslangToSpvTraverser(unsigned int spvVersion,
1775    const glslang::TIntermediate* glslangIntermediate,
1776    spv::SpvBuildLogger* buildLogger, glslang::SpvOptions& options) :
1777        TIntermTraverser(true, false, true),
1778        options(options),
1779        shaderEntry(nullptr), currentFunction(nullptr),
1780        sequenceDepth(0),
1781        opacityMicromapEnableConstId(spv::NoResult),
1782        logger(buildLogger),
1783        builder(spvVersion, (glslang::GetKhronosToolId() << 16) | glslang::GetSpirvGeneratorVersion(), logger),
1784        inEntryPoint(false), entryPointTerminated(false), linkageOnly(false),
1785        glslangIntermediate(glslangIntermediate),
1786        nanMinMaxClamp(glslangIntermediate->getNanMinMaxClamp()),
1787        nonSemanticDebugPrintf(0),
1788        taskPayloadID(0),
1789        descHeapLayout(*this)
1790{
1791    bool isMeshShaderExt = (glslangIntermediate->getRequestedExtensions().find(glslang::E_GL_EXT_mesh_shader) !=
1792                            glslangIntermediate->getRequestedExtensions().end());
1793    spv::ExecutionModel executionModel = TranslateExecutionModel(glslangIntermediate->getStage(), isMeshShaderExt);
1794
1795    builder.clearAccessChain();
1796    builder.setSource(TranslateSourceLanguage(glslangIntermediate->getSource(), glslangIntermediate->getProfile()),
1797                      glslangIntermediate->getVersion());
1798
1799    if (options.emitNonSemanticShaderDebugSource)
1800            this->options.emitNonSemanticShaderDebugInfo = true;
1801    if (options.emitNonSemanticShaderDebugInfo)
1802            this->options.generateDebugInfo = true;
1803
1804    if (this->options.generateDebugInfo) {
1805        if (this->options.emitNonSemanticShaderDebugInfo) {
1806            builder.setEmitNonSemanticShaderDebugInfo(this->options.emitNonSemanticShaderDebugSource);
1807        }
1808        else {
1809            builder.setEmitSpirvDebugInfo();
1810        }
1811        builder.setDebugMainSourceFile(glslangIntermediate->getSourceFile());
1812
1813        // Set the source shader's text
1814        std::string text;
1815
1816        // If for SPV version 1.0, include
1817        // a preamble in comments stating the OpModuleProcessed instructions.
1818        // Otherwise, emit those as actual instructions.
1819        //
1820        // ...Except when using ShaderDebugInfo we DON'T want these as it will mess up the line
1821        // number, instead the user has opt'ed in for ShaderDebugInfo instead, so they will want
1822        // to parse those instead
1823        // https://github.com/KhronosGroup/glslang/issues/3863
1824        const bool add_comments =
1825            glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_1 && !options.emitNonSemanticShaderDebugSource;
1826
1827        const std::vector<std::string>& processes = glslangIntermediate->getProcesses();
1828        for (int p = 0; p < (int)processes.size(); ++p) {
1829            if (add_comments) {
1830                text.append("// OpModuleProcessed ");
1831                text.append(processes[p]);
1832                text.append("\n");
1833            } else if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_1) {
1834                // OpModuleProcessed added in SPIR-V 1.1
1835                builder.addModuleProcessed(processes[p]);
1836            }
1837        }
1838        if (add_comments && (int)processes.size() > 0)
1839            text.append("#line 1\n");
1840        text.append(glslangIntermediate->getSourceText());
1841        builder.setSourceText(text);
1842        // Pass name and text for all included files
1843        const std::map<std::string, std::string>& include_txt = glslangIntermediate->getIncludeText();
1844        for (auto iItr = include_txt.begin(); iItr != include_txt.end(); ++iItr)
1845            builder.addInclude(iItr->first, iItr->second);
1846    } else if (this->options.emitSourceFileName) {
1847        builder.setEmitSourceFileName(glslangIntermediate->getSourceFile());
1848    }
1849
1850    builder.setUseReplicatedComposites(glslangIntermediate->usingReplicatedComposites());
1851
1852    stdBuiltins = builder.import("GLSL.std.450");
1853
1854    spv::AddressingModel addressingModel = spv::AddressingModel::Logical;
1855    spv::MemoryModel memoryModel = spv::MemoryModel::GLSL450;
1856
1857    if (glslangIntermediate->usingPhysicalStorageBuffer()) {
1858        addressingModel = spv::AddressingModel::PhysicalStorageBuffer64EXT;
1859        builder.addIncorporatedExtension(spv::E_SPV_KHR_physical_storage_buffer, spv::Spv_1_5);
1860        builder.addCapability(spv::Capability::PhysicalStorageBufferAddressesEXT);
1861    }
1862    if (glslangIntermediate->usingVulkanMemoryModel()) {
1863        memoryModel = spv::MemoryModel::VulkanKHR;
1864        builder.addCapability(spv::Capability::VulkanMemoryModelKHR);
1865        builder.addIncorporatedExtension(spv::E_SPV_KHR_vulkan_memory_model, spv::Spv_1_5);
1866    }
1867    builder.setMemoryModel(addressingModel, memoryModel);
1868
1869    if (glslangIntermediate->usingVariablePointers()) {
1870        builder.addCapability(spv::Capability::VariablePointers);
1871    }
1872
1873    // If not linking, there is no entry point
1874    if (!options.compileOnly) {
1875        shaderEntry = builder.makeEntryPoint(glslangIntermediate->getEntryPointName().c_str());
1876        entryPoint =
1877            builder.addEntryPoint(executionModel, shaderEntry, glslangIntermediate->getEntryPointName().c_str());
1878    }
1879
1880    // Add the source extensions
1881    const auto& sourceExtensions = glslangIntermediate->getRequestedExtensions();
1882    for (auto it = sourceExtensions.begin(); it != sourceExtensions.end(); ++it)
1883        builder.addSourceExtension(it->c_str());
1884
1885    // Add the top-level modes for this shader.
1886
1887    if (glslangIntermediate->getXfbMode()) {
1888        builder.addCapability(spv::Capability::TransformFeedback);
1889        builder.addExecutionMode(shaderEntry, spv::ExecutionMode::Xfb);
1890    }
1891
1892    if (glslangIntermediate->getLayoutPrimitiveCulling()) {
1893        builder.addCapability(spv::Capability::RayTraversalPrimitiveCullingKHR);
1894    }
1895
1896    // gl_RayFlagsForceOpacityMicromap2StateEXT was referenced. The SPIR-V RayFlags
1897    // bit ForceOpacityMicromap2StateKHR requires the RayTracingOpacityMicromapEXT
1898    // capability. Emit it here so it applies uniformly to ray-tracing-pipeline
1899    // stages and any stage running ray queries.
1900    if (glslangIntermediate->getUsesOpacityMicromap2StateFlag()) {
1901        builder.addCapability(spv::Capability::RayTracingOpacityMicromapEXT);
1902        builder.addExtension("SPV_EXT_opacity_micromap");
1903    }
1904
1905    if (glslangIntermediate->getSubgroupUniformControlFlow()) {
1906        builder.addExtension(spv::E_SPV_KHR_subgroup_uniform_control_flow);
1907        builder.addExecutionMode(shaderEntry, spv::ExecutionMode::SubgroupUniformControlFlowKHR);
1908    }
1909    if (glslangIntermediate->getMaximallyReconverges()) {
1910        builder.addExtension(spv::E_SPV_KHR_maximal_reconvergence);
1911        builder.addExecutionMode(shaderEntry, spv::ExecutionMode::MaximallyReconvergesKHR);
1912    }
1913
1914    if (glslangIntermediate->getQuadDerivMode())
1915    {
1916        builder.addCapability(spv::Capability::QuadControlKHR);
1917        builder.addExtension(spv::E_SPV_KHR_quad_control);
1918        builder.addExecutionMode(shaderEntry, spv::ExecutionMode::QuadDerivativesKHR);
1919    }
1920
1921    if (glslangIntermediate->getReqFullQuadsMode())
1922    {
1923        builder.addCapability(spv::Capability::QuadControlKHR);
1924        builder.addExtension(spv::E_SPV_KHR_quad_control);
1925        builder.addExecutionMode(shaderEntry, spv::ExecutionMode::RequireFullQuadsKHR);
1926    }
1927
1928    if (glslangIntermediate->usingShader64BitIndexing())
1929    {
1930        builder.addCapability(spv::Capability::Shader64BitIndexingEXT);
1931        builder.addExtension(spv::E_SPV_EXT_shader_64bit_indexing);
1932        builder.addExecutionMode(shaderEntry, spv::ExecutionMode::Shader64BitIndexingEXT);
1933    }
1934
1935    spv::ExecutionMode mode;
1936    switch (glslangIntermediate->getStage()) {
1937    case EShLangVertex:
1938        builder.addCapability(spv::Capability::Shader);
1939        break;
1940
1941    case EShLangFragment:
1942        builder.addCapability(spv::Capability::Shader);
1943        if (glslangIntermediate->getPixelCenterInteger())
1944            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::PixelCenterInteger);
1945
1946        if (glslangIntermediate->getOriginUpperLeft())
1947            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::OriginUpperLeft);
1948        else
1949            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::OriginLowerLeft);
1950
1951        if (glslangIntermediate->getEarlyFragmentTests())
1952            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::EarlyFragmentTests);
1953
1954        if (glslangIntermediate->getEarlyAndLateFragmentTestsAMD())
1955        {
1956            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::EarlyAndLateFragmentTestsAMD);
1957            builder.addExtension(spv::E_SPV_AMD_shader_early_and_late_fragment_tests);
1958        }
1959
1960        if (glslangIntermediate->getPostDepthCoverage()) {
1961            builder.addCapability(spv::Capability::SampleMaskPostDepthCoverage);
1962            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::PostDepthCoverage);
1963            builder.addExtension(spv::E_SPV_KHR_post_depth_coverage);
1964        }
1965
1966        if (glslangIntermediate->getNonCoherentColorAttachmentReadEXT()) {
1967            builder.addCapability(spv::Capability::TileImageColorReadAccessEXT);
1968            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::NonCoherentColorAttachmentReadEXT);
1969            builder.addExtension(spv::E_SPV_EXT_shader_tile_image);
1970        }
1971
1972        if (glslangIntermediate->getNonCoherentDepthAttachmentReadEXT()) {
1973            builder.addCapability(spv::Capability::TileImageDepthReadAccessEXT);
1974            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::NonCoherentDepthAttachmentReadEXT);
1975            builder.addExtension(spv::E_SPV_EXT_shader_tile_image);
1976        }
1977
1978        if (glslangIntermediate->getNonCoherentStencilAttachmentReadEXT()) {
1979            builder.addCapability(spv::Capability::TileImageStencilReadAccessEXT);
1980            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::NonCoherentStencilAttachmentReadEXT);
1981            builder.addExtension(spv::E_SPV_EXT_shader_tile_image);
1982        }
1983
1984        if (glslangIntermediate->getNonCoherentTileAttachmentReadQCOM()) {
1985            builder.addCapability(spv::Capability::TileShadingQCOM);
1986            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::NonCoherentTileAttachmentReadQCOM);
1987            builder.addExtension(spv::E_SPV_QCOM_tile_shading);
1988        }
1989
1990        if (glslangIntermediate->isDepthReplacing())
1991            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::DepthReplacing);
1992
1993        if (glslangIntermediate->isStencilReplacing())
1994            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::StencilRefReplacingEXT);
1995
1996        switch(glslangIntermediate->getDepth()) {
1997        case glslang::EldGreater:   mode = spv::ExecutionMode::DepthGreater;   break;
1998        case glslang::EldLess:      mode = spv::ExecutionMode::DepthLess;      break;
1999        case glslang::EldUnchanged: mode = spv::ExecutionMode::DepthUnchanged; break;
2000        default:                    mode = spv::ExecutionMode::Max;            break;
2001        }
2002
2003        if (mode != spv::ExecutionMode::Max)
2004            builder.addExecutionMode(shaderEntry, mode);
2005
2006        switch (glslangIntermediate->getStencil()) {
2007        case glslang::ElsRefUnchangedFrontAMD:  mode = spv::ExecutionMode::StencilRefUnchangedFrontAMD; break;
2008        case glslang::ElsRefGreaterFrontAMD:    mode = spv::ExecutionMode::StencilRefGreaterFrontAMD;   break;
2009        case glslang::ElsRefLessFrontAMD:       mode = spv::ExecutionMode::StencilRefLessFrontAMD;      break;
2010        case glslang::ElsRefUnchangedBackAMD:   mode = spv::ExecutionMode::StencilRefUnchangedBackAMD;  break;
2011        case glslang::ElsRefGreaterBackAMD:     mode = spv::ExecutionMode::StencilRefGreaterBackAMD;    break;
2012        case glslang::ElsRefLessBackAMD:        mode = spv::ExecutionMode::StencilRefLessBackAMD;       break;
2013        default:                       mode = spv::ExecutionMode::Max;                         break;
2014        }
2015
2016        if (mode != spv::ExecutionMode::Max)
2017            builder.addExecutionMode(shaderEntry, (spv::ExecutionMode)mode);
2018        switch (glslangIntermediate->getInterlockOrdering()) {
2019        case glslang::EioPixelInterlockOrdered:         mode = spv::ExecutionMode::PixelInterlockOrderedEXT;
2020            break;
2021        case glslang::EioPixelInterlockUnordered:       mode = spv::ExecutionMode::PixelInterlockUnorderedEXT;
2022            break;
2023        case glslang::EioSampleInterlockOrdered:        mode = spv::ExecutionMode::SampleInterlockOrderedEXT;
2024            break;
2025        case glslang::EioSampleInterlockUnordered:      mode = spv::ExecutionMode::SampleInterlockUnorderedEXT;
2026            break;
2027        case glslang::EioShadingRateInterlockOrdered:   mode = spv::ExecutionMode::ShadingRateInterlockOrderedEXT;
2028            break;
2029        case glslang::EioShadingRateInterlockUnordered: mode = spv::ExecutionMode::ShadingRateInterlockUnorderedEXT;
2030            break;
2031        default:                                        mode = spv::ExecutionMode::Max;
2032            break;
2033        }
2034        if (mode != spv::ExecutionMode::Max) {
2035            builder.addExecutionMode(shaderEntry, (spv::ExecutionMode)mode);
2036            if (mode == spv::ExecutionMode::ShadingRateInterlockOrderedEXT ||
2037                mode == spv::ExecutionMode::ShadingRateInterlockUnorderedEXT) {
2038                builder.addCapability(spv::Capability::FragmentShaderShadingRateInterlockEXT);
2039            } else if (mode == spv::ExecutionMode::PixelInterlockOrderedEXT ||
2040                       mode == spv::ExecutionMode::PixelInterlockUnorderedEXT) {
2041                builder.addCapability(spv::Capability::FragmentShaderPixelInterlockEXT);
2042            } else {
2043                builder.addCapability(spv::Capability::FragmentShaderSampleInterlockEXT);
2044            }
2045            builder.addExtension(spv::E_SPV_EXT_fragment_shader_interlock);
2046        }
2047    break;
2048
2049    case EShLangCompute: {
2050        builder.addCapability(spv::Capability::Shader);
2051        bool needSizeId = false;
2052        for (int dim = 0; dim < 3; ++dim) {
2053            if ((glslangIntermediate->getLocalSizeSpecId(dim) != glslang::TQualifier::layoutNotSet)) {
2054                needSizeId = true;
2055                break;
2056            }
2057        }
2058        if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6 && needSizeId) {
2059            std::vector<spv::Id> dimConstId;
2060            for (int dim = 0; dim < 3; ++dim) {
2061                bool specConst = (glslangIntermediate->getLocalSizeSpecId(dim) != glslang::TQualifier::layoutNotSet);
2062                dimConstId.push_back(builder.makeUintConstant(glslangIntermediate->getLocalSize(dim), specConst));
2063                if (specConst) {
2064                    builder.addDecoration(dimConstId.back(), spv::Decoration::SpecId,
2065                                          glslangIntermediate->getLocalSizeSpecId(dim));
2066                    needSizeId = true;
2067                }
2068            }
2069            builder.addExecutionModeId(shaderEntry, spv::ExecutionMode::LocalSizeId, dimConstId);
2070        } else {
2071            if (glslangIntermediate->getTileShadingRateQCOM(0) >= 1 || glslangIntermediate->getTileShadingRateQCOM(1) >= 1 || glslangIntermediate->getTileShadingRateQCOM(2) >= 1) {
2072                auto rate_x = glslangIntermediate->getTileShadingRateQCOM(0);
2073                auto rate_y = glslangIntermediate->getTileShadingRateQCOM(1);
2074                auto rate_z = glslangIntermediate->getTileShadingRateQCOM(2);
2075                rate_x = ( rate_x == 0 ? 1 : rate_x );
2076                rate_y = ( rate_y == 0 ? 1 : rate_y );
2077                rate_z = ( rate_z == 0 ? 1 : rate_z );
2078                builder.addExecutionMode(shaderEntry, spv::ExecutionMode::TileShadingRateQCOM, rate_x, rate_y, rate_z);
2079            } else {
2080                builder.addExecutionMode(shaderEntry, spv::ExecutionMode::LocalSize, glslangIntermediate->getLocalSize(0),
2081                                                                                   glslangIntermediate->getLocalSize(1),
2082                                                                                   glslangIntermediate->getLocalSize(2));
2083            }
2084        }
2085        addDerivativeGroupExecutionMode(builder, *glslangIntermediate, shaderEntry);
2086
2087        if (glslangIntermediate->getNonCoherentTileAttachmentReadQCOM()) {
2088            builder.addCapability(spv::Capability::TileShadingQCOM);
2089            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::NonCoherentTileAttachmentReadQCOM);
2090            builder.addExtension(spv::E_SPV_QCOM_tile_shading);
2091        }
2092
2093        break;
2094    }
2095    case EShLangTessEvaluation:
2096    case EShLangTessControl:
2097        builder.addCapability(spv::Capability::Tessellation);
2098
2099        glslang::TLayoutGeometry primitive;
2100
2101        if (glslangIntermediate->getStage() == EShLangTessControl) {
2102            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::OutputVertices,
2103                glslangIntermediate->getVertices());
2104            primitive = glslangIntermediate->getOutputPrimitive();
2105        } else {
2106            primitive = glslangIntermediate->getInputPrimitive();
2107        }
2108
2109        switch (primitive) {
2110        case glslang::ElgTriangles:           mode = spv::ExecutionMode::Triangles;     break;
2111        case glslang::ElgQuads:               mode = spv::ExecutionMode::Quads;         break;
2112        case glslang::ElgIsolines:            mode = spv::ExecutionMode::Isolines;      break;
2113        default:                              mode = spv::ExecutionMode::Max;           break;
2114        }
2115        if (mode != spv::ExecutionMode::Max)
2116            builder.addExecutionMode(shaderEntry, mode);
2117
2118        switch (glslangIntermediate->getVertexSpacing()) {
2119        case glslang::EvsEqual:            mode = spv::ExecutionMode::SpacingEqual;          break;
2120        case glslang::EvsFractionalEven:   mode = spv::ExecutionMode::SpacingFractionalEven; break;
2121        case glslang::EvsFractionalOdd:    mode = spv::ExecutionMode::SpacingFractionalOdd;  break;
2122        default:                           mode = spv::ExecutionMode::Max;                   break;
2123        }
2124        if (mode != spv::ExecutionMode::Max)
2125            builder.addExecutionMode(shaderEntry, mode);
2126
2127        switch (glslangIntermediate->getVertexOrder()) {
2128        case glslang::EvoCw:     mode = spv::ExecutionMode::VertexOrderCw;  break;
2129        case glslang::EvoCcw:    mode = spv::ExecutionMode::VertexOrderCcw; break;
2130        default:                 mode = spv::ExecutionMode::Max;            break;
2131        }
2132        if (mode != spv::ExecutionMode::Max)
2133            builder.addExecutionMode(shaderEntry, mode);
2134
2135        if (glslangIntermediate->getPointMode())
2136            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::PointMode);
2137        break;
2138
2139    case EShLangGeometry:
2140        builder.addCapability(spv::Capability::Geometry);
2141        switch (glslangIntermediate->getInputPrimitive()) {
2142        case glslang::ElgPoints:             mode = spv::ExecutionMode::InputPoints;             break;
2143        case glslang::ElgLines:              mode = spv::ExecutionMode::InputLines;              break;
2144        case glslang::ElgLinesAdjacency:     mode = spv::ExecutionMode::InputLinesAdjacency;     break;
2145        case glslang::ElgTriangles:          mode = spv::ExecutionMode::Triangles;               break;
2146        case glslang::ElgTrianglesAdjacency: mode = spv::ExecutionMode::InputTrianglesAdjacency; break;
2147        default:                             mode = spv::ExecutionMode::Max;                     break;
2148        }
2149        if (mode != spv::ExecutionMode::Max)
2150            builder.addExecutionMode(shaderEntry, mode);
2151
2152        builder.addExecutionMode(shaderEntry, spv::ExecutionMode::Invocations, glslangIntermediate->getInvocations());
2153
2154        switch (glslangIntermediate->getOutputPrimitive()) {
2155        case glslang::ElgPoints:        mode = spv::ExecutionMode::OutputPoints;                 break;
2156        case glslang::ElgLineStrip:     mode = spv::ExecutionMode::OutputLineStrip;              break;
2157        case glslang::ElgTriangleStrip: mode = spv::ExecutionMode::OutputTriangleStrip;          break;
2158        default:                        mode = spv::ExecutionMode::Max;                          break;
2159        }
2160        if (mode != spv::ExecutionMode::Max)
2161            builder.addExecutionMode(shaderEntry, mode);
2162        builder.addExecutionMode(shaderEntry, spv::ExecutionMode::OutputVertices, glslangIntermediate->getVertices());
2163        break;
2164
2165    case EShLangRayGen:
2166    case EShLangIntersect:
2167    case EShLangAnyHit:
2168    case EShLangClosestHit:
2169    case EShLangMiss:
2170    case EShLangCallable:
2171    {
2172        auto& extensions = glslangIntermediate->getRequestedExtensions();
2173        if (extensions.find("GL_NV_ray_tracing") == extensions.end()) {
2174            builder.addCapability(spv::Capability::RayTracingKHR);
2175            builder.addExtension("SPV_KHR_ray_tracing");
2176        }
2177        else {
2178            builder.addCapability(spv::Capability::RayTracingNV);
2179            builder.addExtension("SPV_NV_ray_tracing");
2180        }
2181        if (glslangIntermediate->getStage() != EShLangRayGen && glslangIntermediate->getStage() != EShLangCallable) {
2182            if (extensions.find("GL_EXT_ray_cull_mask") != extensions.end()) {
2183                builder.addCapability(spv::Capability::RayCullMaskKHR);
2184                builder.addExtension("SPV_KHR_ray_cull_mask");
2185            }
2186            if (extensions.find("GL_EXT_ray_tracing_position_fetch") != extensions.end()) {
2187                builder.addCapability(spv::Capability::RayTracingPositionFetchKHR);
2188                builder.addExtension("SPV_KHR_ray_tracing_position_fetch");
2189            }
2190        }
2191        break;
2192    }
2193    case EShLangTask:
2194    case EShLangMesh:
2195        if(isMeshShaderExt) {
2196            builder.addCapability(spv::Capability::MeshShadingEXT);
2197            builder.addExtension(spv::E_SPV_EXT_mesh_shader);
2198        } else {
2199            builder.addCapability(spv::Capability::MeshShadingNV);
2200            builder.addExtension(spv::E_SPV_NV_mesh_shader);
2201        }
2202        if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6) {
2203            std::vector<spv::Id> dimConstId;
2204            for (int dim = 0; dim < 3; ++dim) {
2205                bool specConst = (glslangIntermediate->getLocalSizeSpecId(dim) != glslang::TQualifier::layoutNotSet);
2206                dimConstId.push_back(builder.makeUintConstant(glslangIntermediate->getLocalSize(dim), specConst));
2207                if (specConst) {
2208                    builder.addDecoration(dimConstId.back(), spv::Decoration::SpecId,
2209                                          glslangIntermediate->getLocalSizeSpecId(dim));
2210                }
2211            }
2212            builder.addExecutionModeId(shaderEntry, spv::ExecutionMode::LocalSizeId, dimConstId);
2213        } else {
2214            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::LocalSize, glslangIntermediate->getLocalSize(0),
2215                                                                               glslangIntermediate->getLocalSize(1),
2216                                                                               glslangIntermediate->getLocalSize(2));
2217        }
2218        addDerivativeGroupExecutionMode(builder, *glslangIntermediate, shaderEntry);
2219        if (glslangIntermediate->getStage() == EShLangMesh) {
2220            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::OutputVertices,
2221                glslangIntermediate->getVertices());
2222            builder.addExecutionMode(shaderEntry, spv::ExecutionMode::OutputPrimitivesNV,
2223                glslangIntermediate->getPrimitives());
2224
2225            switch (glslangIntermediate->getOutputPrimitive()) {
2226            case glslang::ElgPoints:        mode = spv::ExecutionMode::OutputPoints;      break;
2227            case glslang::ElgLines:         mode = spv::ExecutionMode::OutputLinesNV;     break;
2228            case glslang::ElgTriangles:     mode = spv::ExecutionMode::OutputTrianglesNV; break;
2229            default:                        mode = spv::ExecutionMode::Max;               break;
2230            }
2231            if (mode != spv::ExecutionMode::Max)
2232                builder.addExecutionMode(shaderEntry, (spv::ExecutionMode)mode);
2233        }
2234        break;
2235
2236    default:
2237        break;
2238    }
2239
2240    //
2241    // Add SPIR-V requirements (GL_EXT_spirv_intrinsics)
2242    //
2243    if (glslangIntermediate->hasSpirvRequirement()) {
2244        const glslang::TSpirvRequirement& spirvRequirement = glslangIntermediate->getSpirvRequirement();
2245
2246        // Add SPIR-V extension requirement
2247        for (auto& extension : spirvRequirement.extensions)
2248            builder.addExtension(extension.c_str());
2249
2250        // Add SPIR-V capability requirement
2251        for (auto capability : spirvRequirement.capabilities)
2252            builder.addCapability(static_cast<spv::Capability>(capability));
2253    }
2254
2255    //
2256    // Add SPIR-V execution mode qualifiers (GL_EXT_spirv_intrinsics)
2257    //
2258    if (glslangIntermediate->hasSpirvExecutionMode()) {
2259        const glslang::TSpirvExecutionMode spirvExecutionMode = glslangIntermediate->getSpirvExecutionMode();
2260
2261        // Add spirv_execution_mode
2262        for (auto& mode : spirvExecutionMode.modes) {
2263            if (!mode.second.empty()) {
2264                std::vector<unsigned> literals;
2265                TranslateLiterals(mode.second, literals);
2266                builder.addExecutionMode(shaderEntry, static_cast<spv::ExecutionMode>(mode.first), literals);
2267            } else
2268                builder.addExecutionMode(shaderEntry, static_cast<spv::ExecutionMode>(mode.first));
2269        }
2270
2271        // Add spirv_execution_mode_id
2272        for (auto& modeId : spirvExecutionMode.modeIds) {
2273            std::vector<spv::Id> operandIds;
2274            assert(!modeId.second.empty());
2275            for (auto extraOperand : modeId.second) {
2276                if (extraOperand->getType().getQualifier().isSpecConstant())
2277                    operandIds.push_back(getSymbolId(extraOperand->getAsSymbolNode()));
2278                else
2279                    operandIds.push_back(createSpvConstant(*extraOperand));
2280            }
2281            builder.addExecutionModeId(shaderEntry, static_cast<spv::ExecutionMode>(modeId.first), operandIds);
2282        }
2283    }
2284}
2285
2286// Finish creating SPV, after the traversal is complete.
2287void TGlslangToSpvTraverser::finishSpv(bool compileOnly)
2288{
2289    // If not linking, an entry point is not expected
2290    if (!compileOnly) {
2291        // Finish the entry point function
2292        if (!entryPointTerminated) {
2293            builder.setBuildPoint(shaderEntry->getLastBlock());
2294            builder.leaveFunction();
2295        }
2296
2297        // finish off the entry-point SPV instruction by adding the Input/Output <id>
2298        entryPoint->reserveOperands(iOSet.size());
2299        for (auto id : iOSet)
2300            entryPoint->addIdOperand(id);
2301
2302        // GL_EXT_opacity_micromap_ray_query_mode: the OpacityMicromapIdKHR execution mode is emitted
2303        // whenever the extension is enabled (enabling the extension is the declaration), referencing the
2304        // <id> of gl_EnableOpacityMicromapEXT. The operand constant depends on how the built-in was
2305        // (re)declared: a constant_id redeclaration -> OpSpecConstantFalse (decorated with the SpecId);
2306        // 'const bool = true' -> OpConstantTrue; otherwise (default or '= false') -> OpConstantFalse. If
2307        // the built-in was read in the shader its spec constant is reused (opacityMicromapEnableConstId)
2308        // so only one SpecId is emitted. This path targets SPV_KHR_opacity_micromap; the
2309        // ForceOpacityMicromap2State ray flag keeps using SPV_EXT_opacity_micromap for compatibility.
2310        if (glslangIntermediate->IsRequestedExtension(glslang::E_GL_EXT_opacity_micromap_ray_query_mode)) {
2311            spv::Id enableId = opacityMicromapEnableConstId;
2312            if (enableId == spv::NoResult) {
2313                int specId = glslangIntermediate->getEnableOpacityMicromapSpecId();
2314                if (specId != glslang::TQualifier::layoutNotSet) {
2315                    enableId = builder.makeBoolConstant(false, true);
2316                    builder.addDecoration(enableId, spv::Decoration::SpecId, specId);
2317                } else {
2318                    enableId = builder.makeBoolConstant(glslangIntermediate->getEnableOpacityMicromapDefault(), false);
2319                }
2320            }
2321            builder.addCapability(spv::Capability::RayTracingOpacityMicromapExecutionModeKHR);
2322            builder.addExtension(spv::E_SPV_KHR_opacity_micromap);
2323            builder.addExecutionModeId(shaderEntry, spv::ExecutionMode::OpacityMicromapIdKHR, { enableId });
2324        }
2325    }
2326
2327    // Add capabilities, extensions, remove unneeded decorations, etc.,
2328    // based on the resulting SPIR-V.
2329    // Note: WebGPU code generation must have the opportunity to aggressively
2330    // prune unreachable merge blocks and continue targets.
2331    builder.postProcess(compileOnly);
2332}
2333
2334// Write the SPV into 'out'.
2335void TGlslangToSpvTraverser::dumpSpv(std::vector<unsigned int>& out)
2336{
2337    builder.dump(out);
2338}
2339
2340//
2341// Implement the traversal functions.
2342//
2343// Return true from interior nodes to have the external traversal
2344// continue on to children.  Return false if children were
2345// already processed.
2346//
2347
2348//
2349// Symbols can turn into
2350//  - uniform/input reads
2351//  - output writes
2352//  - complex lvalue base setups:  foo.bar[3]....  , where we see foo and start up an access chain
2353//  - something simple that degenerates into the last bullet
2354//
2355void TGlslangToSpvTraverser::visitSymbol(glslang::TIntermSymbol* symbol)
2356{
2357    // We update the line information even though no code might be generated here
2358    // This is helpful to yield correct lines for control flow instructions
2359    if (!linkageOnly) {
2360        builder.setDebugSourceLocation(symbol->getLoc().line, symbol->getLoc().getFilename());
2361    }
2362
2363    if (symbol->getBasicType() == glslang::EbtFunction) {
2364        return;
2365    }
2366
2367    SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
2368
2369    if (symbol->getType().getQualifier().isSpecConstant())
2370        spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2371#ifdef ENABLE_HLSL
2372    // Skip symbol handling if it is string-typed
2373    if (symbol->getBasicType() == glslang::EbtString)
2374        return;
2375#endif
2376
2377    // getSymbolId() will set up all the IO decorations on the first call.
2378    // Formal function parameters were mapped during makeFunctions().
2379    spv::Id id = getSymbolId(symbol);
2380
2381    if (symbol->getType().getQualifier().isTaskPayload())
2382        taskPayloadID = id; // cache the taskPayloadID to be used it as operand for OpEmitMeshTasksEXT
2383
2384    if (builder.isPointer(id)) {
2385        if (!symbol->getType().getQualifier().isParamInput() &&
2386            !symbol->getType().getQualifier().isParamOutput()) {
2387            // Include all "static use" and "linkage only" interface variables on the OpEntryPoint instruction
2388            // Consider adding to the OpEntryPoint interface list.
2389            // Only looking at structures if they have at least one member.
2390            if (!symbol->getType().isStruct() || symbol->getType().getStruct()->size() > 0) {
2391                spv::StorageClass sc = builder.getStorageClass(id);
2392                // Before SPIR-V 1.4, we only want to include Input and Output.
2393                // Starting with SPIR-V 1.4, we want all globals.
2394                if ((glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4 && builder.isGlobalVariable(id)) ||
2395                    (sc == spv::StorageClass::Input || sc == spv::StorageClass::Output)) {
2396                    iOSet.insert(id);
2397                }
2398            }
2399        }
2400
2401        // If the SPIR-V type is required to be different than the AST type
2402        // (for ex SubgroupMasks or 3x4 ObjectToWorld/WorldToObject matrices),
2403        // translate now from the SPIR-V type to the AST type, for the consuming
2404        // operation.
2405        // Note this turns it from an l-value to an r-value.
2406        // Currently, all symbols needing this are inputs; avoid the map lookup when non-input.
2407        if (symbol->getType().getQualifier().storage == glslang::EvqVaryingIn)
2408            id = translateForcedType(id, symbol->getType());
2409    }
2410
2411    // Only process non-linkage-only nodes for generating actual static uses
2412    if (! linkageOnly || symbol->getQualifier().isSpecConstant()) {
2413        // Prepare to generate code for the access
2414
2415        // L-value chains will be computed left to right.  We're on the symbol now,
2416        // which is the left-most part of the access chain, so now is "clear" time,
2417        // followed by setting the base.
2418        builder.clearAccessChain();
2419
2420        // For now, we consider all user variables as being in memory, so they are pointers,
2421        // except for
2422        // A) R-Value arguments to a function, which are an intermediate object.
2423        //    See comments in handleUserFunctionCall().
2424        // B) Specialization constants (normal constants don't even come in as a variable),
2425        //    These are also pure R-values.
2426        // C) R-Values from type translation, see above call to translateForcedType()
2427        glslang::TQualifier qualifier = symbol->getQualifier();
2428        if (qualifier.isSpecConstant() || rValueParameters.find(symbol->getId()) != rValueParameters.end() ||
2429            (!builder.isPointerType(builder.getTypeId(id)) && !builder.isUntypedPointer(id)))
2430            builder.setAccessChainRValue(id);
2431        else
2432            builder.setAccessChainLValue(id);
2433
2434        if (qualifier.builtIn == glslang::EbvResourceHeapEXT ||
2435            qualifier.builtIn == glslang::EbvSamplerHeapEXT) {
2436            const glslang::TType& symbolType = symbol->getType();
2437            // Direct descriptor-heap image arrays have no variable or member to
2438            // carry NonReadable/NonWritable, so wrap them in a one-member block.
2439            const bool wrapDescHeapImageArray =
2440                symbolType.getQualifier().layoutDescriptorHeap &&
2441                symbolType.isArray() &&
2442                symbolType.isImage() &&
2443                (symbolType.getQualifier().isReadOnly() || symbolType.getQualifier().isWriteOnly());
2444
2445            if (builder.getAccessChainDescHeapBaseType() == spv::NoResult) {
2446                const long long symbolId = symbol->getId();
2447                auto cachedBaseType = heapDescHeapBaseType.find(symbolId);
2448                if (cachedBaseType == heapDescHeapBaseType.end()) {
2449                    spv::Id baseType = convertGlslangToSpvType(symbolType);
2450                    if (wrapDescHeapImageArray)
2451                        baseType = makeDescHeapImageArrayWrapperType(*symbol, baseType);
2452                    cachedBaseType = heapDescHeapBaseType.emplace(symbolId, baseType).first;
2453                }
2454                builder.setAccessChainDescHeapBaseType(cachedBaseType->second);
2455            }
2456            if (wrapDescHeapImageArray)
2457                builder.accessChainPushDescHeapIndex(builder.makeIntConstant(0));
2458            spv::Id heapOffset = makeHeapOffsetId(symbolType);
2459            if (heapOffset != spv::NoResult)
2460                builder.setAccessChainDescHeapBaseOffset(heapOffset);
2461        }
2462    }
2463
2464#ifdef ENABLE_HLSL
2465    // Process linkage-only nodes for any special additional interface work.
2466    if (linkageOnly) {
2467        if (glslangIntermediate->getHlslFunctionality1()) {
2468            // Map implicit counter buffers to their originating buffers, which should have been
2469            // seen by now, given earlier pruning of unused counters, and preservation of order
2470            // of declaration.
2471            if (symbol->getType().getQualifier().isUniformOrBuffer()) {
2472                if (!glslangIntermediate->hasCounterBufferName(symbol->getName())) {
2473                    // Save possible originating buffers for counter buffers, keyed by
2474                    // making the potential counter-buffer name.
2475                    std::string keyName = symbol->getName().c_str();
2476                    keyName = glslangIntermediate->addCounterBufferName(keyName);
2477                    counterOriginator[keyName] = symbol;
2478                } else {
2479                    // Handle a counter buffer, by finding the saved originating buffer.
2480                    std::string keyName = symbol->getName().c_str();
2481                    auto it = counterOriginator.find(keyName);
2482                    if (it != counterOriginator.end()) {
2483                        id = getSymbolId(it->second);
2484                        if (id != spv::NoResult) {
2485                            spv::Id counterId = getSymbolId(symbol);
2486                            if (counterId != spv::NoResult) {
2487                                builder.addExtension("SPV_GOOGLE_hlsl_functionality1");
2488                                builder.addDecorationId(id, spv::Decoration::HlslCounterBufferGOOGLE, counterId);
2489                            }
2490                        }
2491                    }
2492                }
2493            }
2494        }
2495    }
2496#endif
2497}
2498
2499// Create a one-member heap block so image memory qualifiers can be expressed as
2500// legal member decorations instead of decorating an OpLoad result.
2501spv::Id TGlslangToSpvTraverser::makeDescHeapImageArrayWrapperType(const glslang::TIntermSymbol& symbol,
2502                                                                  spv::Id arrayType)
2503{
2504    const glslang::TQualifier& qualifier = symbol.getType().getQualifier();
2505    spv::Id memberOffset = builder.makeUintConstant(0);
2506    const std::vector<spv::Id> members = { arrayType };
2507    const std::string wrapperName = std::string(symbol.getName().c_str()) + "_heap";
2508    spv::Id wrapperType = builder.makeStructType(members, {}, wrapperName.c_str(), false);
2509
2510    builder.addMemberName(wrapperType, 0, symbol.getName().c_str());
2511    builder.addDecoration(wrapperType, spv::Decoration::Block);
2512    builder.addMemberDecorationIdEXT(wrapperType, 0, spv::Decoration::OffsetIdEXT, {memberOffset});
2513
2514    if (qualifier.isReadOnly())
2515        builder.addMemberDecoration(wrapperType, 0, spv::Decoration::NonWritable);
2516    if (qualifier.isWriteOnly())
2517        builder.addMemberDecoration(wrapperType, 0, spv::Decoration::NonReadable);
2518
2519    return wrapperType;
2520}
2521
2522// Create new untyped access chain instruction to descriptor heap, based on EXT_descriptor_heap extension.
2523void TGlslangToSpvTraverser::recordDescHeapAccessChainInfo(glslang::TIntermBinary* node)
2524{
2525    // Descriptor heap indexing is represented separately from the normal access chain.
2526    // Move the just-pushed heap index out of indexChain and into descHeapIndexChain.
2527    if (node->getQualifier().layoutDescriptorHeap) {
2528        if (builder.hasAccessChainIndex())
2529            builder.moveAccessChainIndexToDescHeapIndexChain();
2530    }
2531
2532    // Descriptor leaf nodes need the real resource type for later untyped loads.
2533    // Block payload types need explicit caching because SpvBuilder intentionally
2534    // does not deduplicate general OpTypeStruct declarations.
2535    if (node->getQualifier().descriptorHeapDescriptorNode) {
2536        glslang::TType nodeTy;
2537        nodeTy.shallowCopy(node->getType());
2538
2539        const bool useLeftDescriptorType =
2540            node->getLeft()->getType().isArray() &&
2541            node->getLeft()->getQualifier().isUniformOrBuffer();
2542
2543        if (useLeftDescriptorType)
2544            nodeTy.shallowCopy(node->getLeft()->getType());
2545
2546        if (nodeTy.isArray())
2547            nodeTy.clearArraySizes();
2548        nodeTy.getQualifier().layoutDescriptorHeap = false;
2549
2550        spv::Id descType = spv::NoResult;
2551        if (nodeTy.isStruct()) {
2552            const glslang::TTypeList* descStruct = nodeTy.getStruct();
2553            auto cachedDescType = heapDescHeapBlockDescriptorTypes.find(descStruct);
2554            if (cachedDescType == heapDescHeapBlockDescriptorTypes.end())
2555                cachedDescType = heapDescHeapBlockDescriptorTypes.emplace(descStruct, convertGlslangToSpvType(nodeTy)).first;
2556            descType = cachedDescType->second;
2557        } else {
2558            descType = convertGlslangToSpvType(nodeTy);
2559        }
2560
2561        spv::StorageClass storageClass = TranslateStorageClass(nodeTy);
2562        if (nodeTy.getBasicType() == glslang::EbtAccStruct)
2563            storageClass = spv::StorageClass::UniformConstant;
2564
2565        builder.setAccessChainDescHeapDescriptorType(descType, storageClass, nodeTy.getQualifier().isReadOnly(),
2566                                                     nodeTy.getQualifier().isWriteOnly());
2567    }
2568}
2569
2570bool TGlslangToSpvTraverser::visitBinary(glslang::TVisit /* visit */, glslang::TIntermBinary* node)
2571{
2572    builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
2573
2574    SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
2575    if (node->getType().getQualifier().isSpecConstant())
2576        spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2577
2578    // First, handle special cases
2579    switch (node->getOp()) {
2580    case glslang::EOpAssign:
2581    case glslang::EOpAddAssign:
2582    case glslang::EOpSubAssign:
2583    case glslang::EOpMulAssign:
2584    case glslang::EOpVectorTimesMatrixAssign:
2585    case glslang::EOpVectorTimesScalarAssign:
2586    case glslang::EOpMatrixTimesScalarAssign:
2587    case glslang::EOpMatrixTimesMatrixAssign:
2588    case glslang::EOpDivAssign:
2589    case glslang::EOpModAssign:
2590    case glslang::EOpAndAssign:
2591    case glslang::EOpInclusiveOrAssign:
2592    case glslang::EOpExclusiveOrAssign:
2593    case glslang::EOpLeftShiftAssign:
2594    case glslang::EOpRightShiftAssign:
2595        // A bin-op assign "a += b" means the same thing as "a = a + b"
2596        // where a is evaluated before b. For a simple assignment, GLSL
2597        // says to evaluate the left before the right.  So, always, left
2598        // node then right node.
2599        {
2600            // get the left l-value, save it away
2601            builder.clearAccessChain();
2602            node->getLeft()->traverse(this);
2603            spv::Builder::AccessChain lValue = builder.getAccessChain();
2604
2605            // evaluate the right
2606            builder.clearAccessChain();
2607            node->getRight()->traverse(this);
2608            spv::Id rValue = accessChainLoad(node->getRight()->getType());
2609
2610            // reset line number for assignment
2611            builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
2612
2613            if (node->getOp() != glslang::EOpAssign) {
2614                // the left is also an r-value
2615                builder.setAccessChain(lValue);
2616                spv::Id leftRValue = accessChainLoad(node->getLeft()->getType());
2617
2618                // do the operation
2619                spv::Builder::AccessChain::CoherentFlags coherentFlags = TranslateCoherent(node->getLeft()->getType());
2620                coherentFlags |= TranslateCoherent(node->getRight()->getType());
2621                OpDecorations decorations = { TranslatePrecisionDecoration(node->getOperationPrecision()),
2622                                              TranslateNoContractionDecoration(node->getType().getQualifier()),
2623                                              TranslateNonUniformDecoration(coherentFlags) };
2624                rValue = createBinaryOperation(node->getOp(), decorations,
2625                                               convertGlslangToSpvType(node->getType()), leftRValue, rValue,
2626                                               node->getType().getBasicType());
2627
2628                // these all need their counterparts in createBinaryOperation()
2629                assert(rValue != spv::NoResult);
2630            }
2631
2632            // store the result
2633            builder.setAccessChain(lValue);
2634            multiTypeStore(node->getLeft()->getType(), rValue);
2635
2636            // assignments are expressions having an rValue after they are evaluated...
2637            builder.clearAccessChain();
2638            builder.setAccessChainRValue(rValue);
2639        }
2640        return false;
2641    case glslang::EOpIndexDirect:
2642    case glslang::EOpIndexDirectStruct:
2643        {
2644            // Structure, array, matrix, or vector indirection with statically known index.
2645            // Get the left part of the access chain.
2646            node->getLeft()->traverse(this);
2647
2648            // Add the next element in the chain
2649
2650            const int glslangIndex = node->getRight()->getAsConstantUnion()->getConstArray()[0].getIConst();
2651            if (! node->getLeft()->getType().isArray() &&
2652                node->getLeft()->getType().isVector() &&
2653                node->getOp() == glslang::EOpIndexDirect) {
2654                // Swizzle is uniform so propagate uniform into access chain
2655                spv::Builder::AccessChain::CoherentFlags coherentFlags = TranslateCoherent(node->getLeft()->getType());
2656                coherentFlags.nonUniform = 0;
2657                // This is essentially a hard-coded vector swizzle of size 1,
2658                // so short circuit the access-chain stuff with a swizzle.
2659                std::vector<unsigned> swizzle;
2660                swizzle.push_back(glslangIndex);
2661                int dummySize;
2662                builder.accessChainPushSwizzle(swizzle, convertGlslangToSpvType(node->getLeft()->getType()),
2663                                               coherentFlags,
2664                                               glslangIntermediate->getBaseAlignmentScalar(
2665                                                   node->getLeft()->getType(), dummySize));
2666            } else {
2667
2668                // Load through a block reference is performed with a dot operator that
2669                // is mapped to EOpIndexDirectStruct. When we get to the actual reference,
2670                // do a load and reset the access chain.
2671                if (node->getLeft()->isReference() &&
2672                    !node->getLeft()->getType().isArray() &&
2673                    node->getOp() == glslang::EOpIndexDirectStruct)
2674                {
2675                    spv::Id left = accessChainLoad(node->getLeft()->getType());
2676                    builder.clearAccessChain();
2677                    builder.setAccessChainLValue(left);
2678                }
2679
2680                int spvIndex = glslangIndex;
2681                if (node->getLeft()->getBasicType() == glslang::EbtBlock &&
2682                    node->getOp() == glslang::EOpIndexDirectStruct)
2683                {
2684                    // This may be, e.g., an anonymous block-member selection, which generally need
2685                    // index remapping due to hidden members in anonymous blocks.
2686                    const glslang::TTypeList* glslangMembers = node->getLeft()->getType().getStruct();
2687                    if (memberRemapper.find(glslangMembers) != memberRemapper.end()) {
2688                        std::vector<int>& remapper = memberRemapper[glslangMembers];
2689                        assert(remapper.size() > 0);
2690                        spvIndex = remapper[glslangIndex];
2691                    }
2692                }
2693
2694                // Struct reference propagates uniform lvalue
2695                spv::Builder::AccessChain::CoherentFlags coherentFlags =
2696                        TranslateCoherent(node->getLeft()->getType());
2697                coherentFlags.nonUniform = 0;
2698
2699                // normal case for indexing array or structure or block
2700                if ((node->getRight()->getType().getBasicType() == glslang::EbtUint && glslangIntermediate->usingPromoteUint32Indices()) ||
2701                     node->getRight()->getType().contains64BitInt()) {
2702                    int64_t idx = node->getRight()->getType().contains64BitInt() ?
2703                                    node->getRight()->getAsConstantUnion()->getConstArray()[0].getI64Const() :
2704                                    node->getRight()->getAsConstantUnion()->getConstArray()[0].getUConst();
2705                    builder.accessChainPush(builder.makeInt64Constant(idx),
2706                            coherentFlags,
2707                            node->getLeft()->getType().getBufferReferenceAlignment());
2708
2709                } else {
2710                    builder.accessChainPush(builder.makeIntConstant(spvIndex),
2711                            coherentFlags,
2712                            node->getLeft()->getType().getBufferReferenceAlignment());
2713                }
2714                // Add capabilities here for accessing PointSize and clip/cull distance.
2715                // We have deferred generation of associated capabilities until now.
2716                if (node->getLeft()->getType().isStruct() && ! node->getLeft()->getType().isArray())
2717                    declareUseOfStructMember(*(node->getLeft()->getType().getStruct()), glslangIndex);
2718
2719                // EXT_descriptor_heap
2720                // Record untyped descriptor heap access info.
2721                if (node->getQualifier().layoutDescriptorHeap) {
2722                    recordDescHeapAccessChainInfo(node);
2723                }
2724            }
2725        }
2726        return false;
2727    case glslang::EOpIndexIndirect:
2728        {
2729            // Array, matrix, or vector indirection with variable index.
2730            // Will use native SPIR-V access-chain for and array indirection;
2731            // matrices are arrays of vectors, so will also work for a matrix.
2732            // Will use the access chain's 'component' for variable index into a vector.
2733
2734            // This adapter is building access chains left to right.
2735            // Set up the access chain to the left.
2736            node->getLeft()->traverse(this);
2737
2738            // save it so that computing the right side doesn't trash it
2739            spv::Builder::AccessChain partial = builder.getAccessChain();
2740
2741            // compute the next index in the chain
2742            builder.clearAccessChain();
2743            node->getRight()->traverse(this);
2744            spv::Id index = accessChainLoad(node->getRight()->getType());
2745
2746            // Zero-extend smaller unsigned integer types for array indexing.
2747            // SPIR-V OpAccessChain treats indices as signed, so we need to zero-extend
2748            // unsigned types to preserve their values (signed types are fine as-is).
2749            spv::Id indexType = builder.getTypeId(index);
2750            if (builder.isUintType(indexType) && builder.getScalarTypeWidth(indexType) < 32) {
2751                // Zero-extend unsigned types to preserve their values
2752                spv::Id uintType = builder.makeUintType(32);
2753                index = builder.createUnaryOp(spv::Op::OpUConvert, uintType, index);
2754            }
2755
2756            addIndirectionIndexCapabilities(node->getLeft()->getType(), node->getRight()->getType());
2757
2758            // restore the saved access chain
2759            builder.setAccessChain(partial);
2760
2761            // Only if index is nonUniform should we propagate nonUniform into access chain
2762            spv::Builder::AccessChain::CoherentFlags index_flags = TranslateCoherent(node->getRight()->getType());
2763            spv::Builder::AccessChain::CoherentFlags coherent_flags = TranslateCoherent(node->getLeft()->getType());
2764            coherent_flags.nonUniform = index_flags.nonUniform;
2765
2766            if (! node->getLeft()->getType().isArray() && node->getLeft()->getType().isVector()) {
2767                int dummySize;
2768                builder.accessChainPushComponent(
2769                    index, convertGlslangToSpvType(node->getLeft()->getType()), coherent_flags,
2770                                                glslangIntermediate->getBaseAlignmentScalar(node->getLeft()->getType(),
2771                                                dummySize));
2772            } else {
2773                if (glslangIntermediate->usingPromoteUint32Indices() &&
2774                    node->getRight()->getType().getBasicType() == glslang::EbtUint) {
2775                    index = createIntWidthConversion(index, 0, builder.makeIntegerType(64, true), glslang::EbtInt64, node->getRight()->getType().getBasicType());
2776                }
2777
2778                builder.accessChainPush(index, coherent_flags,
2779                                        node->getLeft()->getType().getBufferReferenceAlignment());
2780            }
2781
2782            // EXT_descriptor_heap
2783            // Record untyped descriptor heap access info.
2784            if (node->getQualifier().layoutDescriptorHeap) {
2785                recordDescHeapAccessChainInfo(node);
2786            }
2787        }
2788        return false;
2789    case glslang::EOpVectorSwizzle:
2790        {
2791            node->getLeft()->traverse(this);
2792            std::vector<unsigned> swizzle;
2793            convertSwizzle(*node->getRight()->getAsAggregate(), swizzle);
2794            int dummySize;
2795            builder.accessChainPushSwizzle(swizzle, convertGlslangToSpvType(node->getLeft()->getType()),
2796                                           TranslateCoherent(node->getLeft()->getType()),
2797                                           glslangIntermediate->getBaseAlignmentScalar(node->getLeft()->getType(),
2798                                               dummySize));
2799        }
2800        return false;
2801    case glslang::EOpMatrixSwizzle:
2802        logger->missingFunctionality("matrix swizzle");
2803        return true;
2804    case glslang::EOpLogicalOr:
2805    case glslang::EOpLogicalAnd:
2806        {
2807
2808            // These may require short circuiting, but can sometimes be done as straight
2809            // binary operations.  The right operand must be short circuited if it has
2810            // side effects, and should probably be if it is complex.
2811            if (node->getType().getQualifier().isSpecConstant() || isTrivial(node->getRight()->getAsTyped()))
2812                break; // handle below as a normal binary operation
2813            // otherwise, we need to do dynamic short circuiting on the right operand
2814            spv::Id result = createShortCircuit(node->getOp(), *node->getLeft()->getAsTyped(),
2815                *node->getRight()->getAsTyped());
2816            builder.clearAccessChain();
2817            builder.setAccessChainRValue(result);
2818        }
2819        return false;
2820    default:
2821        break;
2822    }
2823
2824    // Assume generic binary op...
2825
2826    // get right operand
2827    builder.clearAccessChain();
2828    node->getLeft()->traverse(this);
2829    spv::Id left = accessChainLoad(node->getLeft()->getType());
2830
2831    // get left operand
2832    builder.clearAccessChain();
2833    node->getRight()->traverse(this);
2834    spv::Id right = accessChainLoad(node->getRight()->getType());
2835
2836    // get result
2837    OpDecorations decorations = { TranslatePrecisionDecoration(node->getOperationPrecision()),
2838                                  TranslateNoContractionDecoration(node->getType().getQualifier()),
2839                                  TranslateNonUniformDecoration(node->getType().getQualifier()) };
2840    spv::Id result = createBinaryOperation(node->getOp(), decorations,
2841                                           convertGlslangToSpvType(node->getType()), left, right,
2842                                           node->getLeft()->getType().getBasicType());
2843
2844    builder.clearAccessChain();
2845    if (! result) {
2846        logger->missingFunctionality("unknown glslang binary operation");
2847        return true;  // pick up a child as the place-holder result
2848    } else {
2849        builder.setAccessChainRValue(result);
2850        return false;
2851    }
2852}
2853
2854spv::Id TGlslangToSpvTraverser::convertLoadedBoolInUniformToUint(const glslang::TType& type,
2855                                                                 spv::Id nominalTypeId,
2856                                                                 spv::Id loadedId)
2857{
2858    if (builder.isScalarType(nominalTypeId)) {
2859        // Conversion for bool
2860        spv::Id boolType = builder.makeBoolType();
2861        if (nominalTypeId != boolType)
2862            return builder.createBinOp(spv::Op::OpINotEqual, boolType, loadedId, builder.makeUintConstant(0));
2863    } else if (builder.isVectorType(nominalTypeId)) {
2864        // Conversion for bvec
2865        int vecSize = builder.getNumTypeComponents(nominalTypeId);
2866        spv::Id bvecType = builder.makeVectorType(builder.makeBoolType(), vecSize);
2867        if (nominalTypeId != bvecType)
2868            loadedId = builder.createBinOp(spv::Op::OpINotEqual, bvecType, loadedId,
2869                makeSmearedConstant(builder.makeUintConstant(0), vecSize));
2870    } else if (builder.isArrayType(nominalTypeId)) {
2871        // Conversion for bool array
2872        spv::Id boolArrayTypeId = convertGlslangToSpvType(type);
2873        if (nominalTypeId != boolArrayTypeId)
2874        {
2875            // Use OpCopyLogical from SPIR-V 1.4 if available.
2876            if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4)
2877                return builder.createUnaryOp(spv::Op::OpCopyLogical, boolArrayTypeId, loadedId);
2878
2879            glslang::TType glslangElementType(type, 0);
2880            spv::Id elementNominalTypeId = builder.getContainedTypeId(nominalTypeId);
2881            std::vector<spv::Id> constituents;
2882            for (int index = 0; index < type.getOuterArraySize(); ++index) {
2883                // get the element
2884                spv::Id elementValue = builder.createCompositeExtract(loadedId, elementNominalTypeId, index);
2885
2886                // recursively convert it
2887                spv::Id elementConvertedValue = convertLoadedBoolInUniformToUint(glslangElementType, elementNominalTypeId, elementValue);
2888                constituents.push_back(elementConvertedValue);
2889            }
2890            return builder.createCompositeConstruct(boolArrayTypeId, constituents);
2891        }
2892    }
2893
2894    return loadedId;
2895}
2896
2897// Figure out what, if any, type changes are needed when accessing a specific built-in.
2898// Returns <the type SPIR-V requires for declarion, the type to translate to on use>.
2899// Also see comment for 'forceType', regarding tracking SPIR-V-required types.
2900std::pair<spv::Id, spv::Id> TGlslangToSpvTraverser::getForcedType(glslang::TBuiltInVariable glslangBuiltIn,
2901    const glslang::TType& glslangType)
2902{
2903    switch(glslangBuiltIn)
2904    {
2905        case glslang::EbvSubGroupEqMask:
2906        case glslang::EbvSubGroupGeMask:
2907        case glslang::EbvSubGroupGtMask:
2908        case glslang::EbvSubGroupLeMask:
2909        case glslang::EbvSubGroupLtMask: {
2910            // these require changing a 64-bit scaler -> a vector of 32-bit components
2911            if (glslangType.isVector())
2912                break;
2913            spv::Id ivec4_type = builder.makeVectorType(builder.makeUintType(32), 4);
2914            spv::Id uint64_type = builder.makeUintType(64);
2915            std::pair<spv::Id, spv::Id> ret(ivec4_type, uint64_type);
2916            return ret;
2917        }
2918        // There are no SPIR-V builtins defined for these and map onto original non-transposed
2919        // builtins. During visitBinary we insert a transpose
2920        case glslang::EbvWorldToObject3x4:
2921        case glslang::EbvObjectToWorld3x4: {
2922            spv::Id mat43 = builder.makeMatrixType(builder.makeFloatType(32), 4, 3);
2923            spv::Id mat34 = builder.makeMatrixType(builder.makeFloatType(32), 3, 4);
2924            std::pair<spv::Id, spv::Id> ret(mat43, mat34);
2925            return ret;
2926        }
2927        default:
2928            break;
2929    }
2930
2931    std::pair<spv::Id, spv::Id> ret(spv::NoType, spv::NoType);
2932    return ret;
2933}
2934
2935// For an object previously identified (see getForcedType() and forceType)
2936// as needing type translations, do the translation needed for a load, turning
2937// an L-value into in R-value.
2938spv::Id TGlslangToSpvTraverser::translateForcedType(spv::Id object, const glslang::TType& type)
2939{
2940    const auto forceIt = forceType.find(object);
2941    if (forceIt == forceType.end())
2942        return object;
2943
2944    spv::Id desiredTypeId = forceIt->second;
2945    spv::Id objectTypeId = builder.getTypeId(object);
2946    assert(builder.isPointerType(objectTypeId));
2947    objectTypeId = builder.getContainedTypeId(objectTypeId);
2948
2949    // The load performed below needs to preserve the same Volatile/Coherent memory access
2950    // that TGlslangToSpvTraverser::accessChainLoad() would apply
2951    spv::Builder::AccessChain::CoherentFlags coherentFlags = TranslateCoherent(type);
2952    spv::MemoryAccessMask accessMask =
2953        spv::MemoryAccessMask(TranslateMemoryAccess(coherentFlags) & ~spv::MemoryAccessMask::MakePointerAvailableKHR);
2954    spv::Scope memoryScope = TranslateMemoryScope(coherentFlags);
2955
2956    if (builder.isVectorType(objectTypeId) &&
2957        builder.getScalarTypeWidth(builder.getContainedTypeId(objectTypeId)) == 32) {
2958        if (builder.getScalarTypeWidth(desiredTypeId) == 64) {
2959            // handle 32-bit v.xy* -> 64-bit
2960            builder.clearAccessChain();
2961            builder.setAccessChainLValue(object);
2962            object = builder.accessChainLoad(spv::NoPrecision, spv::Decoration::Max, spv::Decoration::Max, objectTypeId,
2963                                             accessMask, memoryScope);
2964            std::vector<spv::Id> components;
2965            components.push_back(builder.createCompositeExtract(object, builder.getContainedTypeId(objectTypeId), 0));
2966            components.push_back(builder.createCompositeExtract(object, builder.getContainedTypeId(objectTypeId), 1));
2967
2968            spv::Id vecType = builder.makeVectorType(builder.getContainedTypeId(objectTypeId), 2);
2969            return builder.createUnaryOp(spv::Op::OpBitcast, desiredTypeId,
2970                                         builder.createCompositeConstruct(vecType, components));
2971        } else {
2972            logger->missingFunctionality("forcing 32-bit vector type to non 64-bit scalar");
2973        }
2974    } else if (builder.isMatrixType(objectTypeId)) {
2975            // There are no SPIR-V builtins defined for 3x4 variants of ObjectToWorld/WorldToObject
2976            // and we insert a transpose after loading the original non-transposed builtins
2977            builder.clearAccessChain();
2978            builder.setAccessChainLValue(object);
2979            object = builder.accessChainLoad(spv::NoPrecision, spv::Decoration::Max, spv::Decoration::Max, objectTypeId,
2980                                             accessMask, memoryScope);
2981            return builder.createUnaryOp(spv::Op::OpTranspose, desiredTypeId, object);
2982
2983    } else  {
2984        logger->missingFunctionality("forcing non 32-bit vector type");
2985    }
2986
2987    return object;
2988}
2989
2990bool TGlslangToSpvTraverser::visitUnary(glslang::TVisit /* visit */, glslang::TIntermUnary* node)
2991{
2992    builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
2993
2994    SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
2995    if (node->getType().getQualifier().isSpecConstant())
2996        spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2997
2998    spv::Id result = spv::NoResult;
2999
3000    // try texturing first
3001    result = createImageTextureFunctionCall(node);
3002    if (result != spv::NoResult) {
3003        builder.clearAccessChain();
3004        builder.setAccessChainRValue(result);
3005
3006        return false; // done with this node
3007    }
3008
3009    // Non-texturing.
3010
3011    if (node->getOp() == glslang::EOpArrayLength) {
3012        // Quite special; won't want to evaluate the operand.
3013
3014        // Currently, the front-end does not allow .length() on an array until it is sized,
3015        // except for the last block membeor of an SSBO.
3016        // TODO: If this changes, link-time sized arrays might show up here, and need their
3017        // size extracted.
3018
3019        // Normal .length() would have been constant folded by the front-end.
3020        // So, this has to be block.lastMember.length().
3021        // SPV wants "block" and member number as the operands, go get them.
3022
3023        uint32_t bits = node->getType().contains64BitInt() ? 64 : 32;
3024
3025        spv::Id length;
3026        if (node->getOperand()->getType().isCoopMat()) {
3027            spv::Id typeId = convertGlslangToSpvType(node->getOperand()->getType());
3028            assert(builder.isCooperativeMatrixType(typeId));
3029
3030            if (node->getOperand()->getType().isCoopMatKHR()) {
3031                length = builder.createCooperativeMatrixLengthKHR(typeId);
3032            } else {
3033                spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
3034                length = builder.createCooperativeMatrixLengthNV(typeId);
3035            }
3036        } else if (node->getOperand()->getType().isCoopVecOrLongVector() &&
3037                   !node->getOperand()->getType().isArray()) {
3038            // Long/cooperative vectors support v.length() as a compile-time
3039            // component count. For arrays of such vectors, .length() must use
3040            // OpArrayLength and not the vector-component path.
3041            spv::Id typeId = convertGlslangToSpvType(node->getOperand()->getType());
3042            if (builder.isCooperativeVectorType(typeId)) {
3043                length = builder.getCooperativeVectorNumComponents(typeId);
3044            } else {
3045                length = builder.makeIntConstant(builder.getNumTypeConstituents(typeId));
3046            }
3047        } else {
3048            glslang::TIntermTyped* block = node->getOperand()->getAsBinaryNode()->getLeft();
3049            block->traverse(this);
3050            unsigned int member = node->getOperand()->getAsBinaryNode()->getRight()->getAsConstantUnion()
3051                ->getConstArray()[0].getUConst();
3052            length = builder.createArrayLength(builder.accessChainGetLValue(), member, bits);
3053        }
3054
3055        // GLSL semantics say the result of .length() is an int, while SPIR-V says
3056        // signedness must be 0. So, convert from SPIR-V unsigned back to GLSL's
3057        // AST expectation of a signed result.
3058        if (glslangIntermediate->getSource() == glslang::EShSourceGlsl) {
3059            if (builder.isInSpecConstCodeGenMode()) {
3060                length = builder.createBinOp(spv::Op::OpIAdd, builder.makeIntType(bits), length, builder.makeIntConstant(0));
3061            } else {
3062                length = builder.createUnaryOp(spv::Op::OpBitcast, builder.makeIntType(bits), length);
3063            }
3064        }
3065
3066        builder.clearAccessChain();
3067        builder.setAccessChainRValue(length);
3068
3069        return false;
3070    }
3071
3072    // Force variable declaration - Debug Mode Only
3073    if (node->getOp() == glslang::EOpDeclare) {
3074        builder.clearAccessChain();
3075        node->getOperand()->traverse(this);
3076        builder.clearAccessChain();
3077        return false;
3078    }
3079
3080    // Start by evaluating the operand
3081
3082    // Does it need a swizzle inversion?  If so, evaluation is inverted;
3083    // operate first on the swizzle base, then apply the swizzle.
3084    spv::Id invertedType = spv::NoType;
3085    auto resultType = [&invertedType, &node, this](){ return invertedType != spv::NoType ?
3086        invertedType : convertGlslangToSpvType(node->getType()); };
3087    if (node->getOp() == glslang::EOpInterpolateAtCentroid)
3088        invertedType = getInvertedSwizzleType(*node->getOperand());
3089
3090    builder.clearAccessChain();
3091    TIntermNode *operandNode;
3092    if (invertedType != spv::NoType)
3093        operandNode = node->getOperand()->getAsBinaryNode()->getLeft();
3094    else
3095        operandNode = node->getOperand();
3096
3097    operandNode->traverse(this);
3098
3099    spv::Id operand = spv::NoResult;
3100
3101    spv::Builder::AccessChain::CoherentFlags lvalueCoherentFlags;
3102
3103    const auto hitObjectOpsWithLvalue = [](glslang::TOperator op) {
3104        switch(op) {
3105            case glslang::EOpReorderThreadNV:
3106            case glslang::EOpHitObjectGetCurrentTimeNV:
3107            case glslang::EOpHitObjectGetHitKindNV:
3108            case glslang::EOpHitObjectGetPrimitiveIndexNV:
3109            case glslang::EOpHitObjectGetGeometryIndexNV:
3110            case glslang::EOpHitObjectGetInstanceIdNV:
3111            case glslang::EOpHitObjectGetInstanceCustomIndexNV:
3112            case glslang::EOpHitObjectGetObjectRayDirectionNV:
3113            case glslang::EOpHitObjectGetObjectRayOriginNV:
3114            case glslang::EOpHitObjectGetWorldRayDirectionNV:
3115            case glslang::EOpHitObjectGetWorldRayOriginNV:
3116            case glslang::EOpHitObjectGetWorldToObjectNV:
3117            case glslang::EOpHitObjectGetObjectToWorldNV:
3118            case glslang::EOpHitObjectGetRayTMaxNV:
3119            case glslang::EOpHitObjectGetRayTMinNV:
3120            case glslang::EOpHitObjectIsEmptyNV:
3121            case glslang::EOpHitObjectIsHitNV:
3122            case glslang::EOpHitObjectIsMissNV:
3123            case glslang::EOpHitObjectRecordEmptyNV:
3124            case glslang::EOpHitObjectGetShaderBindingTableRecordIndexNV:
3125            case glslang::EOpHitObjectGetShaderRecordBufferHandleNV:
3126            case glslang::EOpHitObjectGetClusterIdNV:
3127            case glslang::EOpHitObjectGetSpherePositionNV:
3128            case glslang::EOpHitObjectGetSphereRadiusNV:
3129            case glslang::EOpHitObjectIsSphereHitNV:
3130            case glslang::EOpHitObjectIsLSSHitNV:
3131            case glslang::EOpReorderThreadEXT:
3132            case glslang::EOpHitObjectGetCurrentTimeEXT:
3133            case glslang::EOpHitObjectGetHitKindEXT:
3134            case glslang::EOpHitObjectGetPrimitiveIndexEXT:
3135            case glslang::EOpHitObjectGetGeometryIndexEXT:
3136            case glslang::EOpHitObjectGetInstanceIdEXT:
3137            case glslang::EOpHitObjectGetInstanceCustomIndexEXT:
3138            case glslang::EOpHitObjectGetObjectRayDirectionEXT:
3139            case glslang::EOpHitObjectGetObjectRayOriginEXT:
3140            case glslang::EOpHitObjectGetWorldRayDirectionEXT:
3141            case glslang::EOpHitObjectGetWorldRayOriginEXT:
3142            case glslang::EOpHitObjectGetWorldToObjectEXT:
3143            case glslang::EOpHitObjectGetObjectToWorldEXT:
3144            case glslang::EOpHitObjectGetRayTMaxEXT:
3145            case glslang::EOpHitObjectGetRayTMinEXT:
3146            case glslang::EOpHitObjectGetRayFlagsEXT:
3147            case glslang::EOpHitObjectIsEmptyEXT:
3148            case glslang::EOpHitObjectIsHitEXT:
3149            case glslang::EOpHitObjectIsMissEXT:
3150            case glslang::EOpHitObjectRecordEmptyEXT:
3151            case glslang::EOpHitObjectGetShaderBindingTableRecordIndexEXT:
3152            case glslang::EOpHitObjectGetShaderRecordBufferHandleEXT:
3153                return true;
3154            default:
3155                return false;
3156        }
3157    };
3158
3159    if (node->getOp() == glslang::EOpAtomicCounterIncrement ||
3160        node->getOp() == glslang::EOpAtomicCounterDecrement ||
3161        node->getOp() == glslang::EOpAtomicCounter          ||
3162        (node->getOp() == glslang::EOpInterpolateAtCentroid &&
3163          glslangIntermediate->getSource() != glslang::EShSourceHlsl)  ||
3164        node->getOp() == glslang::EOpRayQueryProceed        ||
3165        node->getOp() == glslang::EOpRayQueryGetRayTMin     ||
3166        node->getOp() == glslang::EOpRayQueryGetRayFlags    ||
3167        node->getOp() == glslang::EOpRayQueryGetWorldRayOrigin ||
3168        node->getOp() == glslang::EOpRayQueryGetWorldRayDirection ||
3169        node->getOp() == glslang::EOpRayQueryGetIntersectionCandidateAABBOpaque ||
3170        node->getOp() == glslang::EOpRayQueryTerminate ||
3171        node->getOp() == glslang::EOpRayQueryConfirmIntersection ||
3172        (node->getOp() == glslang::EOpSpirvInst && operandNode->getAsTyped()->getQualifier().isSpirvByReference()) ||
3173        hitObjectOpsWithLvalue(node->getOp())) {
3174        operand = builder.accessChainGetLValue(); // Special case l-value operands
3175        lvalueCoherentFlags = builder.getAccessChain().coherentFlags;
3176        lvalueCoherentFlags |= TranslateCoherent(operandNode->getAsTyped()->getType());
3177    } else if (operandNode->getAsTyped()->getQualifier().isSpirvLiteral()) {
3178        // Will be translated to a literal value, make a placeholder here
3179        operand = spv::NoResult;
3180    } else {
3181        operand = accessChainLoad(node->getOperand()->getType());
3182    }
3183
3184    OpDecorations decorations = { TranslatePrecisionDecoration(node->getOperationPrecision()),
3185                                  TranslateNoContractionDecoration(node->getType().getQualifier()),
3186                                  TranslateNonUniformDecoration(node->getType().getQualifier()) };
3187
3188    // it could be a conversion
3189    if (! result) {
3190        bool sameUse = !node->getType().isCoopMatKHR() || !node->getOperand()->getAsTyped()->getType().isCoopMatKHR() ||
3191                       node->getAsTyped()->getType().sameCoopMatUse(node->getOperand()->getAsTyped()->getType());
3192        result = createConversion(node->getOp(), decorations, resultType(), operand,
3193            node->getType().getBasicType(), node->getOperand()->getBasicType(), sameUse);
3194    }
3195
3196    // if not, then possibly an operation
3197    if (! result)
3198        result = createUnaryOperation(node->getOp(), decorations, resultType(), operand,
3199            node->getOperand()->getBasicType(), lvalueCoherentFlags, node->getType());
3200
3201    // it could be attached to a SPIR-V intruction
3202    if (!result) {
3203        if (node->getOp() == glslang::EOpSpirvInst) {
3204            const auto& spirvInst = node->getSpirvInstruction();
3205            spv::IdImmediate idImmOp = {true, operand};
3206            if (operandNode->getAsTyped()->getQualifier().isSpirvLiteral()) {
3207                // Translate the constant to a literal value
3208                std::vector<unsigned> literals;
3209                glslang::TVector<const glslang::TIntermConstantUnion*> constants;
3210                constants.push_back(operandNode->getAsConstantUnion());
3211                TranslateLiterals(constants, literals);
3212                idImmOp = {false, literals[0]};
3213            }
3214
3215            if (spirvInst.set == "") {
3216                if (node->getBasicType() == glslang::EbtVoid)
3217                    builder.createNoResultOp(static_cast<spv::Op>(spirvInst.id), {idImmOp});
3218                else
3219                    result = builder.createOp(static_cast<spv::Op>(spirvInst.id), resultType(), {idImmOp});
3220            } else {
3221                result = builder.createBuiltinCall(
3222                    resultType(), spirvInst.set == "GLSL.std.450" ? stdBuiltins : getExtBuiltins(spirvInst.set.c_str()),
3223                    spirvInst.id, {idImmOp});
3224            }
3225
3226            if (node->getBasicType() == glslang::EbtVoid)
3227                return false; // done with this node
3228        }
3229    }
3230
3231    if (result) {
3232        if (invertedType) {
3233            result = createInvertedSwizzle(decorations.precision, *node->getOperand(), result);
3234            decorations.addNonUniform(builder, result);
3235        }
3236
3237        builder.clearAccessChain();
3238        builder.setAccessChainRValue(result);
3239
3240        return false; // done with this node
3241    }
3242
3243    // it must be a special case, check...
3244    switch (node->getOp()) {
3245    case glslang::EOpPostIncrement:
3246    case glslang::EOpPostDecrement:
3247    case glslang::EOpPreIncrement:
3248    case glslang::EOpPreDecrement:
3249        {
3250            // we need the integer value "1" or the floating point "1.0" to add/subtract
3251            spv::Id one = 0;
3252            if (node->getBasicType() == glslang::EbtFloat)
3253                one = builder.makeFloatConstant(1.0F);
3254            else if (node->getBasicType() == glslang::EbtDouble)
3255                one = builder.makeDoubleConstant(1.0);
3256            else if (node->getBasicType() == glslang::EbtFloat16)
3257                one = builder.makeFloat16Constant(1.0F);
3258            else if (node->getBasicType() == glslang::EbtBFloat16)
3259                one = builder.makeBFloat16Constant(1.0F);
3260            else if (node->getBasicType() == glslang::EbtFloatE5M2)
3261                one = builder.makeFloatE5M2Constant(1.0F);
3262            else if (node->getBasicType() == glslang::EbtFloatE4M3)
3263                one = builder.makeFloatE4M3Constant(1.0F);
3264            else if (node->getBasicType() == glslang::EbtFloatE2M1)
3265                one = builder.makeFloatE2M1Constant(1.0F);
3266            else if (node->getBasicType() == glslang::EbtFloatE3M2)
3267                one = builder.makeFloatE3M2Constant(1.0F);
3268            else if (node->getBasicType() == glslang::EbtFloatE2M3)
3269                one = builder.makeFloatE2M3Constant(1.0F);
3270            else if (node->getBasicType() == glslang::EbtFloatUE8M0)
3271                one = builder.makeFloatUE8M0Constant(1.0F);
3272            else if (node->getBasicType() == glslang::EbtFloatMXINT8)
3273                one = builder.makeFloatMXINT8Constant(1.0F);
3274            else if (node->getBasicType() == glslang::EbtInt8  || node->getBasicType() == glslang::EbtUint8)
3275                one = builder.makeInt8Constant(1);
3276            else if (node->getBasicType() == glslang::EbtInt16 || node->getBasicType() == glslang::EbtUint16)
3277                one = builder.makeInt16Constant(1);
3278            else if (node->getBasicType() == glslang::EbtInt64 || node->getBasicType() == glslang::EbtUint64)
3279                one = builder.makeInt64Constant(1);
3280            else
3281                one = builder.makeIntConstant(1);
3282            glslang::TOperator op;
3283            if (node->getOp() == glslang::EOpPreIncrement ||
3284                node->getOp() == glslang::EOpPostIncrement)
3285                op = glslang::EOpAdd;
3286            else
3287                op = glslang::EOpSub;
3288
3289            spv::Id result = createBinaryOperation(op, decorations,
3290                                                   convertGlslangToSpvType(node->getType()), operand, one,
3291                                                   node->getType().getBasicType());
3292            assert(result != spv::NoResult);
3293
3294            // The result of operation is always stored, but conditionally the
3295            // consumed result.  The consumed result is always an r-value.
3296            accessChainStore(node->getOperand()->getType(), result);
3297            builder.clearAccessChain();
3298            if (node->getOp() == glslang::EOpPreIncrement ||
3299                node->getOp() == glslang::EOpPreDecrement)
3300                builder.setAccessChainRValue(result);
3301            else
3302                builder.setAccessChainRValue(operand);
3303        }
3304
3305        return false;
3306
3307    case glslang::EOpAssumeEXT:
3308        builder.addCapability(spv::Capability::ExpectAssumeKHR);
3309        builder.addExtension(spv::E_SPV_KHR_expect_assume);
3310        builder.createNoResultOp(spv::Op::OpAssumeTrueKHR, operand);
3311        return false;
3312    case glslang::EOpEmitStreamVertex:
3313        builder.createNoResultOp(spv::Op::OpEmitStreamVertex, operand);
3314        return false;
3315    case glslang::EOpEndStreamPrimitive:
3316        builder.createNoResultOp(spv::Op::OpEndStreamPrimitive, operand);
3317        return false;
3318    case glslang::EOpRayQueryTerminate:
3319        builder.createNoResultOp(spv::Op::OpRayQueryTerminateKHR, operand);
3320        return false;
3321    case glslang::EOpRayQueryConfirmIntersection:
3322        builder.createNoResultOp(spv::Op::OpRayQueryConfirmIntersectionKHR, operand);
3323        return false;
3324    case glslang::EOpReorderThreadNV:
3325        builder.createNoResultOp(spv::Op::OpReorderThreadWithHitObjectNV, operand);
3326        return false;
3327    case glslang::EOpReorderThreadEXT:
3328        builder.createNoResultOp(spv::Op::OpReorderThreadWithHitObjectEXT, operand);
3329        return false;
3330    case glslang::EOpHitObjectRecordEmptyNV:
3331        builder.createNoResultOp(spv::Op::OpHitObjectRecordEmptyNV, operand);
3332        return false;
3333    case glslang::EOpHitObjectRecordEmptyEXT:
3334        builder.createNoResultOp(spv::Op::OpHitObjectRecordEmptyEXT, operand);
3335        return false;
3336
3337    case glslang::EOpCreateTensorLayoutNV:
3338        result = builder.createOp(spv::Op::OpCreateTensorLayoutNV, resultType(), std::vector<spv::Id>{});
3339        builder.clearAccessChain();
3340        builder.setAccessChainRValue(result);
3341        return false;
3342
3343    case glslang::EOpCreateTensorViewNV:
3344        result = builder.createOp(spv::Op::OpCreateTensorViewNV, resultType(), std::vector<spv::Id>{});
3345        builder.clearAccessChain();
3346        builder.setAccessChainRValue(result);
3347        return false;
3348
3349    default:
3350        logger->missingFunctionality("unknown glslang unary");
3351        return true;  // pick up operand as placeholder result
3352    }
3353}
3354
3355// Construct a composite object, recursively copying members if their types don't match
3356spv::Id TGlslangToSpvTraverser::createCompositeConstruct(spv::Id resultTypeId, std::vector<spv::Id> constituents)
3357{
3358    for (int c = 0; c < (int)constituents.size(); ++c) {
3359        spv::Id& constituent = constituents[c];
3360        spv::Id lType = builder.getContainedTypeId(resultTypeId, c);
3361        spv::Id rType = builder.getTypeId(constituent);
3362        if (lType != rType) {
3363            if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) {
3364                constituent = builder.createUnaryOp(spv::Op::OpCopyLogical, lType, constituent);
3365            } else if (builder.isStructType(rType)) {
3366                std::vector<spv::Id> rTypeConstituents;
3367                int numrTypeConstituents = builder.getNumTypeConstituents(rType);
3368                for (int i = 0; i < numrTypeConstituents; ++i) {
3369                    rTypeConstituents.push_back(builder.createCompositeExtract(constituent,
3370                        builder.getContainedTypeId(rType, i), i));
3371                }
3372                constituents[c] = createCompositeConstruct(lType, rTypeConstituents);
3373            } else {
3374                assert(builder.isArrayType(rType));
3375                std::vector<spv::Id> rTypeConstituents;
3376                int numrTypeConstituents = builder.getNumTypeConstituents(rType);
3377
3378                spv::Id elementRType = builder.getContainedTypeId(rType);
3379                for (int i = 0; i < numrTypeConstituents; ++i) {
3380                    rTypeConstituents.push_back(builder.createCompositeExtract(constituent, elementRType, i));
3381                }
3382                constituents[c] = createCompositeConstruct(lType, rTypeConstituents);
3383            }
3384        }
3385    }
3386    return builder.createCompositeConstruct(resultTypeId, constituents);
3387}
3388
3389void TGlslangToSpvTraverser::createAbortEXT(const glslang::TIntermSequence &glslangOperands)
3390{
3391    bool isEmptyMsg =
3392        glslangOperands.empty() ||
3393        glslangOperands[0]->getAsConstantUnion()->getConstArray()[0].getSConst()->empty();
3394    // Add Capability and extensions.
3395    builder.addCapability(spv::Capability::AbortKHR);
3396    builder.addCapability(spv::Capability::ConstantDataKHR);
3397    builder.addExtension(spv::E_SPV_KHR_constant_data);
3398    builder.addExtension(spv::E_SPV_KHR_abort);
3399
3400    // 1. Get the message string.
3401    const auto emptyMsg = glslang::TString("\0");
3402    const glslang::TString* msg =
3403        isEmptyMsg ? &emptyMsg : glslangOperands[0]->getAsConstantUnion()->getConstArray()[0].getSConst();
3404    // 2. Prepare to construct message struct variable, record members' types, data and offsets.
3405    std::vector<int> structMemberOffsets;
3406    std::vector<spv::Id> structMemberType;
3407    std::vector<spv::Id> structLoadMemberType;
3408    std::vector<spv::Id> structMemberData;
3409    structMemberOffsets.push_back(0);
3410    auto charType = builder.makeIntType(8);
3411    // 2.1 Get string's length.
3412    //     If not an empty string, \0 is the final character used for padding.
3413    unsigned int msgLen = isEmptyMsg ? 1 : msg->size() + 1;
3414    unsigned int paddingSize = (4 - msgLen % 4) % 4;
3415    msgLen = msgLen + paddingSize;
3416    spv::Id constLen = builder.makeUintConstant(msgLen);
3417    // 2.2 Get string's array types. The string needs one type per role: the constant data is
3418    //     tightly packed, so its result type must not carry ArrayStride, while the message
3419    //     struct member it feeds does need an explicit layout. One type cannot do both, so
3420    //     make a tightly packed type for the constant and a laid out type for the member.
3421    //     Asking makeArrayType for a stride is what keeps the laid out one a distinct type.
3422    auto msgConstDataType = builder.makeArrayType(charType, constLen, 0);
3423    auto msgArrType = builder.makeArrayType(charType, constLen, 1);
3424    // 2.3 Add string constant data.
3425    //     SPV_KHR_abort does not mandate a form for the message; it is emitted as plain
3426    //     constant data. Its example uses a specialization constant so that an application
3427    //     can swap a format modifier at specialization time, but that needs a SpecId the
3428    //     shader author controls, so it is not something to apply automatically here.
3429    auto msgConstData = builder.createConstData(spv::Op::OpConstantDataKHR, msgConstDataType, {msg->c_str()});
3430    // 2.4 Add decorations for this string. Both types describe the same UTF encoded
3431    //     characters, so both are marked as such; only the laid out one takes the stride.
3432    builder.addDecoration(msgConstDataType, spv::Decoration::UTFEncodedKHR);
3433    builder.addDecoration(msgArrType, spv::Decoration::UTFEncodedKHR);
3434    // Array stride for char is 1 byte per element for explicit layout
3435    builder.addDecoration(msgArrType, spv::Decoration::ArrayStride, 1);
3436    // 2.5 Collect data and type for construct an internal message structure member.
3437    //     The message value is built from the tightly packed type the constant already has;
3438    //     the laid out type only appears in the message type operand of OpAbortKHR.
3439    structMemberType.push_back(msgConstDataType);
3440    structLoadMemberType.push_back(msgArrType);
3441    structMemberOffsets.push_back(msgLen);
3442    structMemberData.push_back(msgConstData);
3443    // 3. Add extra following arguments/variables' types in member structure.
3444    //    Matrix members need their stride recorded so they can be explicitly laid out below;
3445    //    0 means the member is not a matrix.
3446    std::vector<int> structMemberMatrixStrides;
3447    structMemberMatrixStrides.push_back(0);
3448    // The message is always packed with scalar rules, so that a given argument list produces
3449    // one layout regardless of how the rest of the shader is laid out. A consumer parsing the
3450    // message then only ever has to know the one set of rules.
3451    for (unsigned int i = 1; i < glslangOperands.size(); i++) {
3452        spv::Builder::AccessChain save = builder.getAccessChain();
3453        builder.clearAccessChain();
3454        // Align this member's offset, then advance past it to get the (not yet aligned)
3455        // offset of the next one.
3456        const glslang::TType& argType = glslangOperands[i]->getAsTyped()->getType();
3457        int memberSize = 0;
3458        int matrixStride = 0;
3459        int alignment =
3460            glslangIntermediate->getMemberAlignment(argType, memberSize, matrixStride, glslang::ElpScalar, false);
3461        glslang::RoundToPow2(structMemberOffsets.back(), alignment);
3462        structMemberOffsets.push_back(structMemberOffsets.back() + memberSize);
3463        // getMemberAlignment's stride is the array stride for an array argument, so ask for
3464        // the matrix stride separately - an array of matrices needs the stride of one matrix.
3465        structMemberMatrixStrides.push_back(
3466            argType.isMatrix() ? getMatrixStride(argType, glslang::ElpScalar, glslang::ElmColumnMajor) : 0);
3467        glslangOperands[i]->traverse(this);
3468        spv::Id memberValue = accessChainLoad(argType);
3469        // A bool has no in-memory representation, so lay a bool member out as a 32-bit integer
3470        if (argType.getBasicType() == glslang::EbtBool) {
3471            spv::Id uintType = builder.makeUintType(32);
3472            spv::Id one = builder.makeUintConstant(1);
3473            spv::Id zero = builder.makeUintConstant(0);
3474            if (argType.isVector()) {
3475                uintType = builder.makeVectorType(uintType, argType.getVectorSize());
3476                one = builder.smearScalar(spv::NoPrecision, one, uintType);
3477                zero = builder.smearScalar(spv::NoPrecision, zero, uintType);
3478            }
3479            memberValue = builder.createTriOp(spv::Op::OpSelect, uintType, memberValue, one, zero);
3480        }
3481        structMemberData.push_back(memberValue);
3482        // Collect both forms of the member's type. The value is built from the loaded types,
3483        // and the message type operand describes the same members explicitly laid out. An
3484        // aggregate argument carries layout inside itself - ArrayStride on an array, Offset
3485        // and MatrixStride on the members of a nested struct - so only the laid out form has
3486        // those. OpAbortKHR only requires the two to match logically, which disregards
3487        // decorations, so the value never has to be converted to the laid out form.
3488        structMemberType.push_back(builder.getTypeId(structMemberData.back()));
3489        glslang::TQualifier memberQualifier = argType.getQualifier();
3490        memberQualifier.layoutPacking = glslang::ElpScalar;
3491        if (memberQualifier.layoutMatrix == glslang::ElmNone)
3492            memberQualifier.layoutMatrix = glslang::ElmColumnMajor;
3493        structLoadMemberType.push_back(
3494            convertGlslangToSpvType(argType, glslang::ElpScalar, memberQualifier, false));
3495
3496        builder.setAccessChain(save);
3497    }
3498    structMemberOffsets.pop_back();
3499    // 4. Construct struct message variable, add abortExt instruction.
3500    //    Two struct types: the value's own type, and the explicitly laid out type that tells a
3501    //    consumer where each member sits. Only the latter carries the layout decorations.
3502    auto structValueType = builder.makeStructType(structMemberType, {}, "abortMessageValueType");
3503    auto structLoadType = builder.makeStructType(structLoadMemberType, {}, "abortMessageLoadType");
3504    for (unsigned int i = 0; i < structMemberOffsets.size(); i++) {
3505        builder.addMemberDecoration(structLoadType, i, spv::Decoration::Offset, structMemberOffsets[i]);
3506        // A matrix in an explicitly laid out struct must state its majorness and stride.
3507        // The loaded value is always in GLSL's default column-major order.
3508        if (structMemberMatrixStrides[i] != 0) {
3509            builder.addMemberDecoration(structLoadType, i, spv::Decoration::ColMajor);
3510            builder.addMemberDecoration(structLoadType, i, spv::Decoration::MatrixStride,
3511                                        structMemberMatrixStrides[i]);
3512        }
3513    }
3514    // The members of structValueType were taken from these values, so they match exactly and
3515    // the builder's plain construct is enough - the traverser's overload, which would convert
3516    // a mismatched member, has nothing to do here.
3517    auto messageVar = builder.createCompositeConstruct(structValueType, structMemberData);
3518    builder.makeStatementTerminator(spv::Op::OpAbortKHR, {structLoadType, messageVar}, "post-abort");
3519}
3520
3521spv::Id TGlslangToSpvTraverser::createCoopMatConversion(spv::Id destType, spv::Id operand, glslang::TBasicType resultBasicType, glslang::TBasicType operandBasicType, bool sameUse, bool transpose)
3522{
3523    spv::Id result;
3524    if (isTypeInt(operandBasicType) && isTypeInt(resultBasicType) &&
3525        isTypeUnsignedInt(operandBasicType) != isTypeUnsignedInt(resultBasicType) &&
3526        GetNumBits(operandBasicType) != GetNumBits(resultBasicType)) {
3527
3528        // OpSConvert/OpUConvert + OpBitCast
3529        operand = createIntWidthConversion(operand, 0, destType, resultBasicType, operandBasicType);
3530        operandBasicType = resultBasicType;
3531        if (transpose) {
3532            builder.addDecoration(operand, spv::Decoration::CooperativeMatrixTransposeEXT);
3533        }
3534        return builder.createUnaryOp(spv::Op::OpBitcast, destType, operand);
3535    }
3536    if (isTypeFloat(operandBasicType) && isTypeFloat(resultBasicType) &&
3537        operandBasicType != resultBasicType) {
3538        result = builder.createUnaryOp(spv::Op::OpFConvert, destType, operand);
3539    } else if (isTypeSignedInt(operandBasicType) && isTypeFloat(resultBasicType)) {
3540        result = builder.createUnaryOp(spv::Op::OpConvertSToF, destType, operand);
3541    } else if (isTypeUnsignedInt(operandBasicType) && isTypeFloat(resultBasicType)) {
3542        result = builder.createUnaryOp(spv::Op::OpConvertUToF, destType, operand);
3543    } else if (isTypeFloat(operandBasicType) && isTypeSignedInt(resultBasicType)) {
3544        result = builder.createUnaryOp(spv::Op::OpConvertFToS, destType, operand);
3545    } else if (isTypeFloat(operandBasicType) && isTypeUnsignedInt(resultBasicType)) {
3546        result = builder.createUnaryOp(spv::Op::OpConvertFToU, destType, operand);
3547    } else if (isTypeSignedInt(resultBasicType) && GetNumBits(operandBasicType) != GetNumBits(resultBasicType)) {
3548        result = builder.createUnaryOp(spv::Op::OpSConvert, destType, operand);
3549    } else if (isTypeUnsignedInt(resultBasicType) && GetNumBits(operandBasicType) != GetNumBits(resultBasicType)) {
3550        result = builder.createUnaryOp(spv::Op::OpUConvert, destType, operand);
3551    } else if (!sameUse) {
3552        result = builder.createUnaryOp(spv::Op::OpCooperativeMatrixConvertUseEXT, destType, operand);
3553    } else {
3554        result = builder.createUnaryOp(spv::Op::OpBitcast, destType, operand);
3555    }
3556    if (transpose) {
3557        builder.addDecoration(result, spv::Decoration::CooperativeMatrixTransposeEXT);
3558    }
3559    return result;
3560}
3561
3562bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TIntermAggregate* node)
3563{
3564    SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
3565    if (node->getType().getQualifier().isSpecConstant())
3566        spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
3567
3568    spv::Id result = spv::NoResult;
3569    spv::Id invertedType = spv::NoType;                     // to use to override the natural type of the node
3570    std::vector<spv::Builder::AccessChain> complexLvalues;  // for holding swizzling l-values too complex for
3571                                                            // SPIR-V, for an out parameter
3572    std::vector<spv::Id> temporaryLvalues;                  // temporaries to pass, as proxies for complexLValues
3573    std::vector<const glslang::TType*> complexLvalueTypes;
3574    spv::Builder::AccessChain tensorReadResultLValue = {};
3575    tensorReadResultLValue.base = spv::NoResult;            // deferred tensorReadARM out-arg store target
3576
3577    auto resultType = [&invertedType, &node, this](){
3578        if (invertedType != spv::NoType) {
3579            return invertedType;
3580        } else {
3581            auto ret = convertGlslangToSpvType(node->getType());
3582            // convertGlslangToSpvType may clobber the debug location, reset it
3583            builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
3584            return ret;
3585        }
3586    };
3587
3588    // try texturing
3589    result = createImageTextureFunctionCall(node);
3590    if (result != spv::NoResult) {
3591        builder.clearAccessChain();
3592        builder.setAccessChainRValue(result);
3593
3594        return false;
3595    } else if (node->getOp() == glslang::EOpImageStore ||
3596        node->getOp() == glslang::EOpImageStoreLod ||
3597        node->getOp() == glslang::EOpImageAtomicStore) {
3598        // "imageStore" is a special case, which has no result
3599        return false;
3600    }
3601
3602    glslang::TOperator binOp = glslang::EOpNull;
3603    bool reduceComparison = true;
3604    bool isMatrix = false;
3605    bool noReturnValue = false;
3606    bool atomic = false;
3607
3608    spv::Builder::AccessChain::CoherentFlags lvalueCoherentFlags;
3609
3610    assert(node->getOp());
3611
3612    spv::Decoration precision = TranslatePrecisionDecoration(node->getOperationPrecision());
3613
3614    switch (node->getOp()) {
3615    case glslang::EOpScope:
3616    case glslang::EOpSequence:
3617    {
3618        if (visit == glslang::EvPreVisit) {
3619            ++sequenceDepth;
3620            if (sequenceDepth == 1) {
3621                // If this is the parent node of all the functions, we want to see them
3622                // early, so all call points have actual SPIR-V functions to reference.
3623                // In all cases, still let the traverser visit the children for us.
3624                makeFunctions(node->getAsAggregate()->getSequence());
3625
3626                // Global initializers is specific to the shader entry point, which does not exist in compile-only mode
3627                if (!options.compileOnly) {
3628                    // Also, we want all globals initializers to go into the beginning of the entry point, before
3629                    // anything else gets there, so visit out of order, doing them all now.
3630                    makeGlobalInitializers(node->getAsAggregate()->getSequence());
3631                }
3632
3633                //Pre process linker objects for ray tracing stages
3634                if (glslangIntermediate->isRayTracingStage())
3635                  collectRayTracingLinkerObjects();
3636
3637                // Initializers are done, don't want to visit again, but functions and link objects need to be processed,
3638                // so do them manually.
3639                visitFunctions(node->getAsAggregate()->getSequence());
3640
3641                return false;
3642            } else {
3643                if (node->getOp() == glslang::EOpScope) {
3644                    auto loc = node->getLoc();
3645                    builder.enterLexicalBlock(loc.line, loc.column);
3646                }
3647            }
3648        } else {
3649            if (sequenceDepth > 1 && node->getOp() == glslang::EOpScope)
3650                builder.leaveLexicalBlock();
3651            --sequenceDepth;
3652        }
3653
3654        return true;
3655    }
3656    case glslang::EOpLinkerObjects:
3657    {
3658        if (visit == glslang::EvPreVisit)
3659            linkageOnly = true;
3660        else
3661            linkageOnly = false;
3662
3663        return true;
3664    }
3665    case glslang::EOpComma:
3666    {
3667        // processing from left to right naturally leaves the right-most
3668        // lying around in the access chain
3669        glslang::TIntermSequence& glslangOperands = node->getSequence();
3670        for (int i = 0; i < (int)glslangOperands.size(); ++i)
3671            glslangOperands[i]->traverse(this);
3672
3673        return false;
3674    }
3675    case glslang::EOpFunction:
3676        if (visit == glslang::EvPreVisit) {
3677            if (options.generateDebugInfo) {
3678                builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
3679            }
3680            if (isShaderEntryPoint(node)) {
3681                inEntryPoint = true;
3682                builder.setBuildPoint(shaderEntry->getLastBlock());
3683                builder.enterFunction(shaderEntry);
3684                currentFunction = shaderEntry;
3685            } else {
3686                // SPIR-V functions should already be in the functionMap from the prepass
3687                // that called makeFunctions().
3688                currentFunction = functionMap[node->getName().c_str()];
3689                spv::Block* functionBlock = currentFunction->getEntryBlock();
3690                builder.setBuildPoint(functionBlock);
3691                builder.enterFunction(currentFunction);
3692            }
3693            if (options.generateDebugInfo && !options.emitNonSemanticShaderDebugInfo) {
3694                const auto& loc = node->getLoc();
3695                const char* sourceFileName = loc.getFilename();
3696                spv::Id sourceFileId = sourceFileName ? builder.getStringId(sourceFileName) : builder.getMainFileId();
3697                currentFunction->setDebugLineInfo(sourceFileId, loc.line, loc.column);
3698            }
3699        } else {
3700            // Here we have finished visiting the function (post-visit). Finalize it.
3701            if (options.generateDebugInfo) {
3702                if (glslangIntermediate->getSource() == glslang::EShSourceGlsl && node->getSequence().size() > 1) {
3703                    auto endLoc = node->getSequence()[1]->getAsAggregate()->getEndLoc();
3704                    builder.setDebugSourceLocation(endLoc.line, endLoc.getFilename());
3705                }
3706            }
3707            if (inEntryPoint)
3708                entryPointTerminated = true;
3709            builder.leaveFunction();
3710            inEntryPoint = false;
3711            currentFunction = nullptr;
3712        }
3713
3714        return true;
3715    case glslang::EOpParameters:
3716        // Parameters will have been consumed by EOpFunction processing, but not
3717        // the body, so we still visited the function node's children, making this
3718        // child redundant.
3719        return false;
3720    case glslang::EOpFunctionCall:
3721    {
3722        builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
3723        if (node->isUserDefined())
3724            result = handleUserFunctionCall(node);
3725        if (result) {
3726            builder.clearAccessChain();
3727            builder.setAccessChainRValue(result);
3728        } else
3729            logger->missingFunctionality("missing user function; linker needs to catch that");
3730
3731        return false;
3732    }
3733    case glslang::EOpConstructMat2x2:
3734    case glslang::EOpConstructMat2x3:
3735    case glslang::EOpConstructMat2x4:
3736    case glslang::EOpConstructMat3x2:
3737    case glslang::EOpConstructMat3x3:
3738    case glslang::EOpConstructMat3x4:
3739    case glslang::EOpConstructMat4x2:
3740    case glslang::EOpConstructMat4x3:
3741    case glslang::EOpConstructMat4x4:
3742    case glslang::EOpConstructDMat2x2:
3743    case glslang::EOpConstructDMat2x3:
3744    case glslang::EOpConstructDMat2x4:
3745    case glslang::EOpConstructDMat3x2:
3746    case glslang::EOpConstructDMat3x3:
3747    case glslang::EOpConstructDMat3x4:
3748    case glslang::EOpConstructDMat4x2:
3749    case glslang::EOpConstructDMat4x3:
3750    case glslang::EOpConstructDMat4x4:
3751    case glslang::EOpConstructIMat2x2:
3752    case glslang::EOpConstructIMat2x3:
3753    case glslang::EOpConstructIMat2x4:
3754    case glslang::EOpConstructIMat3x2:
3755    case glslang::EOpConstructIMat3x3:
3756    case glslang::EOpConstructIMat3x4:
3757    case glslang::EOpConstructIMat4x2:
3758    case glslang::EOpConstructIMat4x3:
3759    case glslang::EOpConstructIMat4x4:
3760    case glslang::EOpConstructUMat2x2:
3761    case glslang::EOpConstructUMat2x3:
3762    case glslang::EOpConstructUMat2x4:
3763    case glslang::EOpConstructUMat3x2:
3764    case glslang::EOpConstructUMat3x3:
3765    case glslang::EOpConstructUMat3x4:
3766    case glslang::EOpConstructUMat4x2:
3767    case glslang::EOpConstructUMat4x3:
3768    case glslang::EOpConstructUMat4x4:
3769    case glslang::EOpConstructBMat2x2:
3770    case glslang::EOpConstructBMat2x3:
3771    case glslang::EOpConstructBMat2x4:
3772    case glslang::EOpConstructBMat3x2:
3773    case glslang::EOpConstructBMat3x3:
3774    case glslang::EOpConstructBMat3x4:
3775    case glslang::EOpConstructBMat4x2:
3776    case glslang::EOpConstructBMat4x3:
3777    case glslang::EOpConstructBMat4x4:
3778    case glslang::EOpConstructF16Mat2x2:
3779    case glslang::EOpConstructF16Mat2x3:
3780    case glslang::EOpConstructF16Mat2x4:
3781    case glslang::EOpConstructF16Mat3x2:
3782    case glslang::EOpConstructF16Mat3x3:
3783    case glslang::EOpConstructF16Mat3x4:
3784    case glslang::EOpConstructF16Mat4x2:
3785    case glslang::EOpConstructF16Mat4x3:
3786    case glslang::EOpConstructF16Mat4x4:
3787        isMatrix = true;
3788        [[fallthrough]];
3789    case glslang::EOpConstructFloat:
3790    case glslang::EOpConstructVec2:
3791    case glslang::EOpConstructVec3:
3792    case glslang::EOpConstructVec4:
3793    case glslang::EOpConstructDouble:
3794    case glslang::EOpConstructDVec2:
3795    case glslang::EOpConstructDVec3:
3796    case glslang::EOpConstructDVec4:
3797    case glslang::EOpConstructFloat16:
3798    case glslang::EOpConstructF16Vec2:
3799    case glslang::EOpConstructF16Vec3:
3800    case glslang::EOpConstructF16Vec4:
3801    case glslang::EOpConstructBFloat16:
3802    case glslang::EOpConstructBF16Vec2:
3803    case glslang::EOpConstructBF16Vec3:
3804    case glslang::EOpConstructBF16Vec4:
3805    case glslang::EOpConstructFloatE5M2:
3806    case glslang::EOpConstructFloatE5M2Vec2:
3807    case glslang::EOpConstructFloatE5M2Vec3:
3808    case glslang::EOpConstructFloatE5M2Vec4:
3809    case glslang::EOpConstructFloatE4M3:
3810    case glslang::EOpConstructFloatE4M3Vec2:
3811    case glslang::EOpConstructFloatE4M3Vec3:
3812    case glslang::EOpConstructFloatE4M3Vec4:
3813    case glslang::EOpConstructFloatE2M1:
3814    case glslang::EOpConstructFloatE2M1Vec2:
3815    case glslang::EOpConstructFloatE2M1Vec3:
3816    case glslang::EOpConstructFloatE2M1Vec4:
3817    case glslang::EOpConstructFloatE3M2:
3818    case glslang::EOpConstructFloatE3M2Vec2:
3819    case glslang::EOpConstructFloatE3M2Vec3:
3820    case glslang::EOpConstructFloatE3M2Vec4:
3821    case glslang::EOpConstructFloatE2M3:
3822    case glslang::EOpConstructFloatE2M3Vec2:
3823    case glslang::EOpConstructFloatE2M3Vec3:
3824    case glslang::EOpConstructFloatE2M3Vec4:
3825    case glslang::EOpConstructFloatUE8M0:
3826    case glslang::EOpConstructFloatUE8M0Vec2:
3827    case glslang::EOpConstructFloatUE8M0Vec3:
3828    case glslang::EOpConstructFloatUE8M0Vec4:
3829    case glslang::EOpConstructFloatMXINT8:
3830    case glslang::EOpConstructFloatMXINT8Vec2:
3831    case glslang::EOpConstructFloatMXINT8Vec3:
3832    case glslang::EOpConstructFloatMXINT8Vec4:
3833    case glslang::EOpConstructBool:
3834    case glslang::EOpConstructBVec2:
3835    case glslang::EOpConstructBVec3:
3836    case glslang::EOpConstructBVec4:
3837    case glslang::EOpConstructInt8:
3838    case glslang::EOpConstructI8Vec2:
3839    case glslang::EOpConstructI8Vec3:
3840    case glslang::EOpConstructI8Vec4:
3841    case glslang::EOpConstructUint8:
3842    case glslang::EOpConstructU8Vec2:
3843    case glslang::EOpConstructU8Vec3:
3844    case glslang::EOpConstructU8Vec4:
3845    case glslang::EOpConstructInt16:
3846    case glslang::EOpConstructI16Vec2:
3847    case glslang::EOpConstructI16Vec3:
3848    case glslang::EOpConstructI16Vec4:
3849    case glslang::EOpConstructUint16:
3850    case glslang::EOpConstructU16Vec2:
3851    case glslang::EOpConstructU16Vec3:
3852    case glslang::EOpConstructU16Vec4:
3853    case glslang::EOpConstructInt:
3854    case glslang::EOpConstructIVec2:
3855    case glslang::EOpConstructIVec3:
3856    case glslang::EOpConstructIVec4:
3857    case glslang::EOpConstructUint:
3858    case glslang::EOpConstructUVec2:
3859    case glslang::EOpConstructUVec3:
3860    case glslang::EOpConstructUVec4:
3861    case glslang::EOpConstructInt64:
3862    case glslang::EOpConstructI64Vec2:
3863    case glslang::EOpConstructI64Vec3:
3864    case glslang::EOpConstructI64Vec4:
3865    case glslang::EOpConstructUint64:
3866    case glslang::EOpConstructU64Vec2:
3867    case glslang::EOpConstructU64Vec3:
3868    case glslang::EOpConstructU64Vec4:
3869    case glslang::EOpConstructStruct:
3870    case glslang::EOpConstructTextureSampler:
3871    case glslang::EOpConstructReference:
3872    case glslang::EOpConstructCooperativeMatrixNV:
3873    case glslang::EOpConstructCooperativeMatrixKHR:
3874    case glslang::EOpConstructCooperativeVectorNV:
3875    case glslang::EOpConstructSaturated:
3876    {
3877        builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
3878        std::vector<spv::Id> arguments;
3879        translateArguments(*node, arguments, lvalueCoherentFlags);
3880        spv::Id constructed;
3881        if (node->getOp() == glslang::EOpConstructTextureSampler) {
3882            const glslang::TType& texType = node->getSequence()[0]->getAsTyped()->getType();
3883            if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6 &&
3884                texType.getSampler().isBuffer()) {
3885                // SamplerBuffer is not supported in spirv1.6 so
3886                // `samplerBuffer(textureBuffer, sampler)` is a no-op
3887                // and textureBuffer is the result going forward
3888                constructed = arguments[0];
3889            } else
3890                constructed = builder.createOp(spv::Op::OpSampledImage, resultType(), arguments);
3891        } else if (node->getOp() == glslang::EOpConstructCooperativeMatrixKHR &&
3892                   node->getType().isCoopMatKHR() && node->getSequence()[0]->getAsTyped()->getType().isCoopMatKHR() &&
3893                   !node->getAsTyped()->getType().sameCoopMatUse(node->getSequence()[0]->getAsTyped()->getType())) {
3894            enableCoopMatConversions();
3895            constructed = builder.createCooperativeMatrixConversion(resultType(), arguments[0]);
3896        } else if (node->getType().isCoopVecOrLongVector() &&
3897                   arguments.size() == 1 &&
3898                   builder.getTypeId(arguments[0]) == resultType()) {
3899            constructed = arguments[0];
3900        } else if (node->getOp() == glslang::EOpConstructStruct ||
3901                 node->getOp() == glslang::EOpConstructCooperativeMatrixNV ||
3902                 node->getOp() == glslang::EOpConstructCooperativeMatrixKHR ||
3903                 node->getType().isArray() ||
3904                 // Handle constructing coopvec from one component here, to avoid the component
3905                 // getting smeared
3906                 (node->getType().hasSpecConstantVectorComponents() && arguments.size() == 1 && builder.isScalar(arguments[0]))) {
3907            std::vector<spv::Id> constituents;
3908            for (int c = 0; c < (int)arguments.size(); ++c)
3909                constituents.push_back(arguments[c]);
3910            constructed = createCompositeConstruct(resultType(), constituents);
3911        } else if (isMatrix)
3912            constructed = builder.createMatrixConstructor(precision, arguments, resultType());
3913        else if (node->getOp() == glslang::EOpConstructSaturated) {
3914            OpDecorations decorations = { TranslatePrecisionDecoration(node->getOperationPrecision()),
3915                                          TranslateNoContractionDecoration(node->getType().getQualifier()),
3916                                          TranslateNonUniformDecoration(lvalueCoherentFlags) };
3917
3918            auto operand = node->getSequence()[1]->getAsTyped();
3919
3920            bool sameUse = !node->getType().isCoopMatKHR() || !operand->getType().isCoopMatKHR() ||
3921                           node->getAsTyped()->getType().sameCoopMatUse(operand->getType());
3922
3923            constructed = createConversion(node->getOp(), decorations, resultType(), arguments[1],
3924                                           node->getType().getBasicType(), operand->getBasicType(), sameUse);
3925            builder.addDecoration(constructed, spv::Decoration::SaturatedToLargestFloat8NormalConversionEXT);
3926            builder.createStore(constructed, arguments[0]);
3927        }
3928        else
3929            constructed = builder.createConstructor(precision, arguments, resultType());
3930
3931        if (node->getType().getQualifier().isNonUniform()) {
3932            auto& extensions = glslangIntermediate->getRequestedExtensions();
3933            if (extensions.find("GL_EXT_descriptor_heap") == extensions.end()) {
3934                builder.addDecoration(constructed, spv::Decoration::NonUniformEXT);
3935            }
3936        }
3937
3938        builder.clearAccessChain();
3939        builder.setAccessChainRValue(constructed);
3940
3941        return false;
3942    }
3943
3944    // These six are component-wise compares with component-wise results.
3945    // Forward on to createBinaryOperation(), requesting a vector result.
3946    case glslang::EOpLessThan:
3947    case glslang::EOpGreaterThan:
3948    case glslang::EOpLessThanEqual:
3949    case glslang::EOpGreaterThanEqual:
3950    case glslang::EOpVectorEqual:
3951    case glslang::EOpVectorNotEqual:
3952    {
3953        // Map the operation to a binary
3954        binOp = node->getOp();
3955        reduceComparison = false;
3956        switch (node->getOp()) {
3957        case glslang::EOpVectorEqual:     binOp = glslang::EOpVectorEqual;      break;
3958        case glslang::EOpVectorNotEqual:  binOp = glslang::EOpVectorNotEqual;   break;
3959        default:                          binOp = node->getOp();                break;
3960        }
3961
3962        break;
3963    }
3964    case glslang::EOpMul:
3965        // component-wise matrix multiply
3966        binOp = glslang::EOpMul;
3967        break;
3968    case glslang::EOpOuterProduct:
3969        // two vectors multiplied to make a matrix
3970        binOp = glslang::EOpOuterProduct;
3971        break;
3972    case glslang::EOpDot:
3973    {
3974        // for scalar dot product, use multiply
3975        glslang::TIntermSequence& glslangOperands = node->getSequence();
3976        if (!glslangOperands[0]->getAsTyped()->getType().isLongVector() &&
3977            glslangOperands[0]->getAsTyped()->getVectorSize() == 1)
3978            binOp = glslang::EOpMul;
3979        else if (isTypeFloat(node->getType().getBasicType()))
3980            binOp = glslang::EOpDot;
3981        break;
3982    }
3983    case glslang::EOpMod:
3984        // when an aggregate, this is the floating-point mod built-in function,
3985        // which can be emitted by the one in createBinaryOperation()
3986        binOp = glslang::EOpMod;
3987        break;
3988
3989    case glslang::EOpEmitVertex:
3990    case glslang::EOpEndPrimitive:
3991    case glslang::EOpBarrier:
3992    case glslang::EOpControlBarrierArriveEXT:
3993    case glslang::EOpControlBarrierWaitEXT:
3994    case glslang::EOpMemoryBarrier:
3995    case glslang::EOpMemoryBarrierAtomicCounter:
3996    case glslang::EOpMemoryBarrierBuffer:
3997    case glslang::EOpMemoryBarrierImage:
3998    case glslang::EOpMemoryBarrierShared:
3999    case glslang::EOpGroupMemoryBarrier:
4000    case glslang::EOpDeviceMemoryBarrier:
4001    case glslang::EOpAllMemoryBarrierWithGroupSync:
4002    case glslang::EOpDeviceMemoryBarrierWithGroupSync:
4003    case glslang::EOpWorkgroupMemoryBarrier:
4004    case glslang::EOpWorkgroupMemoryBarrierWithGroupSync:
4005    case glslang::EOpSubgroupBarrier:
4006    case glslang::EOpSubgroupMemoryBarrier:
4007    case glslang::EOpSubgroupMemoryBarrierBuffer:
4008    case glslang::EOpSubgroupMemoryBarrierImage:
4009    case glslang::EOpSubgroupMemoryBarrierShared:
4010        noReturnValue = true;
4011        // These all have 0 operands and will naturally finish up in the code below for 0 operands
4012        break;
4013
4014    case glslang::EOpAtomicAdd:
4015    case glslang::EOpAtomicSubtract:
4016    case glslang::EOpAtomicMin:
4017    case glslang::EOpAtomicMax:
4018    case glslang::EOpAtomicAnd:
4019    case glslang::EOpAtomicOr:
4020    case glslang::EOpAtomicXor:
4021    case glslang::EOpAtomicExchange:
4022    case glslang::EOpAtomicCompSwap:
4023        atomic = true;
4024        break;
4025
4026    case glslang::EOpAtomicStore:
4027        noReturnValue = true;
4028        [[fallthrough]];
4029    case glslang::EOpAtomicLoad:
4030        atomic = true;
4031        break;
4032
4033    case glslang::EOpAtomicCounterAdd:
4034    case glslang::EOpAtomicCounterSubtract:
4035    case glslang::EOpAtomicCounterMin:
4036    case glslang::EOpAtomicCounterMax:
4037    case glslang::EOpAtomicCounterAnd:
4038    case glslang::EOpAtomicCounterOr:
4039    case glslang::EOpAtomicCounterXor:
4040    case glslang::EOpAtomicCounterExchange:
4041    case glslang::EOpAtomicCounterCompSwap:
4042        builder.addExtension("SPV_KHR_shader_atomic_counter_ops");
4043        builder.addCapability(spv::Capability::AtomicStorageOps);
4044        atomic = true;
4045        break;
4046
4047    case glslang::EOpAbsDifference:
4048    case glslang::EOpAddSaturate:
4049    case glslang::EOpSubSaturate:
4050    case glslang::EOpAverage:
4051    case glslang::EOpAverageRounded:
4052    case glslang::EOpMul32x16:
4053        builder.addCapability(spv::Capability::IntegerFunctions2INTEL);
4054        builder.addExtension("SPV_INTEL_shader_integer_functions2");
4055        binOp = node->getOp();
4056        break;
4057
4058    case glslang::EOpExpectEXT:
4059        builder.addCapability(spv::Capability::ExpectAssumeKHR);
4060        builder.addExtension(spv::E_SPV_KHR_expect_assume);
4061        binOp = node->getOp();
4062        break;
4063
4064    case glslang::EOpIgnoreIntersectionNV:
4065    case glslang::EOpTerminateRayNV:
4066    case glslang::EOpTraceNV:
4067    case glslang::EOpTraceRayMotionNV:
4068    case glslang::EOpTraceKHR:
4069    case glslang::EOpExecuteCallableNV:
4070    case glslang::EOpExecuteCallableKHR:
4071    case glslang::EOpWritePackedPrimitiveIndices4x8NV:
4072    case glslang::EOpEmitMeshTasksEXT:
4073    case glslang::EOpSetMeshOutputsEXT:
4074        noReturnValue = true;
4075        break;
4076    case glslang::EOpRayQueryInitialize:
4077    case glslang::EOpRayQueryTerminate:
4078    case glslang::EOpRayQueryGenerateIntersection:
4079    case glslang::EOpRayQueryConfirmIntersection:
4080        builder.addExtension("SPV_KHR_ray_query");
4081        builder.addCapability(spv::Capability::RayQueryKHR);
4082        noReturnValue = true;
4083        break;
4084    case glslang::EOpRayQueryProceed:
4085    case glslang::EOpRayQueryGetIntersectionType:
4086    case glslang::EOpRayQueryGetRayTMin:
4087    case glslang::EOpRayQueryGetRayFlags:
4088    case glslang::EOpRayQueryGetIntersectionT:
4089    case glslang::EOpRayQueryGetIntersectionInstanceCustomIndex:
4090    case glslang::EOpRayQueryGetIntersectionInstanceId:
4091    case glslang::EOpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffset:
4092    case glslang::EOpRayQueryGetIntersectionGeometryIndex:
4093    case glslang::EOpRayQueryGetIntersectionPrimitiveIndex:
4094    case glslang::EOpRayQueryGetIntersectionBarycentrics:
4095    case glslang::EOpRayQueryGetIntersectionFrontFace:
4096    case glslang::EOpRayQueryGetIntersectionCandidateAABBOpaque:
4097    case glslang::EOpRayQueryGetIntersectionObjectRayDirection:
4098    case glslang::EOpRayQueryGetIntersectionObjectRayOrigin:
4099    case glslang::EOpRayQueryGetWorldRayDirection:
4100    case glslang::EOpRayQueryGetWorldRayOrigin:
4101    case glslang::EOpRayQueryGetIntersectionObjectToWorld:
4102    case glslang::EOpRayQueryGetIntersectionWorldToObject:
4103        builder.addExtension("SPV_KHR_ray_query");
4104        builder.addCapability(spv::Capability::RayQueryKHR);
4105        break;
4106    case glslang::EOpCooperativeMatrixLoad:
4107    case glslang::EOpCooperativeMatrixStore:
4108    case glslang::EOpCooperativeMatrixLoadNV:
4109    case glslang::EOpCooperativeMatrixStoreNV:
4110    case glslang::EOpCooperativeMatrixLoadTensorNV:
4111    case glslang::EOpCooperativeMatrixStoreTensorNV:
4112    case glslang::EOpCooperativeMatrixReduceNV:
4113    case glslang::EOpCooperativeMatrixPerElementOpNV:
4114    case glslang::EOpCooperativeMatrixTransposeNV:
4115    case glslang::EOpCooperativeVectorMatMulNV:
4116    case glslang::EOpCooperativeVectorMatMulAddNV:
4117    case glslang::EOpCooperativeVectorLoadNV:
4118    case glslang::EOpCooperativeVectorStoreNV:
4119    case glslang::EOpCooperativeVectorOuterProductAccumulateNV:
4120    case glslang::EOpCooperativeVectorReduceSumAccumulateNV:
4121        noReturnValue = true;
4122        break;
4123    case glslang::EOpBeginInvocationInterlock:
4124    case glslang::EOpEndInvocationInterlock:
4125        builder.addExtension(spv::E_SPV_EXT_fragment_shader_interlock);
4126        noReturnValue = true;
4127        break;
4128
4129    case glslang::EOpHitObjectTraceRayNV:
4130    case glslang::EOpHitObjectTraceRayMotionNV:
4131    case glslang::EOpHitObjectGetAttributesNV:
4132    case glslang::EOpHitObjectExecuteShaderNV:
4133    case glslang::EOpHitObjectRecordEmptyNV:
4134    case glslang::EOpHitObjectRecordMissNV:
4135    case glslang::EOpHitObjectRecordMissMotionNV:
4136    case glslang::EOpHitObjectRecordHitNV:
4137    case glslang::EOpHitObjectRecordHitMotionNV:
4138    case glslang::EOpHitObjectRecordHitWithIndexNV:
4139    case glslang::EOpHitObjectRecordHitWithIndexMotionNV:
4140    case glslang::EOpReorderThreadNV:
4141        noReturnValue = true;
4142        [[fallthrough]];
4143    case glslang::EOpHitObjectIsEmptyNV:
4144    case glslang::EOpHitObjectIsMissNV:
4145    case glslang::EOpHitObjectIsHitNV:
4146    case glslang::EOpHitObjectGetRayTMinNV:
4147    case glslang::EOpHitObjectGetRayTMaxNV:
4148    case glslang::EOpHitObjectGetObjectRayOriginNV:
4149    case glslang::EOpHitObjectGetObjectRayDirectionNV:
4150    case glslang::EOpHitObjectGetWorldRayOriginNV:
4151    case glslang::EOpHitObjectGetWorldRayDirectionNV:
4152    case glslang::EOpHitObjectGetObjectToWorldNV:
4153    case glslang::EOpHitObjectGetWorldToObjectNV:
4154    case glslang::EOpHitObjectGetInstanceCustomIndexNV:
4155    case glslang::EOpHitObjectGetInstanceIdNV:
4156    case glslang::EOpHitObjectGetGeometryIndexNV:
4157    case glslang::EOpHitObjectGetPrimitiveIndexNV:
4158    case glslang::EOpHitObjectGetHitKindNV:
4159    case glslang::EOpHitObjectGetCurrentTimeNV:
4160    case glslang::EOpHitObjectGetShaderBindingTableRecordIndexNV:
4161    case glslang::EOpHitObjectGetShaderRecordBufferHandleNV:
4162        builder.addExtension(spv::E_SPV_NV_shader_invocation_reorder);
4163        builder.addCapability(spv::Capability::ShaderInvocationReorderNV);
4164        break;
4165
4166    case glslang::EOpHitObjectGetLSSPositionsNV:
4167    case glslang::EOpHitObjectGetLSSRadiiNV:
4168        builder.addExtension(spv::E_SPV_NV_linear_swept_spheres);
4169        builder.addCapability(spv::Capability::ShaderInvocationReorderNV);
4170        builder.addCapability(spv::Capability::RayTracingLinearSweptSpheresGeometryNV);
4171        noReturnValue = true;
4172        break;
4173
4174    case glslang::EOpRayQueryGetIntersectionLSSPositionsNV:
4175    case glslang::EOpRayQueryGetIntersectionLSSRadiiNV:
4176        builder.addExtension(spv::E_SPV_NV_linear_swept_spheres);
4177        builder.addCapability(spv::Capability::RayQueryKHR);
4178        builder.addCapability(spv::Capability::RayTracingLinearSweptSpheresGeometryNV);
4179        noReturnValue = true;
4180        break;
4181
4182    case glslang::EOpRayQueryGetIntersectionSpherePositionNV:
4183    case glslang::EOpRayQueryGetIntersectionSphereRadiusNV:
4184    case glslang::EOpRayQueryIsSphereHitNV:
4185        builder.addExtension(spv::E_SPV_NV_linear_swept_spheres);
4186        builder.addCapability(spv::Capability::RayQueryKHR);
4187        builder.addCapability(spv::Capability::RayTracingSpheresGeometryNV);
4188        builder.addCapability(spv::Capability::RayTracingLinearSweptSpheresGeometryNV);
4189        break;
4190
4191    case glslang::EOpRayQueryGetIntersectionLSSHitValueNV:
4192    case glslang::EOpRayQueryIsLSSHitNV:
4193        builder.addExtension(spv::E_SPV_NV_linear_swept_spheres);
4194        builder.addCapability(spv::Capability::RayQueryKHR);
4195        builder.addCapability(spv::Capability::RayTracingLinearSweptSpheresGeometryNV);
4196        break;
4197
4198    case glslang::EOpHitObjectTraceRayEXT:
4199    case glslang::EOpHitObjectTraceRayMotionEXT:
4200    case glslang::EOpHitObjectGetAttributesEXT:
4201    case glslang::EOpHitObjectExecuteShaderEXT:
4202    case glslang::EOpHitObjectRecordEmptyEXT:
4203    case glslang::EOpHitObjectRecordMissEXT:
4204    case glslang::EOpHitObjectRecordMissMotionEXT:
4205    case glslang::EOpReorderThreadEXT:
4206    case glslang::EOpHitObjectSetShaderBindingTableRecordIndexEXT:
4207    case glslang::EOpHitObjectReorderExecuteEXT:
4208    case glslang::EOpHitObjectTraceReorderExecuteEXT:
4209    case glslang::EOpHitObjectTraceMotionReorderExecuteEXT:
4210    case glslang::EOpHitObjectRecordFromQueryEXT:
4211    case glslang::EOpHitObjectGetIntersectionTriangleVertexPositionsEXT:
4212        noReturnValue = true;
4213        [[fallthrough]];
4214    case glslang::EOpHitObjectIsEmptyEXT:
4215    case glslang::EOpHitObjectIsMissEXT:
4216    case glslang::EOpHitObjectIsHitEXT:
4217    case glslang::EOpHitObjectGetRayTMinEXT:
4218    case glslang::EOpHitObjectGetRayTMaxEXT:
4219    case glslang::EOpHitObjectGetRayFlagsEXT:
4220    case glslang::EOpHitObjectGetObjectRayOriginEXT:
4221    case glslang::EOpHitObjectGetObjectRayDirectionEXT:
4222    case glslang::EOpHitObjectGetWorldRayOriginEXT:
4223    case glslang::EOpHitObjectGetWorldRayDirectionEXT:
4224    case glslang::EOpHitObjectGetObjectToWorldEXT:
4225    case glslang::EOpHitObjectGetWorldToObjectEXT:
4226    case glslang::EOpHitObjectGetInstanceCustomIndexEXT:
4227    case glslang::EOpHitObjectGetInstanceIdEXT:
4228    case glslang::EOpHitObjectGetGeometryIndexEXT:
4229    case glslang::EOpHitObjectGetPrimitiveIndexEXT:
4230    case glslang::EOpHitObjectGetHitKindEXT:
4231    case glslang::EOpHitObjectGetCurrentTimeEXT:
4232    case glslang::EOpHitObjectGetShaderBindingTableRecordIndexEXT:
4233    case glslang::EOpHitObjectGetShaderRecordBufferHandleEXT:
4234        builder.addExtension(spv::E_SPV_EXT_shader_invocation_reorder);
4235        builder.addCapability(spv::Capability::ShaderInvocationReorderEXT);
4236        break;
4237
4238    case glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT:
4239        builder.addExtension(spv::E_SPV_KHR_ray_tracing_position_fetch);
4240        builder.addCapability(spv::Capability::RayQueryPositionFetchKHR);
4241        noReturnValue = true;
4242        break;
4243    case glslang::EOpImageSampleWeightedQCOM:
4244        builder.addCapability(spv::Capability::TextureSampleWeightedQCOM);
4245        builder.addExtension(spv::E_SPV_QCOM_image_processing);
4246        break;
4247    case glslang::EOpImageBoxFilterQCOM:
4248        builder.addCapability(spv::Capability::TextureBoxFilterQCOM);
4249        builder.addExtension(spv::E_SPV_QCOM_image_processing);
4250        break;
4251    case glslang::EOpImageBlockMatchSADQCOM:
4252    case glslang::EOpImageBlockMatchSSDQCOM:
4253        builder.addCapability(spv::Capability::TextureBlockMatchQCOM);
4254        builder.addExtension(spv::E_SPV_QCOM_image_processing);
4255        break;
4256    case glslang::EOpTensorWriteARM:
4257        noReturnValue = true;
4258        break;
4259
4260    case glslang::EOpImageBlockMatchWindowSSDQCOM:
4261    case glslang::EOpImageBlockMatchWindowSADQCOM:
4262        builder.addCapability(spv::Capability::TextureBlockMatchQCOM);
4263        builder.addExtension(spv::E_SPV_QCOM_image_processing);
4264        builder.addCapability(spv::Capability::TextureBlockMatch2QCOM);
4265        builder.addExtension(spv::E_SPV_QCOM_image_processing2);
4266        break;
4267
4268    case glslang::EOpImageBlockMatchGatherSSDQCOM:
4269    case glslang::EOpImageBlockMatchGatherSADQCOM:
4270        builder.addCapability(spv::Capability::TextureBlockMatchQCOM);
4271        builder.addExtension(spv::E_SPV_QCOM_image_processing);
4272        builder.addCapability(spv::Capability::TextureBlockMatch2QCOM);
4273        builder.addExtension(spv::E_SPV_QCOM_image_processing2);
4274        break;
4275
4276    case glslang::EOpFetchMicroTriangleVertexPositionNV:
4277    case glslang::EOpFetchMicroTriangleVertexBarycentricNV:
4278        builder.addExtension(spv::E_SPV_NV_displacement_micromap);
4279        builder.addCapability(spv::Capability::DisplacementMicromapNV);
4280        break;
4281
4282    case glslang::EOpRayQueryGetIntersectionClusterIdNV:
4283        builder.addExtension(spv::E_SPV_NV_cluster_acceleration_structure);
4284        builder.addCapability(spv::Capability::RayQueryKHR);
4285        builder.addCapability(spv::Capability::RayTracingClusterAccelerationStructureNV);
4286        break;
4287
4288    case glslang::EOpCooperativeMatrixGetCoordinateEXT:
4289        builder.addExtension(spv::E_SPV_EXT_cooperative_matrix_maintenance1);
4290        builder.addCapability(spv::Capability::CooperativeMatrixGetCoordinateEXT);
4291        break;
4292
4293    case glslang::EOpDebugPrintf:
4294        noReturnValue = true;
4295        break;
4296
4297    default:
4298        break;
4299    }
4300
4301    //
4302    // See if it maps to a regular operation.
4303    //
4304    if (binOp != glslang::EOpNull) {
4305        glslang::TIntermTyped* left = node->getSequence()[0]->getAsTyped();
4306        glslang::TIntermTyped* right = node->getSequence()[1]->getAsTyped();
4307        assert(left && right);
4308
4309        builder.clearAccessChain();
4310        left->traverse(this);
4311        spv::Id leftId = accessChainLoad(left->getType());
4312
4313        builder.clearAccessChain();
4314        right->traverse(this);
4315        spv::Id rightId = accessChainLoad(right->getType());
4316
4317        builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
4318        OpDecorations decorations = { precision,
4319                                      TranslateNoContractionDecoration(node->getType().getQualifier()),
4320                                      TranslateNonUniformDecoration(node->getType().getQualifier()) };
4321        result = createBinaryOperation(binOp, decorations,
4322                                       resultType(), leftId, rightId,
4323                                       left->getType().getBasicType(), reduceComparison);
4324
4325        // code above should only make binOp that exists in createBinaryOperation
4326        assert(result != spv::NoResult);
4327        builder.clearAccessChain();
4328        builder.setAccessChainRValue(result);
4329
4330        return false;
4331    }
4332
4333    //
4334    // Create the list of operands.
4335    //
4336    glslang::TIntermSequence& glslangOperands = node->getSequence();
4337    std::vector<spv::Id> operands;
4338    std::vector<spv::IdImmediate> memoryAccessOperands;
4339    if (node->getOp() == glslang::EOpAbortEXT) {
4340        createAbortEXT(glslangOperands);
4341        return false;
4342    }
4343    for (int arg = 0; arg < (int)glslangOperands.size(); ++arg) {
4344        // special case l-value operands; there are just a few
4345        bool lvalue = false;
4346        switch (node->getOp()) {
4347        case glslang::EOpModf:
4348            if (arg == 1)
4349                lvalue = true;
4350            break;
4351
4352
4353
4354        case glslang::EOpHitObjectRecordFromQueryEXT:
4355        case glslang::EOpHitObjectGetIntersectionTriangleVertexPositionsEXT:
4356            if (arg == 0 || arg == 1)
4357                lvalue = true;
4358            break;
4359
4360        case glslang::EOpHitObjectRecordHitNV:
4361        case glslang::EOpHitObjectRecordHitMotionNV:
4362        case glslang::EOpHitObjectRecordHitWithIndexNV:
4363        case glslang::EOpHitObjectRecordHitWithIndexMotionNV:
4364        case glslang::EOpHitObjectTraceRayNV:
4365        case glslang::EOpHitObjectTraceRayMotionNV:
4366        case glslang::EOpHitObjectExecuteShaderNV:
4367        case glslang::EOpHitObjectRecordMissNV:
4368        case glslang::EOpHitObjectRecordMissMotionNV:
4369        case glslang::EOpHitObjectGetAttributesNV:
4370        case glslang::EOpHitObjectGetClusterIdNV:
4371        case glslang::EOpHitObjectTraceRayEXT:
4372        case glslang::EOpHitObjectTraceRayMotionEXT:
4373        case glslang::EOpHitObjectExecuteShaderEXT:
4374        case glslang::EOpHitObjectRecordMissEXT:
4375        case glslang::EOpHitObjectRecordMissMotionEXT:
4376        case glslang::EOpHitObjectGetAttributesEXT:
4377        case glslang::EOpHitObjectSetShaderBindingTableRecordIndexEXT:
4378        case glslang::EOpHitObjectReorderExecuteEXT:
4379        case glslang::EOpHitObjectTraceReorderExecuteEXT:
4380        case glslang::EOpHitObjectTraceMotionReorderExecuteEXT:
4381            if (arg == 0)
4382                lvalue = true;
4383            break;
4384
4385        case glslang::EOpHitObjectGetLSSPositionsNV:
4386        case glslang::EOpHitObjectGetLSSRadiiNV:
4387            lvalue = true;
4388            break;
4389
4390        case glslang::EOpRayQueryInitialize:
4391        case glslang::EOpRayQueryTerminate:
4392        case glslang::EOpRayQueryConfirmIntersection:
4393        case glslang::EOpRayQueryProceed:
4394        case glslang::EOpRayQueryGenerateIntersection:
4395        case glslang::EOpRayQueryGetIntersectionType:
4396        case glslang::EOpRayQueryGetIntersectionT:
4397        case glslang::EOpRayQueryGetIntersectionInstanceCustomIndex:
4398        case glslang::EOpRayQueryGetIntersectionInstanceId:
4399        case glslang::EOpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffset:
4400        case glslang::EOpRayQueryGetIntersectionGeometryIndex:
4401        case glslang::EOpRayQueryGetIntersectionPrimitiveIndex:
4402        case glslang::EOpRayQueryGetIntersectionBarycentrics:
4403        case glslang::EOpRayQueryGetIntersectionFrontFace:
4404        case glslang::EOpRayQueryGetIntersectionObjectRayDirection:
4405        case glslang::EOpRayQueryGetIntersectionObjectRayOrigin:
4406        case glslang::EOpRayQueryGetIntersectionObjectToWorld:
4407        case glslang::EOpRayQueryGetIntersectionWorldToObject:
4408        case glslang::EOpRayQueryGetIntersectionClusterIdNV:
4409        case glslang::EOpRayQueryGetIntersectionSpherePositionNV:
4410        case glslang::EOpRayQueryGetIntersectionSphereRadiusNV:
4411        case glslang::EOpRayQueryGetIntersectionLSSHitValueNV:
4412        case glslang::EOpRayQueryIsSphereHitNV:
4413        case glslang::EOpRayQueryIsLSSHitNV:
4414            if (arg == 0)
4415                lvalue = true;
4416            break;
4417
4418        case glslang::EOpAtomicAdd:
4419        case glslang::EOpAtomicSubtract:
4420        case glslang::EOpAtomicMin:
4421        case glslang::EOpAtomicMax:
4422        case glslang::EOpAtomicAnd:
4423        case glslang::EOpAtomicOr:
4424        case glslang::EOpAtomicXor:
4425        case glslang::EOpAtomicExchange:
4426        case glslang::EOpAtomicCompSwap:
4427            if (arg == 0)
4428                lvalue = true;
4429            break;
4430
4431        case glslang::EOpFrexp:
4432            if (arg == 1)
4433                lvalue = true;
4434            break;
4435        case glslang::EOpInterpolateAtSample:
4436        case glslang::EOpInterpolateAtOffset:
4437        case glslang::EOpInterpolateAtVertex:
4438            if (arg == 0) {
4439                // If GLSL, use the address of the interpolant argument.
4440                // If HLSL, use an internal version of OpInterolates that takes
4441                // the rvalue of the interpolant. A fixup pass in spirv-opt
4442                // legalization will remove the OpLoad and convert to an lvalue.
4443                // Had to do this because legalization will only propagate a
4444                // builtin into an rvalue.
4445                lvalue = glslangIntermediate->getSource() != glslang::EShSourceHlsl;
4446
4447                // Does it need a swizzle inversion?  If so, evaluation is inverted;
4448                // operate first on the swizzle base, then apply the swizzle.
4449                // That is, we transform
4450                //
4451                //    interpolate(v.zy)  ->  interpolate(v).zy
4452                //
4453                if (glslangOperands[0]->getAsOperator() &&
4454                    glslangOperands[0]->getAsOperator()->getOp() == glslang::EOpVectorSwizzle)
4455                    invertedType = convertGlslangToSpvType(
4456                        glslangOperands[0]->getAsBinaryNode()->getLeft()->getType());
4457            }
4458            break;
4459        case glslang::EOpAtomicLoad:
4460        case glslang::EOpAtomicStore:
4461        case glslang::EOpAtomicCounterAdd:
4462        case glslang::EOpAtomicCounterSubtract:
4463        case glslang::EOpAtomicCounterMin:
4464        case glslang::EOpAtomicCounterMax:
4465        case glslang::EOpAtomicCounterAnd:
4466        case glslang::EOpAtomicCounterOr:
4467        case glslang::EOpAtomicCounterXor:
4468        case glslang::EOpAtomicCounterExchange:
4469        case glslang::EOpAtomicCounterCompSwap:
4470            if (arg == 0)
4471                lvalue = true;
4472            break;
4473        case glslang::EOpAddCarry:
4474        case glslang::EOpSubBorrow:
4475            if (arg == 2)
4476                lvalue = true;
4477            break;
4478        case glslang::EOpUMulExtended:
4479        case glslang::EOpIMulExtended:
4480            if (arg >= 2)
4481                lvalue = true;
4482            break;
4483        case glslang::EOpCooperativeMatrixLoad:
4484        case glslang::EOpCooperativeMatrixLoadNV:
4485        case glslang::EOpCooperativeMatrixLoadTensorNV:
4486        case glslang::EOpCooperativeVectorLoadNV:
4487            if (arg == 0 || arg == 1)
4488                lvalue = true;
4489            break;
4490        case glslang::EOpCooperativeMatrixStore:
4491        case glslang::EOpCooperativeMatrixStoreNV:
4492        case glslang::EOpCooperativeMatrixStoreTensorNV:
4493        case glslang::EOpCooperativeVectorStoreNV:
4494            if (arg == 1)
4495                lvalue = true;
4496            break;
4497        case glslang::EOpCooperativeVectorMatMulNV:
4498            if (arg == 0 || arg == 3)
4499                lvalue = true;
4500            break;
4501        case glslang::EOpCooperativeVectorMatMulAddNV:
4502            if (arg == 0 || arg == 3 || arg == 6)
4503                lvalue = true;
4504            break;
4505        case glslang::EOpCooperativeVectorOuterProductAccumulateNV:
4506            if (arg == 2)
4507                lvalue = true;
4508            break;
4509        case glslang::EOpCooperativeVectorReduceSumAccumulateNV:
4510            if (arg == 1)
4511                lvalue = true;
4512            break;
4513        case glslang::EOpCooperativeMatrixReduceNV:
4514        case glslang::EOpCooperativeMatrixPerElementOpNV:
4515        case glslang::EOpCooperativeMatrixTransposeNV:
4516            if (arg == 0)
4517                lvalue = true;
4518            break;
4519        case glslang::EOpSpirvInst:
4520            if (glslangOperands[arg]->getAsTyped()->getQualifier().isSpirvByReference())
4521                lvalue = true;
4522            break;
4523        case glslang::EOpReorderThreadNV:
4524        case glslang::EOpReorderThreadEXT:
4525            //Three variants of reorderThreadNV, two of them use hitObjectNV
4526            if (arg == 0 && glslangOperands.size() != 2)
4527                lvalue = true;
4528            break;
4529        case glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT:
4530        case glslang::EOpRayQueryGetIntersectionLSSPositionsNV:
4531        case glslang::EOpRayQueryGetIntersectionLSSRadiiNV:
4532            if (arg == 0 || arg == 2)
4533                lvalue = true;
4534            break;
4535        case glslang::EOpTensorReadARM:
4536            if (arg == 2)
4537                lvalue = true;
4538            break;
4539        default:
4540            break;
4541        }
4542        builder.clearAccessChain();
4543        if (invertedType != spv::NoType && arg == 0)
4544            glslangOperands[0]->getAsBinaryNode()->getLeft()->traverse(this);
4545        else
4546            glslangOperands[arg]->traverse(this);
4547
4548        bool isCoopMat = node->getOp() == glslang::EOpCooperativeMatrixLoad ||
4549                         node->getOp() == glslang::EOpCooperativeMatrixStore ||
4550                         node->getOp() == glslang::EOpCooperativeMatrixLoadNV ||
4551                         node->getOp() == glslang::EOpCooperativeMatrixStoreNV ||
4552                         node->getOp() == glslang::EOpCooperativeMatrixLoadTensorNV ||
4553                         node->getOp() == glslang::EOpCooperativeMatrixStoreTensorNV;
4554        bool isCoopVec = node->getOp() == glslang::EOpCooperativeVectorLoadNV ||
4555                         node->getOp() == glslang::EOpCooperativeVectorStoreNV;
4556        if (isCoopMat || isCoopVec) {
4557
4558            if (arg == 1) {
4559                spv::Builder::AccessChain::CoherentFlags coherentFlags {};
4560                unsigned int alignment {};
4561                if (isCoopMat) {
4562                    // fold "element" parameter into the access chain
4563                    spv::Builder::AccessChain save = builder.getAccessChain();
4564                    builder.clearAccessChain();
4565                    glslangOperands[2]->traverse(this);
4566
4567                    spv::Id elementId = accessChainLoad(glslangOperands[2]->getAsTyped()->getType());
4568
4569                    builder.setAccessChain(save);
4570
4571                    // Point to the first element of the array.
4572                    builder.accessChainPush(elementId,
4573                        TranslateCoherent(glslangOperands[arg]->getAsTyped()->getType()),
4574                                          glslangOperands[arg]->getAsTyped()->getType().getBufferReferenceAlignment());
4575                    coherentFlags = builder.getAccessChain().coherentFlags;
4576                    alignment = builder.getAccessChain().alignment;
4577                } else {
4578                    coherentFlags = builder.getAccessChain().coherentFlags;
4579                    coherentFlags |= TranslateCoherent(glslangOperands[arg]->getAsTyped()->getType());
4580                    alignment = 16;
4581                }
4582
4583                spv::MemoryAccessMask memoryAccess = TranslateMemoryAccess(coherentFlags);
4584                if (node->getOp() == glslang::EOpCooperativeMatrixLoad ||
4585                    node->getOp() == glslang::EOpCooperativeMatrixLoadNV ||
4586                    node->getOp() == glslang::EOpCooperativeMatrixLoadTensorNV ||
4587                    node->getOp() == glslang::EOpCooperativeVectorLoadNV)
4588                    memoryAccess = (memoryAccess & ~spv::MemoryAccessMask::MakePointerAvailableKHR);
4589                if (node->getOp() == glslang::EOpCooperativeMatrixStore ||
4590                    node->getOp() == glslang::EOpCooperativeMatrixStoreNV ||
4591                    node->getOp() == glslang::EOpCooperativeMatrixStoreTensorNV ||
4592                    node->getOp() == glslang::EOpCooperativeVectorStoreNV)
4593                    memoryAccess = (memoryAccess & ~spv::MemoryAccessMask::MakePointerVisibleKHR);
4594                if (builder.getStorageClass(builder.getAccessChain().base) ==
4595                    spv::StorageClass::PhysicalStorageBufferEXT) {
4596                    memoryAccess = (spv::MemoryAccessMask)(memoryAccess | spv::MemoryAccessMask::Aligned);
4597                }
4598
4599                memoryAccessOperands.push_back(spv::IdImmediate(false, memoryAccess));
4600
4601                if (anySet(memoryAccess, spv::MemoryAccessMask::Aligned)) {
4602                    memoryAccessOperands.push_back(spv::IdImmediate(false, alignment));
4603                }
4604
4605                if (anySet(memoryAccess,
4606                    spv::MemoryAccessMask::MakePointerAvailableKHR | spv::MemoryAccessMask::MakePointerVisibleKHR)) {
4607                    memoryAccessOperands.push_back(spv::IdImmediate(true,
4608                        builder.makeUintConstant(TranslateMemoryScope(coherentFlags))));
4609                }
4610            } else if (isCoopMat && arg == 2) {
4611                continue;
4612            }
4613        }
4614
4615        // for l-values, pass the address, for r-values, pass the value
4616        if (lvalue) {
4617            if (invertedType == spv::NoType && !builder.isSpvLvalue()) {
4618                // SPIR-V cannot represent an l-value containing a swizzle that doesn't
4619                // reduce to a simple access chain.  So, we need a temporary vector to
4620                // receive the result, and must later swizzle that into the original
4621                // l-value.
4622                complexLvalues.push_back(builder.getAccessChain());
4623                complexLvalueTypes.push_back(&glslangOperands[arg]->getAsTyped()->getType());
4624                temporaryLvalues.push_back(builder.createVariable(
4625                    spv::NoPrecision, spv::StorageClass::Function,
4626                    builder.accessChainGetInferredType(), "swizzleTemp"));
4627                operands.push_back(temporaryLvalues.back());
4628            } else {
4629                if (node->getOp() == glslang::EOpTensorReadARM && arg == 2) {
4630                    // tensorReadARM stores the result after emitting the op, so keep the
4631                    // original l-value access chain and avoid materializing a transient
4632                    // pointer that may not preserve descriptor-heap indexing.
4633                    tensorReadResultLValue = builder.getAccessChain();
4634                    // Keep the operand slot so optional tensor operands keep their
4635                    // existing indices in the later lowering logic.
4636                    operands.push_back(spv::NoResult);
4637                } else {
4638                    operands.push_back(builder.accessChainGetLValue());
4639                }
4640            }
4641            lvalueCoherentFlags = builder.getAccessChain().coherentFlags;
4642            lvalueCoherentFlags |= TranslateCoherent(glslangOperands[arg]->getAsTyped()->getType());
4643        } else {
4644            builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
4645            glslang::TOperator glslangOp = node->getOp();
4646            if (arg == 1 &&
4647                (glslangOp == glslang::EOpRayQueryGetIntersectionType ||
4648                 glslangOp == glslang::EOpRayQueryGetIntersectionT ||
4649                 glslangOp == glslang::EOpRayQueryGetIntersectionInstanceCustomIndex ||
4650                 glslangOp == glslang::EOpRayQueryGetIntersectionInstanceId ||
4651                 glslangOp == glslang::EOpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffset ||
4652                 glslangOp == glslang::EOpRayQueryGetIntersectionGeometryIndex ||
4653                 glslangOp == glslang::EOpRayQueryGetIntersectionPrimitiveIndex ||
4654                 glslangOp == glslang::EOpRayQueryGetIntersectionBarycentrics ||
4655                 glslangOp == glslang::EOpRayQueryGetIntersectionFrontFace ||
4656                 glslangOp == glslang::EOpRayQueryGetIntersectionObjectRayDirection ||
4657                 glslangOp == glslang::EOpRayQueryGetIntersectionObjectRayOrigin ||
4658                 glslangOp == glslang::EOpRayQueryGetIntersectionObjectToWorld ||
4659                 glslangOp == glslang::EOpRayQueryGetIntersectionWorldToObject ||
4660                 glslangOp == glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT ||
4661                 glslangOp == glslang::EOpRayQueryGetIntersectionClusterIdNV ||
4662                 glslangOp == glslang::EOpRayQueryGetIntersectionSpherePositionNV ||
4663                 glslangOp == glslang::EOpRayQueryGetIntersectionSphereRadiusNV ||
4664                 glslangOp == glslang::EOpRayQueryGetIntersectionLSSHitValueNV ||
4665                 glslangOp == glslang::EOpRayQueryGetIntersectionLSSPositionsNV ||
4666                 glslangOp == glslang::EOpRayQueryGetIntersectionLSSRadiiNV ||
4667                 glslangOp == glslang::EOpRayQueryIsLSSHitNV ||
4668                 glslangOp == glslang::EOpRayQueryIsSphereHitNV
4669                    )) {
4670                bool cond = glslangOperands[arg]->getAsConstantUnion()->getConstArray()[0].getBConst();
4671                operands.push_back(builder.makeIntConstant(cond ? 1 : 0));
4672             } else if ((arg == 10 && glslangOp == glslang::EOpTraceKHR) ||
4673                        (arg == 11 && glslangOp == glslang::EOpTraceRayMotionNV) ||
4674                        (arg == 1  && glslangOp == glslang::EOpExecuteCallableKHR) ||
4675                        (arg == 1  && glslangOp == glslang::EOpHitObjectExecuteShaderNV) ||
4676                        (arg == 1  && glslangOp == glslang::EOpHitObjectExecuteShaderEXT) ||
4677                        (arg == 11 && glslangOp == glslang::EOpHitObjectTraceRayNV) ||
4678                        (arg == 11 && glslangOp == glslang::EOpHitObjectTraceRayEXT) ||
4679                        (arg == 12 && glslangOp == glslang::EOpHitObjectTraceRayMotionNV) ||
4680                        (arg == 12 && glslangOp == glslang::EOpHitObjectTraceRayMotionEXT) ||
4681                        (arg == 12 && glslangOp == glslang::EOpHitObjectTraceMotionReorderExecuteEXT && glslangOperands.size() == 13) ||
4682                        (arg == 14 && glslangOp == glslang::EOpHitObjectTraceMotionReorderExecuteEXT && glslangOperands.size() == 15) ||
4683                        (arg == 11 && glslangOp == glslang::EOpHitObjectTraceReorderExecuteEXT && glslangOperands.size() == 12) ||
4684                        (arg == 13 && glslangOp == glslang::EOpHitObjectTraceReorderExecuteEXT && glslangOperands.size() == 14) ||
4685                        (arg == 1  && glslangOp == glslang::EOpHitObjectReorderExecuteEXT && glslangOperands.size() == 2) ||
4686                        (arg == 3  && glslangOp == glslang::EOpHitObjectReorderExecuteEXT && glslangOperands.size() == 4)) {
4687                 const int set = glslangOp == glslang::EOpExecuteCallableKHR ? 1 : 0;
4688                 const int location = glslangOperands[arg]->getAsConstantUnion()->getConstArray()[0].getUConst();
4689                 auto itNode = locationToSymbol[set].find(location);
4690                 if (itNode == locationToSymbol[set].end()) {
4691                     logger->missingFunctionality("ray tracing location with no matching payload/callable data declaration");
4692                     break;
4693                 }
4694                 visitSymbol(itNode->second);
4695                 spv::Id symId = getSymbolId(itNode->second);
4696                 operands.push_back(symId);
4697            } else if ((arg == 12 && glslangOp == glslang::EOpHitObjectRecordHitNV) ||
4698                       (arg == 13 && glslangOp == glslang::EOpHitObjectRecordHitMotionNV) ||
4699                       (arg == 11 && glslangOp == glslang::EOpHitObjectRecordHitWithIndexNV) ||
4700                       (arg == 12 && glslangOp == glslang::EOpHitObjectRecordHitWithIndexMotionNV) ||
4701                       (arg == 3  && glslangOp == glslang::EOpHitObjectRecordFromQueryEXT) ||
4702                       (arg == 1  && glslangOp == glslang::EOpHitObjectGetAttributesEXT) ||
4703                       (arg == 1  && glslangOp == glslang::EOpHitObjectGetAttributesNV)) {
4704                 const int location = glslangOperands[arg]->getAsConstantUnion()->getConstArray()[0].getUConst();
4705                 const int set = 2;
4706                 auto itNode = locationToSymbol[set].find(location);
4707                 if (itNode == locationToSymbol[set].end()) {
4708                     logger->missingFunctionality("ray tracing location with no matching hit object attribute declaration");
4709                     break;
4710                 }
4711                 visitSymbol(itNode->second);
4712                 spv::Id symId = getSymbolId(itNode->second);
4713                 operands.push_back(symId);
4714            } else if (glslangOperands[arg]->getAsTyped()->getQualifier().isSpirvLiteral()) {
4715                // Will be translated to a literal value, make a placeholder here
4716                operands.push_back(spv::NoResult);
4717            } else if (glslangOperands[arg]->getAsTyped()->getBasicType() == glslang::EbtFunction) {
4718                spv::Function* function = functionMap[glslangOperands[arg]->getAsSymbolNode()->getMangledName().c_str()];
4719                assert(function);
4720                operands.push_back(function->getId());
4721            } else  {
4722               operands.push_back(accessChainLoad(glslangOperands[arg]->getAsTyped()->getType()));
4723            }
4724        }
4725    }
4726
4727    builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
4728    if (node->getOp() == glslang::EOpCooperativeMatrixLoadTensorNV) {
4729        std::vector<spv::IdImmediate> idImmOps;
4730
4731        builder.addCapability(spv::Capability::CooperativeMatrixTensorAddressingNV);
4732        builder.addExtension(spv::E_SPV_NV_cooperative_matrix2);
4733
4734        spv::Id object = builder.createLoad(operands[0], spv::NoPrecision);
4735
4736        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // Pointer
4737        idImmOps.push_back(spv::IdImmediate(true, object)); // Object
4738        idImmOps.push_back(spv::IdImmediate(true, operands[2])); // tensorLayout
4739
4740        idImmOps.insert(idImmOps.end(), memoryAccessOperands.begin(), memoryAccessOperands.end()); // memoryaccess
4741
4742        // initialize tensor operands to zero, then OR in flags based on the operands
4743        size_t tensorOpIdx = idImmOps.size();
4744        idImmOps.push_back(spv::IdImmediate(false, 0));
4745
4746        // Order in `operands` follows GLSL argument order:
4747        //   [optional tensorView] [optional decodeFunc] [optional decodeVectorFunc]
4748        // The emitted tensor-addressing operand order is the same (bit order:
4749        // TensorView 0x1, DecodeFunc 0x2, DecodeVectorFunc 0x4), so we just
4750        // bin operands by source position.
4751        spv::Id viewId = 0;
4752        spv::Id scalarFuncId = 0;
4753        spv::Id vectorFuncId = 0;
4754        uint32_t i = 3;
4755        if (i < operands.size() && builder.isTensorView(operands[i]))
4756            viewId = operands[i++];
4757        if (i < operands.size())
4758            scalarFuncId = operands[i++];
4759        if (i < operands.size())
4760            vectorFuncId = operands[i++];
4761        assert(i == operands.size());
4762
4763        if (viewId != 0) {
4764            addMask(idImmOps[tensorOpIdx].word, spv::TensorAddressingOperandsMask::TensorView);
4765            idImmOps.push_back(spv::IdImmediate(true, viewId));
4766        }
4767        if (scalarFuncId != 0) {
4768            addMask(idImmOps[tensorOpIdx].word, spv::TensorAddressingOperandsMask::DecodeFunc);
4769            builder.addCapability(spv::Capability::CooperativeMatrixBlockLoadsNV);
4770            idImmOps.push_back(spv::IdImmediate(true, scalarFuncId));
4771        }
4772        if (vectorFuncId != 0) {
4773            addMask(idImmOps[tensorOpIdx].word, spv::TensorAddressingOperandsMask::DecodeVectorFunc);
4774            builder.addCapability(spv::Capability::CooperativeMatrixDecodeVectorNV);
4775            builder.addExtension(spv::E_SPV_NV_cooperative_matrix_decode_vector);
4776            idImmOps.push_back(spv::IdImmediate(true, vectorFuncId));
4777        }
4778
4779        // get the pointee type
4780        spv::Id typeId = builder.getContainedTypeId(builder.getTypeId(operands[0]));
4781        assert(builder.isCooperativeMatrixType(typeId));
4782        // do the op
4783        spv::Id result = builder.createOp(spv::Op::OpCooperativeMatrixLoadTensorNV, typeId, idImmOps);
4784        // store the result to the pointer (out param 'm')
4785        builder.createStore(result, operands[0]);
4786        result = 0;
4787    } else if (node->getOp() == glslang::EOpCooperativeMatrixLoad ||
4788               node->getOp() == glslang::EOpCooperativeMatrixLoadNV) {
4789        std::vector<spv::IdImmediate> idImmOps;
4790
4791        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // buf
4792        if (node->getOp() == glslang::EOpCooperativeMatrixLoad) {
4793            idImmOps.push_back(spv::IdImmediate(true, operands[3])); // matrixLayout
4794            auto layout = (spv::CooperativeMatrixLayout)builder.getConstantScalar(operands[3]);
4795            if (layout == spv::CooperativeMatrixLayout::RowBlockedInterleavedARM ||
4796                layout == spv::CooperativeMatrixLayout::ColumnBlockedInterleavedARM) {
4797                builder.addExtension(spv::E_SPV_ARM_cooperative_matrix_layouts);
4798                builder.addCapability(spv::Capability::CooperativeMatrixLayoutsARM);
4799            }
4800            idImmOps.push_back(spv::IdImmediate(true, operands[2])); // stride
4801        } else {
4802            idImmOps.push_back(spv::IdImmediate(true, operands[2])); // stride
4803            idImmOps.push_back(spv::IdImmediate(true, operands[3])); // colMajor
4804        }
4805        idImmOps.insert(idImmOps.end(), memoryAccessOperands.begin(), memoryAccessOperands.end());
4806        // get the pointee type
4807        spv::Id typeId = builder.getContainedTypeId(builder.getTypeId(operands[0]));
4808        assert(builder.isCooperativeMatrixType(typeId));
4809        // do the op
4810        spv::Id result = node->getOp() == glslang::EOpCooperativeMatrixLoad
4811                       ? builder.createOp(spv::Op::OpCooperativeMatrixLoadKHR, typeId, idImmOps)
4812                       : builder.createOp(spv::Op::OpCooperativeMatrixLoadNV, typeId, idImmOps);
4813        // store the result to the pointer (out param 'm')
4814        builder.createStore(result, operands[0]);
4815        result = 0;
4816    } else if (node->getOp() == glslang::EOpCooperativeMatrixStoreTensorNV) {
4817        std::vector<spv::IdImmediate> idImmOps;
4818
4819        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // buf
4820        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // object
4821
4822        builder.addCapability(spv::Capability::CooperativeMatrixTensorAddressingNV);
4823        builder.addExtension(spv::E_SPV_NV_cooperative_matrix2);
4824
4825        idImmOps.push_back(spv::IdImmediate(true, operands[2])); // tensorLayout
4826
4827        idImmOps.insert(idImmOps.end(), memoryAccessOperands.begin(), memoryAccessOperands.end()); // memoryaccess
4828
4829        if (operands.size() > 3) {
4830            idImmOps.push_back(spv::IdImmediate(false, spv::TensorAddressingOperandsMask::TensorView));
4831            idImmOps.push_back(spv::IdImmediate(true, operands[3])); // tensorView
4832        } else {
4833            idImmOps.push_back(spv::IdImmediate(false, 0));
4834        }
4835
4836        builder.createNoResultOp(spv::Op::OpCooperativeMatrixStoreTensorNV, idImmOps);
4837        result = 0;
4838    } else if (node->getOp() == glslang::EOpCooperativeMatrixStore ||
4839               node->getOp() == glslang::EOpCooperativeMatrixStoreNV) {
4840        std::vector<spv::IdImmediate> idImmOps;
4841
4842        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // buf
4843        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // object
4844        if (node->getOp() == glslang::EOpCooperativeMatrixStore) {
4845            idImmOps.push_back(spv::IdImmediate(true, operands[3])); // matrixLayout
4846            auto layout = (spv::CooperativeMatrixLayout)builder.getConstantScalar(operands[3]);
4847            if (layout == spv::CooperativeMatrixLayout::RowBlockedInterleavedARM ||
4848                layout == spv::CooperativeMatrixLayout::ColumnBlockedInterleavedARM) {
4849                builder.addExtension(spv::E_SPV_ARM_cooperative_matrix_layouts);
4850                builder.addCapability(spv::Capability::CooperativeMatrixLayoutsARM);
4851            }
4852            idImmOps.push_back(spv::IdImmediate(true, operands[2])); // stride
4853        } else {
4854            idImmOps.push_back(spv::IdImmediate(true, operands[2])); // stride
4855            idImmOps.push_back(spv::IdImmediate(true, operands[3])); // colMajor
4856        }
4857        idImmOps.insert(idImmOps.end(), memoryAccessOperands.begin(), memoryAccessOperands.end());
4858
4859        if (node->getOp() == glslang::EOpCooperativeMatrixStore)
4860            builder.createNoResultOp(spv::Op::OpCooperativeMatrixStoreKHR, idImmOps);
4861        else
4862            builder.createNoResultOp(spv::Op::OpCooperativeMatrixStoreNV, idImmOps);
4863        result = 0;
4864    } else if (node->getOp() == glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT) {
4865        std::vector<spv::IdImmediate> idImmOps;
4866
4867        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // q
4868        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // committed
4869
4870        spv::Id typeId = builder.makeArrayType(builder.makeVectorType(builder.makeFloatType(32), 3),
4871                                               builder.makeUintConstant(3), 0);
4872        // do the op
4873
4874        spv::Op spvOp = spv::Op::OpRayQueryGetIntersectionTriangleVertexPositionsKHR;
4875
4876        spv::Id result = builder.createOp(spvOp, typeId, idImmOps);
4877        // store the result to the pointer (out param 'm')
4878        builder.createStore(result, operands[2]);
4879        result = 0;
4880    } else if (node->getOp() == glslang::EOpRayQueryGetIntersectionLSSPositionsNV) {
4881        std::vector<spv::IdImmediate> idImmOps;
4882
4883        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // q
4884        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // committed
4885
4886        spv::Id typeId = builder.makeArrayType(builder.makeVectorType(builder.makeFloatType(32), 3),
4887                                               builder.makeUintConstant(2), 0);
4888        // do the op
4889
4890        spv::Op spvOp = spv::Op::OpRayQueryGetIntersectionLSSPositionsNV;
4891
4892        spv::Id result = builder.createOp(spvOp, typeId, idImmOps);
4893        // store the result to the pointer (out param 'm')
4894        builder.createStore(result, operands[2]);
4895        result = 0;
4896    } else if (node->getOp() == glslang::EOpRayQueryGetIntersectionLSSRadiiNV) {
4897        std::vector<spv::IdImmediate> idImmOps;
4898
4899        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // q
4900        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // committed
4901
4902        spv::Id typeId = builder.makeArrayType(builder.makeFloatType(32),
4903                                               builder.makeUintConstant(2), 0);
4904        // do the op
4905
4906        spv::Op spvOp = spv::Op::OpRayQueryGetIntersectionLSSRadiiNV;
4907
4908        spv::Id result = builder.createOp(spvOp, typeId, idImmOps);
4909        // store the result to the pointer (out param 'm')
4910        builder.createStore(result, operands[2]);
4911        result = 0;
4912    } else if (node->getOp() == glslang::EOpHitObjectGetLSSPositionsNV) {
4913        std::vector<spv::IdImmediate> idImmOps;
4914
4915        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // hitObject
4916
4917        spv::Op spvOp = spv::Op::OpHitObjectGetLSSPositionsNV;
4918        spv::Id typeId = builder.makeArrayType(builder.makeVectorType(builder.makeFloatType(32), 3),
4919                                               builder.makeUintConstant(2), 0);
4920
4921        spv::Id result = builder.createOp(spvOp, typeId, idImmOps);
4922        // store the result to the pointer (out param 'm')
4923        builder.createStore(result, operands[1]);
4924        result = 0;
4925    } else if (node->getOp() == glslang::EOpHitObjectGetLSSRadiiNV) {
4926        std::vector<spv::IdImmediate> idImmOps;
4927
4928        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // hitObject
4929
4930        spv::Op spvOp = spv::Op::OpHitObjectGetLSSRadiiNV;
4931        spv::Id typeId = builder.makeArrayType(builder.makeFloatType(32),
4932                                               builder.makeUintConstant(2), 0);
4933
4934        spv::Id result = builder.createOp(spvOp, typeId, idImmOps);
4935        // store the result to the pointer (out param 'm')
4936        builder.createStore(result, operands[1]);
4937        result = 0;
4938    } else if (node->getOp() == glslang::EOpHitObjectGetIntersectionTriangleVertexPositionsEXT) {
4939        std::vector<spv::IdImmediate> idImmOps;
4940
4941        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // hitObject
4942
4943        spv::Op spvOp = spv::Op::OpHitObjectGetIntersectionTriangleVertexPositionsEXT;
4944        spv::Id typeId = builder.makeArrayType(builder.makeVectorType(builder.makeFloatType(32), 3),
4945                                               builder.makeUintConstant(3), 0);
4946
4947        spv::Id result = builder.createOp(spvOp, typeId, idImmOps);
4948        // store the result to the pointer (out param 'm')
4949        builder.createStore(result, operands[1]);
4950        result = 0;
4951    } else if (node->getOp() == glslang::EOpCooperativeMatrixMulAdd) {
4952        auto matrixOperands = spv::CooperativeMatrixOperandsMask::MaskNone;
4953
4954        // If the optional operand is present, initialize matrixOperands to that value.
4955        if (glslangOperands.size() == 4 && glslangOperands[3]->getAsConstantUnion()) {
4956            matrixOperands = (spv::CooperativeMatrixOperandsMask)glslangOperands[3]->getAsConstantUnion()->getConstArray()[0].getIConst();
4957        }
4958
4959        // Determine Cooperative Matrix Operands bits from the signedness of the types.
4960        if (isTypeSignedInt(glslangOperands[0]->getAsTyped()->getBasicType()))
4961            addMask(matrixOperands, spv::CooperativeMatrixOperandsMask::MatrixASignedComponentsKHR);
4962        if (isTypeSignedInt(glslangOperands[1]->getAsTyped()->getBasicType()))
4963            addMask(matrixOperands, spv::CooperativeMatrixOperandsMask::MatrixBSignedComponentsKHR);
4964        if (isTypeSignedInt(glslangOperands[2]->getAsTyped()->getBasicType()))
4965            addMask(matrixOperands, spv::CooperativeMatrixOperandsMask::MatrixCSignedComponentsKHR);
4966        if (isTypeSignedInt(node->getBasicType()))
4967            addMask(matrixOperands, spv::CooperativeMatrixOperandsMask::MatrixResultSignedComponentsKHR);
4968
4969        std::vector<spv::IdImmediate> idImmOps;
4970        idImmOps.push_back(spv::IdImmediate(true, operands[0]));
4971        idImmOps.push_back(spv::IdImmediate(true, operands[1]));
4972        idImmOps.push_back(spv::IdImmediate(true, operands[2]));
4973        if (matrixOperands != spv::CooperativeMatrixOperandsMask::MaskNone)
4974            idImmOps.push_back(spv::IdImmediate(false, matrixOperands));
4975
4976        result = builder.createOp(spv::Op::OpCooperativeMatrixMulAddKHR, resultType(), idImmOps);
4977    } else if (node->getOp() == glslang::EOpCooperativeMatrixReduceNV) {
4978        if (glslangIntermediate->usingCoopMatMaint1()) {
4979            builder.addCapability(spv::Capability::CooperativeMatrixReductionsEXT);
4980            builder.addExtension(spv::E_SPV_EXT_cooperative_matrix_maintenance1);
4981        } else {
4982            builder.addCapability(spv::Capability::CooperativeMatrixReductionsNV);
4983            builder.addExtension(spv::E_SPV_NV_cooperative_matrix2);
4984        }
4985
4986        spv::Op opcode = spv::Op::OpCooperativeMatrixReduceNV;
4987        unsigned mask = glslangOperands[2]->getAsConstantUnion()->getConstArray()[0].getUConst();
4988
4989        spv::Id typeId = builder.getContainedTypeId(builder.getTypeId(operands[0]));
4990        assert(builder.isCooperativeMatrixType(typeId));
4991
4992        result = builder.createCooperativeMatrixReduce(opcode, typeId, operands[1], mask, operands[3]);
4993        // store the result to the pointer (out param 'm')
4994        builder.createStore(result, operands[0]);
4995        result = 0;
4996    } else if (node->getOp() == glslang::EOpCooperativeMatrixPerElementOpNV) {
4997        if (glslangIntermediate->usingCoopMatMaint1()) {
4998            builder.addCapability(spv::Capability::CooperativeMatrixPerElementOperationsEXT);
4999            builder.addExtension(spv::E_SPV_EXT_cooperative_matrix_maintenance1);
5000        } else {
5001            builder.addCapability(spv::Capability::CooperativeMatrixPerElementOperationsNV);
5002            builder.addExtension(spv::E_SPV_NV_cooperative_matrix2);
5003        }
5004
5005        spv::Id typeId = builder.getContainedTypeId(builder.getTypeId(operands[0]));
5006        assert(builder.isCooperativeMatrixType(typeId));
5007
5008        result = builder.createCooperativeMatrixPerElementOp(typeId, operands);
5009        // store the result to the pointer
5010        builder.createStore(result, operands[0]);
5011        result = 0;
5012    } else if (node->getOp() == glslang::EOpCooperativeMatrixTransposeNV) {
5013        enableCoopMatConversions();
5014
5015        spv::Id typeId = builder.getContainedTypeId(builder.getTypeId(operands[0]));
5016        assert(builder.isCooperativeMatrixType(typeId));
5017
5018        if (glslangIntermediate->usingCoopMatMaint1()) {
5019            auto resultBasicType = glslangOperands[0]->getAsTyped()->getBasicType();
5020            auto operandBasicType = glslangOperands[1]->getAsTyped()->getBasicType();
5021            bool sameUse = glslangOperands[0]->getAsTyped()->getType().sameCoopMatUse(glslangOperands[1]->getAsTyped()->getType());
5022
5023            result = createCoopMatConversion(typeId, operands[1], resultBasicType, operandBasicType, sameUse, true);
5024        } else {
5025            result = builder.createUnaryOp(spv::Op::OpCooperativeMatrixTransposeNV, typeId, operands[1]);
5026        }
5027        // store the result to the pointer
5028        builder.createStore(result, operands[0]);
5029        result = 0;
5030    } else if (node->getOp() == glslang::EOpBitCastArrayQCOM) {
5031        builder.addCapability(spv::Capability::CooperativeMatrixConversionQCOM);
5032        builder.addExtension(spv::E_SPV_QCOM_cooperative_matrix_conversion);
5033        result = builder.createUnaryOp(spv::Op::OpBitCastArrayQCOM, resultType(), operands[0]);
5034    } else if (node->getOp() == glslang::EOpCompositeConstructCoopMatQCOM) {
5035        builder.addCapability(spv::Capability::CooperativeMatrixConversionQCOM);
5036        builder.addExtension(spv::E_SPV_QCOM_cooperative_matrix_conversion);
5037        result = builder.createUnaryOp(spv::Op::OpCompositeConstructCoopMatQCOM, resultType(), operands[0]);
5038    } else if (node->getOp() == glslang::EOpCompositeExtractCoopMatQCOM) {
5039        builder.addCapability(spv::Capability::CooperativeMatrixConversionQCOM);
5040        builder.addExtension(spv::E_SPV_QCOM_cooperative_matrix_conversion);
5041        result = builder.createUnaryOp(spv::Op::OpCompositeExtractCoopMatQCOM, resultType(), operands[0]);
5042    } else if (node->getOp() == glslang::EOpExtractSubArrayQCOM) {
5043        builder.addCapability(spv::Capability::CooperativeMatrixConversionQCOM);
5044        builder.addExtension(spv::E_SPV_QCOM_cooperative_matrix_conversion);
5045
5046        std::vector<spv::Id> arguments { operands[0], operands[1] };;
5047        result = builder.createOp(spv::Op::OpExtractSubArrayQCOM, resultType(), arguments);
5048    } else if (node->getOp() == glslang::EOpCooperativeVectorMatMulNV ||
5049               node->getOp() == glslang::EOpCooperativeVectorMatMulAddNV) {
5050        auto matrixOperands = spv::CooperativeMatrixOperandsMask::MaskNone;
5051
5052        bool isMulAdd = node->getOp() == glslang::EOpCooperativeVectorMatMulAddNV;
5053
5054        // Determine Cooperative Matrix Operands bits from the signedness of the types.
5055
5056        if (isTypeSignedInt(glslangOperands[1]->getAsTyped()->getBasicType()))
5057            addMask(matrixOperands, spv::CooperativeMatrixOperandsMask::MatrixBSignedComponentsKHR);
5058        if (isTypeSignedInt(glslangOperands[0]->getAsTyped()->getBasicType()))
5059            addMask(matrixOperands, spv::CooperativeMatrixOperandsMask::MatrixResultSignedComponentsKHR);
5060
5061        uint32_t opIdx = 1;
5062        std::vector<spv::IdImmediate> idImmOps;
5063        idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // Input
5064        idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // InputInterpretation
5065        idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // Matrix
5066        idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // MatrixOffset
5067        idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // MatrixInterpretation
5068        if (isMulAdd) {
5069            idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // Bias
5070            idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // BiasOffset
5071            idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // BiasInterpretation
5072        }
5073        idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // M
5074        idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // K
5075        idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // MemoryLayout
5076        idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // Transpose
5077        idImmOps.push_back(spv::IdImmediate(true, operands[opIdx++])); // MatrixStride
5078        if (matrixOperands != spv::CooperativeMatrixOperandsMask::MaskNone)
5079            idImmOps.push_back(spv::IdImmediate(false, matrixOperands));  // Cooperative Matrix Operands
5080
5081        // get the pointee type
5082        spv::Id typeId = builder.getContainedTypeId(builder.getTypeId(operands[0]));
5083        assert(builder.isCooperativeVectorType(typeId));
5084        // do the op
5085        spv::Id result = builder.createOp(isMulAdd ? spv::Op::OpCooperativeVectorMatrixMulAddNV : spv::Op::OpCooperativeVectorMatrixMulNV, typeId, idImmOps);
5086        // store the result to the pointer (out param 'res')
5087        builder.createStore(result, operands[0]);
5088        result = 0;
5089    } else if (node->getOp() == glslang::EOpCooperativeVectorLoadNV) {
5090        std::vector<spv::IdImmediate> idImmOps;
5091
5092        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // buf
5093        idImmOps.push_back(spv::IdImmediate(true, operands[2])); // offset
5094        idImmOps.insert(idImmOps.end(), memoryAccessOperands.begin(), memoryAccessOperands.end());
5095        // get the pointee type
5096        spv::Id typeId = builder.getContainedTypeId(builder.getTypeId(operands[0]));
5097        assert(builder.isCooperativeVectorType(typeId));
5098        // do the op
5099        spv::Id result = builder.createOp(spv::Op::OpCooperativeVectorLoadNV, typeId, idImmOps);
5100        // store the result to the pointer (out param 'v')
5101        builder.createStore(result, operands[0]);
5102        result = 0;
5103    } else if (node->getOp() == glslang::EOpCooperativeVectorStoreNV) {
5104        std::vector<spv::IdImmediate> idImmOps;
5105
5106        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // buf
5107        idImmOps.push_back(spv::IdImmediate(true, operands[2])); // offset
5108        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // object
5109        idImmOps.insert(idImmOps.end(), memoryAccessOperands.begin(), memoryAccessOperands.end());
5110        builder.createNoResultOp(spv::Op::OpCooperativeVectorStoreNV, idImmOps);
5111        result = 0;
5112    } else if (node->getOp() == glslang::EOpCooperativeVectorOuterProductAccumulateNV) {
5113        builder.addCapability(spv::Capability::CooperativeVectorTrainingNV);
5114        builder.addExtension(spv::E_SPV_NV_cooperative_vector);
5115
5116        std::vector<spv::IdImmediate> idImmOps;
5117
5118        idImmOps.push_back(spv::IdImmediate(true, operands[2])); // Matrix
5119        idImmOps.push_back(spv::IdImmediate(true, operands[3])); // Offset
5120        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // A
5121        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // B
5122        idImmOps.push_back(spv::IdImmediate(true, operands[5])); // MemoryLayout
5123        idImmOps.push_back(spv::IdImmediate(true, operands[6])); // MatrixInterpretation
5124        idImmOps.push_back(spv::IdImmediate(true, operands[4])); // Stride
5125        builder.createNoResultOp(spv::Op::OpCooperativeVectorOuterProductAccumulateNV, idImmOps);
5126        result = 0;
5127    } else if (node->getOp() == glslang::EOpCooperativeVectorReduceSumAccumulateNV) {
5128        builder.addCapability(spv::Capability::CooperativeVectorTrainingNV);
5129        builder.addExtension(spv::E_SPV_NV_cooperative_vector);
5130
5131        std::vector<spv::IdImmediate> idImmOps;
5132
5133        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // Buf
5134        idImmOps.push_back(spv::IdImmediate(true, operands[2])); // Offset
5135        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // A
5136        builder.createNoResultOp(spv::Op::OpCooperativeVectorReduceSumAccumulateNV, idImmOps);
5137        result = 0;
5138    } else if (node->getOp() == glslang::EOpTensorReadARM ||
5139               node->getOp() == glslang::EOpTensorWriteARM) {
5140        const bool isWrite = node->getOp() == glslang::EOpTensorWriteARM;
5141        const unsigned int tensorMinOperandCount = 3;
5142        assert(operands.size() >= tensorMinOperandCount);
5143        std::vector<spv::IdImmediate> idImmOps;
5144
5145        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // tensor
5146        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // coords
5147        if (isWrite) {
5148            idImmOps.push_back(spv::IdImmediate(true, operands[2])); // value
5149        }
5150
5151        // Analyze the tensor operands
5152        spv::IdImmediate tensorOperands = { false, uint32_t(spv::TensorOperandsMask::MaskNone) };
5153        bool pushExtraArg = false;
5154        if (operands.size() > tensorMinOperandCount) {
5155            auto enumVal = builder.getConstantScalar(operands[tensorMinOperandCount]);
5156
5157            if (enumVal & uint32_t(spv::TensorOperandsMask::NontemporalARM)) {
5158                tensorOperands.word |= uint32_t(spv::TensorOperandsMask::NontemporalARM);
5159            }
5160            if (enumVal & uint32_t(spv::TensorOperandsMask::OutOfBoundsValueARM)) {
5161                tensorOperands.word |= uint32_t(spv::TensorOperandsMask::OutOfBoundsValueARM);
5162                assert(operands.size() >= tensorMinOperandCount + 2 &&
5163                    "TensorOperandsOutOfBoundsValueMask requires an additional value");
5164                pushExtraArg = true;
5165            }
5166        }
5167
5168        // Append optional tensor operands if the mask was non-zero.
5169        if (tensorOperands.word) {
5170            idImmOps.push_back(tensorOperands);
5171            if (pushExtraArg)
5172                idImmOps.push_back(spv::IdImmediate(true, operands[tensorMinOperandCount + 1]));
5173        }
5174
5175        if (isWrite) {
5176            builder.createNoResultOp(spv::Op::OpTensorWriteARM, idImmOps);
5177            result = 0;
5178        } else {
5179            // Use the result argument type as the OpTensorReadARM result type.
5180            const glslang::TType &resArgType = glslangOperands[2]->getAsTyped()->getType();
5181            spv::Id retType = convertGlslangToSpvType(resArgType);
5182            result = builder.createOp(spv::Op::OpTensorReadARM, retType, idImmOps);
5183            // Store the result to the result argument.
5184            assert(tensorReadResultLValue.base != spv::NoResult);
5185            builder.setAccessChain(tensorReadResultLValue);
5186            accessChainStore(resArgType, result);
5187        }
5188    } else if (node->getOp() == glslang::EOpTensorSizeARM) {
5189        // Expected operands are (tensor, dimension)
5190        assert(operands.size() == 2);
5191
5192        spv::Id tensorOp = operands[0];
5193        spv::Id dimOp = operands[1];
5194        assert(builder.isTensorTypeARM(builder.getTypeId(tensorOp)) && "operand #0 must be a tensor");
5195
5196        std::vector<spv::IdImmediate> idImmOps;
5197        idImmOps.push_back(spv::IdImmediate(true, tensorOp));
5198        idImmOps.push_back(spv::IdImmediate(true, dimOp));
5199        result = builder.createOp(spv::Op::OpTensorQuerySizeARM, resultType(), idImmOps);
5200    } else if (atomic) {
5201        // Handle all atomics
5202        glslang::TBasicType typeProxy = (node->getOp() == glslang::EOpAtomicStore)
5203            ? node->getSequence()[0]->getAsTyped()->getBasicType() : node->getBasicType();
5204        result = createAtomicOperation(node->getOp(), precision, resultType(), operands, typeProxy,
5205            lvalueCoherentFlags, node->getType());
5206    } else if (node->getOp() == glslang::EOpSpirvInst) {
5207        const auto& spirvInst = node->getSpirvInstruction();
5208        std::vector<spv::IdImmediate> idImmOps;
5209        for (unsigned int i = 0; i < glslangOperands.size(); ++i) {
5210            if (glslangOperands[i]->getAsTyped()->getQualifier().isSpirvLiteral()) {
5211                // Translate the constant to a literal value
5212                std::vector<unsigned> literals;
5213                glslang::TVector<const glslang::TIntermConstantUnion*> constants;
5214                constants.push_back(glslangOperands[i]->getAsConstantUnion());
5215                TranslateLiterals(constants, literals);
5216                idImmOps.push_back({false, literals[0]});
5217            } else
5218                idImmOps.push_back({true, operands[i]});
5219        }
5220
5221        if (spirvInst.set == "") {
5222            if (node->getBasicType() == glslang::EbtVoid)
5223                builder.createNoResultOp(static_cast<spv::Op>(spirvInst.id), idImmOps);
5224            else
5225                result = builder.createOp(static_cast<spv::Op>(spirvInst.id), resultType(), idImmOps);
5226        } else {
5227            result = builder.createBuiltinCall(
5228                resultType(), spirvInst.set == "GLSL.std.450" ? stdBuiltins : getExtBuiltins(spirvInst.set.c_str()),
5229                spirvInst.id, idImmOps);
5230        }
5231        noReturnValue = node->getBasicType() == glslang::EbtVoid;
5232    } else if (node->getOp() == glslang::EOpDebugPrintf) {
5233        if (!nonSemanticDebugPrintf) {
5234            nonSemanticDebugPrintf = builder.import("NonSemantic.DebugPrintf");
5235        }
5236        result = builder.createBuiltinCall(builder.makeVoidType(), nonSemanticDebugPrintf, spv::NonSemanticDebugPrintfDebugPrintf, operands);
5237        builder.addExtension(spv::E_SPV_KHR_non_semantic_info);
5238    } else {
5239        // Pass through to generic operations.
5240        switch (glslangOperands.size()) {
5241        case 0:
5242            result = createNoArgOperation(node->getOp(), precision, resultType());
5243            break;
5244        case 1:
5245            {
5246                OpDecorations decorations = { precision,
5247                                              TranslateNoContractionDecoration(node->getType().getQualifier()),
5248                                              TranslateNonUniformDecoration(node->getType().getQualifier()) };
5249                result = createUnaryOperation(
5250                    node->getOp(), decorations,
5251                    resultType(), operands.front(),
5252                    glslangOperands[0]->getAsTyped()->getBasicType(), lvalueCoherentFlags, node->getType());
5253            }
5254            break;
5255        default:
5256            result = createMiscOperation(node->getOp(), precision, resultType(), operands, node->getBasicType());
5257            break;
5258        }
5259
5260        if (invertedType != spv::NoResult)
5261            result = createInvertedSwizzle(precision, *glslangOperands[0]->getAsBinaryNode(), result);
5262
5263        for (unsigned int i = 0; i < temporaryLvalues.size(); ++i) {
5264            builder.setAccessChain(complexLvalues[i]);
5265            accessChainStore(*complexLvalueTypes[i], builder.createLoad(temporaryLvalues[i], spv::NoPrecision));
5266        }
5267    }
5268
5269    if (noReturnValue)
5270        return false;
5271
5272    if (! result) {
5273        logger->missingFunctionality("unknown glslang aggregate");
5274        return true;  // pick up a child as a placeholder operand
5275    } else {
5276        builder.clearAccessChain();
5277        builder.setAccessChainRValue(result);
5278        return false;
5279    }
5280}
5281
5282// This path handles both if-then-else and ?:
5283// The if-then-else has a node type of void, while
5284// ?: has either a void or a non-void node type
5285//
5286// Leaving the result, when not void:
5287// GLSL only has r-values as the result of a :?, but
5288// if we have an l-value, that can be more efficient if it will
5289// become the base of a complex r-value expression, because the
5290// next layer copies r-values into memory to use the access-chain mechanism
5291bool TGlslangToSpvTraverser::visitSelection(glslang::TVisit /* visit */, glslang::TIntermSelection* node)
5292{
5293    // see if OpSelect can handle it
5294    const auto isOpSelectable = [&]() {
5295        if (node->getBasicType() == glslang::EbtVoid)
5296            return false;
5297        // OpSelect can do all other types starting with SPV 1.4
5298        if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_4) {
5299            // pre-1.4, only scalars and vectors can be handled
5300            if ((!node->getType().isScalar() && !node->getType().isVector()))
5301                return false;
5302        }
5303        return true;
5304    };
5305
5306    // See if it simple and safe, or required, to execute both sides.
5307    // Crucially, side effects must be either semantically required or avoided,
5308    // and there are performance trade-offs.
5309    // Return true if required or a good idea (and safe) to execute both sides,
5310    // false otherwise.
5311    const auto bothSidesPolicy = [&]() -> bool {
5312        // do we have both sides?
5313        if (node->getTrueBlock()  == nullptr ||
5314            node->getFalseBlock() == nullptr)
5315            return false;
5316
5317        // required? (unless we write additional code to look for side effects
5318        // and make performance trade-offs if none are present)
5319        if (!node->getShortCircuit())
5320            return true;
5321
5322        // if not required to execute both, decide based on performance/practicality...
5323
5324        if (!isOpSelectable())
5325            return false;
5326
5327        assert(node->getType() == node->getTrueBlock() ->getAsTyped()->getType() &&
5328               node->getType() == node->getFalseBlock()->getAsTyped()->getType());
5329
5330        // return true if a single operand to ? : is okay for OpSelect
5331        const auto operandOkay = [](glslang::TIntermTyped* node) {
5332            return node->getAsSymbolNode() || node->getType().getQualifier().isConstant();
5333        };
5334
5335        return operandOkay(node->getTrueBlock() ->getAsTyped()) &&
5336               operandOkay(node->getFalseBlock()->getAsTyped());
5337    };
5338
5339    spv::Id result = spv::NoResult; // upcoming result selecting between trueValue and falseValue
5340    // emit the condition before doing anything with selection
5341    node->getCondition()->traverse(this);
5342    spv::Id condition = accessChainLoad(node->getCondition()->getType());
5343
5344    // Find a way of executing both sides and selecting the right result.
5345    const auto executeBothSides = [&]() -> void {
5346        // execute both sides
5347        spv::Id resultType = convertGlslangToSpvType(node->getType());
5348        node->getTrueBlock()->traverse(this);
5349        spv::Id trueValue = accessChainLoad(node->getTrueBlock()->getAsTyped()->getType());
5350        node->getFalseBlock()->traverse(this);
5351        spv::Id falseValue = accessChainLoad(node->getFalseBlock()->getAsTyped()->getType());
5352
5353        builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
5354
5355        // done if void
5356        if (node->getBasicType() == glslang::EbtVoid)
5357            return;
5358
5359        // emit code to select between trueValue and falseValue
5360        // see if OpSelect can handle the result type, and that the SPIR-V types
5361        // of the inputs match the result type.
5362        if (isOpSelectable()) {
5363            // Emit OpSelect for this selection.
5364
5365            // smear condition to vector, if necessary (AST is always scalar)
5366            // Before 1.4, smear like for mix(), starting with 1.4, keep it scalar
5367            if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_4 && builder.isVector(trueValue)) {
5368                condition = builder.smearScalar(spv::NoPrecision, condition,
5369                                                builder.makeVectorType(builder.makeBoolType(),
5370                                                                       builder.getNumComponents(trueValue)));
5371            }
5372
5373            // If the types do not match, it is because of mismatched decorations on aggregates.
5374            // Since isOpSelectable only lets us get here for SPIR-V >= 1.4, we can use OpCopyObject
5375            // to get matching types.
5376            if (builder.getTypeId(trueValue) != resultType) {
5377                trueValue = builder.createUnaryOp(spv::Op::OpCopyLogical, resultType, trueValue);
5378            }
5379            if (builder.getTypeId(falseValue) != resultType) {
5380                falseValue = builder.createUnaryOp(spv::Op::OpCopyLogical, resultType, falseValue);
5381            }
5382
5383            // OpSelect
5384            result = builder.createTriOp(spv::Op::OpSelect, resultType, condition, trueValue, falseValue);
5385
5386            builder.clearAccessChain();
5387            builder.setAccessChainRValue(result);
5388        } else {
5389            // We need control flow to select the result.
5390            // TODO: Once SPIR-V OpSelect allows arbitrary types, eliminate this path.
5391            result = builder.createVariable(TranslatePrecisionDecoration(node->getType()),
5392                spv::StorageClass::Function, resultType);
5393
5394            // Selection control:
5395            const spv::SelectionControlMask control = TranslateSelectionControl(*node);
5396
5397            // make an "if" based on the value created by the condition
5398            spv::Builder::If ifBuilder(condition, control, builder);
5399
5400            // emit the "then" statement
5401            builder.clearAccessChain();
5402            builder.setAccessChainLValue(result);
5403            multiTypeStore(node->getType(), trueValue);
5404
5405            ifBuilder.makeBeginElse();
5406            // emit the "else" statement
5407            builder.clearAccessChain();
5408            builder.setAccessChainLValue(result);
5409            multiTypeStore(node->getType(), falseValue);
5410
5411            // finish off the control flow
5412            ifBuilder.makeEndIf();
5413
5414            builder.clearAccessChain();
5415            builder.setAccessChainLValue(result);
5416        }
5417    };
5418
5419    // Execute the one side needed, as per the condition
5420    const auto executeOneSide = [&]() {
5421        // Always emit control flow.
5422        if (node->getBasicType() != glslang::EbtVoid) {
5423            result = builder.createVariable(TranslatePrecisionDecoration(node->getType()), spv::StorageClass::Function,
5424                convertGlslangToSpvType(node->getType()));
5425        }
5426
5427        // Selection control:
5428        const spv::SelectionControlMask control = TranslateSelectionControl(*node);
5429
5430        // make an "if" based on the value created by the condition
5431        spv::Builder::If ifBuilder(condition, control, builder);
5432
5433        // emit the "then" statement
5434        if (node->getTrueBlock() != nullptr) {
5435            node->getTrueBlock()->traverse(this);
5436            if (result != spv::NoResult) {
5437                spv::Id load = accessChainLoad(node->getTrueBlock()->getAsTyped()->getType());
5438
5439                builder.clearAccessChain();
5440                builder.setAccessChainLValue(result);
5441                multiTypeStore(node->getType(), load);
5442            }
5443        }
5444
5445        if (node->getFalseBlock() != nullptr) {
5446            ifBuilder.makeBeginElse();
5447            // emit the "else" statement
5448            node->getFalseBlock()->traverse(this);
5449            if (result != spv::NoResult) {
5450                spv::Id load = accessChainLoad(node->getFalseBlock()->getAsTyped()->getType());
5451
5452                builder.clearAccessChain();
5453                builder.setAccessChainLValue(result);
5454                multiTypeStore(node->getType(), load);
5455            }
5456        }
5457
5458        // finish off the control flow
5459        ifBuilder.makeEndIf();
5460
5461        if (result != spv::NoResult) {
5462            builder.clearAccessChain();
5463            builder.setAccessChainLValue(result);
5464        }
5465    };
5466
5467    // Try for OpSelect (or a requirement to execute both sides)
5468    if (bothSidesPolicy()) {
5469        SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
5470        if (node->getType().getQualifier().isSpecConstant())
5471            spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
5472        executeBothSides();
5473    } else
5474        executeOneSide();
5475
5476    return false;
5477}
5478
5479bool TGlslangToSpvTraverser::visitSwitch(glslang::TVisit /* visit */, glslang::TIntermSwitch* node)
5480{
5481    // emit and get the condition before doing anything with switch
5482    node->getCondition()->traverse(this);
5483    spv::Id selector = accessChainLoad(node->getCondition()->getAsTyped()->getType());
5484
5485    // Selection control:
5486    const spv::SelectionControlMask control = TranslateSwitchControl(*node);
5487
5488    // browse the children to sort out code segments
5489    int defaultSegment = -1;
5490    std::vector<TIntermNode*> codeSegments;
5491    glslang::TIntermSequence& sequence = node->getBody()->getSequence();
5492    std::vector<int> caseValues;
5493    std::vector<int> valueIndexToSegment(sequence.size());  // note: probably not all are used, it is an overestimate
5494    for (glslang::TIntermSequence::iterator c = sequence.begin(); c != sequence.end(); ++c) {
5495        TIntermNode* child = *c;
5496        if (child->getAsBranchNode() && child->getAsBranchNode()->getFlowOp() == glslang::EOpDefault)
5497            defaultSegment = (int)codeSegments.size();
5498        else if (child->getAsBranchNode() && child->getAsBranchNode()->getFlowOp() == glslang::EOpCase) {
5499            valueIndexToSegment[caseValues.size()] = (int)codeSegments.size();
5500            caseValues.push_back(child->getAsBranchNode()->getExpression()->getAsConstantUnion()
5501                ->getConstArray()[0].getIConst());
5502        } else
5503            codeSegments.push_back(child);
5504    }
5505
5506    // handle the case where the last code segment is missing, due to no code
5507    // statements between the last case and the end of the switch statement
5508    if ((caseValues.size() && (int)codeSegments.size() == valueIndexToSegment[caseValues.size() - 1]) ||
5509        (int)codeSegments.size() == defaultSegment)
5510        codeSegments.push_back(nullptr);
5511
5512    // make the switch statement
5513    std::vector<spv::Block*> segmentBlocks; // returned, as the blocks allocated in the call
5514    builder.makeSwitch(selector, control, (int)codeSegments.size(), caseValues, valueIndexToSegment, defaultSegment,
5515        segmentBlocks);
5516
5517    // emit all the code in the segments
5518    breakForLoop.push(false);
5519    for (unsigned int s = 0; s < codeSegments.size(); ++s) {
5520        builder.nextSwitchSegment(segmentBlocks, s);
5521        if (codeSegments[s])
5522            codeSegments[s]->traverse(this);
5523        else
5524            builder.addSwitchBreak(true);
5525    }
5526    breakForLoop.pop();
5527
5528    builder.endSwitch(segmentBlocks);
5529
5530    return false;
5531}
5532
5533void TGlslangToSpvTraverser::visitConstantUnion(glslang::TIntermConstantUnion* node)
5534{
5535    if (node->getQualifier().isSpirvLiteral())
5536        return; // Translated to a literal value, skip further processing
5537
5538    int nextConst = 0;
5539    spv::Id constant = createSpvConstantFromConstUnionArray(node->getType(), node->getConstArray(), nextConst, false);
5540
5541    builder.clearAccessChain();
5542    builder.setAccessChainRValue(constant);
5543}
5544
5545bool TGlslangToSpvTraverser::visitLoop(glslang::TVisit /* visit */, glslang::TIntermLoop* node)
5546{
5547    auto blocks = builder.makeNewLoop();
5548    builder.createBranch(true, &blocks.head);
5549
5550    // Loop control:
5551    std::vector<unsigned int> operands;
5552    const spv::LoopControlMask control = TranslateLoopControl(*node, operands);
5553
5554    // Spec requires back edges to target header blocks, and every header block
5555    // must dominate its merge block.  Make a header block first to ensure these
5556    // conditions are met.  By definition, it will contain OpLoopMerge, followed
5557    // by a block-ending branch.  But we don't want to put any other body/test
5558    // instructions in it, since the body/test may have arbitrary instructions,
5559    // including merges of its own.
5560    builder.setBuildPoint(&blocks.head);
5561    builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
5562    builder.createLoopMerge(&blocks.merge, &blocks.continue_target, control, operands);
5563    if (node->testFirst() && node->getTest()) {
5564        spv::Block& test = builder.makeNewBlock();
5565        builder.createBranch(true, &test);
5566
5567        builder.setBuildPoint(&test);
5568        node->getTest()->traverse(this);
5569        spv::Id condition = accessChainLoad(node->getTestExpr()->getType());
5570        builder.createConditionalBranch(condition, &blocks.body, &blocks.merge);
5571
5572        builder.setBuildPoint(&blocks.body);
5573        breakForLoop.push(true);
5574        if (node->getBody())
5575            node->getBody()->traverse(this);
5576        builder.createBranch(true, &blocks.continue_target);
5577        breakForLoop.pop();
5578
5579        builder.setBuildPoint(&blocks.continue_target);
5580        if (node->getTerminal())
5581            node->getTerminal()->traverse(this);
5582        builder.createBranch(true, &blocks.head);
5583    } else {
5584        builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
5585        builder.createBranch(true, &blocks.body);
5586
5587        breakForLoop.push(true);
5588        builder.setBuildPoint(&blocks.body);
5589        if (node->getBody())
5590            node->getBody()->traverse(this);
5591        builder.createBranch(true, &blocks.continue_target);
5592        breakForLoop.pop();
5593
5594        builder.setBuildPoint(&blocks.continue_target);
5595        if (node->getTerminal())
5596            node->getTerminal()->traverse(this);
5597        if (node->getTest()) {
5598            node->getTest()->traverse(this);
5599            spv::Id condition =
5600                accessChainLoad(node->getTestExpr()->getType());
5601            builder.createConditionalBranch(condition, &blocks.head, &blocks.merge);
5602        } else {
5603            // TODO: unless there was a break/return/discard instruction
5604            // somewhere in the body, this is an infinite loop, so we should
5605            // issue a warning.
5606            builder.createBranch(true, &blocks.head);
5607        }
5608    }
5609    builder.setBuildPoint(&blocks.merge);
5610    builder.closeLoop();
5611    return false;
5612}
5613
5614bool TGlslangToSpvTraverser::visitBranch(glslang::TVisit /* visit */, glslang::TIntermBranch* node)
5615{
5616    if (node->getExpression())
5617        node->getExpression()->traverse(this);
5618
5619    builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
5620
5621    switch (node->getFlowOp()) {
5622    case glslang::EOpKill:
5623        if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6) {
5624            if (glslangIntermediate->getDiscardIsTerminate() && glslangIntermediate->getSource() == glslang::EShSourceGlsl) {
5625                builder.makeStatementTerminator(spv::Op::OpTerminateInvocation, "post-terminate-invocation");
5626            } else {
5627                builder.addCapability(spv::Capability::DemoteToHelperInvocation);
5628                builder.createNoResultOp(spv::Op::OpDemoteToHelperInvocationEXT);
5629            }
5630        } else {
5631            builder.makeStatementTerminator(spv::Op::OpKill, "post-discard");
5632        }
5633        break;
5634    case glslang::EOpTerminateInvocation:
5635        builder.addExtension(spv::E_SPV_KHR_terminate_invocation);
5636        builder.makeStatementTerminator(spv::Op::OpTerminateInvocation, "post-terminate-invocation");
5637        break;
5638    case glslang::EOpBreak:
5639        if (breakForLoop.top())
5640            builder.createLoopExit();
5641        else
5642            builder.addSwitchBreak(false);
5643        break;
5644    case glslang::EOpContinue:
5645        builder.createLoopContinue();
5646        break;
5647    case glslang::EOpReturn:
5648        if (node->getExpression() != nullptr) {
5649            const glslang::TType& glslangReturnType = node->getExpression()->getType();
5650            spv::Id returnId = accessChainLoad(glslangReturnType);
5651            if (builder.getTypeId(returnId) != currentFunction->getReturnType() ||
5652                TranslatePrecisionDecoration(glslangReturnType) != currentFunction->getReturnPrecision()) {
5653                builder.clearAccessChain();
5654                spv::Id copyId = builder.createVariable(currentFunction->getReturnPrecision(),
5655                    spv::StorageClass::Function, currentFunction->getReturnType());
5656                builder.setAccessChainLValue(copyId);
5657                multiTypeStore(glslangReturnType, returnId);
5658                returnId = builder.createLoad(copyId, currentFunction->getReturnPrecision());
5659            }
5660            builder.makeReturn(false, returnId);
5661        } else
5662            builder.makeReturn(false);
5663
5664        builder.clearAccessChain();
5665        break;
5666
5667    case glslang::EOpDemote:
5668        builder.createNoResultOp(spv::Op::OpDemoteToHelperInvocationEXT);
5669        builder.addExtension(spv::E_SPV_EXT_demote_to_helper_invocation);
5670        builder.addCapability(spv::Capability::DemoteToHelperInvocationEXT);
5671        break;
5672    case glslang::EOpTerminateRayKHR:
5673        builder.makeStatementTerminator(spv::Op::OpTerminateRayKHR, "post-terminateRayKHR");
5674        break;
5675    case glslang::EOpIgnoreIntersectionKHR:
5676        builder.makeStatementTerminator(spv::Op::OpIgnoreIntersectionKHR, "post-ignoreIntersectionKHR");
5677        break;
5678
5679    default:
5680        assert(0);
5681        break;
5682    }
5683
5684    return false;
5685}
5686
5687bool TGlslangToSpvTraverser::visitVariableDecl(glslang::TVisit visit, glslang::TIntermVariableDecl* node)
5688{
5689    if (visit == glslang::EvPreVisit) {
5690        builder.setDebugSourceLocation(node->getDeclSymbol()->getLoc().line, node->getDeclSymbol()->getLoc().getFilename());
5691        // We touch the symbol once here to create the debug info.
5692        getSymbolId(node->getDeclSymbol());
5693    }
5694
5695    return true;
5696}
5697
5698
5699spv::Id TGlslangToSpvTraverser::createSpvVariable(const glslang::TIntermSymbol* node, spv::Id forcedType)
5700{
5701    // First, steer off constants, which are not SPIR-V variables, but
5702    // can still have a mapping to a SPIR-V Id.
5703    // This includes specialization constants.
5704    if (node->getQualifier().isConstant()) {
5705        spv::Id result = createSpvConstant(*node);
5706        if (result != spv::NoResult) {
5707            auto name = node->getAsSymbolNode()->getAccessName().c_str();
5708            auto typeId = convertGlslangToSpvType(node->getType());
5709            builder.createConstVariable(typeId, name, result, currentFunction == nullptr);
5710            return result;
5711        }
5712    }
5713
5714    // Now, handle actual variables
5715    spv::StorageClass storageClass = TranslateStorageClass(node->getType());
5716    spv::Id spvType = forcedType == spv::NoType ? convertGlslangToSpvType(node->getType())
5717                                                : forcedType;
5718
5719    const bool contains16BitType = node->getType().contains16BitFloat() ||
5720                                   node->getType().contains16BitInt();
5721    if (contains16BitType) {
5722        switch (storageClass) {
5723        case spv::StorageClass::Input:
5724        case spv::StorageClass::Output:
5725            builder.addIncorporatedExtension(spv::E_SPV_KHR_16bit_storage, spv::Spv_1_3);
5726            builder.addCapability(spv::Capability::StorageInputOutput16);
5727            break;
5728        case spv::StorageClass::Uniform:
5729            builder.addIncorporatedExtension(spv::E_SPV_KHR_16bit_storage, spv::Spv_1_3);
5730            if (node->getType().getQualifier().storage == glslang::EvqBuffer)
5731                builder.addCapability(spv::Capability::StorageUniformBufferBlock16);
5732            else
5733                builder.addCapability(spv::Capability::StorageUniform16);
5734            break;
5735        case spv::StorageClass::PushConstant:
5736            builder.addIncorporatedExtension(spv::E_SPV_KHR_16bit_storage, spv::Spv_1_3);
5737            builder.addCapability(spv::Capability::StoragePushConstant16);
5738            break;
5739        case spv::StorageClass::StorageBuffer:
5740        case spv::StorageClass::PhysicalStorageBufferEXT:
5741            builder.addIncorporatedExtension(spv::E_SPV_KHR_16bit_storage, spv::Spv_1_3);
5742            builder.addCapability(spv::Capability::StorageUniformBufferBlock16);
5743            break;
5744        case spv::StorageClass::TileAttachmentQCOM:
5745            builder.addCapability(spv::Capability::TileShadingQCOM);
5746            break;
5747        default:
5748            if (storageClass == spv::StorageClass::Workgroup &&
5749                node->getType().getBasicType() == glslang::EbtBlock) {
5750                builder.addCapability(spv::Capability::WorkgroupMemoryExplicitLayout16BitAccessKHR);
5751                break;
5752            }
5753            if (node->getType().contains16BitFloat())
5754                builder.addCapability(spv::Capability::Float16);
5755            if (node->getType().contains16BitInt())
5756                builder.addCapability(spv::Capability::Int16);
5757            break;
5758        }
5759    }
5760
5761    if (node->getType().contains8BitInt()) {
5762        if (storageClass == spv::StorageClass::PushConstant) {
5763            builder.addIncorporatedExtension(spv::E_SPV_KHR_8bit_storage, spv::Spv_1_5);
5764            builder.addCapability(spv::Capability::StoragePushConstant8);
5765        } else if (storageClass == spv::StorageClass::Uniform) {
5766            builder.addIncorporatedExtension(spv::E_SPV_KHR_8bit_storage, spv::Spv_1_5);
5767            builder.addCapability(spv::Capability::UniformAndStorageBuffer8BitAccess);
5768        } else if (storageClass == spv::StorageClass::StorageBuffer) {
5769            builder.addIncorporatedExtension(spv::E_SPV_KHR_8bit_storage, spv::Spv_1_5);
5770            builder.addCapability(spv::Capability::StorageBuffer8BitAccess);
5771        } else if (storageClass == spv::StorageClass::Workgroup &&
5772                   node->getType().getBasicType() == glslang::EbtBlock) {
5773            builder.addCapability(spv::Capability::WorkgroupMemoryExplicitLayout8BitAccessKHR);
5774        } else {
5775            builder.addCapability(spv::Capability::Int8);
5776        }
5777    }
5778
5779    const char* name = node->getName().c_str();
5780    if (glslang::IsAnonymous(name))
5781        name = "";
5782
5783    spv::Id initializer = spv::NoResult;
5784
5785    if (node->getType().getQualifier().storage == glslang::EvqUniform && !node->getConstArray().empty()) {
5786        int nextConst = 0;
5787        initializer = createSpvConstantFromConstUnionArray(node->getType(),
5788                                                           node->getConstArray(),
5789                                                           nextConst,
5790                                                           false /* specConst */);
5791    } else if (node->getType().getQualifier().isNullInit()) {
5792        initializer = builder.makeNullConstant(spvType);
5793    }
5794
5795    spv::Id var = builder.createVariable(spv::NoPrecision, storageClass, spvType, name, initializer, false);
5796
5797    if (options.emitNonSemanticShaderDebugInfo && storageClass != spv::StorageClass::Function) {
5798        // Create variable alias for retargeted symbols if any.
5799        // Notably, this is only applicable to built-in variables so that it is okay to only use name as the key.
5800        auto [itBegin, itEnd] = glslangIntermediate->getBuiltinAliasLookup().equal_range(name);
5801        for (auto it = itBegin; it != itEnd; ++it) {
5802            builder.createDebugGlobalVariable(builder.getDebugType(spvType), it->second.c_str(), var);
5803        }
5804    }
5805
5806    std::vector<spv::Decoration> topLevelDecorations;
5807    glslang::TQualifier typeQualifier = node->getType().getQualifier();
5808    TranslateMemoryDecoration(typeQualifier, topLevelDecorations, glslangIntermediate->usingVulkanMemoryModel());
5809    for (auto deco : topLevelDecorations) {
5810        builder.addDecoration(var, deco);
5811    }
5812    return var;
5813}
5814
5815// Return type Id of the sampled type.
5816spv::Id TGlslangToSpvTraverser::getSampledType(const glslang::TSampler& sampler)
5817{
5818    switch (sampler.type) {
5819        case glslang::EbtInt:      return builder.makeIntType(32);
5820        case glslang::EbtUint:     return builder.makeUintType(32);
5821        case glslang::EbtFloat:    return builder.makeFloatType(32);
5822        case glslang::EbtFloat16:
5823            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float_fetch);
5824            builder.addCapability(spv::Capability::Float16ImageAMD);
5825            return builder.makeFloatType(16);
5826        case glslang::EbtInt64:
5827            builder.addExtension(spv::E_SPV_EXT_shader_image_int64);
5828            builder.addCapability(spv::Capability::Int64ImageEXT);
5829            return builder.makeIntType(64);
5830        case glslang::EbtUint64:
5831            builder.addExtension(spv::E_SPV_EXT_shader_image_int64);
5832            builder.addCapability(spv::Capability::Int64ImageEXT);
5833            return builder.makeUintType(64);
5834        default:
5835            assert(0);
5836            return builder.makeFloatType(32);
5837    }
5838}
5839
5840// If node is a swizzle operation, return the type that should be used if
5841// the swizzle base is first consumed by another operation, before the swizzle
5842// is applied.
5843spv::Id TGlslangToSpvTraverser::getInvertedSwizzleType(const glslang::TIntermTyped& node)
5844{
5845    if (node.getAsOperator() &&
5846        node.getAsOperator()->getOp() == glslang::EOpVectorSwizzle)
5847        return convertGlslangToSpvType(node.getAsBinaryNode()->getLeft()->getType());
5848    else
5849        return spv::NoType;
5850}
5851
5852// When inverting a swizzle with a parent op, this function
5853// will apply the swizzle operation to a completed parent operation.
5854spv::Id TGlslangToSpvTraverser::createInvertedSwizzle(spv::Decoration precision, const glslang::TIntermTyped& node,
5855    spv::Id parentResult)
5856{
5857    std::vector<unsigned> swizzle;
5858    convertSwizzle(*node.getAsBinaryNode()->getRight()->getAsAggregate(), swizzle);
5859    return builder.createRvalueSwizzle(precision, convertGlslangToSpvType(node.getType()), parentResult, swizzle);
5860}
5861
5862// Convert a glslang AST swizzle node to a swizzle vector for building SPIR-V.
5863void TGlslangToSpvTraverser::convertSwizzle(const glslang::TIntermAggregate& node, std::vector<unsigned>& swizzle)
5864{
5865    const glslang::TIntermSequence& swizzleSequence = node.getSequence();
5866    for (int i = 0; i < (int)swizzleSequence.size(); ++i)
5867        swizzle.push_back(swizzleSequence[i]->getAsConstantUnion()->getConstArray()[0].getIConst());
5868}
5869
5870// Convert from a glslang type to an SPV type, by calling into a
5871// recursive version of this function. This establishes the inherited
5872// layout state rooted from the top-level type.
5873spv::Id TGlslangToSpvTraverser::convertGlslangToSpvType(const glslang::TType& type, bool forwardReferenceOnly)
5874{
5875    return convertGlslangToSpvType(type, getExplicitLayout(type), type.getQualifier(), false, forwardReferenceOnly);
5876}
5877
5878spv::LinkageType TGlslangToSpvTraverser::convertGlslangLinkageToSpv(glslang::TLinkType linkType)
5879{
5880    switch (linkType) {
5881    case glslang::ELinkExport:
5882        return spv::LinkageType::Export;
5883    default:
5884        return spv::LinkageType::Max;
5885    }
5886}
5887
5888// Do full recursive conversion of an arbitrary glslang type to a SPIR-V Id.
5889// explicitLayout can be kept the same throughout the hierarchical recursive walk.
5890// Mutually recursive with convertGlslangStructToSpvType().
5891spv::Id TGlslangToSpvTraverser::convertGlslangToSpvType(const glslang::TType& type,
5892    glslang::TLayoutPacking explicitLayout, const glslang::TQualifier& qualifier,
5893    bool lastBufferBlockMember, bool forwardReferenceOnly)
5894{
5895    spv::Id spvType = spv::NoResult;
5896
5897    switch (type.getBasicType()) {
5898    case glslang::EbtVoid:
5899        spvType = builder.makeVoidType();
5900        assert (! type.isArray());
5901        break;
5902    case glslang::EbtBool:
5903        // "transparent" bool doesn't exist in SPIR-V.  The GLSL convention is
5904        // a 32-bit int where non-0 means true.
5905        if (explicitLayout != glslang::ElpNone)
5906            spvType = builder.makeUintType(32);
5907        else
5908            spvType = builder.makeBoolType();
5909        break;
5910    case glslang::EbtInt:
5911        spvType = builder.makeIntType(32);
5912        break;
5913    case glslang::EbtUint:
5914        spvType = builder.makeUintType(32);
5915        break;
5916    case glslang::EbtFloat:
5917        spvType = builder.makeFloatType(32);
5918        break;
5919    case glslang::EbtDouble:
5920        spvType = builder.makeFloatType(64);
5921        break;
5922    case glslang::EbtFloat16:
5923        spvType = builder.makeFloatType(16);
5924        break;
5925    case glslang::EbtBFloat16:
5926        spvType = builder.makeBFloat16Type();
5927        break;
5928    case glslang::EbtFloatE5M2:
5929        spvType = builder.makeFloatE5M2Type();
5930        break;
5931    case glslang::EbtFloatE4M3:
5932        spvType = builder.makeFloatE4M3Type();
5933        break;
5934    case glslang::EbtFloatE2M1:
5935        spvType = builder.makeFloatE2M1Type();
5936        break;
5937    case glslang::EbtFloatE3M2:
5938        spvType = builder.makeFloatE3M2Type();
5939        break;
5940    case glslang::EbtFloatE2M3:
5941        spvType = builder.makeFloatE2M3Type();
5942        break;
5943    case glslang::EbtFloatUE8M0:
5944        spvType = builder.makeFloatUE8M0Type();
5945        break;
5946    case glslang::EbtFloatMXINT8:
5947        spvType = builder.makeFloatMXINT8Type();
5948        break;
5949    case glslang::EbtInt8:
5950        spvType = builder.makeIntType(8);
5951        break;
5952    case glslang::EbtUint8:
5953        spvType = builder.makeUintType(8);
5954        break;
5955    case glslang::EbtInt16:
5956        spvType = builder.makeIntType(16);
5957        break;
5958    case glslang::EbtUint16:
5959        spvType = builder.makeUintType(16);
5960        break;
5961    case glslang::EbtInt64:
5962        spvType = builder.makeIntType(64);
5963        break;
5964    case glslang::EbtUint64:
5965        spvType = builder.makeUintType(64);
5966        break;
5967    case glslang::EbtAtomicUint:
5968        builder.addCapability(spv::Capability::AtomicStorage);
5969        spvType = builder.makeUintType(32);
5970        break;
5971    case glslang::EbtAccStruct:
5972        switch (glslangIntermediate->getStage()) {
5973        case EShLangRayGen:
5974        case EShLangIntersect:
5975        case EShLangAnyHit:
5976        case EShLangClosestHit:
5977        case EShLangMiss:
5978        case EShLangCallable:
5979            // these all should have the RayTracingNV/KHR capability already
5980            break;
5981        default:
5982            {
5983                auto& extensions = glslangIntermediate->getRequestedExtensions();
5984                if (extensions.find("GL_EXT_ray_query") != extensions.end()) {
5985                    builder.addExtension(spv::E_SPV_KHR_ray_query);
5986                    builder.addCapability(spv::Capability::RayQueryKHR);
5987                }
5988            }
5989            break;
5990        }
5991        spvType = builder.makeAccelerationStructureType();
5992        break;
5993    case glslang::EbtRayQuery:
5994        {
5995            auto& extensions = glslangIntermediate->getRequestedExtensions();
5996            if (extensions.find("GL_EXT_ray_query") != extensions.end()) {
5997                builder.addExtension(spv::E_SPV_KHR_ray_query);
5998                builder.addCapability(spv::Capability::RayQueryKHR);
5999            }
6000            spvType = builder.makeRayQueryType();
6001        }
6002        break;
6003    case glslang::EbtReference:
6004        {
6005            // Make the forward pointer, then recurse to convert the structure type, then
6006            // patch up the forward pointer with a real pointer type.
6007            if (forwardPointers.find(type.getReferentType()) == forwardPointers.end()) {
6008                spv::Id forwardId = builder.makeForwardPointer(spv::StorageClass::PhysicalStorageBufferEXT);
6009                forwardPointers[type.getReferentType()] = forwardId;
6010            }
6011            spvType = forwardPointers[type.getReferentType()];
6012            if (!forwardReferenceOnly) {
6013                spv::Id referentType = convertGlslangToSpvType(*type.getReferentType());
6014                builder.makePointerFromForwardPointer(spv::StorageClass::PhysicalStorageBufferEXT,
6015                                                      forwardPointers[type.getReferentType()],
6016                                                      referentType);
6017            }
6018        }
6019        break;
6020    case glslang::EbtSampler:
6021        {
6022            const glslang::TSampler& sampler = type.getSampler();
6023            std::string debugName;
6024
6025            if (sampler.isPureSampler()) {
6026                if (options.emitNonSemanticShaderDebugInfo) {
6027                    if (glslangIntermediate->getSource() == glslang::EShSourceGlsl) {
6028                        debugName = sampler.getString();
6029                    }
6030                    else {
6031                        debugName = "type.sampler";
6032                    }
6033                }
6034                spvType = builder.makeSamplerType(debugName.c_str());
6035            } else {
6036                // an image is present, make its type
6037                if (options.emitNonSemanticShaderDebugInfo) {
6038                    if (glslangIntermediate->getSource() == glslang::EShSourceGlsl) {
6039                        debugName = sampler.removeCombined().getString();
6040                    }
6041                    else {
6042                        switch (sampler.dim) {
6043                        case glslang::Esd1D:           debugName = "type.1d.image"; break;
6044                        case glslang::Esd2D:           debugName = "type.2d.image"; break;
6045                        case glslang::Esd3D:           debugName = "type.3d.image"; break;
6046                        case glslang::EsdCube:         debugName = "type.cube.image"; break;
6047                        default:                       debugName = "type.image"; break;
6048                        }
6049                    }
6050                }
6051                spvType = builder.makeImageType(getSampledType(sampler), TranslateDimensionality(sampler),
6052                                                sampler.isShadow(), sampler.isArrayed(), sampler.isMultiSample(),
6053                                                sampler.isImageClass() ? 2 : 1, TranslateImageFormat(type), debugName.c_str());
6054                if (sampler.isCombined() &&
6055                    (!sampler.isBuffer() || glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_6)) {
6056                    // Already has both image and sampler, make the combined type. Only combine sampler to
6057                    // buffer if before SPIR-V 1.6.
6058                    if (options.emitNonSemanticShaderDebugInfo) {
6059                        if (glslangIntermediate->getSource() == glslang::EShSourceGlsl) {
6060                            debugName = sampler.getString();
6061                        }
6062                        else {
6063                            debugName = "type.sampled.image";
6064                        }
6065                    }
6066                    spvType = builder.makeSampledImageType(spvType, debugName.c_str());
6067                }
6068            }
6069        }
6070        break;
6071    case glslang::EbtStruct:
6072    case glslang::EbtBlock:
6073        {
6074            // If it's an inner declared heap buffer or uniform block, using OpTypeBufferEXT
6075            // to convert it to an untyped type.
6076            if (type.getQualifier().layoutDescriptorHeap && type.getQualifier().descriptorHeapDescriptorNode) {
6077                spvType = builder.makeUntypedPointer(TranslateStorageClass(type), true);
6078                break;
6079            }
6080
6081            // If we've seen this struct type, return it
6082            const glslang::TTypeList* glslangMembers = type.getStruct();
6083
6084            // Try to share structs for different layouts, but not yet for other
6085            // kinds of qualification (primarily not yet including interpolant qualification).
6086            if (! HasNonLayoutQualifiers(type, qualifier))
6087                spvType = structMap[explicitLayout][qualifier.layoutMatrix][glslangMembers];
6088            if (spvType != spv::NoResult)
6089                break;
6090
6091            // else, we haven't seen it...
6092            if (type.getBasicType() == glslang::EbtBlock)
6093                memberRemapper[glslangMembers].resize(glslangMembers->size());
6094            spvType = convertGlslangStructToSpvType(type, glslangMembers, explicitLayout, qualifier);
6095        }
6096        break;
6097    case glslang::EbtString:
6098        // no type used for OpString
6099        return 0;
6100
6101    case glslang::EbtHitObjectNV: {
6102        builder.addExtension(spv::E_SPV_NV_shader_invocation_reorder);
6103        builder.addCapability(spv::Capability::ShaderInvocationReorderNV);
6104        spvType = builder.makeHitObjectNVType();
6105    }
6106    break;
6107
6108    case glslang::EbtHitObjectEXT: {
6109        builder.addExtension(spv::E_SPV_EXT_shader_invocation_reorder);
6110        builder.addCapability(spv::Capability::ShaderInvocationReorderEXT);
6111        spvType = builder.makeHitObjectEXTType();
6112    }
6113    break;
6114    case glslang::EbtSpirvType: {
6115        // GL_EXT_spirv_intrinsics
6116        const auto& spirvType = type.getSpirvType();
6117        const auto& spirvInst = spirvType.spirvInst;
6118
6119        std::vector<spv::IdImmediate> operands;
6120        for (const auto& typeParam : spirvType.typeParams) {
6121            if (typeParam.getAsConstant() != nullptr) {
6122                // Constant expression
6123                auto constant = typeParam.getAsConstant();
6124                if (constant->isLiteral()) {
6125                    if (constant->getBasicType() == glslang::EbtFloat) {
6126                        float floatValue = static_cast<float>(constant->getConstArray()[0].getDConst());
6127                        unsigned literal;
6128                        static_assert(sizeof(literal) == sizeof(floatValue), "sizeof(unsigned) != sizeof(float)");
6129                        memcpy(&literal, &floatValue, sizeof(literal));
6130                        operands.push_back({false, literal});
6131                    } else if (constant->getBasicType() == glslang::EbtInt) {
6132                        unsigned literal = constant->getConstArray()[0].getIConst();
6133                        operands.push_back({false, literal});
6134                    } else if (constant->getBasicType() == glslang::EbtUint) {
6135                        unsigned literal = constant->getConstArray()[0].getUConst();
6136                        operands.push_back({false, literal});
6137                    } else if (constant->getBasicType() == glslang::EbtBool) {
6138                        unsigned literal = constant->getConstArray()[0].getBConst();
6139                        operands.push_back({false, literal});
6140                    } else if (constant->getBasicType() == glslang::EbtString) {
6141                        auto str = constant->getConstArray()[0].getSConst()->c_str();
6142                        unsigned literal = 0;
6143                        char* literalPtr = reinterpret_cast<char*>(&literal);
6144                        unsigned charCount = 0;
6145                        char ch = 0;
6146                        do {
6147                            ch = *(str++);
6148                            *(literalPtr++) = ch;
6149                            ++charCount;
6150                            if (charCount == 4) {
6151                                operands.push_back({false, literal});
6152                                literalPtr = reinterpret_cast<char*>(&literal);
6153                                charCount = 0;
6154                            }
6155                        } while (ch != 0);
6156
6157                        // Partial literal is padded with 0
6158                        if (charCount > 0) {
6159                            for (; charCount < 4; ++charCount)
6160                                *(literalPtr++) = 0;
6161                            operands.push_back({false, literal});
6162                        }
6163                    } else
6164                        assert(0); // Unexpected type
6165                } else
6166                    operands.push_back({true, createSpvConstant(*constant)});
6167            } else {
6168                // Type specifier
6169                assert(typeParam.getAsType() != nullptr);
6170                operands.push_back({true, convertGlslangToSpvType(*typeParam.getAsType())});
6171            }
6172        }
6173
6174        assert(spirvInst.set == ""); // Currently, couldn't be extended instructions.
6175        spvType = builder.makeGenericType(static_cast<spv::Op>(spirvInst.id), operands);
6176
6177        break;
6178    }
6179    case glslang::EbtTensorLayoutNV:
6180    {
6181        builder.addCapability(spv::Capability::TensorAddressingNV);
6182        builder.addExtension(spv::E_SPV_NV_tensor_addressing);
6183
6184        std::vector<spv::IdImmediate> operands;
6185        for (uint32_t i = 0; i < 2; ++i) {
6186            operands.push_back({true, makeArraySizeId(*type.getTypeParameters()->arraySizes, i, true)});
6187        }
6188        spvType = builder.makeGenericType(spv::Op::OpTypeTensorLayoutNV, operands);
6189        break;
6190    }
6191    case glslang::EbtTensorViewNV:
6192    {
6193        builder.addCapability(spv::Capability::TensorAddressingNV);
6194        builder.addExtension(spv::E_SPV_NV_tensor_addressing);
6195
6196        uint32_t dim = type.getTypeParameters()->arraySizes->getDimSize(0);
6197        assert(dim >= 1 && dim <= 5);
6198        std::vector<spv::IdImmediate> operands;
6199        for (uint32_t i = 0; i < dim + 2; ++i) {
6200            operands.push_back({true, makeArraySizeId(*type.getTypeParameters()->arraySizes, i, true, i==1)});
6201        }
6202        spvType = builder.makeGenericType(spv::Op::OpTypeTensorViewNV, operands);
6203        break;
6204    }
6205    default:
6206        assert(0);
6207        break;
6208    }
6209
6210    if (type.isMatrix())
6211        spvType = builder.makeMatrixType(spvType, type.getMatrixCols(), type.getMatrixRows());
6212    else {
6213        // If this variable has a vector element count greater than 1, create a SPIR-V vector
6214        if (type.getVectorSize() > 1)
6215            spvType = builder.makeVectorType(spvType, type.getVectorSize());
6216    }
6217
6218    if (type.isCoopMatNV()) {
6219        builder.addCapability(spv::Capability::CooperativeMatrixNV);
6220        builder.addExtension(spv::E_SPV_NV_cooperative_matrix);
6221
6222        if (type.getBasicType() == glslang::EbtFloat16)
6223            builder.addCapability(spv::Capability::Float16);
6224        if (type.getBasicType() == glslang::EbtUint8 ||
6225            type.getBasicType() == glslang::EbtInt8) {
6226            builder.addCapability(spv::Capability::Int8);
6227        }
6228
6229        spv::Id scope = makeArraySizeId(*type.getTypeParameters()->arraySizes, 1);
6230        spv::Id rows = makeArraySizeId(*type.getTypeParameters()->arraySizes, 2);
6231        spv::Id cols = makeArraySizeId(*type.getTypeParameters()->arraySizes, 3);
6232
6233        spvType = builder.makeCooperativeMatrixTypeNV(spvType, scope, rows, cols);
6234    }
6235
6236    if (type.isCoopMatKHR()) {
6237        builder.addCapability(spv::Capability::CooperativeMatrixKHR);
6238        builder.addExtension(spv::E_SPV_KHR_cooperative_matrix);
6239
6240        if (type.getBasicType() == glslang::EbtBFloat16) {
6241            builder.addExtension(spv::E_SPV_KHR_bfloat16);
6242            builder.addCapability(spv::Capability::BFloat16CooperativeMatrixKHR);
6243        }
6244
6245        if (type.getBasicType() == glslang::EbtFloatE5M2 || type.getBasicType() == glslang::EbtFloatE4M3) {
6246            builder.addExtension(spv::E_SPV_EXT_float8);
6247            builder.addCapability(spv::Capability::Float8CooperativeMatrixEXT);
6248        }
6249
6250        if (type.getBasicType() == glslang::EbtFloat16)
6251            builder.addCapability(spv::Capability::Float16);
6252        if (type.getBasicType() == glslang::EbtUint8 || type.getBasicType() == glslang::EbtInt8) {
6253            builder.addCapability(spv::Capability::Int8);
6254        }
6255
6256        spv::Id scope = makeArraySizeId(*type.getTypeParameters()->arraySizes, 0);
6257        spv::Id rows = makeArraySizeId(*type.getTypeParameters()->arraySizes, 1);
6258        spv::Id cols = makeArraySizeId(*type.getTypeParameters()->arraySizes, 2);
6259        spv::Id use = makeArraySizeId(*type.getTypeParameters()->arraySizes, 3, true);
6260
6261        spvType = builder.makeCooperativeMatrixTypeKHR(spvType, scope, rows, cols, use);
6262    }
6263    else if (type.isTensorARM()) {
6264        builder.addCapability(spv::Capability::TensorsARM);
6265        builder.addExtension(spv::E_SPV_ARM_tensors);
6266        if (type.getBasicType() == glslang::EbtInt8 || type.getBasicType() == glslang::EbtUint8) {
6267            builder.addCapability(spv::Capability::Int8);
6268        } else if (type.getBasicType() == glslang::EbtInt16 ||
6269                   type.getBasicType() == glslang::EbtUint16) {
6270            builder.addCapability(spv::Capability::Int16);
6271        } else if (type.getBasicType() == glslang::EbtInt64 ||
6272                   type.getBasicType() == glslang::EbtUint64) {
6273            builder.addCapability(spv::Capability::Int64);
6274        } else if (type.getBasicType() == glslang::EbtFloat16) {
6275            builder.addCapability(spv::Capability::Float16);
6276        }
6277
6278        spv::Id rank = makeArraySizeId(*type.getTypeParameters()->arraySizes, 0);
6279
6280        spvType = builder.makeTensorTypeARM(spvType, rank);
6281    }
6282
6283    if (type.isCoopVecNV()) {
6284        builder.addCapability(spv::Capability::CooperativeVectorNV);
6285        builder.addExtension(spv::E_SPV_NV_cooperative_vector);
6286
6287        if (type.getBasicType() == glslang::EbtFloat16)
6288            builder.addCapability(spv::Capability::Float16);
6289        if (type.getBasicType() == glslang::EbtUint8 || type.getBasicType() == glslang::EbtInt8) {
6290            builder.addCapability(spv::Capability::Int8);
6291        }
6292
6293        spv::Id components = makeArraySizeId(*type.getTypeParameters()->arraySizes, 0);
6294
6295        spvType = builder.makeCooperativeVectorTypeNV(spvType, components);
6296    }
6297
6298    if (type.isLongVector()) {
6299        // SPIR-V LongVectorEXT not needed when component count is literal 2-4.
6300        const bool needLongVectorCap = type.hasSpecConstantVectorComponents() ||
6301            (type.getTypeParameters()->arraySizes->getDimSize(0) < 2 ||
6302             type.getTypeParameters()->arraySizes->getDimSize(0) > 4);
6303        if (needLongVectorCap) {
6304            builder.addCapability(spv::Capability::LongVectorEXT);
6305            builder.addExtension(spv::E_SPV_EXT_long_vector);
6306        }
6307
6308        if (type.getBasicType() == glslang::EbtFloat16)
6309            builder.addCapability(spv::Capability::Float16);
6310        if (type.getBasicType() == glslang::EbtUint8 || type.getBasicType() == glslang::EbtInt8) {
6311            builder.addCapability(spv::Capability::Int8);
6312        }
6313
6314        if (type.hasSpecConstantVectorComponents()) {
6315            spv::Id components = makeArraySizeId(*type.getTypeParameters()->arraySizes, 0);
6316            spvType = builder.makeCooperativeVectorTypeNV(spvType, components);
6317        } else {
6318            spvType = builder.makeVectorType(spvType, type.getTypeParameters()->arraySizes->getDimSize(0));
6319        }
6320    }
6321
6322    if (type.isArray()) {
6323        if (type.getQualifier().layoutDescriptorHeap && containsDescHeapDescriptorType(type)) {
6324            std::vector<spv::Id> arrayStrides = descHeapLayout.getOrCreateArrayStrides(type);
6325            assert((int)arrayStrides.size() == type.getArraySizes()->getNumDims());
6326            int strideIndex = 0;
6327
6328            for (int dim = type.getArraySizes()->getNumDims() - 1; dim > 0; --dim) {
6329                spvType = builder.makeArrayType(spvType, makeArraySizeId(*type.getArraySizes(), dim), 1);
6330                builder.addDecorationId(spvType, spv::Decoration::ArrayStrideIdEXT, arrayStrides[strideIndex++]);
6331            }
6332
6333            if (type.isSizedArray() && !type.isImplicitlySizedArray())
6334                spvType = builder.makeArrayType(spvType, makeArraySizeId(*type.getArraySizes(), 0), 1);
6335            else
6336                spvType = builder.makeRuntimeArray(spvType);
6337            builder.addDecorationId(spvType, spv::Decoration::ArrayStrideIdEXT, arrayStrides[strideIndex]);
6338        } else {
6339            glslang::TLayoutPacking arrayLayout = type.getQualifier().layoutDescriptorHeap ? glslang::ElpScalar : explicitLayout;
6340            int stride = 0;  // keep this 0 unless doing an explicit layout; 0 will mean no decoration, no stride
6341
6342            // Do all but the outer dimension
6343            if (type.getArraySizes()->getNumDims() > 1) {
6344                // We need to decorate array strides for types needing explicit layout, except blocks.
6345                if (arrayLayout != glslang::ElpNone && type.getBasicType() != glslang::EbtBlock) {
6346                    // Use a dummy glslang type for querying internal strides of
6347                    // arrays of arrays, but using just a one-dimensional array.
6348                    glslang::TType simpleArrayType(type, 0); // deference type of the array
6349                    while (simpleArrayType.getArraySizes()->getNumDims() > 1)
6350                        simpleArrayType.getArraySizes()->dereference();
6351
6352                    // Will compute the higher-order strides here, rather than making a whole
6353                    // pile of types and doing repetitive recursion on their contents.
6354                    stride = getArrayStride(simpleArrayType, arrayLayout, qualifier.layoutMatrix);
6355                }
6356
6357                // make the arrays
6358                for (int dim = type.getArraySizes()->getNumDims() - 1; dim > 0; --dim) {
6359                    spvType = builder.makeArrayType(spvType, makeArraySizeId(*type.getArraySizes(), dim), stride);
6360                    if (stride > 0)
6361                        builder.addDecoration(spvType, spv::Decoration::ArrayStride, stride);
6362                    stride *= type.getArraySizes()->getDimSize(dim);
6363                }
6364            } else {
6365                // single-dimensional array, and don't yet have stride
6366                // We need to decorate array strides for types needing explicit layout, except blocks.
6367                if (arrayLayout != glslang::ElpNone && type.getBasicType() != glslang::EbtBlock)
6368                    stride = getArrayStride(type, arrayLayout, qualifier.layoutMatrix);
6369            }
6370
6371            // Do the outer dimension, which might not be known for a runtime-sized array.
6372            // (Unsized arrays that survive through linking will be runtime-sized arrays)
6373            const bool useRuntimeDescHeapArray =
6374                type.getQualifier().layoutDescriptorHeap && type.isImplicitlySizedArray();
6375            if (type.isSizedArray() && !useRuntimeDescHeapArray)
6376                spvType = builder.makeArrayType(spvType, makeArraySizeId(*type.getArraySizes(), 0), stride);
6377            else {
6378                // If we see an runtime array in a buffer_reference, it is not a descriptor
6379                if (!lastBufferBlockMember && type.getBasicType() != glslang::EbtReference &&
6380                    !type.getQualifier().layoutDescriptorHeap) {
6381                    builder.addIncorporatedExtension("SPV_EXT_descriptor_indexing", spv::Spv_1_5);
6382                    builder.addCapability(spv::Capability::RuntimeDescriptorArrayEXT);
6383                }
6384                spvType = builder.makeRuntimeArray(spvType);
6385            }
6386            if (stride > 0)
6387                builder.addDecoration(spvType, spv::Decoration::ArrayStride, stride);
6388        }
6389    }
6390
6391    return spvType;
6392}
6393
6394// Apply SPIR-V decorations to the SPIR-V object (provided by SPIR-V ID). If member index is provided, the
6395// decorations are applied to this member.
6396void TGlslangToSpvTraverser::applySpirvDecorate(const glslang::TType& type, spv::Id id, std::optional<int> member)
6397{
6398    assert(type.getQualifier().hasSpirvDecorate());
6399
6400    const glslang::TSpirvDecorate& spirvDecorate = type.getQualifier().getSpirvDecorate();
6401
6402    // Add spirv_decorate
6403    for (auto& decorate : spirvDecorate.decorates) {
6404        if (!decorate.second.empty()) {
6405            std::vector<unsigned> literals;
6406            TranslateLiterals(decorate.second, literals);
6407            if (member.has_value())
6408                builder.addMemberDecoration(id, *member, static_cast<spv::Decoration>(decorate.first), literals);
6409            else
6410                builder.addDecoration(id, static_cast<spv::Decoration>(decorate.first), literals);
6411        } else {
6412            if (member.has_value())
6413                builder.addMemberDecoration(id, *member, static_cast<spv::Decoration>(decorate.first));
6414            else
6415                builder.addDecoration(id, static_cast<spv::Decoration>(decorate.first));
6416        }
6417    }
6418
6419    // Add spirv_decorate_id
6420    if (member.has_value()) {
6421        // spirv_decorate_id not applied to members
6422        assert(spirvDecorate.decorateIds.empty());
6423    } else {
6424        for (auto& decorateId : spirvDecorate.decorateIds) {
6425            std::vector<spv::Id> operandIds;
6426            assert(!decorateId.second.empty());
6427            for (auto extraOperand : decorateId.second) {
6428                if (extraOperand->getQualifier().isFrontEndConstant())
6429                    operandIds.push_back(createSpvConstant(*extraOperand));
6430                else
6431                    operandIds.push_back(getSymbolId(extraOperand->getAsSymbolNode()));
6432            }
6433            builder.addDecorationId(id, static_cast<spv::Decoration>(decorateId.first), operandIds);
6434        }
6435    }
6436
6437    // Add spirv_decorate_string
6438    for (auto& decorateString : spirvDecorate.decorateStrings) {
6439        std::vector<const char*> strings;
6440        assert(!decorateString.second.empty());
6441        for (auto extraOperand : decorateString.second) {
6442            const char* string = extraOperand->getConstArray()[0].getSConst()->c_str();
6443            strings.push_back(string);
6444        }
6445        if (member.has_value())
6446            builder.addMemberDecoration(id, *member, static_cast<spv::Decoration>(decorateString.first), strings);
6447        else
6448            builder.addDecoration(id, static_cast<spv::Decoration>(decorateString.first), strings);
6449    }
6450}
6451
6452// TODO: this functionality should exist at a higher level, in creating the AST
6453//
6454// Identify interface members that don't have their required extension turned on.
6455//
6456bool TGlslangToSpvTraverser::filterMember(const glslang::TType& member)
6457{
6458    auto& extensions = glslangIntermediate->getRequestedExtensions();
6459
6460    if (member.getFieldName() == "gl_SecondaryViewportMaskNV" &&
6461        extensions.find("GL_NV_stereo_view_rendering") == extensions.end())
6462        return true;
6463    if (member.getFieldName() == "gl_SecondaryPositionNV" &&
6464        extensions.find("GL_NV_stereo_view_rendering") == extensions.end())
6465        return true;
6466
6467    if (glslangIntermediate->getStage() == EShLangMesh) {
6468        if (member.getFieldName() == "gl_PrimitiveShadingRateEXT" &&
6469            extensions.find("GL_EXT_fragment_shading_rate") == extensions.end())
6470            return true;
6471    }
6472
6473    if (glslangIntermediate->getStage() != EShLangMesh) {
6474        if ((member.getFieldName() == "gl_ViewportMask" || member.getFieldName() == "gl_ViewportIndex" ||
6475             member.getFieldName() == "gl_Layer") &&
6476            extensions.find("GL_NV_viewport_array2") == extensions.end())
6477            return true;
6478        if (member.getFieldName() == "gl_PositionPerViewNV" &&
6479            extensions.find("GL_NVX_multiview_per_view_attributes") == extensions.end())
6480            return true;
6481        if (member.getFieldName() == "gl_ViewportMaskPerViewNV" &&
6482            extensions.find("GL_NVX_multiview_per_view_attributes") == extensions.end())
6483            return true;
6484        // The front end hides these where it owns a copy of the block. One it never had to
6485        // copy up, like the sized tessellation gl_in, still has the shared type.
6486        if (member.getFieldName() == "gl_ClipDistance" || member.getFieldName() == "gl_CullDistance") {
6487            if (glslangIntermediate->getProfile() == EEsProfile &&
6488                extensions.find("GL_EXT_clip_cull_distance") == extensions.end())
6489                return true;
6490            if (member.getFieldName() == "gl_CullDistance" &&
6491                glslangIntermediate->getProfile() != EEsProfile && glslangIntermediate->getVersion() < 450 &&
6492                extensions.find("GL_ARB_cull_distance") == extensions.end())
6493                return true;
6494        }
6495    }
6496
6497    return false;
6498}
6499
6500// Do full recursive conversion of a glslang structure (or block) type to a SPIR-V Id.
6501// explicitLayout can be kept the same throughout the hierarchical recursive walk.
6502// Mutually recursive with convertGlslangToSpvType().
6503spv::Id TGlslangToSpvTraverser::convertGlslangStructToSpvType(const glslang::TType& type,
6504                                                              const glslang::TTypeList* glslangMembers,
6505                                                              glslang::TLayoutPacking explicitLayout,
6506                                                              const glslang::TQualifier& qualifier)
6507{
6508    // Create a vector of struct types for SPIR-V to consume
6509    std::vector<spv::Id> spvMembers;
6510    int memberDelta = 0;  // how much the member's index changes from glslang to SPIR-V, normally 0,
6511                          // except sometimes for blocks
6512    std::vector<std::pair<glslang::TType*, glslang::TQualifier> > deferredForwardPointers;
6513    std::vector<spv::StructMemberDebugInfo> memberDebugInfo;
6514    for (int i = 0; i < (int)glslangMembers->size(); i++) {
6515        auto& glslangMember = (*glslangMembers)[i];
6516        if (glslangMember.type->hiddenMember()) {
6517            ++memberDelta;
6518            if (type.getBasicType() == glslang::EbtBlock)
6519                memberRemapper[glslangMembers][i] = -1;
6520        } else {
6521            if (type.getBasicType() == glslang::EbtBlock) {
6522                if (filterMember(*glslangMember.type)) {
6523                    memberDelta++;
6524                    memberRemapper[glslangMembers][i] = -1;
6525                    continue;
6526                }
6527                memberRemapper[glslangMembers][i] = i - memberDelta;
6528            }
6529            // modify just this child's view of the qualifier
6530            glslang::TQualifier memberQualifier = glslangMember.type->getQualifier();
6531            InheritQualifiers(memberQualifier, qualifier);
6532
6533            // manually inherit location
6534            if (! memberQualifier.hasLocation() && qualifier.hasLocation())
6535                memberQualifier.layoutLocation = qualifier.layoutLocation;
6536
6537            // recurse
6538            bool lastBufferBlockMember = qualifier.storage == glslang::EvqBuffer &&
6539                                         i == (int)glslangMembers->size() - 1;
6540
6541            // Make forward pointers for any pointer members.
6542            if (glslangMember.type->isReference() &&
6543                forwardPointers.find(glslangMember.type->getReferentType()) == forwardPointers.end()) {
6544                deferredForwardPointers.push_back(std::make_pair(glslangMember.type, memberQualifier));
6545            }
6546
6547            // Create the member type.
6548            auto const spvMember = convertGlslangToSpvType(*glslangMember.type, explicitLayout, memberQualifier, lastBufferBlockMember,
6549                glslangMember.type->isReference());
6550            spvMembers.push_back(spvMember);
6551
6552            // Update the builder with the type's location so that we can create debug types for the structure members.
6553            // There doesn't exist a "clean" entry point for this information to be passed along to the builder so, for now,
6554            // it is stored in the builder and consumed during the construction of composite debug types.
6555            // TODO: This probably warrants further investigation. This approach was decided to be the least ugly of the
6556            // quick and dirty approaches that were tried.
6557            // Advantages of this approach:
6558            //  + Relatively clean. No direct calls into debug type system.
6559            //  + Handles nested recursive structures.
6560            // Disadvantages of this approach:
6561            //  + Not as clean as desired. Traverser queries/sets persistent state. This is fragile.
6562            //  + Table lookup during creation of composite debug types. This really shouldn't be necessary.
6563            if(options.emitNonSemanticShaderDebugInfo) {
6564                spv::StructMemberDebugInfo debugInfo{};
6565                debugInfo.name = glslangMember.type->getFieldName();
6566                debugInfo.line = glslangMember.loc.line;
6567                debugInfo.column = glslangMember.loc.column;
6568
6569                // Per the GLSL spec, bool variables inside of a uniform or buffer block are generated as uint.
6570                // But for debug info, we want to represent them as bool because that is the original type in
6571                // the source code. The bool type can be nested within a vector or a multidimensional array,
6572                // so we must construct the chain of types up from the scalar bool.
6573                if (glslangIntermediate->getSource() == glslang::EShSourceGlsl && explicitLayout != glslang::ElpNone &&
6574                    glslangMember.type->getBasicType() == glslang::EbtBool) {
6575                    auto typeId = builder.makeBoolType();
6576                    if (glslangMember.type->isVector()) {
6577                        typeId = builder.makeVectorType(typeId, glslangMember.type->getVectorSize());
6578                    }
6579                    if (glslangMember.type->isArray()) {
6580                        const auto* arraySizes = glslangMember.type->getArraySizes();
6581                        int dims = arraySizes->getNumDims();
6582                        for (int i = dims - 1; i >= 0; --i) {
6583                            spv::Id size = builder.makeIntConstant(arraySizes->getDimSize(i));
6584                            typeId = builder.makeArrayType(typeId, size, 0);
6585                        }
6586                    }
6587                    debugInfo.debugTypeOverride = builder.getDebugType(typeId);
6588                } else if (glslangMember.type->getQualifier().builtIn != glslang::EbvNone) {
6589                    // TODO - The built-in currently are not provide the correct line/column and spirv-val will validate these when using shaderDebugInfo
6590                    //
6591                    // There is a larger issue because even defining the builtIn such as
6592                    //
6593                    //      out gl_PerVertex {
6594                    //         vec4 gl_Position;
6595                    //     };
6596                    //
6597                    // in the shader also doesn't produce the correct line/column
6598                    // So for now, provide zero, as that is a valid value here
6599                    debugInfo.line = 0;
6600                    debugInfo.column = 0;
6601                }
6602
6603                memberDebugInfo.push_back(debugInfo);
6604            }
6605        }
6606    }
6607
6608    // OffsetIdEXT must only be applied to members of structs where the struct
6609    // contains at least one <<Descriptor,descriptor>> type.
6610    const bool useDescHeapIdDecorations = qualifier.layoutDescriptorHeap &&
6611        !type.getQualifier().descriptorHeapDescriptorNode && containsDescHeapDescriptorType(type);
6612
6613    // Prepare the layout ids for the descriptor heap struct.
6614    std::vector<spv::Id> descHeapMemberOffsets;
6615    if (useDescHeapIdDecorations)
6616        descHeapMemberOffsets = descHeapLayout.getOrCreateStructMemberOffsets(type);
6617
6618    // Make the SPIR-V type
6619    spv::Id spvType = builder.makeStructType(spvMembers, memberDebugInfo, type.getTypeName().c_str(), false);
6620    if (! HasNonLayoutQualifiers(type, qualifier))
6621        structMap[explicitLayout][qualifier.layoutMatrix][glslangMembers] = spvType;
6622
6623    // Decorate it
6624    if (useDescHeapIdDecorations) {
6625        // Only the top-level heap layout struct represents a resourceheap/samplerheap block.
6626        // If its last member is a runtime array, Vulkan still requires the struct type to
6627        // carry Block/BufferBlock; nested heap member structs should not get this decoration.
6628        const bool isTopLevelHeapStruct =
6629            qualifier.storage == glslang::EvqResourceHeap ||
6630            qualifier.storage == glslang::EvqSamplerHeap;
6631        if (isTopLevelHeapStruct && glslangIntermediate->getSpv().vulkan > 0 && !spvMembers.empty() &&
6632            builder.getOpCode(spvMembers.back()) == spv::Op::OpTypeRuntimeArray) {
6633            builder.addDecoration(spvType, TranslateBlockDecoration(qualifier.storage,
6634                                  glslangIntermediate->usingStorageBuffer()));
6635        }
6636
6637        assert(descHeapMemberOffsets.size() == glslangMembers->size());
6638        for (int member = 0; member < (int)glslangMembers->size() && member < (int)spvMembers.size(); ++member) {
6639            glslang::TType& glslangMember = *(*glslangMembers)[member].type;
6640
6641            // Decorate offset.
6642            builder.addMemberDecorationIdEXT(spvType, member, spv::Decoration::OffsetIdEXT,
6643                                             {descHeapMemberOffsets[member]});
6644
6645            // Decorate matrix layout.
6646            glslang::TQualifier memberQualifier = glslangMember.getQualifier();
6647            InheritQualifiers(memberQualifier, qualifier);
6648            if (!glslangMember.isMatrix() || memberQualifier.layoutMatrix == glslang::ElmNone)
6649                continue;
6650
6651            builder.addMemberDecoration(spvType, member,
6652                                        TranslateLayoutDecoration(glslangMember, memberQualifier.layoutMatrix));
6653            builder.addMemberDecoration(spvType, member, spv::Decoration::MatrixStride,
6654                                        getMatrixStride(glslangMember, glslang::ElpScalar, memberQualifier.layoutMatrix));
6655        }
6656    } else if (qualifier.layoutDescriptorHeap) {
6657        if (!type.getQualifier().descriptorHeapDescriptorNode)
6658            decorateStructType(type, glslangMembers, glslang::ElpScalar, qualifier, spvType, spvMembers);
6659    } else {
6660        decorateStructType(type, glslangMembers, explicitLayout, qualifier, spvType, spvMembers);
6661    }
6662
6663    for (int i = 0; i < (int)deferredForwardPointers.size(); ++i) {
6664        auto it = deferredForwardPointers[i];
6665        convertGlslangToSpvType(*it.first, explicitLayout, it.second, false);
6666    }
6667
6668    return spvType;
6669}
6670
6671// Keep access to the traverser state needed to emit layout ids.
6672DescHeapLayoutEmitter::DescHeapLayoutEmitter(TGlslangToSpvTraverser& traverser)
6673    : traverser(traverser)
6674{
6675}
6676
6677// Return true when this type is a descriptor payload in a descriptor heap.
6678bool DescHeapLayoutEmitter::isDescriptorPayload(const glslang::TType& type) const
6679{
6680    if (type.isArray()) {
6681        glslang::TType elementTy(type, 0);
6682        return isDescriptorPayload(elementTy);
6683    }
6684
6685    return type.getQualifier().descriptorHeapDescriptorNode;
6686}
6687
6688// Return true when this type or a nested member contains a descriptor payload.
6689bool DescHeapLayoutEmitter::containsDescriptorPayload(const glslang::TType& type) const
6690{
6691    if (isDescriptorPayload(type))
6692        return true;
6693
6694    if (type.isArray()) {
6695        // The element can be a struct containing descriptor payload members.
6696        glslang::TType elementTy(type, 0);
6697        return containsDescriptorPayload(elementTy);
6698    }
6699
6700    if (type.isStruct()) {
6701        const glslang::TTypeList* structTyList = type.getStruct();
6702        for (const auto& member : *structTyList) {
6703            if (containsDescriptorPayload(*member.type))
6704                return true;
6705        }
6706    }
6707
6708    return false;
6709}
6710
6711// Return the heap size id for one non-array value.
6712spv::Id DescHeapLayoutEmitter::getOrCreateValueSize(const glslang::TType& type)
6713{
6714    assert(!type.isArray());
6715    if (type.isStruct() && !isDescriptorPayload(type) && containsDescriptorPayload(type))
6716        return getOrCreateStructSize(type);
6717
6718    auto cachedSize = valueSizeCache.find(&type);
6719    if (cachedSize != valueSizeCache.end())
6720        return cachedSize->second;
6721
6722    spv::Id size = computeValueSize(type);
6723    return valueSizeCache.emplace(&type, size).first->second;
6724}
6725
6726// Return the placement alignment id for one non-array value.
6727spv::Id DescHeapLayoutEmitter::getOrCreateValueAlignment(const glslang::TType& type)
6728{
6729    assert(!type.isArray());
6730    if (type.isStruct() && !isDescriptorPayload(type) && containsDescriptorPayload(type))
6731        return getOrCreateStructAlignment(type);
6732
6733    auto cachedAlignment = valueAlignmentCache.find(&type);
6734    if (cachedAlignment != valueAlignmentCache.end())
6735        return cachedAlignment->second;
6736
6737    spv::Id alignment = computeValueAlignment(type);
6738    return valueAlignmentCache.emplace(&type, alignment).first->second;
6739}
6740
6741// Return the total heap size id for a sized array.
6742spv::Id DescHeapLayoutEmitter::getOrCreateArraySize(const glslang::TType& arrayType)
6743{
6744    assert(arrayType.isArray());
6745    auto cachedSize = arraySizeCache.find(&arrayType);
6746    if (cachedSize != arraySizeCache.end())
6747        return cachedSize->second;
6748
6749    spv::Id size = spv::NoResult;
6750    if (!containsDescriptorPayload(arrayType)) {
6751        int scalarSize;
6752        int stride;
6753        traverser.glslangIntermediate->getMemberAlignment(
6754            arrayType, scalarSize, stride, glslang::ElpScalar,
6755            arrayType.getQualifier().layoutMatrix == glslang::ElmRowMajor);
6756        size = makeUint(scalarSize);
6757    } else if (!arrayType.isImplicitlySizedArray()) {
6758        std::vector<spv::Id> strides = getOrCreateArrayStrides(arrayType);
6759        assert(!strides.empty());
6760        size = makeSpecOp(spv::Op::OpIMul,
6761                          {strides.back(), traverser.makeArraySizeId(*arrayType.getArraySizes(), 0)});
6762    }
6763    // Runtime arrays do not have a finite total size.
6764
6765    return arraySizeCache.emplace(&arrayType, size).first->second;
6766}
6767
6768// Return the placement alignment id for an array member.
6769spv::Id DescHeapLayoutEmitter::getOrCreateArrayAlignment(const glslang::TType& arrayType)
6770{
6771    assert(arrayType.isArray());
6772    auto cachedAlignment = arrayAlignmentCache.find(&arrayType);
6773    if (cachedAlignment != arrayAlignmentCache.end())
6774        return cachedAlignment->second;
6775
6776    spv::Id alignment = spv::NoResult;
6777    if (!containsDescriptorPayload(arrayType)) {
6778        int size;
6779        int stride;
6780        int scalarAlignment = traverser.glslangIntermediate->getMemberAlignment(
6781            arrayType, size, stride, glslang::ElpScalar,
6782            arrayType.getQualifier().layoutMatrix == glslang::ElmRowMajor);
6783        alignment = makeUint(scalarAlignment);
6784    } else {
6785        glslang::TType elementTy;
6786        getInnermostArrayElementType(arrayType, elementTy);
6787        alignment = computeValueAlignment(elementTy);
6788    }
6789
6790    return arrayAlignmentCache.emplace(&arrayType, alignment).first->second;
6791}
6792
6793// Return ArrayStrideIdEXT operand ids from inner dimension to outer dimension.
6794std::vector<spv::Id> DescHeapLayoutEmitter::getOrCreateArrayStrides(
6795    const glslang::TType& arrayType)
6796{
6797    assert(arrayType.isArray());
6798    auto cachedStrides = arrayStridesCache.find(&arrayType);
6799    if (cachedStrides != arrayStridesCache.end())
6800        return cachedStrides->second;
6801
6802    std::vector<spv::Id> strides;
6803    if (!containsDescriptorPayload(arrayType)) {
6804        int size;
6805        int stride;
6806        traverser.glslangIntermediate->getMemberAlignment(
6807            arrayType, size, stride, glslang::ElpScalar,
6808            arrayType.getQualifier().layoutMatrix == glslang::ElmRowMajor);
6809        if (stride > 0)
6810            strides.push_back(makeUint(stride));
6811    } else {
6812        glslang::TType elementTy;
6813        getInnermostArrayElementType(arrayType, elementTy);
6814        spv::Id stride = spv::NoResult;
6815        if (isDescriptorPayload(arrayType) &&
6816            arrayType.getQualifier().layoutDescriptorStride != glslang::TQualifier::layoutDescriptorStrideEnd)
6817            stride = makeUint(arrayType.getQualifier().layoutDescriptorStride);
6818        else if (isDescriptorPayload(arrayType) &&
6819                 arrayType.getQualifier().layoutDescriptorSize != glslang::TQualifier::layoutDescriptorSizeEnd)
6820            stride = makeUint(arrayType.getQualifier().layoutDescriptorSize);
6821        else
6822            stride = computeValueSize(elementTy);
6823        const glslang::TArraySizes* arraySizes = arrayType.getArraySizes();
6824        for (int dim = arraySizes->getNumDims() - 1; dim >= 0; --dim) {
6825            strides.push_back(stride);
6826            if (dim > 0)
6827                stride = makeSpecOp(spv::Op::OpIMul, {stride, traverser.makeArraySizeId(*arraySizes, dim)});
6828        }
6829    }
6830
6831    return arrayStridesCache.emplace(&arrayType, strides).first->second;
6832}
6833
6834// Create a 32-bit unsigned constant id.
6835spv::Id DescHeapLayoutEmitter::makeUint(unsigned value)
6836{
6837    return traverser.builder.makeUintConstant(value);
6838}
6839
6840// Create a 32-bit unsigned OpSpecConstantOp id.
6841spv::Id DescHeapLayoutEmitter::makeSpecOp(spv::Op op,
6842    const std::vector<spv::Id>& operands)
6843{
6844    return traverser.builder.createSpecConstantOp(op, traverser.builder.makeUintType(32), operands, {});
6845}
6846
6847// Return the byte footprint id for a struct member.
6848spv::Id DescHeapLayoutEmitter::getMemberSize(const glslang::TType& memberType)
6849{
6850    if (!memberType.isArray())
6851        return getOrCreateValueSize(memberType);
6852
6853    spv::Id arraySize = getOrCreateArraySize(memberType);
6854    if (arraySize != spv::NoResult)
6855        return arraySize;
6856
6857    // Runtime arrays have no finite footprint. The stride fallback is only
6858    // valid for the legal last-member case.
6859    assert(memberType.isImplicitlySizedArray());
6860    std::vector<spv::Id> strides = getOrCreateArrayStrides(memberType);
6861    assert(!strides.empty());
6862    return strides.back();
6863}
6864
6865// Return the placement alignment id for a struct member.
6866spv::Id DescHeapLayoutEmitter::getMemberAlignment(const glslang::TType& memberType)
6867{
6868    return memberType.isArray() ? getOrCreateArrayAlignment(memberType) :
6869                                  getOrCreateValueAlignment(memberType);
6870}
6871
6872// Return member offset ids relative to the start of this struct.
6873std::vector<spv::Id> DescHeapLayoutEmitter::getOrCreateStructMemberOffsets(
6874    const glslang::TType& structType)
6875{
6876    assert(structType.isStruct());
6877
6878    const glslang::TTypeList* structTyList = structType.getStruct();
6879    auto cachedOffsets = structMemberOffsetsCache.find(structTyList);
6880    if (cachedOffsets != structMemberOffsetsCache.end())
6881        return cachedOffsets->second;
6882
6883    std::vector<spv::Id> memberOffsets;
6884    memberOffsets.reserve(structTyList->size());
6885
6886    const glslang::TType* previousMemberTy = nullptr;
6887    spv::Id currentOffset = makeUint(0);
6888    for (int i = 0; i < (int)structTyList->size(); ++i) {
6889        const glslang::TType& memberTy = *(*structTyList)[i].type;
6890
6891        if (memberTy.getQualifier().hasOffset()) {
6892            currentOffset = makeUint(memberTy.getQualifier().layoutOffset);
6893        } else if (previousMemberTy != nullptr) {
6894            spv::Id offsetNotAligned = makeSpecOp(spv::Op::OpIAdd, {currentOffset, getMemberSize(*previousMemberTy)});
6895            currentOffset = traverser.builder.createSpecConstantAlignTo(offsetNotAligned, getMemberAlignment(memberTy));
6896        }
6897
6898        memberOffsets.push_back(currentOffset);
6899
6900        previousMemberTy = &memberTy;
6901    }
6902
6903    return structMemberOffsetsCache.emplace(structTyList, memberOffsets).first->second;
6904}
6905
6906// Return the max member alignment id for a descriptor-containing struct.
6907spv::Id DescHeapLayoutEmitter::getOrCreateStructAlignment(
6908    const glslang::TType& structType)
6909{
6910    assert(structType.isStruct());
6911
6912    const glslang::TTypeList* structTyList = structType.getStruct();
6913    auto cachedAlignment = structAlignmentCache.find(structTyList);
6914    if (cachedAlignment != structAlignmentCache.end())
6915        return cachedAlignment->second;
6916
6917    spv::Id alignment = spv::NoResult;
6918    for (int i = 0; i < (int)structTyList->size(); ++i) {
6919        spv::Id memberAlignment = getMemberAlignment(*(*structTyList)[i].type);
6920        alignment = alignment == spv::NoResult ? memberAlignment :
6921            traverser.builder.createSpecConstantSelectMax(alignment, memberAlignment);
6922    }
6923
6924    if (alignment == spv::NoResult)
6925        alignment = makeUint(1);
6926
6927    return structAlignmentCache.emplace(structTyList, alignment).first->second;
6928}
6929
6930// Return the padded heap size id for a descriptor-containing struct.
6931spv::Id DescHeapLayoutEmitter::getOrCreateStructSize(const glslang::TType& structType)
6932{
6933    assert(structType.isStruct());
6934
6935    const glslang::TTypeList* structTyList = structType.getStruct();
6936    auto cachedSize = structSizeCache.find(structTyList);
6937    if (cachedSize != structSizeCache.end())
6938        return cachedSize->second;
6939
6940    spv::Id size = spv::NoResult;
6941    if (structTyList->empty()) {
6942        size = makeUint(0);
6943    } else {
6944        std::vector<spv::Id> memberOffsets = getOrCreateStructMemberOffsets(structType);
6945        const glslang::TType& lastMemberTy = *structTyList->back().type;
6946        spv::Id rawSize = makeSpecOp(spv::Op::OpIAdd, {memberOffsets.back(), getMemberSize(lastMemberTy)});
6947        size = traverser.builder.createSpecConstantAlignTo(rawSize, getOrCreateStructAlignment(structType));
6948    }
6949
6950    return structSizeCache.emplace(structTyList, size).first->second;
6951}
6952
6953// Compute the heap size id without writing the TType* value-size cache.
6954spv::Id DescHeapLayoutEmitter::computeValueSize(const glslang::TType& type)
6955{
6956    assert(!type.isArray());
6957
6958    if (isDescriptorPayload(type)) {
6959        if (type.getQualifier().layoutDescriptorSize != glslang::TQualifier::layoutDescriptorSizeEnd)
6960            return makeUint(type.getQualifier().layoutDescriptorSize);
6961        return traverser.builder.createConstantSizeOfEXT(traverser.convertGlslangToSpvType(type));
6962    }
6963
6964    if (type.isStruct() && containsDescriptorPayload(type))
6965        return getOrCreateStructSize(type);
6966
6967    int size;
6968    int stride;
6969    int alignment = traverser.glslangIntermediate->getMemberAlignment(
6970        type, size, stride, glslang::ElpScalar,
6971        type.getQualifier().layoutMatrix == glslang::ElmRowMajor);
6972    glslang::RoundToPow2(size, alignment);
6973    return makeUint(size);
6974}
6975
6976// Compute the alignment id without writing the TType* value-alignment cache.
6977spv::Id DescHeapLayoutEmitter::computeValueAlignment(const glslang::TType& type)
6978{
6979    assert(!type.isArray());
6980
6981    if (isDescriptorPayload(type))
6982        return computeValueSize(type);
6983
6984    if (type.isStruct() && containsDescriptorPayload(type))
6985        return getOrCreateStructAlignment(type);
6986
6987    int size;
6988    int stride;
6989    int alignment = traverser.glslangIntermediate->getMemberAlignment(
6990        type, size, stride, glslang::ElpScalar,
6991        type.getQualifier().layoutMatrix == glslang::ElmRowMajor);
6992    return makeUint(alignment);
6993}
6994
6995// Fill a local non-array element type for an array type.
6996void DescHeapLayoutEmitter::getInnermostArrayElementType(const glslang::TType& arrayType,
6997                                                         glslang::TType& elementType) const
6998{
6999    assert(arrayType.isArray());
7000
7001    // Peel array dimensions on this local copy only.
7002    glslang::TType arrayElementTy(arrayType, 0);
7003    elementType.shallowCopy(arrayElementTy);
7004    if (elementType.isArray())
7005        elementType.clearArraySizes();
7006}
7007
7008// Return true only when the type itself is a descriptor heap payload. Arrays are
7009// unwrapped so descriptor arrays are treated as descriptor payloads too.
7010bool TGlslangToSpvTraverser::isDescHeapDescriptorType(const glslang::TType& type) const
7011{
7012    if (type.isArray()) {
7013        glslang::TType elementTy(type, 0);
7014        return isDescHeapDescriptorType(elementTy);
7015    }
7016
7017    return type.getQualifier().descriptorHeapDescriptorNode;
7018}
7019
7020// Return true when this type or any nested array/struct member contains a
7021// descriptor heap payload. This decides whether heap Id decorations are needed.
7022bool TGlslangToSpvTraverser::containsDescHeapDescriptorType(const glslang::TType& type) const
7023{
7024    if (isDescHeapDescriptorType(type))
7025        return true;
7026
7027    if (type.isArray()) {
7028        glslang::TType elementTy(type, 0);
7029        return containsDescHeapDescriptorType(elementTy);
7030    }
7031
7032    if (type.isStruct()) {
7033        const glslang::TTypeList* structTyList = type.getStruct();
7034        for (const auto& member : *structTyList) {
7035            if (containsDescHeapDescriptorType(*member.type))
7036                return true;
7037        }
7038    }
7039
7040    return false;
7041}
7042
7043void TGlslangToSpvTraverser::decorateStructType(const glslang::TType& type,
7044                                                const glslang::TTypeList* glslangMembers,
7045                                                glslang::TLayoutPacking explicitLayout,
7046                                                const glslang::TQualifier& qualifier,
7047                                                spv::Id spvType,
7048                                                const std::vector<spv::Id>& spvMembers)
7049{
7050    // Name and decorate the non-hidden members
7051    int offset = -1;
7052    bool memberLocationInvalid = type.isArrayOfArrays() ||
7053        (type.isArray() && (type.getQualifier().isArrayedIo(glslangIntermediate->getStage()) == false));
7054    for (int i = 0; i < (int)glslangMembers->size(); i++) {
7055        glslang::TType& glslangMember = *(*glslangMembers)[i].type;
7056        int member = i;
7057        if (type.getBasicType() == glslang::EbtBlock) {
7058            member = memberRemapper[glslangMembers][i];
7059            if (filterMember(glslangMember))
7060                continue;
7061        }
7062
7063        // modify just this child's view of the qualifier
7064        glslang::TQualifier memberQualifier = glslangMember.getQualifier();
7065        InheritQualifiers(memberQualifier, qualifier);
7066
7067        // using -1 above to indicate a hidden member
7068        if (member < 0)
7069            continue;
7070
7071        builder.addMemberName(spvType, member, glslangMember.getFieldName().c_str());
7072        builder.addMemberDecoration(spvType, member,
7073                                    TranslateLayoutDecoration(glslangMember, memberQualifier.layoutMatrix));
7074        builder.addMemberDecoration(spvType, member, TranslatePrecisionDecoration(glslangMember));
7075        // Add interpolation and auxiliary storage decorations only to
7076        // top-level members of Input and Output storage classes
7077        if (type.getQualifier().storage == glslang::EvqVaryingIn ||
7078            type.getQualifier().storage == glslang::EvqVaryingOut) {
7079            if (type.getBasicType() == glslang::EbtBlock ||
7080                glslangIntermediate->getSource() == glslang::EShSourceHlsl) {
7081                builder.addMemberDecoration(spvType, member, TranslateInterpolationDecoration(memberQualifier));
7082                builder.addMemberDecoration(spvType, member, TranslateAuxiliaryStorageDecoration(memberQualifier));
7083                addMeshNVDecoration(spvType, member, memberQualifier);
7084            }
7085        }
7086        builder.addMemberDecoration(spvType, member, TranslateInvariantDecoration(memberQualifier));
7087
7088        if (type.getBasicType() == glslang::EbtBlock &&
7089            qualifier.storage == glslang::EvqBuffer) {
7090            // Add memory decorations only to top-level members of shader storage block
7091            std::vector<spv::Decoration> memory;
7092            TranslateMemoryDecoration(memberQualifier, memory, glslangIntermediate->usingVulkanMemoryModel());
7093            for (unsigned int i = 0; i < memory.size(); ++i)
7094                builder.addMemberDecoration(spvType, member, memory[i]);
7095        }
7096
7097        // Location assignment was already completed correctly by the front end,
7098        // just track whether a member needs to be decorated.
7099        // Ignore member locations if the container is an array, as that's
7100        // ill-specified and decisions have been made to not allow this.
7101        if (!memberLocationInvalid && memberQualifier.hasLocation())
7102            builder.addMemberDecoration(spvType, member, spv::Decoration::Location, memberQualifier.layoutLocation);
7103
7104        // component, XFB, others
7105        if (glslangMember.getQualifier().hasComponent())
7106            builder.addMemberDecoration(spvType, member, spv::Decoration::Component,
7107                                        glslangMember.getQualifier().layoutComponent);
7108        if (glslangMember.getQualifier().hasXfbOffset())
7109            builder.addMemberDecoration(spvType, member, spv::Decoration::Offset,
7110                                        glslangMember.getQualifier().layoutXfbOffset);
7111        else if (glslangMember.getQualifier().hasMemberOffset()) {
7112            builder.addExtension(spv::E_SPV_NV_push_constant_bank);
7113            builder.addCapability(spv::Capability::PushConstantBanksNV);
7114            builder.addMemberDecoration(spvType, member, spv::Decoration::MemberOffsetNV,
7115                                        glslangMember.getQualifier().layoutMemberOffset);
7116        } else if (explicitLayout != glslang::ElpNone) {
7117            // figure out what to do with offset, which is accumulating
7118            int nextOffset;
7119            updateMemberOffset(type, glslangMember, offset, nextOffset, explicitLayout, memberQualifier.layoutMatrix);
7120            if (offset >= 0)
7121                builder.addMemberDecoration(spvType, member, spv::Decoration::Offset, offset);
7122            offset = nextOffset;
7123        }
7124
7125        if (glslangMember.isMatrix() && explicitLayout != glslang::ElpNone)
7126            builder.addMemberDecoration(spvType, member, spv::Decoration::MatrixStride,
7127                                        getMatrixStride(glslangMember, explicitLayout, memberQualifier.layoutMatrix));
7128
7129        // built-in variable decorations
7130        spv::BuiltIn builtIn = TranslateBuiltInDecoration(glslangMember.getQualifier().builtIn, true);
7131        // built-in decoration is used to detect AST nodes, but won't be decorated on member variables.
7132        if (builtIn != spv::BuiltIn::Max &&
7133            builtIn != spv::BuiltIn::ResourceHeapEXT && builtIn != spv::BuiltIn::SamplerHeapEXT)
7134            builder.addMemberDecoration(spvType, member, spv::Decoration::BuiltIn, (int)builtIn);
7135
7136        // nonuniform
7137        builder.addMemberDecoration(spvType, member, TranslateNonUniformDecoration(glslangMember.getQualifier()));
7138
7139        if (glslangIntermediate->getHlslFunctionality1() && memberQualifier.semanticName != nullptr) {
7140            builder.addExtension("SPV_GOOGLE_hlsl_functionality1");
7141            builder.addMemberDecoration(spvType, member, spv::Decoration::HlslSemanticGOOGLE,
7142                                        memberQualifier.semanticName);
7143        }
7144
7145        if (builtIn == spv::BuiltIn::Layer) {
7146            // SPV_NV_viewport_array2 extension
7147            if (glslangMember.getQualifier().layoutViewportRelative){
7148                builder.addMemberDecoration(spvType, member, spv::Decoration::ViewportRelativeNV);
7149                builder.addCapability(spv::Capability::ShaderViewportMaskNV);
7150                builder.addExtension(spv::E_SPV_NV_viewport_array2);
7151            }
7152            if (glslangMember.getQualifier().layoutSecondaryViewportRelativeOffset != -2048){
7153                builder.addMemberDecoration(spvType, member,
7154                                            spv::Decoration::SecondaryViewportRelativeNV,
7155                                            glslangMember.getQualifier().layoutSecondaryViewportRelativeOffset);
7156                builder.addCapability(spv::Capability::ShaderStereoViewNV);
7157                builder.addExtension(spv::E_SPV_NV_stereo_view_rendering);
7158            }
7159        }
7160        if (glslangMember.getQualifier().layoutPassthrough) {
7161            builder.addMemberDecoration(spvType, member, spv::Decoration::PassthroughNV);
7162            builder.addCapability(spv::Capability::GeometryShaderPassthroughNV);
7163            builder.addExtension(spv::E_SPV_NV_geometry_shader_passthrough);
7164        }
7165
7166        // Add SPIR-V decorations (GL_EXT_spirv_intrinsics)
7167        if (glslangMember.getQualifier().hasSpirvDecorate())
7168            applySpirvDecorate(glslangMember, spvType, member);
7169    }
7170
7171    // Decorate the structure
7172    builder.addDecoration(spvType, TranslateLayoutDecoration(type, qualifier.layoutMatrix));
7173    const auto basicType = type.getBasicType();
7174    const auto typeStorageQualifier = type.getQualifier().storage;
7175    if (basicType == glslang::EbtBlock || qualifier.isBufferType()) {
7176        builder.addDecoration(spvType, TranslateBlockDecoration(typeStorageQualifier, glslangIntermediate->usingStorageBuffer()));
7177    } else if (basicType == glslang::EbtStruct && glslangIntermediate->getSpv().vulkan > 0) {
7178        const auto hasRuntimeArray = !spvMembers.empty() && builder.getOpCode(spvMembers.back()) == spv::Op::OpTypeRuntimeArray;
7179        if (hasRuntimeArray) {
7180            builder.addDecoration(spvType, TranslateBlockDecoration(typeStorageQualifier, glslangIntermediate->usingStorageBuffer()));
7181        }
7182    }
7183
7184    if (qualifier.hasHitObjectShaderRecordNV())
7185        builder.addDecoration(spvType, spv::Decoration::HitObjectShaderRecordBufferNV);
7186
7187    if (qualifier.hasBank()) {
7188        builder.addExtension(spv::E_SPV_NV_push_constant_bank);
7189        builder.addCapability(spv::Capability::PushConstantBanksNV);
7190        builder.addDecoration(spvType, spv::Decoration::BankNV, qualifier.layoutBank);
7191    }
7192
7193    if (qualifier.hasHitObjectShaderRecordEXT())
7194        builder.addDecoration(spvType, spv::Decoration::HitObjectShaderRecordBufferEXT);
7195}
7196
7197// Turn the expression forming the array size into an id.
7198// This is not quite trivial, because of specialization constants.
7199// Sometimes, a raw constant is turned into an Id, and sometimes
7200// a specialization constant expression is.
7201spv::Id TGlslangToSpvTraverser::makeArraySizeId(const glslang::TArraySizes& arraySizes, int dim, bool allowZero, bool boolType)
7202{
7203    // First, see if this is sized with a node, meaning a specialization constant:
7204    glslang::TIntermTyped* specNode = arraySizes.getDimNode(dim);
7205    if (specNode != nullptr) {
7206        builder.clearAccessChain();
7207        SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
7208        spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
7209        specNode->traverse(this);
7210        return accessChainLoad(specNode->getAsTyped()->getType());
7211    }
7212
7213    // Otherwise, need a compile-time (front end) size, get it:
7214    int size = arraySizes.getDimSize(dim);
7215
7216    if (!allowZero)
7217        assert(size > 0);
7218
7219    if (boolType) {
7220        return builder.makeBoolConstant(size);
7221    } else {
7222        return builder.makeUintConstant(size);
7223    }
7224}
7225
7226// Turn a structured descriptor heap_offset into an id used to shift the heap base.
7227spv::Id TGlslangToSpvTraverser::makeHeapOffsetId(const glslang::TType& type)
7228{
7229    const glslang::TQualifier& qualifier = type.getQualifier();
7230    if (qualifier.layoutHeapOffsetNode != nullptr) {
7231        spv::Builder::AccessChain savedAccessChain = builder.getAccessChain();
7232        builder.clearAccessChain();
7233
7234        qualifier.layoutHeapOffsetNode->traverse(this);
7235        spv::Id heapOffset = accessChainLoad(qualifier.layoutHeapOffsetNode->getType());
7236
7237        builder.setAccessChain(savedAccessChain);
7238        return heapOffset;
7239    }
7240
7241    if (qualifier.layoutHeapOffset != 0)
7242        return builder.makeUintConstant(qualifier.layoutHeapOffset);
7243
7244    return spv::NoResult;
7245}
7246
7247// Wrap the builder's accessChainLoad to:
7248//  - localize handling of RelaxedPrecision
7249//  - use the SPIR-V inferred type instead of another conversion of the glslang type
7250//    (avoids unnecessary work and possible type punning for structures)
7251//  - do conversion of concrete to abstract type
7252spv::Id TGlslangToSpvTraverser::accessChainLoad(const glslang::TType& type)
7253{
7254    spv::Id nominalTypeId = builder.accessChainGetInferredType();
7255
7256    spv::Builder::AccessChain::CoherentFlags coherentFlags = builder.getAccessChain().coherentFlags;
7257    coherentFlags |= TranslateCoherent(type);
7258
7259    spv::MemoryAccessMask accessMask = spv::MemoryAccessMask(TranslateMemoryAccess(coherentFlags) & ~spv::MemoryAccessMask::MakePointerAvailableKHR);
7260    // If the value being loaded is HelperInvocation, SPIR-V 1.6 is being generated (so that
7261    // SPV_EXT_demote_to_helper_invocation is in core) and the memory model is in use, add
7262    // the Volatile MemoryAccess semantic.
7263    if (type.getQualifier().builtIn == glslang::EbvHelperInvocation &&
7264        glslangIntermediate->usingVulkanMemoryModel() &&
7265        glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6) {
7266        accessMask = spv::MemoryAccessMask(accessMask | spv::MemoryAccessMask::Volatile);
7267    }
7268
7269    unsigned int alignment = builder.getAccessChain().alignment;
7270    alignment |= type.getBufferReferenceAlignment();
7271
7272    spv::Id loadedId = builder.accessChainLoad(TranslatePrecisionDecoration(type),
7273        TranslateNonUniformDecoration(builder.getAccessChain().coherentFlags),
7274        TranslateNonUniformDecoration(type.getQualifier()),
7275        nominalTypeId,
7276        accessMask,
7277        TranslateMemoryScope(coherentFlags),
7278        alignment);
7279
7280    // Need to convert to abstract types when necessary
7281    if (type.getBasicType() == glslang::EbtBool) {
7282        loadedId = convertLoadedBoolInUniformToUint(type, nominalTypeId, loadedId);
7283    }
7284
7285    if (!builder.getAccessChain().descHeapInfo.descHeapInstId.empty()) {
7286        for (auto heapInst : builder.getAccessChain().descHeapInfo.descHeapInstId)
7287            heapInst->setTypeId(convertGlslangToSpvType(type));
7288        builder.clearAccessChain();
7289    }
7290
7291    return loadedId;
7292}
7293
7294// Wrap the builder's accessChainStore to:
7295//  - do conversion of concrete to abstract type
7296//
7297// Implicitly uses the existing builder.accessChain as the storage target.
7298void TGlslangToSpvTraverser::accessChainStore(const glslang::TType& type, spv::Id rvalue)
7299{
7300    // Need to convert to abstract types when necessary
7301    if (type.getBasicType() == glslang::EbtBool) {
7302        spv::Id nominalTypeId = builder.accessChainGetInferredType();
7303
7304        if (builder.isScalarType(nominalTypeId)) {
7305            // Conversion for bool
7306            spv::Id boolType = builder.makeBoolType();
7307            if (nominalTypeId != boolType) {
7308                // keep these outside arguments, for determinant order-of-evaluation
7309                spv::Id one = builder.makeUintConstant(1);
7310                spv::Id zero = builder.makeUintConstant(0);
7311                rvalue = builder.createTriOp(spv::Op::OpSelect, nominalTypeId, rvalue, one, zero);
7312            } else if (builder.getTypeId(rvalue) != boolType)
7313                rvalue = builder.createBinOp(spv::Op::OpINotEqual, boolType, rvalue, builder.makeUintConstant(0));
7314        } else if (builder.isVectorType(nominalTypeId)) {
7315            // Conversion for bvec
7316            int vecSize = builder.getNumTypeComponents(nominalTypeId);
7317            spv::Id bvecType = builder.makeVectorType(builder.makeBoolType(), vecSize);
7318            if (nominalTypeId != bvecType) {
7319                // keep these outside arguments, for determinant order-of-evaluation
7320                spv::Id one = makeSmearedConstant(builder.makeUintConstant(1), vecSize);
7321                spv::Id zero = makeSmearedConstant(builder.makeUintConstant(0), vecSize);
7322                rvalue = builder.createTriOp(spv::Op::OpSelect, nominalTypeId, rvalue, one, zero);
7323            } else if (builder.getTypeId(rvalue) != bvecType)
7324                rvalue = builder.createBinOp(spv::Op::OpINotEqual, bvecType, rvalue,
7325                                             makeSmearedConstant(builder.makeUintConstant(0), vecSize));
7326        }
7327    }
7328
7329    spv::Builder::AccessChain::CoherentFlags coherentFlags = builder.getAccessChain().coherentFlags;
7330    coherentFlags |= TranslateCoherent(type);
7331
7332    unsigned int alignment = builder.getAccessChain().alignment;
7333    alignment |= type.getBufferReferenceAlignment();
7334
7335    builder.accessChainStore(rvalue, TranslateNonUniformDecoration(builder.getAccessChain().coherentFlags),
7336                             spv::MemoryAccessMask(TranslateMemoryAccess(coherentFlags) &
7337                                ~spv::MemoryAccessMask::MakePointerVisibleKHR),
7338                             TranslateMemoryScope(coherentFlags), alignment);
7339}
7340
7341// For storing when types match at the glslang level, but not might match at the
7342// SPIR-V level.
7343//
7344// This especially happens when a single glslang type expands to multiple
7345// SPIR-V types, like a struct that is used in a member-undecorated way as well
7346// as in a member-decorated way.
7347//
7348// NOTE: This function can handle any store request; if it's not special it
7349// simplifies to a simple OpStore.
7350//
7351// Implicitly uses the existing builder.accessChain as the storage target.
7352void TGlslangToSpvTraverser::multiTypeStore(const glslang::TType& type, spv::Id rValue)
7353{
7354    // we only do the complex path here if it's an aggregate
7355    if (! type.isStruct() && ! type.isArray()) {
7356        accessChainStore(type, rValue);
7357        return;
7358    }
7359
7360    // and, it has to be a case of type aliasing
7361    spv::Id rType = builder.getTypeId(rValue);
7362    spv::Id lValue = builder.accessChainGetLValue();
7363    spv::Id lType = builder.getContainedTypeId(builder.getTypeId(lValue));
7364    if (lType == rType) {
7365        accessChainStore(type, rValue);
7366        return;
7367    }
7368
7369    // Recursively (as needed) copy an aggregate type to a different aggregate type,
7370    // where the two types were the same type in GLSL. This requires member
7371    // by member copy, recursively.
7372
7373    // SPIR-V 1.4 added an instruction to do help do this.
7374    if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) {
7375        // However, bool in uniform space is changed to int, so
7376        // OpCopyLogical does not work for that.
7377        // TODO: It would be more robust to do a full recursive verification of the types satisfying SPIR-V rules.
7378        bool rBool = builder.containsType(builder.getTypeId(rValue), spv::Op::OpTypeBool, 0);
7379        bool lBool = builder.containsType(lType, spv::Op::OpTypeBool, 0);
7380        if (lBool == rBool) {
7381            spv::Id logicalCopy = builder.createUnaryOp(spv::Op::OpCopyLogical, lType, rValue);
7382            accessChainStore(type, logicalCopy);
7383            return;
7384        }
7385    }
7386
7387    // If an array, copy element by element.
7388    if (type.isArray()) {
7389        glslang::TType glslangElementType(type, 0);
7390        spv::Id elementRType = builder.getContainedTypeId(rType);
7391        for (int index = 0; index < type.getOuterArraySize(); ++index) {
7392            // get the source member
7393            spv::Id elementRValue = builder.createCompositeExtract(rValue, elementRType, index);
7394
7395            // set up the target storage
7396            builder.clearAccessChain();
7397            builder.setAccessChainLValue(lValue);
7398            builder.accessChainPush(builder.makeIntConstant(index), TranslateCoherent(type),
7399                type.getBufferReferenceAlignment());
7400
7401            // store the member
7402            multiTypeStore(glslangElementType, elementRValue);
7403        }
7404    } else {
7405        assert(type.isStruct());
7406
7407        // loop over structure members
7408        const glslang::TTypeList& members = *type.getStruct();
7409        for (int m = 0; m < (int)members.size(); ++m) {
7410            const glslang::TType& glslangMemberType = *members[m].type;
7411
7412            // get the source member
7413            spv::Id memberRType = builder.getContainedTypeId(rType, m);
7414            spv::Id memberRValue = builder.createCompositeExtract(rValue, memberRType, m);
7415
7416            // set up the target storage
7417            builder.clearAccessChain();
7418            builder.setAccessChainLValue(lValue);
7419            builder.accessChainPush(builder.makeIntConstant(m), TranslateCoherent(type),
7420                type.getBufferReferenceAlignment());
7421
7422            // store the member
7423            multiTypeStore(glslangMemberType, memberRValue);
7424        }
7425    }
7426}
7427
7428// Decide whether or not this type should be
7429// decorated with offsets and strides, and if so
7430// whether std140 or std430 rules should be applied.
7431glslang::TLayoutPacking TGlslangToSpvTraverser::getExplicitLayout(const glslang::TType& type) const
7432{
7433    // Has to be a block. buffer_type is represented as a struct use in the AST,
7434    // but its payload still follows uniform/buffer block layout rules.
7435    if (type.getBasicType() != glslang::EbtBlock &&
7436        !type.getQualifier().isBufferType())
7437        return glslang::ElpNone;
7438
7439    // has to be a uniform or buffer block or task in/out blocks
7440    if (type.getQualifier().storage != glslang::EvqUniform &&
7441        type.getQualifier().storage != glslang::EvqBuffer &&
7442        type.getQualifier().storage != glslang::EvqShared &&
7443        !type.getQualifier().isTaskMemory())
7444        return glslang::ElpNone;
7445
7446    // return the layout to use
7447    switch (type.getQualifier().layoutPacking) {
7448    case glslang::ElpStd140:
7449    case glslang::ElpStd430:
7450    case glslang::ElpScalar:
7451        return type.getQualifier().layoutPacking;
7452    default:
7453        return glslang::ElpNone;
7454    }
7455}
7456
7457// Given an array type, returns the integer stride required for that array
7458int TGlslangToSpvTraverser::getArrayStride(const glslang::TType& arrayType, glslang::TLayoutPacking explicitLayout,
7459    glslang::TLayoutMatrix matrixLayout)
7460{
7461    int size;
7462    int stride;
7463    glslangIntermediate->getMemberAlignment(arrayType, size, stride, explicitLayout,
7464        matrixLayout == glslang::ElmRowMajor);
7465
7466    return stride;
7467}
7468
7469// Given a matrix type, or array (of array) of matrixes type, returns the integer stride required for that matrix
7470// when used as a member of an interface block
7471int TGlslangToSpvTraverser::getMatrixStride(const glslang::TType& matrixType, glslang::TLayoutPacking explicitLayout,
7472    glslang::TLayoutMatrix matrixLayout)
7473{
7474    glslang::TType elementType;
7475    elementType.shallowCopy(matrixType);
7476    elementType.clearArraySizes();
7477
7478    int size;
7479    int stride;
7480    glslangIntermediate->getMemberAlignment(elementType, size, stride, explicitLayout,
7481        matrixLayout == glslang::ElmRowMajor);
7482
7483    return stride;
7484}
7485
7486// Given a member type of a struct, realign the current offset for it, and compute
7487// the next (not yet aligned) offset for the next member, which will get aligned
7488// on the next call.
7489// 'currentOffset' should be passed in already initialized, ready to modify, and reflecting
7490// the migration of data from nextOffset -> currentOffset.  It should be -1 on the first call.
7491// -1 means a non-forced member offset (no decoration needed).
7492void TGlslangToSpvTraverser::updateMemberOffset(const glslang::TType& structType, const glslang::TType& memberType,
7493    int& currentOffset, int& nextOffset, glslang::TLayoutPacking explicitLayout, glslang::TLayoutMatrix matrixLayout)
7494{
7495    // this will get a positive value when deemed necessary
7496    nextOffset = -1;
7497
7498    // override anything in currentOffset with user-set offset
7499    if (memberType.getQualifier().hasOffset())
7500        currentOffset = memberType.getQualifier().layoutOffset;
7501
7502    // It could be that current linker usage in glslang updated all the layoutOffset,
7503    // in which case the following code does not matter.  But, that's not quite right
7504    // once cross-compilation unit GLSL validation is done, as the original user
7505    // settings are needed in layoutOffset, and then the following will come into play.
7506
7507    if (explicitLayout == glslang::ElpNone) {
7508        if (! memberType.getQualifier().hasOffset())
7509            currentOffset = -1;
7510
7511        return;
7512    }
7513
7514    // Getting this far means we need explicit offsets
7515    if (currentOffset < 0)
7516        currentOffset = 0;
7517
7518    // Now, currentOffset is valid (either 0, or from a previous nextOffset),
7519    // but possibly not yet correctly aligned.
7520
7521    int memberSize;
7522    int dummyStride;
7523    int memberAlignment = glslangIntermediate->getMemberAlignment(memberType, memberSize, dummyStride, explicitLayout,
7524        matrixLayout == glslang::ElmRowMajor);
7525
7526    bool isVectorLike = memberType.isVector();
7527    if (memberType.isMatrix()) {
7528        if (matrixLayout == glslang::ElmRowMajor)
7529            isVectorLike = memberType.getMatrixRows() == 1;
7530        else
7531            isVectorLike = memberType.getMatrixCols() == 1;
7532    }
7533
7534    // Adjust alignment for HLSL rules
7535    // TODO: make this consistent in early phases of code:
7536    //       adjusting this late means inconsistencies with earlier code, which for reflection is an issue
7537    // Until reflection is brought in sync with these adjustments, don't apply to $Global,
7538    // which is the most likely to rely on reflection, and least likely to rely implicit layouts
7539    if (glslangIntermediate->usingHlslOffsets() &&
7540        ! memberType.isStruct() && structType.getTypeName().compare("$Global") != 0) {
7541        int componentSize;
7542        int componentAlignment = glslangIntermediate->getBaseAlignmentScalar(memberType, componentSize);
7543        if (! memberType.isArray() && isVectorLike && componentAlignment <= 4)
7544            memberAlignment = componentAlignment;
7545
7546        // Don't add unnecessary padding after this member
7547        // (undo std140 bumping size to a mutliple of vec4)
7548        if (explicitLayout == glslang::ElpStd140) {
7549            if (memberType.isMatrix()) {
7550                if (matrixLayout == glslang::ElmRowMajor)
7551                    memberSize -= componentSize * (4 - memberType.getMatrixCols());
7552                else
7553                    memberSize -= componentSize * (4 - memberType.getMatrixRows());
7554            } else if (memberType.isArray())
7555                memberSize -= componentSize * (4 - memberType.getVectorSize());
7556        }
7557    }
7558
7559    // Bump up to member alignment
7560    glslang::RoundToPow2(currentOffset, memberAlignment);
7561
7562    // Bump up to vec4 if there is a bad straddle
7563    if (explicitLayout != glslang::ElpScalar && glslangIntermediate->improperStraddle(memberType, memberSize,
7564        currentOffset, isVectorLike))
7565        glslang::RoundToPow2(currentOffset, 16);
7566
7567    nextOffset = currentOffset + memberSize;
7568}
7569
7570void TGlslangToSpvTraverser::declareUseOfStructMember(const glslang::TTypeList& members, int glslangMember)
7571{
7572    const glslang::TBuiltInVariable glslangBuiltIn = members[glslangMember].type->getQualifier().builtIn;
7573    switch (glslangBuiltIn)
7574    {
7575    case glslang::EbvPointSize:
7576    case glslang::EbvClipDistance:
7577    case glslang::EbvCullDistance:
7578    case glslang::EbvViewportMaskNV:
7579    case glslang::EbvSecondaryPositionNV:
7580    case glslang::EbvSecondaryViewportMaskNV:
7581    case glslang::EbvPositionPerViewNV:
7582    case glslang::EbvViewportMaskPerViewNV:
7583    case glslang::EbvTaskCountNV:
7584    case glslang::EbvPrimitiveCountNV:
7585    case glslang::EbvPrimitiveIndicesNV:
7586    case glslang::EbvClipDistancePerViewNV:
7587    case glslang::EbvCullDistancePerViewNV:
7588    case glslang::EbvLayerPerViewNV:
7589    case glslang::EbvMeshViewCountNV:
7590    case glslang::EbvMeshViewIndicesNV:
7591        // Generate the associated capability.  Delegate to TranslateBuiltInDecoration.
7592        // Alternately, we could just call this for any glslang built-in, since the
7593        // capability already guards against duplicates.
7594        TranslateBuiltInDecoration(glslangBuiltIn, false);
7595        break;
7596    default:
7597        // Capabilities were already generated when the struct was declared.
7598        break;
7599    }
7600}
7601
7602bool TGlslangToSpvTraverser::isShaderEntryPoint(const glslang::TIntermAggregate* node)
7603{
7604    return node->getName().compare(glslangIntermediate->getEntryPointMangledName().c_str()) == 0;
7605}
7606
7607// Does parameter need a place to keep writes, separate from the original?
7608// Assumes called after originalParam(), which filters out block/buffer/opaque-based
7609// qualifiers such that we should have only in/out/inout/constreadonly here.
7610bool TGlslangToSpvTraverser::writableParam(glslang::TStorageQualifier qualifier) const
7611{
7612    assert(qualifier == glslang::EvqIn ||
7613           qualifier == glslang::EvqOut ||
7614           qualifier == glslang::EvqInOut ||
7615           qualifier == glslang::EvqUniform ||
7616           qualifier == glslang::EvqConstReadOnly);
7617    return qualifier != glslang::EvqConstReadOnly &&
7618           qualifier != glslang::EvqUniform;
7619}
7620
7621// Is parameter pass-by-original?
7622bool TGlslangToSpvTraverser::originalParam(glslang::TStorageQualifier qualifier, const glslang::TType& paramType,
7623                                           bool implicitThisParam)
7624{
7625    if (implicitThisParam)                                                                     // implicit this
7626        return true;
7627    if (glslangIntermediate->getSource() == glslang::EShSourceHlsl)
7628        return paramType.getBasicType() == glslang::EbtBlock;
7629    return (paramType.containsOpaque() && !glslangIntermediate->getBindlessMode()) ||       // sampler, etc.
7630           paramType.getQualifier().isSpirvByReference() ||                                    // spirv_by_reference
7631           (paramType.getBasicType() == glslang::EbtBlock && qualifier == glslang::EvqBuffer); // SSBO
7632}
7633
7634// Make all the functions, skeletally, without actually visiting their bodies.
7635void TGlslangToSpvTraverser::makeFunctions(const glslang::TIntermSequence& glslFunctions)
7636{
7637    const auto getParamDecorations = [&](std::vector<spv::Decoration>& decorations, const glslang::TType& type,
7638        bool useVulkanMemoryModel) {
7639        spv::Decoration paramPrecision = TranslatePrecisionDecoration(type);
7640        if (paramPrecision != spv::NoPrecision)
7641            decorations.push_back(paramPrecision);
7642        TranslateMemoryDecoration(type.getQualifier(), decorations, useVulkanMemoryModel);
7643        if (type.isReference()) {
7644            // Original and non-writable params pass the pointer directly and
7645            // use restrict/aliased, others are stored to a pointer in Function
7646            // memory and use RestrictPointer/AliasedPointer.
7647            if (originalParam(type.getQualifier().storage, type, false) ||
7648                !writableParam(type.getQualifier().storage)) {
7649                // TranslateMemoryDecoration added Restrict decoration already.
7650                if (!type.getQualifier().isRestrict()) {
7651                    decorations.push_back(spv::Decoration::Aliased);
7652                }
7653            } else {
7654                decorations.push_back(type.getQualifier().isRestrict() ? spv::Decoration::RestrictPointerEXT :
7655                                                                         spv::Decoration::AliasedPointerEXT);
7656            }
7657        }
7658    };
7659
7660    for (int f = 0; f < (int)glslFunctions.size(); ++f) {
7661        glslang::TIntermAggregate* glslFunction = glslFunctions[f]->getAsAggregate();
7662        if (! glslFunction || glslFunction->getOp() != glslang::EOpFunction)
7663            continue;
7664
7665        builder.setDebugSourceLocation(glslFunction->getLoc().line, glslFunction->getLoc().getFilename());
7666
7667        if (isShaderEntryPoint(glslFunction)) {
7668            shaderEntry->setFunctionControl(TranslateFunctionControl(glslFunction->getFunctionControl()));
7669            // For HLSL, the entry function is actually a compiler generated function to resolve the difference of
7670            // entry function signature between HLSL and SPIR-V. So we don't emit debug information for that.
7671            if (glslangIntermediate->getSource() != glslang::EShSourceHlsl) {
7672                builder.setupFunctionDebugInfo(shaderEntry, glslangIntermediate->getEntryPointMangledName().c_str(),
7673                                               std::vector<spv::Id>(), // main function has no param
7674                                               std::vector<char const*>());
7675                if (options.emitNonSemanticShaderDebugInfo &&
7676                    options.compilerSignature != nullptr && options.commandLineArguments != nullptr) {
7677                    builder.makeDebugEntryPoint(shaderEntry, options.compilerSignature, options.commandLineArguments,
7678                                                options.currentWorkingDirectory);
7679                }
7680            }
7681            continue;
7682        }
7683        // We're on a user function.  Set up the basic interface for the function now,
7684        // so that it's available to call.  Translating the body will happen later.
7685        //
7686        // Typically (except for a "const in" parameter), an address will be passed to the
7687        // function.  What it is an address of varies:
7688        //
7689        // - "in" parameters not marked as "const" can be written to without modifying the calling
7690        //   argument so that write needs to be to a copy, hence the address of a copy works.
7691        //
7692        // - "const in" parameters can just be the r-value, as no writes need occur.
7693        //
7694        // - "out" and "inout" arguments can't be done as pointers to the calling argument, because
7695        //   GLSL has copy-in/copy-out semantics.  They can be handled though with a pointer to a copy.
7696
7697        std::vector<spv::Id> paramTypes;
7698        std::vector<char const*> paramNames;
7699        std::vector<std::vector<spv::Decoration>> paramDecorations; // list of decorations per parameter
7700        glslang::TIntermSequence& parameters = glslFunction->getSequence()[0]->getAsAggregate()->getSequence();
7701
7702#ifdef ENABLE_HLSL
7703        bool implicitThis = (int)parameters.size() > 0 && parameters[0]->getAsSymbolNode()->getName() ==
7704                                                          glslangIntermediate->implicitThisName;
7705#else
7706        bool implicitThis = false;
7707#endif
7708
7709        paramDecorations.resize(parameters.size());
7710        for (int p = 0; p < (int)parameters.size(); ++p) {
7711            const glslang::TType& paramType = parameters[p]->getAsTyped()->getType();
7712            spv::Id typeId = convertGlslangToSpvType(paramType);
7713            if (originalParam(paramType.getQualifier().storage, paramType, implicitThis && p == 0))
7714                typeId = builder.makePointer(TranslateStorageClass(paramType), typeId);
7715            else if (writableParam(paramType.getQualifier().storage))
7716                typeId = builder.makePointer(spv::StorageClass::Function, typeId);
7717            else
7718                rValueParameters.insert(parameters[p]->getAsSymbolNode()->getId());
7719            getParamDecorations(paramDecorations[p], paramType, glslangIntermediate->usingVulkanMemoryModel());
7720            paramTypes.push_back(typeId);
7721        }
7722
7723        for (auto const parameter:parameters) {
7724            paramNames.push_back(parameter->getAsSymbolNode()->getName().c_str());
7725        }
7726
7727        spv::Block* functionBlock;
7728        spv::Function* function = builder.makeFunctionEntry(
7729            TranslatePrecisionDecoration(glslFunction->getType()), convertGlslangToSpvType(glslFunction->getType()),
7730            glslFunction->getName().c_str(), convertGlslangLinkageToSpv(glslFunction->getLinkType()), paramTypes,
7731            paramDecorations, &functionBlock);
7732        function->setFunctionControl(TranslateFunctionControl(glslFunction->getFunctionControl()));
7733        builder.setupFunctionDebugInfo(function, glslFunction->getName().c_str(), paramTypes, paramNames);
7734        if (implicitThis)
7735            function->setImplicitThis();
7736
7737        // Track function to emit/call later
7738        functionMap[glslFunction->getName().c_str()] = function;
7739
7740        // Set the parameter id's
7741        for (int p = 0; p < (int)parameters.size(); ++p) {
7742            symbolValues[parameters[p]->getAsSymbolNode()->getId()] = function->getParamId(p);
7743            // give a name too
7744            builder.addName(function->getParamId(p), parameters[p]->getAsSymbolNode()->getName().c_str());
7745
7746            const glslang::TType& paramType = parameters[p]->getAsTyped()->getType();
7747            if (paramType.contains8BitInt())
7748                builder.addCapability(spv::Capability::Int8);
7749            if (paramType.contains16BitInt())
7750                builder.addCapability(spv::Capability::Int16);
7751            if (paramType.contains16BitFloat())
7752                builder.addCapability(spv::Capability::Float16);
7753        }
7754    }
7755}
7756
7757// Process all the initializers, while skipping the functions and link objects
7758void TGlslangToSpvTraverser::makeGlobalInitializers(const glslang::TIntermSequence& initializers)
7759{
7760    builder.setBuildPoint(shaderEntry->getLastBlock());
7761    for (int i = 0; i < (int)initializers.size(); ++i) {
7762        glslang::TIntermAggregate* initializer = initializers[i]->getAsAggregate();
7763        if (initializer && initializer->getOp() != glslang::EOpFunction && initializer->getOp() !=
7764            glslang::EOpLinkerObjects) {
7765
7766            // We're on a top-level node that's not a function.  Treat as an initializer, whose
7767            // code goes into the beginning of the entry point.
7768            initializer->traverse(this);
7769        }
7770    }
7771}
7772// Walk over all linker objects to create a map for payload and callable data linker objects
7773// and their location to be used during codegen for OpTraceKHR and OpExecuteCallableKHR
7774// This is done here since it is possible that these linker objects are not be referenced in the AST
7775void TGlslangToSpvTraverser::collectRayTracingLinkerObjects()
7776{
7777    glslang::TIntermAggregate* linkerObjects = glslangIntermediate->findLinkerObjects();
7778    for (auto& objSeq : linkerObjects->getSequence()) {
7779        auto objNode = objSeq->getAsSymbolNode();
7780        if (objNode != nullptr) {
7781            if (objNode->getQualifier().hasLocation()) {
7782                unsigned int location = objNode->getQualifier().layoutLocation;
7783                auto st = objNode->getQualifier().storage;
7784                int set;
7785                switch (st)
7786                {
7787                case glslang::EvqPayload:
7788                case glslang::EvqPayloadIn:
7789                    set = 0;
7790                    break;
7791                case glslang::EvqCallableData:
7792                case glslang::EvqCallableDataIn:
7793                    set = 1;
7794                    break;
7795
7796                case glslang::EvqHitObjectAttrNV:
7797                case glslang::EvqHitObjectAttrEXT:
7798                    set = 2;
7799                    break;
7800
7801                default:
7802                    set = -1;
7803                }
7804                if (set != -1)
7805                    locationToSymbol[set].insert(std::make_pair(location, objNode));
7806            }
7807        }
7808    }
7809}
7810// Process all the functions, while skipping initializers.
7811void TGlslangToSpvTraverser::visitFunctions(const glslang::TIntermSequence& glslFunctions)
7812{
7813    for (int f = 0; f < (int)glslFunctions.size(); ++f) {
7814        glslang::TIntermAggregate* node = glslFunctions[f]->getAsAggregate();
7815        if (node && (node->getOp() == glslang::EOpFunction || node->getOp() == glslang::EOpLinkerObjects))
7816            node->traverse(this);
7817    }
7818}
7819
7820void TGlslangToSpvTraverser::translateArguments(const glslang::TIntermAggregate& node, std::vector<spv::Id>& arguments,
7821    spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags)
7822{
7823    const glslang::TIntermSequence& glslangArguments = node.getSequence();
7824
7825    glslang::TSampler sampler = {};
7826    bool cubeCompare = false;
7827    bool f16ShadowCompare = false;
7828    if (node.isTexture() || node.isImage()) {
7829        sampler = glslangArguments[0]->getAsTyped()->getType().getSampler();
7830        cubeCompare = sampler.dim == glslang::EsdCube && sampler.arrayed && sampler.shadow;
7831        f16ShadowCompare = sampler.shadow &&
7832            glslangArguments[1]->getAsTyped()->getType().getBasicType() == glslang::EbtFloat16;
7833    }
7834
7835    for (int i = 0; i < (int)glslangArguments.size(); ++i) {
7836        builder.clearAccessChain();
7837        glslangArguments[i]->traverse(this);
7838
7839        // Special case l-value operands
7840        bool lvalue = false;
7841        switch (node.getOp()) {
7842        case glslang::EOpImageAtomicAdd:
7843        case glslang::EOpImageAtomicMin:
7844        case glslang::EOpImageAtomicMax:
7845        case glslang::EOpImageAtomicAnd:
7846        case glslang::EOpImageAtomicOr:
7847        case glslang::EOpImageAtomicXor:
7848        case glslang::EOpImageAtomicExchange:
7849        case glslang::EOpImageAtomicCompSwap:
7850        case glslang::EOpImageAtomicLoad:
7851        case glslang::EOpImageAtomicStore:
7852            if (i == 0)
7853                lvalue = true;
7854            break;
7855        case glslang::EOpSparseImageLoad:
7856            if ((sampler.ms && i == 3) || (! sampler.ms && i == 2))
7857                lvalue = true;
7858            break;
7859        case glslang::EOpSparseTexture:
7860            if (((cubeCompare || f16ShadowCompare) && i == 3) || (! (cubeCompare || f16ShadowCompare) && i == 2))
7861                lvalue = true;
7862            break;
7863        case glslang::EOpSparseTextureClamp:
7864            if (((cubeCompare || f16ShadowCompare) && i == 4) || (! (cubeCompare || f16ShadowCompare) && i == 3))
7865                lvalue = true;
7866            break;
7867        case glslang::EOpSparseTextureLod:
7868        case glslang::EOpSparseTextureOffset:
7869            if  ((f16ShadowCompare && i == 4) || (! f16ShadowCompare && i == 3))
7870                lvalue = true;
7871            break;
7872        case glslang::EOpSparseTextureFetch:
7873            if ((sampler.dim != glslang::EsdRect && i == 3) || (sampler.dim == glslang::EsdRect && i == 2))
7874                lvalue = true;
7875            break;
7876        case glslang::EOpSparseTextureFetchOffset:
7877            if ((sampler.dim != glslang::EsdRect && i == 4) || (sampler.dim == glslang::EsdRect && i == 3))
7878                lvalue = true;
7879            break;
7880        case glslang::EOpSparseTextureLodOffset:
7881        case glslang::EOpSparseTextureGrad:
7882        case glslang::EOpSparseTextureOffsetClamp:
7883            if ((f16ShadowCompare && i == 5) || (! f16ShadowCompare && i == 4))
7884                lvalue = true;
7885            break;
7886        case glslang::EOpSparseTextureGradOffset:
7887        case glslang::EOpSparseTextureGradClamp:
7888            if ((f16ShadowCompare && i == 6) || (! f16ShadowCompare && i == 5))
7889                lvalue = true;
7890            break;
7891        case glslang::EOpSparseTextureGradOffsetClamp:
7892            if ((f16ShadowCompare && i == 7) || (! f16ShadowCompare && i == 6))
7893                lvalue = true;
7894            break;
7895        case glslang::EOpSparseTextureGather:
7896            if ((sampler.shadow && i == 3) || (! sampler.shadow && i == 2))
7897                lvalue = true;
7898            break;
7899        case glslang::EOpSparseTextureGatherOffset:
7900        case glslang::EOpSparseTextureGatherOffsets:
7901            if ((sampler.shadow && i == 4) || (! sampler.shadow && i == 3))
7902                lvalue = true;
7903            break;
7904        case glslang::EOpSparseTextureGatherLod:
7905            if (i == 3)
7906                lvalue = true;
7907            break;
7908        case glslang::EOpSparseTextureGatherLodOffset:
7909        case glslang::EOpSparseTextureGatherLodOffsets:
7910            if (i == 4)
7911                lvalue = true;
7912            break;
7913        case glslang::EOpSparseImageLoadLod:
7914            if (i == 3)
7915                lvalue = true;
7916            break;
7917        case glslang::EOpImageSampleFootprintNV:
7918            if (i == 4)
7919                lvalue = true;
7920            break;
7921        case glslang::EOpImageSampleFootprintClampNV:
7922        case glslang::EOpImageSampleFootprintLodNV:
7923            if (i == 5)
7924                lvalue = true;
7925            break;
7926        case glslang::EOpImageSampleFootprintGradNV:
7927            if (i == 6)
7928                lvalue = true;
7929            break;
7930        case glslang::EOpImageSampleFootprintGradClampNV:
7931            if (i == 7)
7932                lvalue = true;
7933            break;
7934        case glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT:
7935        case glslang::EOpRayQueryGetIntersectionLSSPositionsNV:
7936        case glslang::EOpRayQueryGetIntersectionLSSRadiiNV:
7937            if (i == 2)
7938                lvalue = true;
7939            break;
7940        case glslang::EOpConstructSaturated:
7941            if (i == 0)
7942                lvalue = true;
7943            break;
7944        default:
7945            break;
7946        }
7947
7948        if (lvalue) {
7949            spv::Id lvalue_id = builder.accessChainGetLValue();
7950            arguments.push_back(lvalue_id);
7951            lvalueCoherentFlags = builder.getAccessChain().coherentFlags;
7952            builder.addDecoration(lvalue_id, TranslateNonUniformDecoration(lvalueCoherentFlags));
7953            lvalueCoherentFlags |= TranslateCoherent(glslangArguments[i]->getAsTyped()->getType());
7954        } else {
7955            if (i > 0 &&
7956                glslangArguments[i]->getAsSymbolNode() && glslangArguments[i-1]->getAsSymbolNode() &&
7957                glslangArguments[i]->getAsSymbolNode()->getId() == glslangArguments[i-1]->getAsSymbolNode()->getId()) {
7958                // Reuse the id if possible
7959                spv::Id id = arguments[i-1];
7960                arguments.push_back(id);
7961            } else {
7962                arguments.push_back(accessChainLoad(glslangArguments[i]->getAsTyped()->getType()));
7963            }
7964        }
7965    }
7966}
7967
7968void TGlslangToSpvTraverser::translateArguments(glslang::TIntermUnary& node, std::vector<spv::Id>& arguments)
7969{
7970    builder.clearAccessChain();
7971    node.getOperand()->traverse(this);
7972    arguments.push_back(accessChainLoad(node.getOperand()->getType()));
7973}
7974
7975spv::Id TGlslangToSpvTraverser::createImageTextureFunctionCall(glslang::TIntermOperator* node)
7976{
7977    if (! node->isImage() && ! node->isTexture())
7978        return spv::NoResult;
7979
7980    builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
7981
7982    // Process a GLSL texturing op (will be SPV image)
7983
7984    const glslang::TType &imageType = node->getAsAggregate()
7985                                        ? node->getAsAggregate()->getSequence()[0]->getAsTyped()->getType()
7986                                        : node->getAsUnaryNode()->getOperand()->getAsTyped()->getType();
7987    const glslang::TSampler sampler = imageType.getSampler();
7988    bool f16ShadowCompare = (sampler.shadow && node->getAsAggregate())
7989            ? node->getAsAggregate()->getSequence()[1]->getAsTyped()->getType().getBasicType() == glslang::EbtFloat16
7990            : false;
7991
7992    const auto signExtensionMask = [&]() {
7993        if (builder.getSpvVersion() >= spv::Spv_1_4) {
7994            if (sampler.type == glslang::EbtUint)
7995                return spv::ImageOperandsMask::ZeroExtend;
7996            else if (sampler.type == glslang::EbtInt)
7997                return spv::ImageOperandsMask::SignExtend;
7998        }
7999        return spv::ImageOperandsMask::MaskNone;
8000    };
8001
8002    spv::Builder::AccessChain::CoherentFlags lvalueCoherentFlags;
8003
8004    std::vector<spv::Id> arguments;
8005    if (node->getAsAggregate())
8006        translateArguments(*node->getAsAggregate(), arguments, lvalueCoherentFlags);
8007    else
8008        translateArguments(*node->getAsUnaryNode(), arguments);
8009    spv::Decoration precision = TranslatePrecisionDecoration(node->getType());
8010
8011    spv::Builder::TextureParameters params = { };
8012    params.sampler = arguments[0];
8013
8014    glslang::TCrackedTextureOp cracked;
8015    node->crackTexture(sampler, cracked);
8016
8017    const bool isUnsignedResult = node->getType().getBasicType() == glslang::EbtUint;
8018
8019    if (builder.isSampledImage(params.sampler) &&
8020        ((cracked.query && node->getOp() != glslang::EOpTextureQueryLod) || cracked.fragMask || cracked.fetch)) {
8021        params.sampler = builder.createUnaryOp(spv::Op::OpImage, builder.getImageType(params.sampler), params.sampler);
8022        if (imageType.getQualifier().isNonUniform()) {
8023            auto& extensions = glslangIntermediate->getRequestedExtensions();
8024            if (extensions.find("GL_EXT_descriptor_heap") == extensions.end()) {
8025                builder.addDecoration(params.sampler, spv::Decoration::NonUniformEXT);
8026            }
8027        }
8028    }
8029    // Check for queries
8030    if (cracked.query) {
8031        switch (node->getOp()) {
8032        case glslang::EOpImageQuerySize:
8033        case glslang::EOpTextureQuerySize:
8034            if (arguments.size() > 1) {
8035                params.lod = arguments[1];
8036                return builder.createTextureQueryCall(spv::Op::OpImageQuerySizeLod, params, isUnsignedResult);
8037            } else
8038                return builder.createTextureQueryCall(spv::Op::OpImageQuerySize, params, isUnsignedResult);
8039        case glslang::EOpImageQuerySamples:
8040        case glslang::EOpTextureQuerySamples:
8041            return builder.createTextureQueryCall(spv::Op::OpImageQuerySamples, params, isUnsignedResult);
8042        case glslang::EOpTextureQueryLod:
8043            params.coords = arguments[1];
8044            return builder.createTextureQueryCall(spv::Op::OpImageQueryLod, params, isUnsignedResult);
8045        case glslang::EOpTextureQueryLevels:
8046            return builder.createTextureQueryCall(spv::Op::OpImageQueryLevels, params, isUnsignedResult);
8047        case glslang::EOpSparseTexelsResident:
8048            return builder.createUnaryOp(spv::Op::OpImageSparseTexelsResident, builder.makeBoolType(), arguments[0]);
8049        default:
8050            assert(0);
8051            break;
8052        }
8053    }
8054
8055    int components = node->getType().getVectorSize();
8056
8057    if (node->getOp() == glslang::EOpImageLoad ||
8058        node->getOp() == glslang::EOpImageLoadLod ||
8059        node->getOp() == glslang::EOpTextureFetch ||
8060        node->getOp() == glslang::EOpTextureFetchOffset) {
8061        // These must produce 4 components, per SPIR-V spec.  We'll add a conversion constructor if needed.
8062        // This will only happen through the HLSL path for operator[], so we do not have to handle e.g.
8063        // the EOpTexture/Proj/Lod/etc family.  It would be harmless to do so, but would need more logic
8064        // here around e.g. which ones return scalars or other types.
8065        components = 4;
8066    }
8067
8068    glslang::TType returnType(node->getType().getBasicType(), glslang::EvqTemporary, components);
8069
8070    auto resultType = [&returnType,this]{ return convertGlslangToSpvType(returnType); };
8071
8072    // Check for image functions other than queries
8073    if (node->isImage()) {
8074        std::vector<spv::IdImmediate> operands;
8075        auto opIt = arguments.begin();
8076        spv::IdImmediate image = { true, *(opIt++) };
8077        operands.push_back(image);
8078
8079        // Handle subpass operations
8080        // TODO: GLSL should change to have the "MS" only on the type rather than the
8081        // built-in function.
8082        if (cracked.subpass) {
8083            // add on the (0,0) coordinate
8084            spv::Id zero = builder.makeIntConstant(0);
8085            std::vector<spv::Id> comps;
8086            comps.push_back(zero);
8087            comps.push_back(zero);
8088            spv::IdImmediate coord = { true,
8089                builder.makeCompositeConstant(builder.makeVectorType(builder.makeIntType(32), 2), comps) };
8090            operands.push_back(coord);
8091            spv::IdImmediate imageOperands = { false, spv::ImageOperandsMask::MaskNone };
8092            imageOperands.word = imageOperands.word | (unsigned)signExtensionMask();
8093            if (sampler.isMultiSample()) {
8094                imageOperands.word = imageOperands.word | (unsigned)spv::ImageOperandsMask::Sample;
8095            }
8096            if (imageOperands.word != (unsigned)spv::ImageOperandsMask::MaskNone) {
8097                operands.push_back(imageOperands);
8098                if (sampler.isMultiSample()) {
8099                    spv::IdImmediate imageOperand = { true, *(opIt++) };
8100                    operands.push_back(imageOperand);
8101                }
8102            }
8103            spv::Id result = builder.createOp(spv::Op::OpImageRead, resultType(), operands);
8104            builder.setPrecision(result, precision);
8105            return result;
8106        }
8107
8108        if (cracked.attachmentEXT) {
8109            if (opIt != arguments.end()) {
8110                spv::IdImmediate sample = { true, *opIt };
8111                operands.push_back(sample);
8112            }
8113            spv::Id result = builder.createOp(spv::Op::OpColorAttachmentReadEXT, resultType(), operands);
8114            builder.addExtension(spv::E_SPV_EXT_shader_tile_image);
8115            builder.setPrecision(result, precision);
8116            return result;
8117        }
8118
8119        spv::IdImmediate coord = { true, *(opIt++) };
8120        operands.push_back(coord);
8121        if (node->getOp() == glslang::EOpImageLoad || node->getOp() == glslang::EOpImageLoadLod) {
8122            spv::ImageOperandsMask mask = spv::ImageOperandsMask::MaskNone;
8123            if (sampler.isMultiSample()) {
8124                mask = mask | spv::ImageOperandsMask::Sample;
8125            }
8126            if (cracked.lod) {
8127                builder.addExtension(spv::E_SPV_AMD_shader_image_load_store_lod);
8128                builder.addCapability(spv::Capability::ImageReadWriteLodAMD);
8129                mask = mask | spv::ImageOperandsMask::Lod;
8130            }
8131            mask = mask | TranslateImageOperands(TranslateCoherent(imageType));
8132            mask = (spv::ImageOperandsMask)(mask & ~spv::ImageOperandsMask::MakeTexelAvailableKHR);
8133            mask = mask | signExtensionMask();
8134            if (mask != spv::ImageOperandsMask::MaskNone) {
8135                spv::IdImmediate imageOperands = { false, (unsigned int)mask };
8136                operands.push_back(imageOperands);
8137            }
8138            if (anySet(mask, spv::ImageOperandsMask::Sample)) {
8139                spv::IdImmediate imageOperand = { true, *opIt++ };
8140                operands.push_back(imageOperand);
8141            }
8142            if (anySet(mask, spv::ImageOperandsMask::Lod)) {
8143                spv::IdImmediate imageOperand = { true, *opIt++ };
8144                operands.push_back(imageOperand);
8145            }
8146            if (anySet(mask, spv::ImageOperandsMask::MakeTexelVisibleKHR)) {
8147                spv::IdImmediate imageOperand = { true,
8148                                    builder.makeUintConstant(TranslateMemoryScope(TranslateCoherent(imageType))) };
8149                operands.push_back(imageOperand);
8150            }
8151
8152            if (builder.getImageTypeFormat(builder.getImageType(operands.front().word)) == spv::ImageFormat::Unknown)
8153                builder.addCapability(spv::Capability::StorageImageReadWithoutFormat);
8154
8155            std::vector<spv::Id> result(1, builder.createOp(spv::Op::OpImageRead, resultType(), operands));
8156            builder.setPrecision(result[0], precision);
8157
8158            // If needed, add a conversion constructor to the proper size.
8159            if (components != node->getType().getVectorSize())
8160                result[0] = builder.createConstructor(precision, result, convertGlslangToSpvType(node->getType()));
8161
8162            return result[0];
8163        } else if (node->getOp() == glslang::EOpImageStore || node->getOp() == glslang::EOpImageStoreLod) {
8164
8165            // Push the texel value before the operands
8166            if (sampler.isMultiSample() || cracked.lod) {
8167                spv::IdImmediate texel = { true, *(opIt + 1) };
8168                operands.push_back(texel);
8169            } else {
8170                spv::IdImmediate texel = { true, *opIt };
8171                operands.push_back(texel);
8172            }
8173
8174            spv::ImageOperandsMask mask = spv::ImageOperandsMask::MaskNone;
8175            if (sampler.isMultiSample()) {
8176                mask = mask | spv::ImageOperandsMask::Sample;
8177            }
8178            if (cracked.lod) {
8179                builder.addExtension(spv::E_SPV_AMD_shader_image_load_store_lod);
8180                builder.addCapability(spv::Capability::ImageReadWriteLodAMD);
8181                mask = mask | spv::ImageOperandsMask::Lod;
8182            }
8183            mask = mask | TranslateImageOperands(TranslateCoherent(imageType));
8184            mask = (spv::ImageOperandsMask)(mask & ~spv::ImageOperandsMask::MakeTexelVisibleKHR);
8185            mask = mask | signExtensionMask();
8186            if (mask != spv::ImageOperandsMask::MaskNone) {
8187                spv::IdImmediate imageOperands = { false, (unsigned int)mask };
8188                operands.push_back(imageOperands);
8189            }
8190            if (anySet(mask, spv::ImageOperandsMask::Sample)) {
8191                spv::IdImmediate imageOperand = { true, *opIt++ };
8192                operands.push_back(imageOperand);
8193            }
8194            if (anySet(mask, spv::ImageOperandsMask::Lod)) {
8195                spv::IdImmediate imageOperand = { true, *opIt++ };
8196                operands.push_back(imageOperand);
8197            }
8198            if (anySet(mask, spv::ImageOperandsMask::MakeTexelAvailableKHR)) {
8199                spv::IdImmediate imageOperand = { true,
8200                    builder.makeUintConstant(TranslateMemoryScope(TranslateCoherent(imageType))) };
8201                operands.push_back(imageOperand);
8202            }
8203
8204            builder.createNoResultOp(spv::Op::OpImageWrite, operands);
8205            if (builder.getImageTypeFormat(builder.getImageType(operands.front().word)) == spv::ImageFormat::Unknown)
8206                builder.addCapability(spv::Capability::StorageImageWriteWithoutFormat);
8207            return spv::NoResult;
8208        } else if (node->getOp() == glslang::EOpSparseImageLoad ||
8209                   node->getOp() == glslang::EOpSparseImageLoadLod) {
8210            builder.addCapability(spv::Capability::SparseResidency);
8211            if (builder.getImageTypeFormat(builder.getImageType(operands.front().word)) == spv::ImageFormat::Unknown)
8212                builder.addCapability(spv::Capability::StorageImageReadWithoutFormat);
8213
8214            spv::ImageOperandsMask mask = spv::ImageOperandsMask::MaskNone;
8215            if (sampler.isMultiSample()) {
8216                mask = mask | spv::ImageOperandsMask::Sample;
8217            }
8218            if (cracked.lod) {
8219                builder.addExtension(spv::E_SPV_AMD_shader_image_load_store_lod);
8220                builder.addCapability(spv::Capability::ImageReadWriteLodAMD);
8221
8222                mask = mask | spv::ImageOperandsMask::Lod;
8223            }
8224            mask = mask | TranslateImageOperands(TranslateCoherent(imageType));
8225            mask = (spv::ImageOperandsMask)(mask & ~spv::ImageOperandsMask::MakeTexelAvailableKHR);
8226            mask = mask | signExtensionMask();
8227            if (mask != spv::ImageOperandsMask::MaskNone) {
8228                spv::IdImmediate imageOperands = { false, (unsigned int)mask };
8229                operands.push_back(imageOperands);
8230            }
8231            if (anySet(mask, spv::ImageOperandsMask::Sample)) {
8232                spv::IdImmediate imageOperand = { true, *opIt++ };
8233                operands.push_back(imageOperand);
8234            }
8235            if (anySet(mask, spv::ImageOperandsMask::Lod)) {
8236                spv::IdImmediate imageOperand = { true, *opIt++ };
8237                operands.push_back(imageOperand);
8238            }
8239            if (anySet(mask, spv::ImageOperandsMask::MakeTexelVisibleKHR)) {
8240                spv::IdImmediate imageOperand = { true, builder.makeUintConstant(TranslateMemoryScope(
8241                    TranslateCoherent(imageType))) };
8242                operands.push_back(imageOperand);
8243            }
8244
8245            // Create the return type that was a special structure
8246            spv::Id texelOut = *opIt;
8247            spv::Id typeId0 = resultType();
8248            spv::Id typeId1 = builder.getDerefTypeId(texelOut);
8249            spv::Id resultTypeId = builder.makeStructResultType(typeId0, typeId1);
8250
8251            spv::Id resultId = builder.createOp(spv::Op::OpImageSparseRead, resultTypeId, operands);
8252
8253            // Decode the return type
8254            builder.createStore(builder.createCompositeExtract(resultId, typeId1, 1), texelOut);
8255            return builder.createCompositeExtract(resultId, typeId0, 0);
8256        } else {
8257            // Process image atomic operations
8258
8259            // GLSL "IMAGE_PARAMS" will involve in constructing an image texel pointer and this pointer,
8260            // as the first source operand, is required by SPIR-V atomic operations.
8261            // For non-MS, the sample value should be 0
8262            spv::IdImmediate sample = { true, sampler.isMultiSample() ? *(opIt++) : builder.makeUintConstant(0) };
8263            operands.push_back(sample);
8264
8265            spv::Id resultTypeId;
8266            glslang::TBasicType typeProxy = node->getBasicType();
8267            // imageAtomicStore has a void return type so base the pointer type on
8268            // the type of the value operand.
8269            if (node->getOp() == glslang::EOpImageAtomicStore) {
8270                resultTypeId = builder.makePointer(spv::StorageClass::Image, builder.getTypeId(*opIt));
8271                typeProxy = node->getAsAggregate()->getSequence()[0]->getAsTyped()->getType().getSampler().type;
8272            } else {
8273                resultTypeId = builder.makePointer(spv::StorageClass::Image, resultType());
8274            }
8275
8276            // EXT_descriptor_heap
8277            // For image atomic parameter, using untyped image texel pointer to carry on type metadata.
8278            spv::Op imgTexelOp = spv::Op::OpImageTexelPointer;
8279            if (node->getQualifier().isUsedByAtomic() &&
8280                imageType.getQualifier().builtIn == glslang::EbvResourceHeapEXT) {
8281                operands.insert(operands.begin(), {true, convertGlslangToSpvType(imageType)});
8282                imgTexelOp = spv::Op::OpUntypedImageTexelPointerEXT;
8283                resultTypeId = builder.makeUntypedPointer(spv::StorageClass::Image);
8284            }
8285
8286            spv::Id pointer = builder.createOp(imgTexelOp, resultTypeId, operands);
8287            if (imageType.getQualifier().nonUniform) {
8288                auto& extensions = glslangIntermediate->getRequestedExtensions();
8289                if (extensions.find("GL_EXT_descriptor_heap") == extensions.end()) {
8290                    builder.addDecoration(pointer, spv::Decoration::NonUniformEXT);
8291                }
8292            }
8293
8294            std::vector<spv::Id> operands;
8295            operands.push_back(pointer);
8296            for (; opIt != arguments.end(); ++opIt)
8297                operands.push_back(*opIt);
8298
8299            return createAtomicOperation(node->getOp(), precision, resultType(), operands, typeProxy,
8300                lvalueCoherentFlags, node->getType());
8301        }
8302    }
8303
8304    // Check for fragment mask functions other than queries
8305    if (cracked.fragMask) {
8306        assert(sampler.ms);
8307
8308        auto opIt = arguments.begin();
8309        std::vector<spv::Id> operands;
8310
8311        operands.push_back(params.sampler);
8312        ++opIt;
8313
8314        if (sampler.isSubpass()) {
8315            // add on the (0,0) coordinate
8316            spv::Id zero = builder.makeIntConstant(0);
8317            std::vector<spv::Id> comps;
8318            comps.push_back(zero);
8319            comps.push_back(zero);
8320            operands.push_back(builder.makeCompositeConstant(
8321                builder.makeVectorType(builder.makeIntType(32), 2), comps));
8322        }
8323
8324        for (; opIt != arguments.end(); ++opIt)
8325            operands.push_back(*opIt);
8326
8327        spv::Op fragMaskOp = spv::Op::OpNop;
8328        if (node->getOp() == glslang::EOpFragmentMaskFetch)
8329            fragMaskOp = spv::Op::OpFragmentMaskFetchAMD;
8330        else if (node->getOp() == glslang::EOpFragmentFetch)
8331            fragMaskOp = spv::Op::OpFragmentFetchAMD;
8332
8333        builder.addExtension(spv::E_SPV_AMD_shader_fragment_mask);
8334        builder.addCapability(spv::Capability::FragmentMaskAMD);
8335        return builder.createOp(fragMaskOp, resultType(), operands);
8336    }
8337
8338    // Check for texture functions other than queries
8339    bool sparse = node->isSparseTexture();
8340    bool imageFootprint = node->isImageFootprint();
8341    bool cubeCompare = sampler.dim == glslang::EsdCube && sampler.isArrayed() && sampler.isShadow();
8342
8343    // check for bias argument
8344    bool bias = false;
8345    if (! cracked.lod && ! cracked.grad && ! cracked.fetch && ! cubeCompare) {
8346        int nonBiasArgCount = 2;
8347        if (cracked.gather)
8348            ++nonBiasArgCount; // comp argument should be present when bias argument is present
8349
8350        if (f16ShadowCompare)
8351            ++nonBiasArgCount;
8352        if (cracked.offset)
8353            ++nonBiasArgCount;
8354        else if (cracked.offsets)
8355            ++nonBiasArgCount;
8356        if (cracked.grad)
8357            nonBiasArgCount += 2;
8358        if (cracked.lodClamp)
8359            ++nonBiasArgCount;
8360        if (sparse)
8361            ++nonBiasArgCount;
8362        if (imageFootprint)
8363            //Following three extra arguments
8364            // int granularity, bool coarse, out gl_TextureFootprint2DNV footprint
8365            nonBiasArgCount += 3;
8366        if ((int)arguments.size() > nonBiasArgCount)
8367            bias = true;
8368    }
8369
8370    if (cracked.gather) {
8371        const auto& sourceExtensions = glslangIntermediate->getRequestedExtensions();
8372        if (bias || cracked.lod ||
8373            sourceExtensions.find(glslang::E_GL_AMD_texture_gather_bias_lod) != sourceExtensions.end()) {
8374            builder.addExtension(spv::E_SPV_AMD_texture_gather_bias_lod);
8375            builder.addCapability(spv::Capability::ImageGatherBiasLodAMD);
8376        }
8377    }
8378
8379    // set the rest of the arguments
8380
8381    params.coords = arguments[1];
8382    int extraArgs = 0;
8383    bool noImplicitLod = false;
8384
8385    // sort out where Dref is coming from
8386    if (cubeCompare || f16ShadowCompare) {
8387        params.Dref = arguments[2];
8388        ++extraArgs;
8389    } else if (sampler.shadow && cracked.gather) {
8390        params.Dref = arguments[2];
8391        ++extraArgs;
8392    } else if (sampler.shadow) {
8393        std::vector<spv::Id> indexes;
8394        int dRefComp;
8395        if (cracked.proj)
8396            dRefComp = 2;  // "The resulting 3rd component of P in the shadow forms is used as Dref"
8397        else
8398            dRefComp = builder.getNumComponents(params.coords) - 1;
8399        indexes.push_back(dRefComp);
8400        params.Dref = builder.createCompositeExtract(params.coords,
8401            builder.getScalarTypeId(builder.getTypeId(params.coords)), indexes);
8402    }
8403
8404    // lod
8405    if (cracked.lod) {
8406        params.lod = arguments[2 + extraArgs];
8407        ++extraArgs;
8408    } else if (glslangIntermediate->getStage() != EShLangFragment &&
8409               !(glslangIntermediate->getStage() == EShLangCompute &&
8410                 glslangIntermediate->hasLayoutDerivativeModeNone())) {
8411        // we need to invent the default lod for an explicit lod instruction for a non-fragment stage
8412        noImplicitLod = true;
8413    }
8414
8415    // multisample
8416    if (sampler.isMultiSample()) {
8417        params.sample = arguments[2 + extraArgs]; // For MS, "sample" should be specified
8418        ++extraArgs;
8419    }
8420
8421    // gradient
8422    if (cracked.grad) {
8423        params.gradX = arguments[2 + extraArgs];
8424        params.gradY = arguments[3 + extraArgs];
8425        extraArgs += 2;
8426    }
8427
8428    // offset and offsets
8429    if (cracked.offset) {
8430        params.offset = arguments[2 + extraArgs];
8431        ++extraArgs;
8432    } else if (cracked.offsets) {
8433        params.offsets = arguments[2 + extraArgs];
8434        ++extraArgs;
8435    }
8436
8437    // lod clamp
8438    if (cracked.lodClamp) {
8439        params.lodClamp = arguments[2 + extraArgs];
8440        ++extraArgs;
8441    }
8442    // sparse
8443    if (sparse) {
8444        params.texelOut = arguments[2 + extraArgs];
8445        ++extraArgs;
8446    }
8447    // gather component
8448    if (cracked.gather && ! sampler.shadow) {
8449        // default component is 0, if missing, otherwise an argument
8450        if (2 + extraArgs < (int)arguments.size()) {
8451            params.component = arguments[2 + extraArgs];
8452            ++extraArgs;
8453        } else
8454            params.component = builder.makeIntConstant(0);
8455    }
8456    spv::Id  resultStruct = spv::NoResult;
8457    if (imageFootprint) {
8458        //Following three extra arguments
8459        // int granularity, bool coarse, out gl_TextureFootprint2DNV footprint
8460        params.granularity = arguments[2 + extraArgs];
8461        params.coarse = arguments[3 + extraArgs];
8462        resultStruct = arguments[4 + extraArgs];
8463        extraArgs += 3;
8464    }
8465
8466    // bias
8467    if (bias) {
8468        params.bias = arguments[2 + extraArgs];
8469        ++extraArgs;
8470    }
8471
8472    if (imageFootprint) {
8473        builder.addExtension(spv::E_SPV_NV_shader_image_footprint);
8474        builder.addCapability(spv::Capability::ImageFootprintNV);
8475
8476
8477        //resultStructType(OpenGL type) contains 5 elements:
8478        //struct gl_TextureFootprint2DNV {
8479        //    uvec2 anchor;
8480        //    uvec2 offset;
8481        //    uvec2 mask;
8482        //    uint  lod;
8483        //    uint  granularity;
8484        //};
8485        //or
8486        //struct gl_TextureFootprint3DNV {
8487        //    uvec3 anchor;
8488        //    uvec3 offset;
8489        //    uvec2 mask;
8490        //    uint  lod;
8491        //    uint  granularity;
8492        //};
8493        spv::Id resultStructType = builder.getContainedTypeId(builder.getTypeId(resultStruct));
8494        assert(builder.isStructType(resultStructType));
8495
8496        //resType (SPIR-V type) contains 6 elements:
8497        //Member 0 must be a Boolean type scalar(LOD),
8498        //Member 1 must be a vector of integer type, whose Signedness operand is 0(anchor),
8499        //Member 2 must be a vector of integer type, whose Signedness operand is 0(offset),
8500        //Member 3 must be a vector of integer type, whose Signedness operand is 0(mask),
8501        //Member 4 must be a scalar of integer type, whose Signedness operand is 0(lod),
8502        //Member 5 must be a scalar of integer type, whose Signedness operand is 0(granularity).
8503        std::vector<spv::Id> members;
8504        members.push_back(resultType());
8505        for (int i = 0; i < 5; i++) {
8506            members.push_back(builder.getContainedTypeId(resultStructType, i));
8507        }
8508        spv::Id resType = builder.makeStructType(members, {}, "ResType");
8509
8510        //call ImageFootprintNV
8511        spv::Id res = builder.createTextureCall(precision, resType, sparse, cracked.fetch, cracked.proj,
8512                                                cracked.gather, noImplicitLod, params, signExtensionMask());
8513
8514        //copy resType (SPIR-V type) to resultStructType(OpenGL type)
8515        for (int i = 0; i < 5; i++) {
8516            builder.clearAccessChain();
8517            builder.setAccessChainLValue(resultStruct);
8518
8519            //Accessing to a struct we created, no coherent flag is set
8520            spv::Builder::AccessChain::CoherentFlags flags;
8521            flags.clear();
8522
8523            builder.accessChainPush(builder.makeIntConstant(i), flags, 0);
8524            builder.accessChainStore(builder.createCompositeExtract(res, builder.getContainedTypeId(resType, i+1),
8525                i+1), TranslateNonUniformDecoration(imageType.getQualifier()));
8526        }
8527        return builder.createCompositeExtract(res, resultType(), 0);
8528    }
8529
8530    // projective component (might not to move)
8531    // GLSL: "The texture coordinates consumed from P, not including the last component of P,
8532    //       are divided by the last component of P."
8533    // SPIR-V:  "... (u [, v] [, w], q)... It may be a vector larger than needed, but all
8534    //          unused components will appear after all used components."
8535    if (cracked.proj) {
8536        int projSourceComp = builder.getNumComponents(params.coords) - 1;
8537        int projTargetComp;
8538        switch (sampler.dim) {
8539        case glslang::Esd1D:   projTargetComp = 1;              break;
8540        case glslang::Esd2D:   projTargetComp = 2;              break;
8541        case glslang::EsdRect: projTargetComp = 2;              break;
8542        default:               projTargetComp = projSourceComp; break;
8543        }
8544        // copy the projective coordinate if we have to
8545        if (projTargetComp != projSourceComp) {
8546            spv::Id projComp = builder.createCompositeExtract(params.coords,
8547                                    builder.getScalarTypeId(builder.getTypeId(params.coords)), projSourceComp);
8548            params.coords = builder.createCompositeInsert(projComp, params.coords,
8549                                    builder.getTypeId(params.coords), projTargetComp);
8550        }
8551    }
8552
8553    if (cracked.gather) {
8554        spv::GatherModes mode = spv::GatherModes::Max;
8555        switch (node->getOp()) {
8556        case glslang::EOpTextureGather4x1QCOM:
8557        case glslang::EOpTextureGather4x1OffsetQCOM:
8558            mode = spv::GatherModes::Gather4x1QCOM;
8559            break;
8560        case glslang::EOpTextureGatherDQCOM:
8561        case glslang::EOpTextureGatherDOffsetQCOM:
8562            mode = spv::GatherModes::GatherDQCOM;
8563            break;
8564        case glslang::EOpTextureGatherH2QCOM:
8565        case glslang::EOpTextureGatherH2OffsetQCOM:
8566            mode = spv::GatherModes::GatherH2QCOM;
8567            break;
8568        case glslang::EOpTextureGatherV2QCOM:
8569        case glslang::EOpTextureGatherV2OffsetQCOM:
8570            mode = spv::GatherModes::GatherV2QCOM;
8571            break;
8572        default:
8573            break;
8574        }
8575
8576        if (mode != spv::GatherModes::Max) {
8577          if (mode == spv::GatherModes::Gather4x1QCOM) {
8578            builder.addCapability(spv::Capability::ImageGatherLinearQCOM);
8579          } else {
8580            builder.addCapability(spv::Capability::ImageGatherExtendedModesQCOM);
8581          }
8582          builder.addExtension(spv::E_SPV_QCOM_image_processing3);
8583          params.gatherMode = builder.makeIntConstant(unsigned(mode));
8584        }
8585    }
8586
8587    // nonprivate
8588    if (imageType.getQualifier().nonprivate) {
8589        params.nonprivate = true;
8590    }
8591
8592    // volatile
8593    if (imageType.getQualifier().volatil) {
8594        params.volatil = true;
8595    }
8596
8597    if (imageType.getQualifier().nontemporal) {
8598        params.nontemporal = true;
8599    }
8600
8601    std::vector<spv::Id> result( 1,
8602        builder.createTextureCall(precision, resultType(), sparse, cracked.fetch, cracked.proj, cracked.gather,
8603                                  noImplicitLod, params, signExtensionMask())
8604    );
8605
8606    if (components != node->getType().getVectorSize())
8607        result[0] = builder.createConstructor(precision, result, convertGlslangToSpvType(node->getType()));
8608
8609    return result[0];
8610}
8611
8612spv::Id TGlslangToSpvTraverser::handleUserFunctionCall(const glslang::TIntermAggregate* node)
8613{
8614    // Grab the function's pointer from the previously created function
8615    spv::Function* function = functionMap[node->getName().c_str()];
8616    if (! function)
8617        return 0;
8618
8619    const glslang::TIntermSequence& glslangArgs = node->getSequence();
8620    const glslang::TQualifierList& qualifiers = node->getQualifierList();
8621
8622    //  See comments in makeFunctions() for details about the semantics for parameter passing.
8623    //
8624    // These imply we need a four step process:
8625    // 1. Evaluate the arguments
8626    // 2. Allocate and make copies of in, out, and inout arguments
8627    // 3. Make the call
8628    // 4. Copy back the results
8629
8630    // 1. Evaluate the arguments and their types
8631    std::vector<spv::Builder::AccessChain> lValues;
8632    std::vector<spv::Id> rValues;
8633    std::vector<const glslang::TType*> argTypes;
8634    for (int a = 0; a < (int)glslangArgs.size(); ++a) {
8635        argTypes.push_back(&glslangArgs[a]->getAsTyped()->getType());
8636        // build l-value
8637        builder.clearAccessChain();
8638        glslangArgs[a]->traverse(this);
8639        // keep outputs and pass-by-originals as l-values, evaluate others as r-values
8640        if (originalParam(qualifiers[a], *argTypes[a], function->hasImplicitThis() && a == 0) ||
8641            writableParam(qualifiers[a])) {
8642            // save l-value
8643            lValues.push_back(builder.getAccessChain());
8644        } else {
8645            // process r-value
8646            rValues.push_back(accessChainLoad(*argTypes.back()));
8647        }
8648    }
8649
8650    // Reset source location to the function call location after argument evaluation
8651    builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename());
8652
8653    // 2. Allocate space for anything needing a copy, and if it's "in" or "inout"
8654    // copy the original into that space.
8655    //
8656    // Also, build up the list of actual arguments to pass in for the call
8657    int lValueCount = 0;
8658    int rValueCount = 0;
8659    std::vector<spv::Id> spvArgs;
8660    for (int a = 0; a < (int)glslangArgs.size(); ++a) {
8661        spv::Id arg;
8662        if (originalParam(qualifiers[a], *argTypes[a], function->hasImplicitThis() && a == 0)) {
8663            builder.setAccessChain(lValues[lValueCount]);
8664            arg = builder.accessChainGetLValue();
8665            ++lValueCount;
8666        } else if (writableParam(qualifiers[a])) {
8667            // need space to hold the copy
8668            arg = builder.createVariable(function->getParamPrecision(a), spv::StorageClass::Function,
8669                builder.getContainedTypeId(function->getParamType(a)), "param");
8670            if (qualifiers[a] == glslang::EvqIn || qualifiers[a] == glslang::EvqInOut) {
8671                // need to copy the input into output space
8672                builder.setAccessChain(lValues[lValueCount]);
8673                spv::Id copy = accessChainLoad(*argTypes[a]);
8674                builder.clearAccessChain();
8675                builder.setAccessChainLValue(arg);
8676                multiTypeStore(*argTypes[a], copy);
8677            }
8678            ++lValueCount;
8679        } else {
8680            // process r-value, which involves a copy for a type mismatch
8681            if (function->getParamType(a) != builder.getTypeId(rValues[rValueCount]) ||
8682                TranslatePrecisionDecoration(*argTypes[a]) != function->getParamPrecision(a))
8683            {
8684                spv::Id argCopy = builder.createVariable(function->getParamPrecision(a), spv::StorageClass::Function, function->getParamType(a), "arg");
8685                builder.clearAccessChain();
8686                builder.setAccessChainLValue(argCopy);
8687                multiTypeStore(*argTypes[a], rValues[rValueCount]);
8688                arg = builder.createLoad(argCopy, function->getParamPrecision(a));
8689            } else
8690                arg = rValues[rValueCount];
8691            ++rValueCount;
8692        }
8693        spvArgs.push_back(arg);
8694    }
8695
8696    // 3. Make the call.
8697    spv::Id result = builder.createFunctionCall(function, spvArgs);
8698    builder.setPrecision(result, TranslatePrecisionDecoration(node->getType()));
8699    builder.addDecoration(result, TranslateNonUniformDecoration(node->getType().getQualifier()));
8700
8701    // 4. Copy back out an "out" arguments.
8702    lValueCount = 0;
8703    for (int a = 0; a < (int)glslangArgs.size(); ++a) {
8704        if (originalParam(qualifiers[a], *argTypes[a], function->hasImplicitThis() && a == 0))
8705            ++lValueCount;
8706        else if (writableParam(qualifiers[a])) {
8707            if (qualifiers[a] == glslang::EvqOut || qualifiers[a] == glslang::EvqInOut) {
8708                spv::Id copy = builder.createLoad(spvArgs[a], spv::NoPrecision);
8709                builder.addDecoration(copy, TranslateNonUniformDecoration(argTypes[a]->getQualifier()));
8710                builder.setAccessChain(lValues[lValueCount]);
8711                multiTypeStore(*argTypes[a], copy);
8712            }
8713            ++lValueCount;
8714        }
8715    }
8716
8717    return result;
8718}
8719
8720// Translate AST operation to SPV operation, already having SPV-based operands/types.
8721spv::Id TGlslangToSpvTraverser::createBinaryOperation(glslang::TOperator op, OpDecorations& decorations,
8722                                                      spv::Id typeId, spv::Id left, spv::Id right,
8723                                                      glslang::TBasicType typeProxy, bool reduceComparison)
8724{
8725    bool isUnsigned = isTypeUnsignedInt(typeProxy);
8726    bool isFloat = isTypeFloat(typeProxy);
8727    bool isBool = typeProxy == glslang::EbtBool;
8728
8729    spv::Op binOp = spv::Op::OpNop;
8730    bool needMatchingVectors = true;  // for non-matrix ops, would a scalar need to smear to match a vector?
8731    bool comparison = false;
8732
8733    switch (op) {
8734    case glslang::EOpAdd:
8735    case glslang::EOpAddAssign:
8736        if (isFloat)
8737            binOp = spv::Op::OpFAdd;
8738        else
8739            binOp = spv::Op::OpIAdd;
8740        break;
8741    case glslang::EOpSub:
8742    case glslang::EOpSubAssign:
8743        if (isFloat)
8744            binOp = spv::Op::OpFSub;
8745        else
8746            binOp = spv::Op::OpISub;
8747        break;
8748    case glslang::EOpMul:
8749    case glslang::EOpMulAssign:
8750        if (isFloat)
8751            binOp = spv::Op::OpFMul;
8752        else
8753            binOp = spv::Op::OpIMul;
8754        break;
8755    case glslang::EOpVectorTimesScalar:
8756    case glslang::EOpVectorTimesScalarAssign:
8757        if (isFloat && (builder.isVector(left) || builder.isVector(right) || builder.isCooperativeVector(left) || builder.isCooperativeVector(right))) {
8758            if (builder.isVector(right) || builder.isCooperativeVector(right))
8759                std::swap(left, right);
8760            assert(builder.isScalar(right));
8761            needMatchingVectors = false;
8762            binOp = spv::Op::OpVectorTimesScalar;
8763        } else if (isFloat) {
8764            binOp = spv::Op::OpFMul;
8765        } else if (builder.isCooperativeVector(left) || builder.isCooperativeVector(right)) {
8766            if (builder.isCooperativeVector(right))
8767                std::swap(left, right);
8768            assert(builder.isScalar(right));
8769            // Construct a cooperative vector from the scalar
8770            right = builder.createCompositeConstruct(builder.getTypeId(left), { right });
8771            binOp = spv::Op::OpIMul;
8772        } else {
8773            binOp = spv::Op::OpIMul;
8774        }
8775        break;
8776    case glslang::EOpVectorTimesMatrix:
8777    case glslang::EOpVectorTimesMatrixAssign:
8778        binOp = spv::Op::OpVectorTimesMatrix;
8779        break;
8780    case glslang::EOpMatrixTimesVector:
8781        binOp = spv::Op::OpMatrixTimesVector;
8782        break;
8783    case glslang::EOpMatrixTimesScalar:
8784    case glslang::EOpMatrixTimesScalarAssign:
8785        binOp = spv::Op::OpMatrixTimesScalar;
8786        break;
8787    case glslang::EOpMatrixTimesMatrix:
8788    case glslang::EOpMatrixTimesMatrixAssign:
8789        binOp = spv::Op::OpMatrixTimesMatrix;
8790        break;
8791    case glslang::EOpOuterProduct:
8792        binOp = spv::Op::OpOuterProduct;
8793        needMatchingVectors = false;
8794        break;
8795    case glslang::EOpDot:
8796        if (typeProxy == glslang::EbtBFloat16) {
8797            builder.addExtension(spv::E_SPV_KHR_bfloat16);
8798            builder.addCapability(spv::Capability::BFloat16DotProductKHR);
8799        }
8800        binOp = spv::Op::OpDot;
8801        break;
8802
8803    case glslang::EOpDiv:
8804    case glslang::EOpDivAssign:
8805        if (isFloat)
8806            binOp = spv::Op::OpFDiv;
8807        else if (isUnsigned)
8808            binOp = spv::Op::OpUDiv;
8809        else
8810            binOp = spv::Op::OpSDiv;
8811        break;
8812    case glslang::EOpMod:
8813    case glslang::EOpModAssign:
8814        if (isFloat)
8815            binOp = spv::Op::OpFMod;
8816        else if (isUnsigned)
8817            binOp = spv::Op::OpUMod;
8818        else
8819            binOp = spv::Op::OpSMod;
8820        break;
8821    case glslang::EOpRightShift:
8822    case glslang::EOpRightShiftAssign:
8823        if (isUnsigned)
8824            binOp = spv::Op::OpShiftRightLogical;
8825        else
8826            binOp = spv::Op::OpShiftRightArithmetic;
8827        break;
8828    case glslang::EOpLeftShift:
8829    case glslang::EOpLeftShiftAssign:
8830        binOp = spv::Op::OpShiftLeftLogical;
8831        break;
8832    case glslang::EOpAnd:
8833    case glslang::EOpAndAssign:
8834        binOp = spv::Op::OpBitwiseAnd;
8835        break;
8836    case glslang::EOpLogicalAnd:
8837        needMatchingVectors = false;
8838        binOp = spv::Op::OpLogicalAnd;
8839        break;
8840    case glslang::EOpInclusiveOr:
8841    case glslang::EOpInclusiveOrAssign:
8842        binOp = spv::Op::OpBitwiseOr;
8843        break;
8844    case glslang::EOpLogicalOr:
8845        needMatchingVectors = false;
8846        binOp = spv::Op::OpLogicalOr;
8847        break;
8848    case glslang::EOpExclusiveOr:
8849    case glslang::EOpExclusiveOrAssign:
8850        binOp = spv::Op::OpBitwiseXor;
8851        break;
8852    case glslang::EOpLogicalXor:
8853        needMatchingVectors = false;
8854        binOp = spv::Op::OpLogicalNotEqual;
8855        break;
8856
8857    case glslang::EOpAbsDifference:
8858        binOp = isUnsigned ? spv::Op::OpAbsUSubINTEL : spv::Op::OpAbsISubINTEL;
8859        break;
8860
8861    case glslang::EOpAddSaturate:
8862        binOp = isUnsigned ? spv::Op::OpUAddSatINTEL : spv::Op::OpIAddSatINTEL;
8863        break;
8864
8865    case glslang::EOpSubSaturate:
8866        binOp = isUnsigned ? spv::Op::OpUSubSatINTEL : spv::Op::OpISubSatINTEL;
8867        break;
8868
8869    case glslang::EOpAverage:
8870        binOp = isUnsigned ? spv::Op::OpUAverageINTEL : spv::Op::OpIAverageINTEL;
8871        break;
8872
8873    case glslang::EOpAverageRounded:
8874        binOp = isUnsigned ? spv::Op::OpUAverageRoundedINTEL : spv::Op::OpIAverageRoundedINTEL;
8875        break;
8876
8877    case glslang::EOpMul32x16:
8878        binOp = isUnsigned ? spv::Op::OpUMul32x16INTEL : spv::Op::OpIMul32x16INTEL;
8879        break;
8880
8881    case glslang::EOpExpectEXT:
8882        binOp = spv::Op::OpExpectKHR;
8883        break;
8884
8885    case glslang::EOpLessThan:
8886    case glslang::EOpGreaterThan:
8887    case glslang::EOpLessThanEqual:
8888    case glslang::EOpGreaterThanEqual:
8889    case glslang::EOpEqual:
8890    case glslang::EOpNotEqual:
8891    case glslang::EOpVectorEqual:
8892    case glslang::EOpVectorNotEqual:
8893        comparison = true;
8894        break;
8895    default:
8896        break;
8897    }
8898
8899    // handle mapped binary operations (should be non-comparison)
8900    if (binOp != spv::Op::OpNop) {
8901        assert(comparison == false);
8902        if (builder.isMatrix(left) || builder.isMatrix(right) ||
8903            builder.isCooperativeMatrix(left) || builder.isCooperativeMatrix(right))
8904            return createBinaryMatrixOperation(binOp, decorations, typeId, left, right);
8905
8906        // No matrix involved; make both operands be the same number of components, if needed
8907        if (needMatchingVectors)
8908            builder.promoteScalar(decorations.precision, left, right);
8909
8910        spv::Id result = builder.createBinOp(binOp, typeId, left, right);
8911        decorations.addNoContraction(builder, result);
8912        decorations.addNonUniform(builder, result);
8913        return builder.setPrecision(result, decorations.precision);
8914    }
8915
8916    if (! comparison)
8917        return 0;
8918
8919    // Handle comparison instructions
8920
8921    if (reduceComparison && (op == glslang::EOpEqual || op == glslang::EOpNotEqual)
8922                         && (builder.isVector(left) || builder.isMatrix(left) || builder.isAggregate(left))) {
8923        spv::Id result = builder.createCompositeCompare(decorations.precision, left, right, op == glslang::EOpEqual);
8924        decorations.addNonUniform(builder, result);
8925        return result;
8926    }
8927
8928    switch (op) {
8929    case glslang::EOpLessThan:
8930        if (isFloat)
8931            binOp = spv::Op::OpFOrdLessThan;
8932        else if (isUnsigned)
8933            binOp = spv::Op::OpULessThan;
8934        else
8935            binOp = spv::Op::OpSLessThan;
8936        break;
8937    case glslang::EOpGreaterThan:
8938        if (isFloat)
8939            binOp = spv::Op::OpFOrdGreaterThan;
8940        else if (isUnsigned)
8941            binOp = spv::Op::OpUGreaterThan;
8942        else
8943            binOp = spv::Op::OpSGreaterThan;
8944        break;
8945    case glslang::EOpLessThanEqual:
8946        if (isFloat)
8947            binOp = spv::Op::OpFOrdLessThanEqual;
8948        else if (isUnsigned)
8949            binOp = spv::Op::OpULessThanEqual;
8950        else
8951            binOp = spv::Op::OpSLessThanEqual;
8952        break;
8953    case glslang::EOpGreaterThanEqual:
8954        if (isFloat)
8955            binOp = spv::Op::OpFOrdGreaterThanEqual;
8956        else if (isUnsigned)
8957            binOp = spv::Op::OpUGreaterThanEqual;
8958        else
8959            binOp = spv::Op::OpSGreaterThanEqual;
8960        break;
8961    case glslang::EOpEqual:
8962    case glslang::EOpVectorEqual:
8963        if (isFloat)
8964            binOp = spv::Op::OpFOrdEqual;
8965        else if (isBool)
8966            binOp = spv::Op::OpLogicalEqual;
8967        else
8968            binOp = spv::Op::OpIEqual;
8969        break;
8970    case glslang::EOpNotEqual:
8971    case glslang::EOpVectorNotEqual:
8972        if (isFloat)
8973            binOp = spv::Op::OpFUnordNotEqual;
8974        else if (isBool)
8975            binOp = spv::Op::OpLogicalNotEqual;
8976        else
8977            binOp = spv::Op::OpINotEqual;
8978        break;
8979    default:
8980        break;
8981    }
8982
8983    if (binOp != spv::Op::OpNop) {
8984        spv::Id result = builder.createBinOp(binOp, typeId, left, right);
8985        decorations.addNoContraction(builder, result);
8986        decorations.addNonUniform(builder, result);
8987        return builder.setPrecision(result, decorations.precision);
8988    }
8989
8990    return 0;
8991}
8992
8993//
8994// Translate AST matrix operation to SPV operation, already having SPV-based operands/types.
8995// These can be any of:
8996//
8997//   matrix * scalar
8998//   scalar * matrix
8999//   matrix * matrix     linear algebraic
9000//   matrix * vector
9001//   vector * matrix
9002//   matrix * matrix     componentwise
9003//   matrix op matrix    op in {+, -, /}
9004//   matrix op scalar    op in {+, -, /}
9005//   scalar op matrix    op in {+, -, /}
9006//
9007spv::Id TGlslangToSpvTraverser::createBinaryMatrixOperation(spv::Op op, OpDecorations& decorations, spv::Id typeId,
9008                                                            spv::Id left, spv::Id right)
9009{
9010    bool firstClass = true;
9011
9012    // First, handle first-class matrix operations (* and matrix/scalar)
9013    switch (op) {
9014    case spv::Op::OpFDiv:
9015        if (builder.isMatrix(left) && builder.isScalar(right)) {
9016            // turn matrix / scalar into a multiply...
9017            spv::Id resultType = builder.getTypeId(right);
9018            right = builder.createBinOp(spv::Op::OpFDiv, resultType, builder.makeFpConstant(resultType, 1.0), right);
9019            op = spv::Op::OpMatrixTimesScalar;
9020        } else
9021            firstClass = false;
9022        break;
9023    case spv::Op::OpMatrixTimesScalar:
9024        if (builder.isMatrix(right) || builder.isCooperativeMatrix(right))
9025            std::swap(left, right);
9026        assert(builder.isScalar(right));
9027        break;
9028    case spv::Op::OpVectorTimesMatrix:
9029        assert(builder.isVector(left));
9030        assert(builder.isMatrix(right));
9031        break;
9032    case spv::Op::OpMatrixTimesVector:
9033        assert(builder.isMatrix(left));
9034        assert(builder.isVector(right));
9035        break;
9036    case spv::Op::OpMatrixTimesMatrix:
9037        assert(builder.isMatrix(left));
9038        assert(builder.isMatrix(right));
9039        break;
9040    default:
9041        firstClass = false;
9042        break;
9043    }
9044
9045    if (builder.isCooperativeMatrix(left) || builder.isCooperativeMatrix(right))
9046        firstClass = true;
9047
9048    if (firstClass) {
9049        spv::Id result = builder.createBinOp(op, typeId, left, right);
9050        decorations.addNoContraction(builder, result);
9051        decorations.addNonUniform(builder, result);
9052        return builder.setPrecision(result, decorations.precision);
9053    }
9054
9055    // Handle component-wise +, -, *, %, and / for all combinations of type.
9056    // The result type of all of them is the same type as the (a) matrix operand.
9057    // The algorithm is to:
9058    //   - break the matrix(es) into vectors
9059    //   - smear any scalar to a vector
9060    //   - do vector operations
9061    //   - make a matrix out the vector results
9062    switch (op) {
9063    case spv::Op::OpFAdd:
9064    case spv::Op::OpFSub:
9065    case spv::Op::OpFDiv:
9066    case spv::Op::OpFMod:
9067    case spv::Op::OpFMul:
9068    {
9069        // one time set up...
9070        bool  leftMat = builder.isMatrix(left);
9071        bool rightMat = builder.isMatrix(right);
9072        unsigned int numCols = leftMat ? builder.getNumColumns(left) : builder.getNumColumns(right);
9073        int numRows = leftMat ? builder.getNumRows(left) : builder.getNumRows(right);
9074        spv::Id scalarType = builder.getScalarTypeId(typeId);
9075        spv::Id vecType = builder.makeVectorType(scalarType, numRows);
9076        std::vector<spv::Id> results;
9077        spv::Id smearVec = spv::NoResult;
9078        if (builder.isScalar(left))
9079            smearVec = builder.smearScalar(decorations.precision, left, vecType);
9080        else if (builder.isScalar(right))
9081            smearVec = builder.smearScalar(decorations.precision, right, vecType);
9082
9083        // do each vector op
9084        for (unsigned int c = 0; c < numCols; ++c) {
9085            std::vector<unsigned int> indexes;
9086            indexes.push_back(c);
9087            spv::Id  leftVec =  leftMat ? builder.createCompositeExtract( left, vecType, indexes) : smearVec;
9088            spv::Id rightVec = rightMat ? builder.createCompositeExtract(right, vecType, indexes) : smearVec;
9089            spv::Id result = builder.createBinOp(op, vecType, leftVec, rightVec);
9090            decorations.addNoContraction(builder, result);
9091            decorations.addNonUniform(builder, result);
9092            results.push_back(builder.setPrecision(result, decorations.precision));
9093        }
9094
9095        // put the pieces together
9096        spv::Id result = builder.setPrecision(builder.createCompositeConstruct(typeId, results), decorations.precision);
9097        decorations.addNonUniform(builder, result);
9098        return result;
9099    }
9100    default:
9101        assert(0);
9102        return spv::NoResult;
9103    }
9104}
9105
9106spv::Id TGlslangToSpvTraverser::createUnaryOperation(glslang::TOperator op, OpDecorations& decorations, spv::Id typeId,
9107    spv::Id operand, glslang::TBasicType typeProxy, const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags,
9108    const glslang::TType &opType)
9109{
9110    spv::Op unaryOp = spv::Op::OpNop;
9111    int extBuiltins = -1;
9112    int libCall = -1;
9113    bool isUnsigned = isTypeUnsignedInt(typeProxy);
9114    bool isFloat = isTypeFloat(typeProxy);
9115
9116    switch (op) {
9117    case glslang::EOpNegative:
9118        if (isFloat) {
9119            unaryOp = spv::Op::OpFNegate;
9120            if (builder.isMatrixType(typeId))
9121                return createUnaryMatrixOperation(unaryOp, decorations, typeId, operand, typeProxy);
9122        } else
9123            unaryOp = spv::Op::OpSNegate;
9124        break;
9125
9126    case glslang::EOpLogicalNot:
9127    case glslang::EOpVectorLogicalNot:
9128        unaryOp = spv::Op::OpLogicalNot;
9129        break;
9130    case glslang::EOpBitwiseNot:
9131        unaryOp = spv::Op::OpNot;
9132        break;
9133
9134    case glslang::EOpDeterminant:
9135        libCall = spv::GLSLstd450Determinant;
9136        break;
9137    case glslang::EOpMatrixInverse:
9138        libCall = spv::GLSLstd450MatrixInverse;
9139        break;
9140    case glslang::EOpTranspose:
9141        unaryOp = spv::Op::OpTranspose;
9142        break;
9143
9144    case glslang::EOpRadians:
9145        libCall = spv::GLSLstd450Radians;
9146        break;
9147    case glslang::EOpDegrees:
9148        libCall = spv::GLSLstd450Degrees;
9149        break;
9150    case glslang::EOpSin:
9151        libCall = spv::GLSLstd450Sin;
9152        break;
9153    case glslang::EOpCos:
9154        libCall = spv::GLSLstd450Cos;
9155        break;
9156    case glslang::EOpTan:
9157        libCall = spv::GLSLstd450Tan;
9158        break;
9159    case glslang::EOpAcos:
9160        libCall = spv::GLSLstd450Acos;
9161        break;
9162    case glslang::EOpAsin:
9163        libCall = spv::GLSLstd450Asin;
9164        break;
9165    case glslang::EOpAtan:
9166        libCall = spv::GLSLstd450Atan;
9167        break;
9168
9169    case glslang::EOpAcosh:
9170        libCall = spv::GLSLstd450Acosh;
9171        break;
9172    case glslang::EOpAsinh:
9173        libCall = spv::GLSLstd450Asinh;
9174        break;
9175    case glslang::EOpAtanh:
9176        libCall = spv::GLSLstd450Atanh;
9177        break;
9178    case glslang::EOpTanh:
9179        libCall = spv::GLSLstd450Tanh;
9180        break;
9181    case glslang::EOpCosh:
9182        libCall = spv::GLSLstd450Cosh;
9183        break;
9184    case glslang::EOpSinh:
9185        libCall = spv::GLSLstd450Sinh;
9186        break;
9187
9188    case glslang::EOpLength:
9189        libCall = spv::GLSLstd450Length;
9190        break;
9191    case glslang::EOpNormalize:
9192        libCall = spv::GLSLstd450Normalize;
9193        break;
9194
9195    case glslang::EOpExp:
9196        libCall = spv::GLSLstd450Exp;
9197        break;
9198    case glslang::EOpLog:
9199        libCall = spv::GLSLstd450Log;
9200        break;
9201    case glslang::EOpExp2:
9202        libCall = spv::GLSLstd450Exp2;
9203        break;
9204    case glslang::EOpLog2:
9205        libCall = spv::GLSLstd450Log2;
9206        break;
9207    case glslang::EOpSqrt:
9208        libCall = spv::GLSLstd450Sqrt;
9209        break;
9210    case glslang::EOpInverseSqrt:
9211        libCall = spv::GLSLstd450InverseSqrt;
9212        break;
9213
9214    case glslang::EOpFloor:
9215        libCall = spv::GLSLstd450Floor;
9216        break;
9217    case glslang::EOpTrunc:
9218        libCall = spv::GLSLstd450Trunc;
9219        break;
9220    case glslang::EOpRound:
9221        libCall = spv::GLSLstd450Round;
9222        break;
9223    case glslang::EOpRoundEven:
9224        libCall = spv::GLSLstd450RoundEven;
9225        break;
9226    case glslang::EOpCeil:
9227        libCall = spv::GLSLstd450Ceil;
9228        break;
9229    case glslang::EOpFract:
9230        libCall = spv::GLSLstd450Fract;
9231        break;
9232
9233    case glslang::EOpIsNan:
9234        unaryOp = spv::Op::OpIsNan;
9235        break;
9236    case glslang::EOpIsInf:
9237        unaryOp = spv::Op::OpIsInf;
9238        break;
9239    case glslang::EOpIsFinite:
9240        unaryOp = spv::Op::OpIsFinite;
9241        break;
9242
9243    case glslang::EOpFloatBitsToInt:
9244    case glslang::EOpFloatBitsToUint:
9245    case glslang::EOpIntBitsToFloat:
9246    case glslang::EOpUintBitsToFloat:
9247    case glslang::EOpDoubleBitsToInt64:
9248    case glslang::EOpDoubleBitsToUint64:
9249    case glslang::EOpInt64BitsToDouble:
9250    case glslang::EOpUint64BitsToDouble:
9251    case glslang::EOpFloat16BitsToInt16:
9252    case glslang::EOpFloat16BitsToUint16:
9253    case glslang::EOpInt16BitsToFloat16:
9254    case glslang::EOpUint16BitsToFloat16:
9255        unaryOp = spv::Op::OpBitcast;
9256        break;
9257
9258    case glslang::EOpPackSnorm2x16:
9259        libCall = spv::GLSLstd450PackSnorm2x16;
9260        break;
9261    case glslang::EOpUnpackSnorm2x16:
9262        libCall = spv::GLSLstd450UnpackSnorm2x16;
9263        break;
9264    case glslang::EOpPackUnorm2x16:
9265        libCall = spv::GLSLstd450PackUnorm2x16;
9266        break;
9267    case glslang::EOpUnpackUnorm2x16:
9268        libCall = spv::GLSLstd450UnpackUnorm2x16;
9269        break;
9270    case glslang::EOpPackHalf2x16:
9271        libCall = spv::GLSLstd450PackHalf2x16;
9272        break;
9273    case glslang::EOpUnpackHalf2x16:
9274        libCall = spv::GLSLstd450UnpackHalf2x16;
9275        break;
9276    case glslang::EOpPackSnorm4x8:
9277        libCall = spv::GLSLstd450PackSnorm4x8;
9278        break;
9279    case glslang::EOpUnpackSnorm4x8:
9280        libCall = spv::GLSLstd450UnpackSnorm4x8;
9281        break;
9282    case glslang::EOpPackUnorm4x8:
9283        libCall = spv::GLSLstd450PackUnorm4x8;
9284        break;
9285    case glslang::EOpUnpackUnorm4x8:
9286        libCall = spv::GLSLstd450UnpackUnorm4x8;
9287        break;
9288    case glslang::EOpPackDouble2x32:
9289        libCall = spv::GLSLstd450PackDouble2x32;
9290        break;
9291    case glslang::EOpUnpackDouble2x32:
9292        libCall = spv::GLSLstd450UnpackDouble2x32;
9293        break;
9294
9295    case glslang::EOpPackInt2x32:
9296    case glslang::EOpUnpackInt2x32:
9297    case glslang::EOpPackUint2x32:
9298    case glslang::EOpUnpackUint2x32:
9299    case glslang::EOpPack16:
9300    case glslang::EOpPack32:
9301    case glslang::EOpPack64:
9302    case glslang::EOpUnpack32:
9303    case glslang::EOpUnpack16:
9304    case glslang::EOpUnpack8:
9305    case glslang::EOpPackInt2x16:
9306    case glslang::EOpUnpackInt2x16:
9307    case glslang::EOpPackUint2x16:
9308    case glslang::EOpUnpackUint2x16:
9309    case glslang::EOpPackInt4x16:
9310    case glslang::EOpUnpackInt4x16:
9311    case glslang::EOpPackUint4x16:
9312    case glslang::EOpUnpackUint4x16:
9313    case glslang::EOpPackFloat2x16:
9314    case glslang::EOpUnpackFloat2x16:
9315    case glslang::EOpUnpackFloat2xE2M1:
9316    case glslang::EOpUnpackFloat4xE2M1:
9317    case glslang::EOpUnpackFloat8xE2M1:
9318    case glslang::EOpUnpackFloat16xE2M1:
9319    case glslang::EOpPackFloat2xE2M1:
9320    case glslang::EOpPackFloat4xE2M1:
9321    case glslang::EOpPackFloat8xE2M1:
9322    case glslang::EOpPackFloat16xE2M1:
9323    case glslang::EOpUnpackFloat4xE3M2:
9324    case glslang::EOpUnpackFloat8xE3M2:
9325    case glslang::EOpUnpackFloat16xE3M2:
9326    case glslang::EOpPackFloat4xE3M2:
9327    case glslang::EOpPackFloat8xE3M2:
9328    case glslang::EOpPackFloat16xE3M2:
9329    case glslang::EOpUnpackFloat4xE2M3:
9330    case glslang::EOpUnpackFloat8xE2M3:
9331    case glslang::EOpUnpackFloat16xE2M3:
9332    case glslang::EOpPackFloat4xE2M3:
9333    case glslang::EOpPackFloat8xE2M3:
9334    case glslang::EOpPackFloat16xE2M3:
9335
9336        unaryOp = spv::Op::OpBitcast;
9337        break;
9338
9339    case glslang::EOpDPdx:
9340        if (typeProxy == glslang::EbtFloat16)
9341            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
9342        unaryOp = spv::Op::OpDPdx;
9343        break;
9344    case glslang::EOpDPdy:
9345        if (typeProxy == glslang::EbtFloat16)
9346            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
9347        unaryOp = spv::Op::OpDPdy;
9348        break;
9349    case glslang::EOpFwidth:
9350        if (typeProxy == glslang::EbtFloat16)
9351            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
9352        unaryOp = spv::Op::OpFwidth;
9353        break;
9354
9355    case glslang::EOpAny:
9356        unaryOp = spv::Op::OpAny;
9357        break;
9358    case glslang::EOpAll:
9359        unaryOp = spv::Op::OpAll;
9360        break;
9361
9362    case glslang::EOpAbs:
9363        if (isFloat)
9364            libCall = spv::GLSLstd450FAbs;
9365        else
9366            libCall = spv::GLSLstd450SAbs;
9367        break;
9368    case glslang::EOpSign:
9369        if (isFloat)
9370            libCall = spv::GLSLstd450FSign;
9371        else
9372            libCall = spv::GLSLstd450SSign;
9373        break;
9374
9375    case glslang::EOpDPdxFine:
9376        if (typeProxy == glslang::EbtFloat16)
9377            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
9378        unaryOp = spv::Op::OpDPdxFine;
9379        break;
9380    case glslang::EOpDPdyFine:
9381        if (typeProxy == glslang::EbtFloat16)
9382            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
9383        unaryOp = spv::Op::OpDPdyFine;
9384        break;
9385    case glslang::EOpFwidthFine:
9386        if (typeProxy == glslang::EbtFloat16)
9387            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
9388        unaryOp = spv::Op::OpFwidthFine;
9389        break;
9390    case glslang::EOpDPdxCoarse:
9391        if (typeProxy == glslang::EbtFloat16)
9392            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
9393        unaryOp = spv::Op::OpDPdxCoarse;
9394        break;
9395    case glslang::EOpDPdyCoarse:
9396        if (typeProxy == glslang::EbtFloat16)
9397            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
9398        unaryOp = spv::Op::OpDPdyCoarse;
9399        break;
9400    case glslang::EOpFwidthCoarse:
9401        if (typeProxy == glslang::EbtFloat16)
9402            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
9403        unaryOp = spv::Op::OpFwidthCoarse;
9404        break;
9405    case glslang::EOpRayQueryProceed:
9406        unaryOp = spv::Op::OpRayQueryProceedKHR;
9407        break;
9408    case glslang::EOpRayQueryGetRayTMin:
9409        unaryOp = spv::Op::OpRayQueryGetRayTMinKHR;
9410        break;
9411    case glslang::EOpRayQueryGetRayFlags:
9412        unaryOp = spv::Op::OpRayQueryGetRayFlagsKHR;
9413        break;
9414    case glslang::EOpRayQueryGetWorldRayOrigin:
9415        unaryOp = spv::Op::OpRayQueryGetWorldRayOriginKHR;
9416        break;
9417    case glslang::EOpRayQueryGetWorldRayDirection:
9418        unaryOp = spv::Op::OpRayQueryGetWorldRayDirectionKHR;
9419        break;
9420    case glslang::EOpRayQueryGetIntersectionCandidateAABBOpaque:
9421        unaryOp = spv::Op::OpRayQueryGetIntersectionCandidateAABBOpaqueKHR;
9422        break;
9423    case glslang::EOpInterpolateAtCentroid:
9424        if (typeProxy == glslang::EbtFloat16)
9425            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
9426        libCall = spv::GLSLstd450InterpolateAtCentroid;
9427        break;
9428    case glslang::EOpAtomicCounterIncrement:
9429    case glslang::EOpAtomicCounterDecrement:
9430    case glslang::EOpAtomicCounter:
9431    {
9432        // Handle all of the atomics in one place, in createAtomicOperation()
9433        std::vector<spv::Id> operands;
9434        operands.push_back(operand);
9435        return createAtomicOperation(op, decorations.precision, typeId, operands, typeProxy, lvalueCoherentFlags, opType);
9436    }
9437
9438    case glslang::EOpBitFieldReverse:
9439        unaryOp = spv::Op::OpBitReverse;
9440        break;
9441    case glslang::EOpBitCount:
9442        unaryOp = spv::Op::OpBitCount;
9443        break;
9444    case glslang::EOpFindLSB:
9445        libCall = spv::GLSLstd450FindILsb;
9446        break;
9447    case glslang::EOpFindMSB:
9448        if (isUnsigned)
9449            libCall = spv::GLSLstd450FindUMsb;
9450        else
9451            libCall = spv::GLSLstd450FindSMsb;
9452        break;
9453
9454    case glslang::EOpCountLeadingZeros:
9455        builder.addCapability(spv::Capability::IntegerFunctions2INTEL);
9456        builder.addExtension("SPV_INTEL_shader_integer_functions2");
9457        unaryOp = spv::Op::OpUCountLeadingZerosINTEL;
9458        break;
9459
9460    case glslang::EOpCountTrailingZeros:
9461        builder.addCapability(spv::Capability::IntegerFunctions2INTEL);
9462        builder.addExtension("SPV_INTEL_shader_integer_functions2");
9463        unaryOp = spv::Op::OpUCountTrailingZerosINTEL;
9464        break;
9465
9466    case glslang::EOpBallot:
9467    case glslang::EOpReadFirstInvocation:
9468    case glslang::EOpAnyInvocation:
9469    case glslang::EOpAllInvocations:
9470    case glslang::EOpAllInvocationsEqual:
9471    case glslang::EOpMinInvocations:
9472    case glslang::EOpMaxInvocations:
9473    case glslang::EOpAddInvocations:
9474    case glslang::EOpMinInvocationsNonUniform:
9475    case glslang::EOpMaxInvocationsNonUniform:
9476    case glslang::EOpAddInvocationsNonUniform:
9477    case glslang::EOpMinInvocationsInclusiveScan:
9478    case glslang::EOpMaxInvocationsInclusiveScan:
9479    case glslang::EOpAddInvocationsInclusiveScan:
9480    case glslang::EOpMinInvocationsInclusiveScanNonUniform:
9481    case glslang::EOpMaxInvocationsInclusiveScanNonUniform:
9482    case glslang::EOpAddInvocationsInclusiveScanNonUniform:
9483    case glslang::EOpMinInvocationsExclusiveScan:
9484    case glslang::EOpMaxInvocationsExclusiveScan:
9485    case glslang::EOpAddInvocationsExclusiveScan:
9486    case glslang::EOpMinInvocationsExclusiveScanNonUniform:
9487    case glslang::EOpMaxInvocationsExclusiveScanNonUniform:
9488    case glslang::EOpAddInvocationsExclusiveScanNonUniform:
9489    {
9490        std::vector<spv::Id> operands;
9491        operands.push_back(operand);
9492        return createInvocationsOperation(op, typeId, operands, typeProxy);
9493    }
9494    case glslang::EOpSubgroupAll:
9495    case glslang::EOpSubgroupAny:
9496    case glslang::EOpSubgroupAllEqual:
9497    case glslang::EOpSubgroupBroadcastFirst:
9498    case glslang::EOpSubgroupBallot:
9499    case glslang::EOpSubgroupInverseBallot:
9500    case glslang::EOpSubgroupBallotBitCount:
9501    case glslang::EOpSubgroupBallotInclusiveBitCount:
9502    case glslang::EOpSubgroupBallotExclusiveBitCount:
9503    case glslang::EOpSubgroupBallotFindLSB:
9504    case glslang::EOpSubgroupBallotFindMSB:
9505    case glslang::EOpSubgroupAdd:
9506    case glslang::EOpSubgroupMul:
9507    case glslang::EOpSubgroupMin:
9508    case glslang::EOpSubgroupMax:
9509    case glslang::EOpSubgroupAnd:
9510    case glslang::EOpSubgroupOr:
9511    case glslang::EOpSubgroupXor:
9512    case glslang::EOpSubgroupInclusiveAdd:
9513    case glslang::EOpSubgroupInclusiveMul:
9514    case glslang::EOpSubgroupInclusiveMin:
9515    case glslang::EOpSubgroupInclusiveMax:
9516    case glslang::EOpSubgroupInclusiveAnd:
9517    case glslang::EOpSubgroupInclusiveOr:
9518    case glslang::EOpSubgroupInclusiveXor:
9519    case glslang::EOpSubgroupExclusiveAdd:
9520    case glslang::EOpSubgroupExclusiveMul:
9521    case glslang::EOpSubgroupExclusiveMin:
9522    case glslang::EOpSubgroupExclusiveMax:
9523    case glslang::EOpSubgroupExclusiveAnd:
9524    case glslang::EOpSubgroupExclusiveOr:
9525    case glslang::EOpSubgroupExclusiveXor:
9526    case glslang::EOpSubgroupQuadSwapHorizontal:
9527    case glslang::EOpSubgroupQuadSwapVertical:
9528    case glslang::EOpSubgroupQuadSwapDiagonal:
9529    case glslang::EOpSubgroupQuadAll:
9530    case glslang::EOpSubgroupQuadAny: {
9531        std::vector<spv::Id> operands;
9532        operands.push_back(operand);
9533        return createSubgroupOperation(op, typeId, operands, typeProxy);
9534    }
9535    case glslang::EOpMbcnt:
9536        extBuiltins = getExtBuiltins(spv::E_SPV_AMD_shader_ballot);
9537        libCall = spv::MbcntAMD;
9538        break;
9539
9540    case glslang::EOpCubeFaceIndex:
9541        extBuiltins = getExtBuiltins(spv::E_SPV_AMD_gcn_shader);
9542        libCall = spv::CubeFaceIndexAMD;
9543        break;
9544
9545    case glslang::EOpCubeFaceCoord:
9546        extBuiltins = getExtBuiltins(spv::E_SPV_AMD_gcn_shader);
9547        libCall = spv::CubeFaceCoordAMD;
9548        break;
9549    case glslang::EOpSubgroupPartition:
9550        unaryOp = spv::Op::OpGroupNonUniformPartitionNV;
9551        break;
9552    case glslang::EOpConstructReference:
9553        unaryOp = spv::Op::OpBitcast;
9554        break;
9555
9556    case glslang::EOpConvUint64ToAccStruct:
9557    case glslang::EOpConvUvec2ToAccStruct:
9558        unaryOp = spv::Op::OpConvertUToAccelerationStructureKHR;
9559        break;
9560
9561    case glslang::EOpHitObjectIsEmptyNV:
9562        unaryOp = spv::Op::OpHitObjectIsEmptyNV;
9563        break;
9564
9565    case glslang::EOpHitObjectIsEmptyEXT:
9566        unaryOp = spv::Op::OpHitObjectIsEmptyEXT;
9567        break;
9568
9569    case glslang::EOpHitObjectIsMissNV:
9570        unaryOp = spv::Op::OpHitObjectIsMissNV;
9571        break;
9572
9573    case glslang::EOpHitObjectIsMissEXT:
9574        unaryOp = spv::Op::OpHitObjectIsMissEXT;
9575        break;
9576
9577    case glslang::EOpHitObjectIsHitNV:
9578        unaryOp = spv::Op::OpHitObjectIsHitNV;
9579        break;
9580
9581    case glslang::EOpHitObjectIsHitEXT:
9582        unaryOp = spv::Op::OpHitObjectIsHitEXT;
9583        break;
9584
9585    case glslang::EOpHitObjectGetObjectRayOriginNV:
9586        unaryOp = spv::Op::OpHitObjectGetObjectRayOriginNV;
9587        break;
9588
9589    case glslang::EOpHitObjectGetObjectRayOriginEXT:
9590        unaryOp = spv::Op::OpHitObjectGetObjectRayOriginEXT;
9591        break;
9592
9593    case glslang::EOpHitObjectGetObjectRayDirectionNV:
9594        unaryOp = spv::Op::OpHitObjectGetObjectRayDirectionNV;
9595        break;
9596
9597    case glslang::EOpHitObjectGetObjectRayDirectionEXT:
9598        unaryOp = spv::Op::OpHitObjectGetObjectRayDirectionEXT;
9599        break;
9600
9601    case glslang::EOpHitObjectGetWorldRayOriginNV:
9602        unaryOp = spv::Op::OpHitObjectGetWorldRayOriginNV;
9603        break;
9604
9605    case glslang::EOpHitObjectGetWorldRayOriginEXT:
9606        unaryOp = spv::Op::OpHitObjectGetWorldRayOriginEXT;
9607        break;
9608
9609    case glslang::EOpHitObjectGetWorldRayDirectionNV:
9610        unaryOp = spv::Op::OpHitObjectGetWorldRayDirectionNV;
9611        break;
9612
9613    case glslang::EOpHitObjectGetWorldRayDirectionEXT:
9614        unaryOp = spv::Op::OpHitObjectGetWorldRayDirectionEXT;
9615        break;
9616
9617    case glslang::EOpHitObjectGetObjectToWorldNV:
9618        unaryOp = spv::Op::OpHitObjectGetObjectToWorldNV;
9619        break;
9620
9621    case glslang::EOpHitObjectGetObjectToWorldEXT:
9622        unaryOp = spv::Op::OpHitObjectGetObjectToWorldEXT;
9623        break;
9624
9625    case glslang::EOpHitObjectGetWorldToObjectNV:
9626        unaryOp = spv::Op::OpHitObjectGetWorldToObjectNV;
9627        break;
9628
9629    case glslang::EOpHitObjectGetWorldToObjectEXT:
9630        unaryOp = spv::Op::OpHitObjectGetWorldToObjectEXT;
9631        break;
9632
9633    case glslang::EOpHitObjectGetRayTMinNV:
9634        unaryOp = spv::Op::OpHitObjectGetRayTMinNV;
9635        break;
9636
9637    case glslang::EOpHitObjectGetRayTMinEXT:
9638        unaryOp = spv::Op::OpHitObjectGetRayTMinEXT;
9639        break;
9640
9641    case glslang::EOpHitObjectGetRayTMaxNV:
9642        unaryOp = spv::Op::OpHitObjectGetRayTMaxNV;
9643        break;
9644
9645    case glslang::EOpHitObjectGetRayTMaxEXT:
9646        unaryOp = spv::Op::OpHitObjectGetRayTMaxEXT;
9647        break;
9648
9649    case glslang::EOpHitObjectGetRayFlagsEXT:
9650        unaryOp = spv::Op::OpHitObjectGetRayFlagsEXT;
9651        break;
9652
9653    case glslang::EOpHitObjectGetPrimitiveIndexNV:
9654        unaryOp = spv::Op::OpHitObjectGetPrimitiveIndexNV;
9655        break;
9656
9657    case glslang::EOpHitObjectGetPrimitiveIndexEXT:
9658        unaryOp = spv::Op::OpHitObjectGetPrimitiveIndexEXT;
9659        break;
9660
9661    case glslang::EOpHitObjectGetInstanceIdNV:
9662        unaryOp = spv::Op::OpHitObjectGetInstanceIdNV;
9663        break;
9664
9665    case glslang::EOpHitObjectGetInstanceIdEXT:
9666        unaryOp = spv::Op::OpHitObjectGetInstanceIdEXT;
9667        break;
9668
9669    case glslang::EOpHitObjectGetInstanceCustomIndexNV:
9670        unaryOp = spv::Op::OpHitObjectGetInstanceCustomIndexNV;
9671        break;
9672
9673    case glslang::EOpHitObjectGetInstanceCustomIndexEXT:
9674        unaryOp = spv::Op::OpHitObjectGetInstanceCustomIndexEXT;
9675        break;
9676
9677    case glslang::EOpHitObjectGetGeometryIndexNV:
9678        unaryOp = spv::Op::OpHitObjectGetGeometryIndexNV;
9679        break;
9680
9681    case glslang::EOpHitObjectGetGeometryIndexEXT:
9682        unaryOp = spv::Op::OpHitObjectGetGeometryIndexEXT;
9683        break;
9684
9685    case glslang::EOpHitObjectGetHitKindNV:
9686        unaryOp = spv::Op::OpHitObjectGetHitKindNV;
9687        break;
9688
9689    case glslang::EOpHitObjectGetHitKindEXT:
9690        unaryOp = spv::Op::OpHitObjectGetHitKindEXT;
9691        break;
9692
9693    case glslang::EOpHitObjectGetCurrentTimeNV:
9694        unaryOp = spv::Op::OpHitObjectGetCurrentTimeNV;
9695        break;
9696
9697    case glslang::EOpHitObjectGetCurrentTimeEXT:
9698        unaryOp = spv::Op::OpHitObjectGetCurrentTimeEXT;
9699        break;
9700
9701    case glslang::EOpHitObjectGetShaderBindingTableRecordIndexNV:
9702        unaryOp = spv::Op::OpHitObjectGetShaderBindingTableRecordIndexNV;
9703        break;
9704
9705    case glslang::EOpHitObjectGetShaderBindingTableRecordIndexEXT:
9706        unaryOp = spv::Op::OpHitObjectGetShaderBindingTableRecordIndexEXT;
9707        break;
9708
9709    case glslang::EOpHitObjectGetShaderRecordBufferHandleNV:
9710        unaryOp = spv::Op::OpHitObjectGetShaderRecordBufferHandleNV;
9711        break;
9712
9713    case glslang::EOpHitObjectGetClusterIdNV:
9714        unaryOp = spv::Op::OpHitObjectGetClusterIdNV;
9715        builder.addExtension(spv::E_SPV_NV_cluster_acceleration_structure);
9716        builder.addCapability(spv::Capability::ShaderInvocationReorderNV);
9717        builder.addCapability(spv::Capability::RayTracingClusterAccelerationStructureNV);
9718        break;
9719
9720    case glslang::EOpHitObjectGetSpherePositionNV:
9721        unaryOp = spv::Op::OpHitObjectGetSpherePositionNV;
9722        builder.addExtension(spv::E_SPV_NV_linear_swept_spheres);
9723        builder.addCapability(spv::Capability::ShaderInvocationReorderNV);
9724        builder.addCapability(spv::Capability::RayTracingSpheresGeometryNV);
9725        break;
9726
9727    case glslang::EOpHitObjectGetSphereRadiusNV:
9728        unaryOp = spv::Op::OpHitObjectGetSphereRadiusNV;
9729        builder.addExtension(spv::E_SPV_NV_linear_swept_spheres);
9730        builder.addCapability(spv::Capability::ShaderInvocationReorderNV);
9731        builder.addCapability(spv::Capability::RayTracingSpheresGeometryNV);
9732        break;
9733
9734    case glslang::EOpHitObjectIsSphereHitNV:
9735        unaryOp = spv::Op::OpHitObjectIsSphereHitNV;
9736        builder.addExtension(spv::E_SPV_NV_linear_swept_spheres);
9737        builder.addCapability(spv::Capability::ShaderInvocationReorderNV);
9738        builder.addCapability(spv::Capability::RayTracingSpheresGeometryNV);
9739        break;
9740
9741    case glslang::EOpHitObjectIsLSSHitNV:
9742        unaryOp = spv::Op::OpHitObjectIsLSSHitNV;
9743        builder.addExtension(spv::E_SPV_NV_linear_swept_spheres);
9744        builder.addCapability(spv::Capability::ShaderInvocationReorderNV);
9745        builder.addCapability(spv::Capability::RayTracingLinearSweptSpheresGeometryNV);
9746        break;
9747
9748    case glslang::EOpHitObjectGetShaderRecordBufferHandleEXT:
9749        unaryOp = spv::Op::OpHitObjectGetShaderRecordBufferHandleEXT;
9750        break;
9751
9752    case glslang::EOpFetchMicroTriangleVertexPositionNV:
9753        unaryOp = spv::Op::OpFetchMicroTriangleVertexPositionNV;
9754        break;
9755
9756    case glslang::EOpFetchMicroTriangleVertexBarycentricNV:
9757        unaryOp = spv::Op::OpFetchMicroTriangleVertexBarycentricNV;
9758        break;
9759
9760    case glslang::EOpCopyObject:
9761        unaryOp = spv::Op::OpCopyObject;
9762        break;
9763
9764    case glslang::EOpDepthAttachmentReadEXT:
9765        builder.addExtension(spv::E_SPV_EXT_shader_tile_image);
9766        builder.addCapability(spv::Capability::TileImageDepthReadAccessEXT);
9767        unaryOp = spv::Op::OpDepthAttachmentReadEXT;
9768        decorations.precision = spv::NoPrecision;
9769        break;
9770    case glslang::EOpStencilAttachmentReadEXT:
9771        builder.addExtension(spv::E_SPV_EXT_shader_tile_image);
9772        builder.addCapability(spv::Capability::TileImageStencilReadAccessEXT);
9773        unaryOp = spv::Op::OpStencilAttachmentReadEXT;
9774        decorations.precision = spv::Decoration::RelaxedPrecision;
9775        break;
9776
9777    default:
9778        return 0;
9779    }
9780
9781    spv::Id id;
9782    if (libCall >= 0) {
9783        std::vector<spv::Id> args;
9784        args.push_back(operand);
9785        id = builder.createBuiltinCall(typeId, extBuiltins >= 0 ? extBuiltins : stdBuiltins, libCall, args);
9786    } else {
9787        id = builder.createUnaryOp(unaryOp, typeId, operand);
9788    }
9789
9790    decorations.addNoContraction(builder, id);
9791    decorations.addNonUniform(builder, id);
9792    return builder.setPrecision(id, decorations.precision);
9793}
9794
9795// Create a unary operation on a matrix
9796spv::Id TGlslangToSpvTraverser::createUnaryMatrixOperation(spv::Op op, OpDecorations& decorations, spv::Id typeId,
9797                                                           spv::Id operand, glslang::TBasicType /* typeProxy */)
9798{
9799    // Handle unary operations vector by vector.
9800    // The result type is the same type as the original type.
9801    // The algorithm is to:
9802    //   - break the matrix into vectors
9803    //   - apply the operation to each vector
9804    //   - make a matrix out the vector results
9805
9806    // get the types sorted out
9807    int numCols = builder.getNumColumns(operand);
9808    int numRows = builder.getNumRows(operand);
9809    spv::Id srcVecType  = builder.makeVectorType(builder.getScalarTypeId(builder.getTypeId(operand)), numRows);
9810    spv::Id destVecType = builder.makeVectorType(builder.getScalarTypeId(typeId), numRows);
9811    std::vector<spv::Id> results;
9812
9813    // do each vector op
9814    for (int c = 0; c < numCols; ++c) {
9815        std::vector<unsigned int> indexes;
9816        indexes.push_back(c);
9817        spv::Id srcVec  = builder.createCompositeExtract(operand, srcVecType, indexes);
9818        spv::Id destVec = builder.createUnaryOp(op, destVecType, srcVec);
9819        decorations.addNoContraction(builder, destVec);
9820        decorations.addNonUniform(builder, destVec);
9821        results.push_back(builder.setPrecision(destVec, decorations.precision));
9822    }
9823
9824    // put the pieces together
9825    spv::Id result = builder.setPrecision(builder.createCompositeConstruct(typeId, results), decorations.precision);
9826    decorations.addNonUniform(builder, result);
9827    return result;
9828}
9829
9830// For converting integers where both the bitwidth and the signedness could
9831// change, but only do the width change here. The caller is still responsible
9832// for the signedness conversion.
9833// destType is the final type that will be converted to, but this function
9834// may only be doing part of that conversion.
9835spv::Id TGlslangToSpvTraverser::createIntWidthConversion(spv::Id operand, int vectorSize, spv::Id destType,
9836                                                         glslang::TBasicType resultBasicType, glslang::TBasicType operandBasicType)
9837{
9838    // Get the result type width, based on the type to convert to.
9839    int width = GetNumBits(resultBasicType);
9840
9841    // Get the conversion operation and result type,
9842    // based on the target width, but the source type.
9843    spv::Id type = spv::NoType;
9844    spv::Op convOp = spv::Op::OpNop;
9845    if (isTypeSignedInt(operandBasicType)) {
9846        convOp = spv::Op::OpSConvert;
9847        type = builder.makeIntType(width);
9848    } else {
9849        convOp = spv::Op::OpUConvert;
9850        type = builder.makeUintType(width);
9851    }
9852
9853    if (builder.getOpCode(destType) == spv::Op::OpTypeCooperativeVectorNV) {
9854        type = builder.makeCooperativeVectorTypeNV(type, builder.getCooperativeVectorNumComponents(destType));
9855    } else if (vectorSize > 0)
9856        type = builder.makeVectorType(type, vectorSize);
9857    else if (builder.getOpCode(destType) == spv::Op::OpTypeCooperativeMatrixKHR ||
9858             builder.getOpCode(destType) == spv::Op::OpTypeCooperativeMatrixNV) {
9859
9860        type = builder.makeCooperativeMatrixTypeWithSameShape(type, destType);
9861    }
9862
9863    if (convOp == spv::Op::OpUConvert && specConstUConvertNeedsWorkaround()) {
9864        return createSpecConstUConvert(type, operand, vectorSize, GetNumBits(operandBasicType), width);
9865    }
9866
9867    return builder.createUnaryOp(convOp, type, operand);
9868}
9869
9870// OpUConvert only became a valid specialization constant operation in SPIR-V 1.4.
9871bool TGlslangToSpvTraverser::specConstUConvertNeedsWorkaround()
9872{
9873    return builder.isInSpecConstCodeGenMode() &&
9874           glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_4;
9875}
9876
9877// OpSConvert is valid in every version, and only differs from OpUConvert when widening,
9878// where it brings in copies of the sign bit that have to be masked off.
9879spv::Id TGlslangToSpvTraverser::createSpecConstUConvert(spv::Id typeId, spv::Id operand, int vectorSize,
9880                                                       int operandWidth, int resultWidth)
9881{
9882    spv::Id result = builder.createUnaryOp(spv::Op::OpSConvert, typeId, operand);
9883    if (resultWidth <= operandWidth)
9884        return result;
9885
9886    const spv::Id maskType = builder.makeUintType(resultWidth);
9887    const unsigned long long maskValue = (1ull << operandWidth) - 1;
9888    spv::Id mask = resultWidth == 64 ? builder.makeInt64Constant(maskType, maskValue, false)
9889                                     : builder.makeIntConstant(maskType, static_cast<unsigned>(maskValue), false);
9890    mask = makeSmearedConstant(mask, vectorSize);
9891    return builder.createBinOp(spv::Op::OpBitwiseAnd, typeId, result, mask);
9892}
9893
9894spv::Id TGlslangToSpvTraverser::createConversion(glslang::TOperator op, OpDecorations& decorations, spv::Id destType,
9895                                                 spv::Id operand, glslang::TBasicType resultBasicType, glslang::TBasicType operandBasicType,
9896                                                 bool sameCoopMatUse)
9897{
9898    spv::Op convOp = spv::Op::OpNop;
9899    spv::Id zero = 0;
9900    spv::Id one = 0;
9901
9902    int vectorSize = builder.isVectorType(destType) ? builder.getNumTypeComponents(destType) : 0;
9903
9904    if (IsOpNumericConv(op) || op == glslang::EOpConstructSaturated) {
9905        if (builder.isCooperativeMatrixType(destType)) {
9906            if (!sameCoopMatUse) {
9907                enableCoopMatConversions();
9908            }
9909            auto result = createCoopMatConversion(destType, operand, resultBasicType, operandBasicType, sameCoopMatUse, false);
9910            return result;
9911        }
9912
9913        if (isTypeSignedInt(operandBasicType) && isTypeFloat(resultBasicType)) {
9914            convOp = spv::Op::OpConvertSToF;
9915        }
9916        if (isTypeUnsignedInt(operandBasicType) && isTypeFloat(resultBasicType)) {
9917            convOp = spv::Op::OpConvertUToF;
9918        }
9919        if (isTypeFloat(operandBasicType) && isTypeSignedInt(resultBasicType)) {
9920            convOp = spv::Op::OpConvertFToS;
9921        }
9922        if (isTypeFloat(operandBasicType) && isTypeUnsignedInt(resultBasicType)) {
9923            convOp = spv::Op::OpConvertFToU;
9924        }
9925        if (isTypeSignedInt(operandBasicType) && isTypeSignedInt(resultBasicType)) {
9926            convOp = spv::Op::OpSConvert;
9927        }
9928        if (isTypeUnsignedInt(operandBasicType) && isTypeUnsignedInt(resultBasicType)) {
9929            convOp = spv::Op::OpUConvert;
9930        }
9931        if (isTypeFloat(operandBasicType) && isTypeFloat(resultBasicType)) {
9932            convOp = spv::Op::OpFConvert;
9933            if (builder.isMatrixType(destType))
9934                return createUnaryMatrixOperation(convOp, decorations, destType, operand, operandBasicType);
9935        }
9936        if (isTypeInt(operandBasicType) && isTypeInt(resultBasicType) &&
9937            isTypeUnsignedInt(operandBasicType) != isTypeUnsignedInt(resultBasicType)) {
9938
9939            if (GetNumBits(operandBasicType) != GetNumBits(resultBasicType)) {
9940                // OpSConvert/OpUConvert + OpBitCast
9941                operand = createIntWidthConversion(operand, vectorSize, destType, resultBasicType, operandBasicType);
9942            }
9943
9944            if (builder.isInSpecConstCodeGenMode()) {
9945                uint32_t bits = GetNumBits(resultBasicType);
9946                spv::Id zeroType = builder.makeUintType(bits);
9947                if (bits == 64) {
9948                    zero = builder.makeInt64Constant(zeroType, 0, false);
9949                } else {
9950                    zero = builder.makeIntConstant(zeroType, 0, false);
9951                }
9952                zero = makeSmearedConstant(zero, vectorSize);
9953                // Use OpIAdd, instead of OpBitcast to do the conversion when
9954                // generating for OpSpecConstantOp instruction.
9955                return builder.createBinOp(spv::Op::OpIAdd, destType, operand, zero);
9956            }
9957            // For normal run-time conversion instruction, use OpBitcast.
9958            convOp = spv::Op::OpBitcast;
9959        }
9960        if (resultBasicType == glslang::EbtBool) {
9961            uint32_t bits = GetNumBits(operandBasicType);
9962            if (isTypeInt(operandBasicType)) {
9963                spv::Id zeroType = builder.makeUintType(bits);
9964                if (bits == 64) {
9965                    zero = builder.makeInt64Constant(zeroType, 0, false);
9966                } else {
9967                    zero = builder.makeIntConstant(zeroType, 0, false);
9968                }
9969                zero = makeSmearedConstant(zero, vectorSize);
9970                return builder.createBinOp(spv::Op::OpINotEqual, destType, operand, zero);
9971            } else {
9972                assert(isTypeFloat(operandBasicType));
9973                if (bits == 64) {
9974                    zero = builder.makeDoubleConstant(0.0);
9975                } else if (bits == 32) {
9976                    zero = builder.makeFloatConstant(0.0);
9977                } else {
9978                    assert(bits == 16);
9979                    zero = builder.makeFloat16Constant(0.0);
9980                }
9981                zero = makeSmearedConstant(zero, vectorSize);
9982                return builder.createBinOp(spv::Op::OpFUnordNotEqual, destType, operand, zero);
9983            }
9984        }
9985        if (operandBasicType == glslang::EbtBool) {
9986            uint32_t bits = GetNumBits(resultBasicType);
9987            convOp = spv::Op::OpSelect;
9988            if (isTypeInt(resultBasicType)) {
9989                spv::Id zeroType = isTypeSignedInt(resultBasicType) ? builder.makeIntType(bits) : builder.makeUintType(bits);
9990                if (bits == 64) {
9991                    zero = builder.makeInt64Constant(zeroType, 0, false);
9992                    one = builder.makeInt64Constant(zeroType, 1, false);
9993                } else {
9994                    zero = builder.makeIntConstant(zeroType, 0, false);
9995                    one = builder.makeIntConstant(zeroType, 1, false);
9996                }
9997            } else {
9998                assert(isTypeFloat(resultBasicType));
9999                if (bits == 64) {
10000                    zero = builder.makeDoubleConstant(0.0);
10001                    one = builder.makeDoubleConstant(1.0);
10002                } else if (bits == 32) {
10003                    zero = builder.makeFloatConstant(0.0);
10004                    one = builder.makeFloatConstant(1.0);
10005                } else {
10006                    assert(bits == 16);
10007                    zero = builder.makeFloat16Constant(0.0);
10008                    one = builder.makeFloat16Constant(1.0);
10009                }
10010            }
10011        }
10012    }
10013
10014    if (convOp == spv::Op::OpNop) {
10015        switch (op) {
10016        case glslang::EOpConvUint64ToPtr:
10017            convOp = spv::Op::OpConvertUToPtr;
10018            break;
10019        case glslang::EOpConvPtrToUint64:
10020            convOp = spv::Op::OpConvertPtrToU;
10021            break;
10022        case glslang::EOpConvPtrToUvec2:
10023        case glslang::EOpConvUvec2ToPtr:
10024            convOp = spv::Op::OpBitcast;
10025            break;
10026
10027        default:
10028            break;
10029        }
10030    }
10031
10032    spv::Id result = 0;
10033    if (convOp == spv::Op::OpNop)
10034        return result;
10035
10036    if (convOp == spv::Op::OpSelect) {
10037        zero = makeSmearedConstant(zero, vectorSize);
10038        one  = makeSmearedConstant(one, vectorSize);
10039        result = builder.createTriOp(convOp, destType, operand, one, zero);
10040    } else if (convOp == spv::Op::OpUConvert && specConstUConvertNeedsWorkaround()) {
10041        result = createSpecConstUConvert(destType, operand, vectorSize, GetNumBits(operandBasicType),
10042                                         GetNumBits(resultBasicType));
10043    } else
10044        result = builder.createUnaryOp(convOp, destType, operand);
10045
10046    result = builder.setPrecision(result, decorations.precision);
10047    decorations.addNonUniform(builder, result);
10048    return result;
10049}
10050
10051spv::Id TGlslangToSpvTraverser::makeSmearedConstant(spv::Id constant, int vectorSize)
10052{
10053    if (vectorSize == 0)
10054        return constant;
10055
10056    spv::Id vectorTypeId = builder.makeVectorType(builder.getTypeId(constant), vectorSize);
10057    std::vector<spv::Id> components;
10058    for (int c = 0; c < vectorSize; ++c)
10059        components.push_back(constant);
10060    return builder.makeCompositeConstant(vectorTypeId, components);
10061}
10062
10063// For glslang ops that map to SPV atomic opCodes
10064spv::Id TGlslangToSpvTraverser::createAtomicOperation(glslang::TOperator op, spv::Decoration /*precision*/,
10065    spv::Id typeId, std::vector<spv::Id>& operands, glslang::TBasicType typeProxy,
10066    const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags, const glslang::TType &opType)
10067{
10068    spv::Op opCode = spv::Op::OpNop;
10069
10070    switch (op) {
10071    case glslang::EOpAtomicAdd:
10072    case glslang::EOpImageAtomicAdd:
10073    case glslang::EOpAtomicCounterAdd:
10074        opCode = spv::Op::OpAtomicIAdd;
10075        if (typeProxy == glslang::EbtFloat16 || typeProxy == glslang::EbtFloat || typeProxy == glslang::EbtDouble) {
10076            opCode = spv::Op::OpAtomicFAddEXT;
10077            if (typeProxy == glslang::EbtFloat16 &&
10078                (opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) {
10079                builder.addExtension(spv::E_SPV_NV_shader_atomic_fp16_vector);
10080                builder.addCapability(spv::Capability::AtomicFloat16VectorNV);
10081            } else {
10082                builder.addExtension(spv::E_SPV_EXT_shader_atomic_float_add);
10083                if (typeProxy == glslang::EbtFloat16) {
10084                    builder.addExtension(spv::E_SPV_EXT_shader_atomic_float16_add);
10085                    builder.addCapability(spv::Capability::AtomicFloat16AddEXT);
10086                } else if (typeProxy == glslang::EbtFloat) {
10087                    builder.addCapability(spv::Capability::AtomicFloat32AddEXT);
10088                } else {
10089                    builder.addCapability(spv::Capability::AtomicFloat64AddEXT);
10090                }
10091            }
10092        }
10093        break;
10094    case glslang::EOpAtomicSubtract:
10095    case glslang::EOpAtomicCounterSubtract:
10096        opCode = spv::Op::OpAtomicISub;
10097        break;
10098    case glslang::EOpAtomicMin:
10099    case glslang::EOpImageAtomicMin:
10100    case glslang::EOpAtomicCounterMin:
10101        if (typeProxy == glslang::EbtFloat16 || typeProxy == glslang::EbtFloat || typeProxy == glslang::EbtDouble) {
10102            opCode = spv::Op::OpAtomicFMinEXT;
10103            if (typeProxy == glslang::EbtFloat16 &&
10104                (opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) {
10105                builder.addExtension(spv::E_SPV_NV_shader_atomic_fp16_vector);
10106                builder.addCapability(spv::Capability::AtomicFloat16VectorNV);
10107            } else {
10108                builder.addExtension(spv::E_SPV_EXT_shader_atomic_float_min_max);
10109                if (typeProxy == glslang::EbtFloat16)
10110                    builder.addCapability(spv::Capability::AtomicFloat16MinMaxEXT);
10111                else if (typeProxy == glslang::EbtFloat)
10112                    builder.addCapability(spv::Capability::AtomicFloat32MinMaxEXT);
10113                else
10114                    builder.addCapability(spv::Capability::AtomicFloat64MinMaxEXT);
10115            }
10116        } else if (typeProxy == glslang::EbtUint || typeProxy == glslang::EbtUint64) {
10117            opCode = spv::Op::OpAtomicUMin;
10118        } else {
10119            opCode = spv::Op::OpAtomicSMin;
10120        }
10121        break;
10122    case glslang::EOpAtomicMax:
10123    case glslang::EOpImageAtomicMax:
10124    case glslang::EOpAtomicCounterMax:
10125        if (typeProxy == glslang::EbtFloat16 || typeProxy == glslang::EbtFloat || typeProxy == glslang::EbtDouble) {
10126            opCode = spv::Op::OpAtomicFMaxEXT;
10127            if (typeProxy == glslang::EbtFloat16 &&
10128                (opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) {
10129                builder.addExtension(spv::E_SPV_NV_shader_atomic_fp16_vector);
10130                builder.addCapability(spv::Capability::AtomicFloat16VectorNV);
10131            } else {
10132                builder.addExtension(spv::E_SPV_EXT_shader_atomic_float_min_max);
10133                if (typeProxy == glslang::EbtFloat16)
10134                    builder.addCapability(spv::Capability::AtomicFloat16MinMaxEXT);
10135                else if (typeProxy == glslang::EbtFloat)
10136                    builder.addCapability(spv::Capability::AtomicFloat32MinMaxEXT);
10137                else
10138                    builder.addCapability(spv::Capability::AtomicFloat64MinMaxEXT);
10139            }
10140        } else if (typeProxy == glslang::EbtUint || typeProxy == glslang::EbtUint64) {
10141            opCode = spv::Op::OpAtomicUMax;
10142        } else {
10143            opCode = spv::Op::OpAtomicSMax;
10144        }
10145        break;
10146    case glslang::EOpAtomicAnd:
10147    case glslang::EOpImageAtomicAnd:
10148    case glslang::EOpAtomicCounterAnd:
10149        opCode = spv::Op::OpAtomicAnd;
10150        break;
10151    case glslang::EOpAtomicOr:
10152    case glslang::EOpImageAtomicOr:
10153    case glslang::EOpAtomicCounterOr:
10154        opCode = spv::Op::OpAtomicOr;
10155        break;
10156    case glslang::EOpAtomicXor:
10157    case glslang::EOpImageAtomicXor:
10158    case glslang::EOpAtomicCounterXor:
10159        opCode = spv::Op::OpAtomicXor;
10160        break;
10161    case glslang::EOpAtomicExchange:
10162    case glslang::EOpImageAtomicExchange:
10163    case glslang::EOpAtomicCounterExchange:
10164        if ((typeProxy == glslang::EbtFloat16) &&
10165            (opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) {
10166                builder.addExtension(spv::E_SPV_NV_shader_atomic_fp16_vector);
10167                builder.addCapability(spv::Capability::AtomicFloat16VectorNV);
10168        }
10169
10170        opCode = spv::Op::OpAtomicExchange;
10171        break;
10172    case glslang::EOpAtomicCompSwap:
10173    case glslang::EOpImageAtomicCompSwap:
10174    case glslang::EOpAtomicCounterCompSwap:
10175        opCode = spv::Op::OpAtomicCompareExchange;
10176        break;
10177    case glslang::EOpAtomicCounterIncrement:
10178        opCode = spv::Op::OpAtomicIIncrement;
10179        break;
10180    case glslang::EOpAtomicCounterDecrement:
10181        opCode = spv::Op::OpAtomicIDecrement;
10182        break;
10183    case glslang::EOpAtomicCounter:
10184    case glslang::EOpImageAtomicLoad:
10185    case glslang::EOpAtomicLoad:
10186        opCode = spv::Op::OpAtomicLoad;
10187        break;
10188    case glslang::EOpAtomicStore:
10189    case glslang::EOpImageAtomicStore:
10190        opCode = spv::Op::OpAtomicStore;
10191        break;
10192    default:
10193        assert(0);
10194        break;
10195    }
10196
10197    if (typeProxy == glslang::EbtInt64 || typeProxy == glslang::EbtUint64)
10198        builder.addCapability(spv::Capability::Int64Atomics);
10199
10200    // Sort out the operands
10201    //  - mapping from glslang -> SPV
10202    //  - there are extra SPV operands that are optional in glslang
10203    //  - compare-exchange swaps the value and comparator
10204    //  - compare-exchange has an extra memory semantics
10205    //  - EOpAtomicCounterDecrement needs a post decrement
10206    spv::Id pointerId = 0, compareId = 0, valueId = 0;
10207    // scope defaults to Device in the old model, QueueFamilyKHR in the new model
10208    spv::Id scopeId;
10209    if (glslangIntermediate->usingVulkanMemoryModel()) {
10210        scopeId = builder.makeUintConstant(spv::Scope::QueueFamilyKHR);
10211    } else {
10212        scopeId = builder.makeUintConstant(spv::Scope::Device);
10213    }
10214    // semantics default to relaxed
10215    spv::Id semanticsId = builder.makeUintConstant(lvalueCoherentFlags.isVolatile() &&
10216        glslangIntermediate->usingVulkanMemoryModel() ?
10217                                                    spv::MemorySemanticsMask::Volatile :
10218                                                    spv::MemorySemanticsMask::MaskNone);
10219    spv::Id semanticsId2 = semanticsId;
10220
10221    pointerId = operands[0];
10222    if (opCode == spv::Op::OpAtomicIIncrement || opCode == spv::Op::OpAtomicIDecrement) {
10223        // no additional operands
10224    } else if (opCode == spv::Op::OpAtomicCompareExchange) {
10225        compareId = operands[1];
10226        valueId = operands[2];
10227        if (operands.size() > 3) {
10228            scopeId = operands[3];
10229            semanticsId = builder.makeUintConstant(
10230                builder.getConstantScalar(operands[4]) | builder.getConstantScalar(operands[5]));
10231            semanticsId2 = builder.makeUintConstant(
10232                builder.getConstantScalar(operands[6]) | builder.getConstantScalar(operands[7]));
10233        }
10234    } else if (opCode == spv::Op::OpAtomicLoad) {
10235        if (operands.size() > 1) {
10236            scopeId = operands[1];
10237            semanticsId = builder.makeUintConstant(
10238                builder.getConstantScalar(operands[2]) | builder.getConstantScalar(operands[3]));
10239        }
10240    } else {
10241        // atomic store or RMW
10242        valueId = operands[1];
10243        if (operands.size() > 2) {
10244            scopeId = operands[2];
10245            semanticsId = builder.makeUintConstant
10246                (builder.getConstantScalar(operands[3]) | builder.getConstantScalar(operands[4]));
10247        }
10248    }
10249
10250    // Check for capabilities
10251    auto const semanticsImmediate = (spv::MemorySemanticsMask)(builder.getConstantScalar(semanticsId) | builder.getConstantScalar(semanticsId2));
10252    if (anySet(semanticsImmediate, spv::MemorySemanticsMask::MakeAvailableKHR |
10253                                   spv::MemorySemanticsMask::MakeVisibleKHR |
10254                                   spv::MemorySemanticsMask::OutputMemoryKHR |
10255                                   spv::MemorySemanticsMask::Volatile)) {
10256        builder.addCapability(spv::Capability::VulkanMemoryModelKHR);
10257    }
10258
10259    auto const scope = (spv::Scope)builder.getConstantScalar(scopeId);
10260    if (scope == spv::Scope::QueueFamily) {
10261        builder.addCapability(spv::Capability::VulkanMemoryModelKHR);
10262    }
10263
10264    if (glslangIntermediate->usingVulkanMemoryModel() && scope == spv::Scope::Device) {
10265        builder.addCapability(spv::Capability::VulkanMemoryModelDeviceScopeKHR);
10266    }
10267
10268    std::vector<spv::Id> spvAtomicOperands;  // hold the spv operands
10269    spvAtomicOperands.reserve(6);
10270    spvAtomicOperands.push_back(pointerId);
10271    spvAtomicOperands.push_back(scopeId);
10272    spvAtomicOperands.push_back(semanticsId);
10273    if (opCode == spv::Op::OpAtomicCompareExchange) {
10274        spvAtomicOperands.push_back(semanticsId2);
10275        spvAtomicOperands.push_back(valueId);
10276        spvAtomicOperands.push_back(compareId);
10277    } else if (opCode != spv::Op::OpAtomicLoad && opCode != spv::Op::OpAtomicIIncrement && opCode != spv::Op::OpAtomicIDecrement) {
10278        spvAtomicOperands.push_back(valueId);
10279    }
10280
10281    if (opCode == spv::Op::OpAtomicStore) {
10282        builder.createNoResultOp(opCode, spvAtomicOperands);
10283        return 0;
10284    } else {
10285        spv::Id resultId = builder.createOp(opCode, typeId, spvAtomicOperands);
10286
10287        // GLSL and HLSL atomic-counter decrement return post-decrement value,
10288        // while SPIR-V returns pre-decrement value. Translate between these semantics.
10289        if (op == glslang::EOpAtomicCounterDecrement)
10290            resultId = builder.createBinOp(spv::Op::OpISub, typeId, resultId, builder.makeIntConstant(1));
10291
10292        return resultId;
10293    }
10294}
10295
10296// Create group invocation operations.
10297spv::Id TGlslangToSpvTraverser::createInvocationsOperation(glslang::TOperator op, spv::Id typeId,
10298    std::vector<spv::Id>& operands, glslang::TBasicType typeProxy)
10299{
10300    bool isUnsigned = isTypeUnsignedInt(typeProxy);
10301    bool isFloat = isTypeFloat(typeProxy);
10302
10303    spv::Op opCode = spv::Op::OpNop;
10304    std::vector<spv::IdImmediate> spvGroupOperands;
10305    spv::GroupOperation groupOperation = spv::GroupOperation::Max;
10306
10307    if (op == glslang::EOpBallot || op == glslang::EOpReadFirstInvocation ||
10308        op == glslang::EOpReadInvocation) {
10309        builder.addExtension(spv::E_SPV_KHR_shader_ballot);
10310        builder.addCapability(spv::Capability::SubgroupBallotKHR);
10311    } else if (op == glslang::EOpAnyInvocation ||
10312        op == glslang::EOpAllInvocations ||
10313        op == glslang::EOpAllInvocationsEqual) {
10314        builder.addExtension(spv::E_SPV_KHR_subgroup_vote);
10315        builder.addCapability(spv::Capability::SubgroupVoteKHR);
10316    } else {
10317        builder.addCapability(spv::Capability::Groups);
10318        if (op == glslang::EOpMinInvocationsNonUniform ||
10319            op == glslang::EOpMaxInvocationsNonUniform ||
10320            op == glslang::EOpAddInvocationsNonUniform ||
10321            op == glslang::EOpMinInvocationsInclusiveScanNonUniform ||
10322            op == glslang::EOpMaxInvocationsInclusiveScanNonUniform ||
10323            op == glslang::EOpAddInvocationsInclusiveScanNonUniform ||
10324            op == glslang::EOpMinInvocationsExclusiveScanNonUniform ||
10325            op == glslang::EOpMaxInvocationsExclusiveScanNonUniform ||
10326            op == glslang::EOpAddInvocationsExclusiveScanNonUniform)
10327            builder.addExtension(spv::E_SPV_AMD_shader_ballot);
10328
10329        switch (op) {
10330        case glslang::EOpMinInvocations:
10331        case glslang::EOpMaxInvocations:
10332        case glslang::EOpAddInvocations:
10333        case glslang::EOpMinInvocationsNonUniform:
10334        case glslang::EOpMaxInvocationsNonUniform:
10335        case glslang::EOpAddInvocationsNonUniform:
10336            groupOperation = spv::GroupOperation::Reduce;
10337            break;
10338        case glslang::EOpMinInvocationsInclusiveScan:
10339        case glslang::EOpMaxInvocationsInclusiveScan:
10340        case glslang::EOpAddInvocationsInclusiveScan:
10341        case glslang::EOpMinInvocationsInclusiveScanNonUniform:
10342        case glslang::EOpMaxInvocationsInclusiveScanNonUniform:
10343        case glslang::EOpAddInvocationsInclusiveScanNonUniform:
10344            groupOperation = spv::GroupOperation::InclusiveScan;
10345            break;
10346        case glslang::EOpMinInvocationsExclusiveScan:
10347        case glslang::EOpMaxInvocationsExclusiveScan:
10348        case glslang::EOpAddInvocationsExclusiveScan:
10349        case glslang::EOpMinInvocationsExclusiveScanNonUniform:
10350        case glslang::EOpMaxInvocationsExclusiveScanNonUniform:
10351        case glslang::EOpAddInvocationsExclusiveScanNonUniform:
10352            groupOperation = spv::GroupOperation::ExclusiveScan;
10353            break;
10354        default:
10355            break;
10356        }
10357        spv::IdImmediate scope = { true, builder.makeUintConstant(spv::Scope::Subgroup) };
10358        spvGroupOperands.push_back(scope);
10359        if (groupOperation != spv::GroupOperation::Max) {
10360            spv::IdImmediate groupOp = { false, (unsigned)groupOperation };
10361            spvGroupOperands.push_back(groupOp);
10362        }
10363    }
10364
10365    for (auto opIt = operands.begin(); opIt != operands.end(); ++opIt) {
10366        spv::IdImmediate op = { true, *opIt };
10367        spvGroupOperands.push_back(op);
10368    }
10369
10370    switch (op) {
10371    case glslang::EOpAnyInvocation:
10372        opCode = spv::Op::OpSubgroupAnyKHR;
10373        break;
10374    case glslang::EOpAllInvocations:
10375        opCode = spv::Op::OpSubgroupAllKHR;
10376        break;
10377    case glslang::EOpAllInvocationsEqual:
10378        opCode = spv::Op::OpSubgroupAllEqualKHR;
10379        break;
10380    case glslang::EOpReadInvocation:
10381        opCode = spv::Op::OpSubgroupReadInvocationKHR;
10382        if (builder.isVectorType(typeId))
10383            return CreateInvocationsVectorOperation(opCode, groupOperation, typeId, operands);
10384        break;
10385    case glslang::EOpReadFirstInvocation:
10386        opCode = spv::Op::OpSubgroupFirstInvocationKHR;
10387        if (builder.isVectorType(typeId))
10388            return CreateInvocationsVectorOperation(opCode, groupOperation, typeId, operands);
10389        break;
10390    case glslang::EOpBallot:
10391    {
10392        // NOTE: According to the spec, the result type of "OpSubgroupBallotKHR" must be a 4 component vector of 32
10393        // bit integer types. The GLSL built-in function "ballotARB()" assumes the maximum number of invocations in
10394        // a subgroup is 64. Thus, we have to convert uvec4.xy to uint64_t as follow:
10395        //
10396        //     result = Bitcast(SubgroupBallotKHR(Predicate).xy)
10397        //
10398        spv::Id uintType  = builder.makeUintType(32);
10399        spv::Id uvec4Type = builder.makeVectorType(uintType, 4);
10400        spv::Id result = builder.createOp(spv::Op::OpSubgroupBallotKHR, uvec4Type, spvGroupOperands);
10401
10402        std::vector<spv::Id> components;
10403        components.push_back(builder.createCompositeExtract(result, uintType, 0));
10404        components.push_back(builder.createCompositeExtract(result, uintType, 1));
10405
10406        spv::Id uvec2Type = builder.makeVectorType(uintType, 2);
10407        return builder.createUnaryOp(spv::Op::OpBitcast, typeId,
10408                                     builder.createCompositeConstruct(uvec2Type, components));
10409    }
10410
10411    case glslang::EOpMinInvocations:
10412    case glslang::EOpMaxInvocations:
10413    case glslang::EOpAddInvocations:
10414    case glslang::EOpMinInvocationsInclusiveScan:
10415    case glslang::EOpMaxInvocationsInclusiveScan:
10416    case glslang::EOpAddInvocationsInclusiveScan:
10417    case glslang::EOpMinInvocationsExclusiveScan:
10418    case glslang::EOpMaxInvocationsExclusiveScan:
10419    case glslang::EOpAddInvocationsExclusiveScan:
10420        if (op == glslang::EOpMinInvocations ||
10421            op == glslang::EOpMinInvocationsInclusiveScan ||
10422            op == glslang::EOpMinInvocationsExclusiveScan) {
10423            if (isFloat)
10424                opCode = spv::Op::OpGroupFMin;
10425            else {
10426                if (isUnsigned)
10427                    opCode = spv::Op::OpGroupUMin;
10428                else
10429                    opCode = spv::Op::OpGroupSMin;
10430            }
10431        } else if (op == glslang::EOpMaxInvocations ||
10432                   op == glslang::EOpMaxInvocationsInclusiveScan ||
10433                   op == glslang::EOpMaxInvocationsExclusiveScan) {
10434            if (isFloat)
10435                opCode = spv::Op::OpGroupFMax;
10436            else {
10437                if (isUnsigned)
10438                    opCode = spv::Op::OpGroupUMax;
10439                else
10440                    opCode = spv::Op::OpGroupSMax;
10441            }
10442        } else {
10443            if (isFloat)
10444                opCode = spv::Op::OpGroupFAdd;
10445            else
10446                opCode = spv::Op::OpGroupIAdd;
10447        }
10448
10449        if (builder.isVectorType(typeId))
10450            return CreateInvocationsVectorOperation(opCode, groupOperation, typeId, operands);
10451
10452        break;
10453    case glslang::EOpMinInvocationsNonUniform:
10454    case glslang::EOpMaxInvocationsNonUniform:
10455    case glslang::EOpAddInvocationsNonUniform:
10456    case glslang::EOpMinInvocationsInclusiveScanNonUniform:
10457    case glslang::EOpMaxInvocationsInclusiveScanNonUniform:
10458    case glslang::EOpAddInvocationsInclusiveScanNonUniform:
10459    case glslang::EOpMinInvocationsExclusiveScanNonUniform:
10460    case glslang::EOpMaxInvocationsExclusiveScanNonUniform:
10461    case glslang::EOpAddInvocationsExclusiveScanNonUniform:
10462        if (op == glslang::EOpMinInvocationsNonUniform ||
10463            op == glslang::EOpMinInvocationsInclusiveScanNonUniform ||
10464            op == glslang::EOpMinInvocationsExclusiveScanNonUniform) {
10465            if (isFloat)
10466                opCode = spv::Op::OpGroupFMinNonUniformAMD;
10467            else {
10468                if (isUnsigned)
10469                    opCode = spv::Op::OpGroupUMinNonUniformAMD;
10470                else
10471                    opCode = spv::Op::OpGroupSMinNonUniformAMD;
10472            }
10473        }
10474        else if (op == glslang::EOpMaxInvocationsNonUniform ||
10475                 op == glslang::EOpMaxInvocationsInclusiveScanNonUniform ||
10476                 op == glslang::EOpMaxInvocationsExclusiveScanNonUniform) {
10477            if (isFloat)
10478                opCode = spv::Op::OpGroupFMaxNonUniformAMD;
10479            else {
10480                if (isUnsigned)
10481                    opCode = spv::Op::OpGroupUMaxNonUniformAMD;
10482                else
10483                    opCode = spv::Op::OpGroupSMaxNonUniformAMD;
10484            }
10485        }
10486        else {
10487            if (isFloat)
10488                opCode = spv::Op::OpGroupFAddNonUniformAMD;
10489            else
10490                opCode = spv::Op::OpGroupIAddNonUniformAMD;
10491        }
10492
10493        if (builder.isVectorType(typeId))
10494            return CreateInvocationsVectorOperation(opCode, groupOperation, typeId, operands);
10495
10496        break;
10497    default:
10498        logger->missingFunctionality("invocation operation");
10499        return spv::NoResult;
10500    }
10501
10502    assert(opCode != spv::Op::OpNop);
10503    return builder.createOp(opCode, typeId, spvGroupOperands);
10504}
10505
10506// Create group invocation operations on a vector
10507spv::Id TGlslangToSpvTraverser::CreateInvocationsVectorOperation(spv::Op op, spv::GroupOperation groupOperation,
10508    spv::Id typeId, std::vector<spv::Id>& operands)
10509{
10510    assert(op == spv::Op::OpGroupFMin || op == spv::Op::OpGroupUMin || op == spv::Op::OpGroupSMin ||
10511           op == spv::Op::OpGroupFMax || op == spv::Op::OpGroupUMax || op == spv::Op::OpGroupSMax ||
10512           op == spv::Op::OpGroupFAdd || op == spv::Op::OpGroupIAdd || op == spv::Op::OpGroupBroadcast ||
10513           op == spv::Op::OpSubgroupReadInvocationKHR || op == spv::Op::OpSubgroupFirstInvocationKHR ||
10514           op == spv::Op::OpGroupFMinNonUniformAMD || op == spv::Op::OpGroupUMinNonUniformAMD ||
10515           op == spv::Op::OpGroupSMinNonUniformAMD ||
10516           op == spv::Op::OpGroupFMaxNonUniformAMD || op == spv::Op::OpGroupUMaxNonUniformAMD ||
10517           op == spv::Op::OpGroupSMaxNonUniformAMD ||
10518           op == spv::Op::OpGroupFAddNonUniformAMD || op == spv::Op::OpGroupIAddNonUniformAMD);
10519
10520    // Handle group invocation operations scalar by scalar.
10521    // The result type is the same type as the original type.
10522    // The algorithm is to:
10523    //   - break the vector into scalars
10524    //   - apply the operation to each scalar
10525    //   - make a vector out the scalar results
10526
10527    // get the types sorted out
10528    int numComponents = builder.getNumComponents(operands[0]);
10529    spv::Id scalarType = builder.getScalarTypeId(builder.getTypeId(operands[0]));
10530    std::vector<spv::Id> results;
10531
10532    // do each scalar op
10533    for (int comp = 0; comp < numComponents; ++comp) {
10534        std::vector<unsigned int> indexes;
10535        indexes.push_back(comp);
10536        spv::IdImmediate scalar = { true, builder.createCompositeExtract(operands[0], scalarType, indexes) };
10537        std::vector<spv::IdImmediate> spvGroupOperands;
10538        if (op == spv::Op::OpSubgroupReadInvocationKHR) {
10539            spvGroupOperands.push_back(scalar);
10540            spv::IdImmediate operand = { true, operands[1] };
10541            spvGroupOperands.push_back(operand);
10542        } else if (op == spv::Op::OpSubgroupFirstInvocationKHR) {
10543            spvGroupOperands.push_back(scalar);
10544        } else if (op == spv::Op::OpGroupBroadcast) {
10545            spv::IdImmediate scope = { true, builder.makeUintConstant(spv::Scope::Subgroup) };
10546            spvGroupOperands.push_back(scope);
10547            spvGroupOperands.push_back(scalar);
10548            spv::IdImmediate operand = { true, operands[1] };
10549            spvGroupOperands.push_back(operand);
10550        } else {
10551            spv::IdImmediate scope = { true, builder.makeUintConstant(spv::Scope::Subgroup) };
10552            spvGroupOperands.push_back(scope);
10553            spv::IdImmediate groupOp = { false, (unsigned)groupOperation };
10554            spvGroupOperands.push_back(groupOp);
10555            spvGroupOperands.push_back(scalar);
10556        }
10557
10558        results.push_back(builder.createOp(op, scalarType, spvGroupOperands));
10559    }
10560
10561    // put the pieces together
10562    return builder.createCompositeConstruct(typeId, results);
10563}
10564
10565// Create subgroup invocation operations.
10566spv::Id TGlslangToSpvTraverser::createSubgroupOperation(glslang::TOperator op, spv::Id typeId,
10567    std::vector<spv::Id>& operands, glslang::TBasicType typeProxy)
10568{
10569    // Add the required capabilities.
10570    switch (op) {
10571    case glslang::EOpSubgroupElect:
10572        builder.addCapability(spv::Capability::GroupNonUniform);
10573        break;
10574    case glslang::EOpSubgroupQuadAll:
10575    case glslang::EOpSubgroupQuadAny:
10576        builder.addExtension(spv::E_SPV_KHR_quad_control);
10577        builder.addCapability(spv::Capability::QuadControlKHR);
10578        [[fallthrough]];
10579    case glslang::EOpSubgroupAll:
10580    case glslang::EOpSubgroupAny:
10581    case glslang::EOpSubgroupAllEqual:
10582        builder.addCapability(spv::Capability::GroupNonUniform);
10583        builder.addCapability(spv::Capability::GroupNonUniformVote);
10584        break;
10585    case glslang::EOpSubgroupBroadcast:
10586    case glslang::EOpSubgroupBroadcastFirst:
10587    case glslang::EOpSubgroupBallot:
10588    case glslang::EOpSubgroupInverseBallot:
10589    case glslang::EOpSubgroupBallotBitExtract:
10590    case glslang::EOpSubgroupBallotBitCount:
10591    case glslang::EOpSubgroupBallotInclusiveBitCount:
10592    case glslang::EOpSubgroupBallotExclusiveBitCount:
10593    case glslang::EOpSubgroupBallotFindLSB:
10594    case glslang::EOpSubgroupBallotFindMSB:
10595        builder.addCapability(spv::Capability::GroupNonUniform);
10596        builder.addCapability(spv::Capability::GroupNonUniformBallot);
10597        break;
10598    case glslang::EOpSubgroupRotate:
10599    case glslang::EOpSubgroupClusteredRotate:
10600        builder.addExtension(spv::E_SPV_KHR_subgroup_rotate);
10601        builder.addCapability(spv::Capability::GroupNonUniformRotateKHR);
10602        break;
10603    case glslang::EOpSubgroupShuffle:
10604    case glslang::EOpSubgroupShuffleXor:
10605        builder.addCapability(spv::Capability::GroupNonUniform);
10606        builder.addCapability(spv::Capability::GroupNonUniformShuffle);
10607        break;
10608    case glslang::EOpSubgroupShuffleUp:
10609    case glslang::EOpSubgroupShuffleDown:
10610        builder.addCapability(spv::Capability::GroupNonUniform);
10611        builder.addCapability(spv::Capability::GroupNonUniformShuffleRelative);
10612        break;
10613    case glslang::EOpSubgroupAdd:
10614    case glslang::EOpSubgroupMul:
10615    case glslang::EOpSubgroupMin:
10616    case glslang::EOpSubgroupMax:
10617    case glslang::EOpSubgroupAnd:
10618    case glslang::EOpSubgroupOr:
10619    case glslang::EOpSubgroupXor:
10620    case glslang::EOpSubgroupInclusiveAdd:
10621    case glslang::EOpSubgroupInclusiveMul:
10622    case glslang::EOpSubgroupInclusiveMin:
10623    case glslang::EOpSubgroupInclusiveMax:
10624    case glslang::EOpSubgroupInclusiveAnd:
10625    case glslang::EOpSubgroupInclusiveOr:
10626    case glslang::EOpSubgroupInclusiveXor:
10627    case glslang::EOpSubgroupExclusiveAdd:
10628    case glslang::EOpSubgroupExclusiveMul:
10629    case glslang::EOpSubgroupExclusiveMin:
10630    case glslang::EOpSubgroupExclusiveMax:
10631    case glslang::EOpSubgroupExclusiveAnd:
10632    case glslang::EOpSubgroupExclusiveOr:
10633    case glslang::EOpSubgroupExclusiveXor:
10634        builder.addCapability(spv::Capability::GroupNonUniform);
10635        builder.addCapability(spv::Capability::GroupNonUniformArithmetic);
10636        break;
10637    case glslang::EOpSubgroupClusteredAdd:
10638    case glslang::EOpSubgroupClusteredMul:
10639    case glslang::EOpSubgroupClusteredMin:
10640    case glslang::EOpSubgroupClusteredMax:
10641    case glslang::EOpSubgroupClusteredAnd:
10642    case glslang::EOpSubgroupClusteredOr:
10643    case glslang::EOpSubgroupClusteredXor:
10644        builder.addCapability(spv::Capability::GroupNonUniform);
10645        builder.addCapability(spv::Capability::GroupNonUniformClustered);
10646        break;
10647    case glslang::EOpSubgroupQuadBroadcast:
10648    case glslang::EOpSubgroupQuadSwapHorizontal:
10649    case glslang::EOpSubgroupQuadSwapVertical:
10650    case glslang::EOpSubgroupQuadSwapDiagonal:
10651        builder.addCapability(spv::Capability::GroupNonUniform);
10652        builder.addCapability(spv::Capability::GroupNonUniformQuad);
10653        break;
10654    case glslang::EOpSubgroupPartitionedAdd:
10655    case glslang::EOpSubgroupPartitionedMul:
10656    case glslang::EOpSubgroupPartitionedMin:
10657    case glslang::EOpSubgroupPartitionedMax:
10658    case glslang::EOpSubgroupPartitionedAnd:
10659    case glslang::EOpSubgroupPartitionedOr:
10660    case glslang::EOpSubgroupPartitionedXor:
10661    case glslang::EOpSubgroupPartitionedInclusiveAdd:
10662    case glslang::EOpSubgroupPartitionedInclusiveMul:
10663    case glslang::EOpSubgroupPartitionedInclusiveMin:
10664    case glslang::EOpSubgroupPartitionedInclusiveMax:
10665    case glslang::EOpSubgroupPartitionedInclusiveAnd:
10666    case glslang::EOpSubgroupPartitionedInclusiveOr:
10667    case glslang::EOpSubgroupPartitionedInclusiveXor:
10668    case glslang::EOpSubgroupPartitionedExclusiveAdd:
10669    case glslang::EOpSubgroupPartitionedExclusiveMul:
10670    case glslang::EOpSubgroupPartitionedExclusiveMin:
10671    case glslang::EOpSubgroupPartitionedExclusiveMax:
10672    case glslang::EOpSubgroupPartitionedExclusiveAnd:
10673    case glslang::EOpSubgroupPartitionedExclusiveOr:
10674    case glslang::EOpSubgroupPartitionedExclusiveXor:
10675        builder.addExtension(spv::E_SPV_NV_shader_subgroup_partitioned);
10676        builder.addCapability(spv::Capability::GroupNonUniformPartitionedNV);
10677        break;
10678    default: assert(0 && "Unhandled subgroup operation!");
10679    }
10680
10681
10682    const bool isUnsigned = isTypeUnsignedInt(typeProxy);
10683    const bool isFloat = isTypeFloat(typeProxy);
10684    const bool isBool = typeProxy == glslang::EbtBool;
10685
10686    spv::Op opCode = spv::Op::OpNop;
10687
10688    // Figure out which opcode to use.
10689    switch (op) {
10690    case glslang::EOpSubgroupElect:                   opCode = spv::Op::OpGroupNonUniformElect; break;
10691    case glslang::EOpSubgroupQuadAll:                 opCode = spv::Op::OpGroupNonUniformQuadAllKHR; break;
10692    case glslang::EOpSubgroupAll:                     opCode = spv::Op::OpGroupNonUniformAll; break;
10693    case glslang::EOpSubgroupQuadAny:                 opCode = spv::Op::OpGroupNonUniformQuadAnyKHR; break;
10694    case glslang::EOpSubgroupAny:                     opCode = spv::Op::OpGroupNonUniformAny; break;
10695    case glslang::EOpSubgroupAllEqual:                opCode = spv::Op::OpGroupNonUniformAllEqual; break;
10696    case glslang::EOpSubgroupBroadcast:               opCode = spv::Op::OpGroupNonUniformBroadcast; break;
10697    case glslang::EOpSubgroupBroadcastFirst:          opCode = spv::Op::OpGroupNonUniformBroadcastFirst; break;
10698    case glslang::EOpSubgroupBallot:                  opCode = spv::Op::OpGroupNonUniformBallot; break;
10699    case glslang::EOpSubgroupInverseBallot:           opCode = spv::Op::OpGroupNonUniformInverseBallot; break;
10700    case glslang::EOpSubgroupBallotBitExtract:        opCode = spv::Op::OpGroupNonUniformBallotBitExtract; break;
10701    case glslang::EOpSubgroupBallotBitCount:
10702    case glslang::EOpSubgroupBallotInclusiveBitCount:
10703    case glslang::EOpSubgroupBallotExclusiveBitCount: opCode = spv::Op::OpGroupNonUniformBallotBitCount; break;
10704    case glslang::EOpSubgroupBallotFindLSB:           opCode = spv::Op::OpGroupNonUniformBallotFindLSB; break;
10705    case glslang::EOpSubgroupBallotFindMSB:           opCode = spv::Op::OpGroupNonUniformBallotFindMSB; break;
10706    case glslang::EOpSubgroupShuffle:                 opCode = spv::Op::OpGroupNonUniformShuffle; break;
10707    case glslang::EOpSubgroupShuffleXor:              opCode = spv::Op::OpGroupNonUniformShuffleXor; break;
10708    case glslang::EOpSubgroupShuffleUp:               opCode = spv::Op::OpGroupNonUniformShuffleUp; break;
10709    case glslang::EOpSubgroupShuffleDown:             opCode = spv::Op::OpGroupNonUniformShuffleDown; break;
10710    case glslang::EOpSubgroupRotate:
10711    case glslang::EOpSubgroupClusteredRotate:         opCode = spv::Op::OpGroupNonUniformRotateKHR; break;
10712    case glslang::EOpSubgroupAdd:
10713    case glslang::EOpSubgroupInclusiveAdd:
10714    case glslang::EOpSubgroupExclusiveAdd:
10715    case glslang::EOpSubgroupClusteredAdd:
10716    case glslang::EOpSubgroupPartitionedAdd:
10717    case glslang::EOpSubgroupPartitionedInclusiveAdd:
10718    case glslang::EOpSubgroupPartitionedExclusiveAdd:
10719        if (isFloat) {
10720            opCode = spv::Op::OpGroupNonUniformFAdd;
10721        } else {
10722            opCode = spv::Op::OpGroupNonUniformIAdd;
10723        }
10724        break;
10725    case glslang::EOpSubgroupMul:
10726    case glslang::EOpSubgroupInclusiveMul:
10727    case glslang::EOpSubgroupExclusiveMul:
10728    case glslang::EOpSubgroupClusteredMul:
10729    case glslang::EOpSubgroupPartitionedMul:
10730    case glslang::EOpSubgroupPartitionedInclusiveMul:
10731    case glslang::EOpSubgroupPartitionedExclusiveMul:
10732        if (isFloat) {
10733            opCode = spv::Op::OpGroupNonUniformFMul;
10734        } else {
10735            opCode = spv::Op::OpGroupNonUniformIMul;
10736        }
10737        break;
10738    case glslang::EOpSubgroupMin:
10739    case glslang::EOpSubgroupInclusiveMin:
10740    case glslang::EOpSubgroupExclusiveMin:
10741    case glslang::EOpSubgroupClusteredMin:
10742    case glslang::EOpSubgroupPartitionedMin:
10743    case glslang::EOpSubgroupPartitionedInclusiveMin:
10744    case glslang::EOpSubgroupPartitionedExclusiveMin:
10745        if (isFloat) {
10746            opCode = spv::Op::OpGroupNonUniformFMin;
10747        } else if (isUnsigned) {
10748            opCode = spv::Op::OpGroupNonUniformUMin;
10749        } else {
10750            opCode = spv::Op::OpGroupNonUniformSMin;
10751        }
10752        break;
10753    case glslang::EOpSubgroupMax:
10754    case glslang::EOpSubgroupInclusiveMax:
10755    case glslang::EOpSubgroupExclusiveMax:
10756    case glslang::EOpSubgroupClusteredMax:
10757    case glslang::EOpSubgroupPartitionedMax:
10758    case glslang::EOpSubgroupPartitionedInclusiveMax:
10759    case glslang::EOpSubgroupPartitionedExclusiveMax:
10760        if (isFloat) {
10761            opCode = spv::Op::OpGroupNonUniformFMax;
10762        } else if (isUnsigned) {
10763            opCode = spv::Op::OpGroupNonUniformUMax;
10764        } else {
10765            opCode = spv::Op::OpGroupNonUniformSMax;
10766        }
10767        break;
10768    case glslang::EOpSubgroupAnd:
10769    case glslang::EOpSubgroupInclusiveAnd:
10770    case glslang::EOpSubgroupExclusiveAnd:
10771    case glslang::EOpSubgroupClusteredAnd:
10772    case glslang::EOpSubgroupPartitionedAnd:
10773    case glslang::EOpSubgroupPartitionedInclusiveAnd:
10774    case glslang::EOpSubgroupPartitionedExclusiveAnd:
10775        if (isBool) {
10776            opCode = spv::Op::OpGroupNonUniformLogicalAnd;
10777        } else {
10778            opCode = spv::Op::OpGroupNonUniformBitwiseAnd;
10779        }
10780        break;
10781    case glslang::EOpSubgroupOr:
10782    case glslang::EOpSubgroupInclusiveOr:
10783    case glslang::EOpSubgroupExclusiveOr:
10784    case glslang::EOpSubgroupClusteredOr:
10785    case glslang::EOpSubgroupPartitionedOr:
10786    case glslang::EOpSubgroupPartitionedInclusiveOr:
10787    case glslang::EOpSubgroupPartitionedExclusiveOr:
10788        if (isBool) {
10789            opCode = spv::Op::OpGroupNonUniformLogicalOr;
10790        } else {
10791            opCode = spv::Op::OpGroupNonUniformBitwiseOr;
10792        }
10793        break;
10794    case glslang::EOpSubgroupXor:
10795    case glslang::EOpSubgroupInclusiveXor:
10796    case glslang::EOpSubgroupExclusiveXor:
10797    case glslang::EOpSubgroupClusteredXor:
10798    case glslang::EOpSubgroupPartitionedXor:
10799    case glslang::EOpSubgroupPartitionedInclusiveXor:
10800    case glslang::EOpSubgroupPartitionedExclusiveXor:
10801        if (isBool) {
10802            opCode = spv::Op::OpGroupNonUniformLogicalXor;
10803        } else {
10804            opCode = spv::Op::OpGroupNonUniformBitwiseXor;
10805        }
10806        break;
10807    case glslang::EOpSubgroupQuadBroadcast:      opCode = spv::Op::OpGroupNonUniformQuadBroadcast; break;
10808    case glslang::EOpSubgroupQuadSwapHorizontal:
10809    case glslang::EOpSubgroupQuadSwapVertical:
10810    case glslang::EOpSubgroupQuadSwapDiagonal:   opCode = spv::Op::OpGroupNonUniformQuadSwap; break;
10811    default: assert(0 && "Unhandled subgroup operation!");
10812    }
10813
10814    // get the right Group Operation
10815    spv::GroupOperation groupOperation = spv::GroupOperation::Max;
10816    switch (op) {
10817    default:
10818        break;
10819    case glslang::EOpSubgroupBallotBitCount:
10820    case glslang::EOpSubgroupAdd:
10821    case glslang::EOpSubgroupMul:
10822    case glslang::EOpSubgroupMin:
10823    case glslang::EOpSubgroupMax:
10824    case glslang::EOpSubgroupAnd:
10825    case glslang::EOpSubgroupOr:
10826    case glslang::EOpSubgroupXor:
10827        groupOperation = spv::GroupOperation::Reduce;
10828        break;
10829    case glslang::EOpSubgroupBallotInclusiveBitCount:
10830    case glslang::EOpSubgroupInclusiveAdd:
10831    case glslang::EOpSubgroupInclusiveMul:
10832    case glslang::EOpSubgroupInclusiveMin:
10833    case glslang::EOpSubgroupInclusiveMax:
10834    case glslang::EOpSubgroupInclusiveAnd:
10835    case glslang::EOpSubgroupInclusiveOr:
10836    case glslang::EOpSubgroupInclusiveXor:
10837        groupOperation = spv::GroupOperation::InclusiveScan;
10838        break;
10839    case glslang::EOpSubgroupBallotExclusiveBitCount:
10840    case glslang::EOpSubgroupExclusiveAdd:
10841    case glslang::EOpSubgroupExclusiveMul:
10842    case glslang::EOpSubgroupExclusiveMin:
10843    case glslang::EOpSubgroupExclusiveMax:
10844    case glslang::EOpSubgroupExclusiveAnd:
10845    case glslang::EOpSubgroupExclusiveOr:
10846    case glslang::EOpSubgroupExclusiveXor:
10847        groupOperation = spv::GroupOperation::ExclusiveScan;
10848        break;
10849    case glslang::EOpSubgroupClusteredAdd:
10850    case glslang::EOpSubgroupClusteredMul:
10851    case glslang::EOpSubgroupClusteredMin:
10852    case glslang::EOpSubgroupClusteredMax:
10853    case glslang::EOpSubgroupClusteredAnd:
10854    case glslang::EOpSubgroupClusteredOr:
10855    case glslang::EOpSubgroupClusteredXor:
10856        groupOperation = spv::GroupOperation::ClusteredReduce;
10857        break;
10858    case glslang::EOpSubgroupPartitionedAdd:
10859    case glslang::EOpSubgroupPartitionedMul:
10860    case glslang::EOpSubgroupPartitionedMin:
10861    case glslang::EOpSubgroupPartitionedMax:
10862    case glslang::EOpSubgroupPartitionedAnd:
10863    case glslang::EOpSubgroupPartitionedOr:
10864    case glslang::EOpSubgroupPartitionedXor:
10865        groupOperation = spv::GroupOperation::PartitionedReduceNV;
10866        break;
10867    case glslang::EOpSubgroupPartitionedInclusiveAdd:
10868    case glslang::EOpSubgroupPartitionedInclusiveMul:
10869    case glslang::EOpSubgroupPartitionedInclusiveMin:
10870    case glslang::EOpSubgroupPartitionedInclusiveMax:
10871    case glslang::EOpSubgroupPartitionedInclusiveAnd:
10872    case glslang::EOpSubgroupPartitionedInclusiveOr:
10873    case glslang::EOpSubgroupPartitionedInclusiveXor:
10874        groupOperation = spv::GroupOperation::PartitionedInclusiveScanNV;
10875        break;
10876    case glslang::EOpSubgroupPartitionedExclusiveAdd:
10877    case glslang::EOpSubgroupPartitionedExclusiveMul:
10878    case glslang::EOpSubgroupPartitionedExclusiveMin:
10879    case glslang::EOpSubgroupPartitionedExclusiveMax:
10880    case glslang::EOpSubgroupPartitionedExclusiveAnd:
10881    case glslang::EOpSubgroupPartitionedExclusiveOr:
10882    case glslang::EOpSubgroupPartitionedExclusiveXor:
10883        groupOperation = spv::GroupOperation::PartitionedExclusiveScanNV;
10884        break;
10885    }
10886
10887    // build the instruction
10888    std::vector<spv::IdImmediate> spvGroupOperands;
10889
10890    // Every operation begins with the Execution Scope operand.
10891    spv::IdImmediate executionScope = { true, builder.makeUintConstant(spv::Scope::Subgroup) };
10892    // All other ops need the execution scope. Quad Control Ops don't need scope, it's always Quad.
10893    if (opCode != spv::Op::OpGroupNonUniformQuadAllKHR && opCode != spv::Op::OpGroupNonUniformQuadAnyKHR) {
10894        spvGroupOperands.push_back(executionScope);
10895    }
10896
10897    // Next, for all operations that use a Group Operation, push that as an operand.
10898    if (groupOperation != spv::GroupOperation::Max) {
10899        spv::IdImmediate groupOperand = { false, (unsigned)groupOperation };
10900        spvGroupOperands.push_back(groupOperand);
10901    }
10902
10903    // Push back the operands next.
10904    for (auto opIt = operands.cbegin(); opIt != operands.cend(); ++opIt) {
10905        spv::IdImmediate operand = { true, *opIt };
10906        spvGroupOperands.push_back(operand);
10907    }
10908
10909    // Some opcodes have additional operands.
10910    spv::Id directionId = spv::NoResult;
10911    switch (op) {
10912    default: break;
10913    case glslang::EOpSubgroupQuadSwapHorizontal: directionId = builder.makeUintConstant(0); break;
10914    case glslang::EOpSubgroupQuadSwapVertical:   directionId = builder.makeUintConstant(1); break;
10915    case glslang::EOpSubgroupQuadSwapDiagonal:   directionId = builder.makeUintConstant(2); break;
10916    }
10917    if (directionId != spv::NoResult) {
10918        spv::IdImmediate direction = { true, directionId };
10919        spvGroupOperands.push_back(direction);
10920    }
10921
10922    return builder.createOp(opCode, typeId, spvGroupOperands);
10923}
10924
10925spv::Id TGlslangToSpvTraverser::createMiscOperation(glslang::TOperator op, spv::Decoration precision,
10926    spv::Id typeId, std::vector<spv::Id>& operands, glslang::TBasicType typeProxy)
10927{
10928    bool isUnsigned = isTypeUnsignedInt(typeProxy);
10929    bool isFloat = isTypeFloat(typeProxy);
10930
10931    spv::Op opCode = spv::Op::OpNop;
10932    int extBuiltins = -1;
10933    int libCall = -1;
10934    size_t consumedOperands = operands.size();
10935    spv::Id typeId0 = 0;
10936    if (consumedOperands > 0)
10937        typeId0 = builder.getTypeId(operands[0]);
10938    spv::Id typeId1 = 0;
10939    if (consumedOperands > 1)
10940        typeId1 = builder.getTypeId(operands[1]);
10941    spv::Id frexpIntType = 0;
10942
10943    switch (op) {
10944    case glslang::EOpMin:
10945        if (isFloat)
10946            libCall = nanMinMaxClamp ? spv::GLSLstd450NMin : spv::GLSLstd450FMin;
10947        else if (isUnsigned)
10948            libCall = spv::GLSLstd450UMin;
10949        else
10950            libCall = spv::GLSLstd450SMin;
10951        builder.promoteScalar(precision, operands.front(), operands.back());
10952        break;
10953    case glslang::EOpModf:
10954        {
10955            libCall = spv::GLSLstd450ModfStruct;
10956            assert(builder.isFloatType(builder.getScalarTypeId(typeId0)));
10957            // The returned struct has two members of the same type as the first argument
10958            typeId = builder.makeStructResultType(typeId0, typeId0);
10959            consumedOperands = 1;
10960        }
10961        break;
10962    case glslang::EOpMax:
10963        if (isFloat)
10964            libCall = nanMinMaxClamp ? spv::GLSLstd450NMax : spv::GLSLstd450FMax;
10965        else if (isUnsigned)
10966            libCall = spv::GLSLstd450UMax;
10967        else
10968            libCall = spv::GLSLstd450SMax;
10969        builder.promoteScalar(precision, operands.front(), operands.back());
10970        break;
10971    case glslang::EOpPow:
10972        libCall = spv::GLSLstd450Pow;
10973        break;
10974    case glslang::EOpDot:
10975    case glslang::EOpDotPackedEXT:
10976    case glslang::EOpDotAccSatEXT:
10977    case glslang::EOpDotPackedAccSatEXT:
10978        {
10979            if (builder.isFloatType(builder.getScalarTypeId(typeId0)) ||
10980                // HLSL supports dot(int,int) which is just a multiply
10981                glslangIntermediate->getSource() == glslang::EShSourceHlsl) {
10982                if (typeProxy == glslang::EbtBFloat16) {
10983                    builder.addExtension(spv::E_SPV_KHR_bfloat16);
10984                    builder.addCapability(spv::Capability::BFloat16DotProductKHR);
10985                }
10986                opCode = spv::Op::OpDot;
10987            } else {
10988                builder.addExtension(spv::E_SPV_KHR_integer_dot_product);
10989                builder.addCapability(spv::Capability::DotProductKHR);
10990                const unsigned int vectorSize = builder.getNumComponents(operands[0]);
10991                if (op == glslang::EOpDotPackedEXT || op == glslang::EOpDotPackedAccSatEXT) {
10992                    builder.addCapability(spv::Capability::DotProductInput4x8BitPackedKHR);
10993                } else if (vectorSize == 4 && builder.getScalarTypeWidth(typeId0) == 8) {
10994                    builder.addCapability(spv::Capability::DotProductInput4x8BitKHR);
10995                } else {
10996                    builder.addCapability(spv::Capability::DotProductInputAllKHR);
10997                }
10998                const bool type0isSigned = builder.isIntType(builder.getScalarTypeId(typeId0));
10999                const bool type1isSigned = builder.isIntType(builder.getScalarTypeId(typeId1));
11000                const bool accSat = (op == glslang::EOpDotAccSatEXT || op == glslang::EOpDotPackedAccSatEXT);
11001                if (!type0isSigned && !type1isSigned) {
11002                    opCode = accSat ? spv::Op::OpUDotAccSatKHR : spv::Op::OpUDotKHR;
11003                } else if (type0isSigned && type1isSigned) {
11004                    opCode = accSat ? spv::Op::OpSDotAccSatKHR : spv::Op::OpSDotKHR;
11005                } else {
11006                    opCode = accSat ? spv::Op::OpSUDotAccSatKHR : spv::Op::OpSUDotKHR;
11007                    // the spir-v opcode assumes the operands to be "signed, unsigned" in that order, so swap if needed
11008                    if (type1isSigned) {
11009                        std::swap(operands[0], operands[1]);
11010                    }
11011                }
11012                std::vector<spv::IdImmediate> operands2;
11013                for (auto &o : operands) {
11014                    operands2.push_back({true, o});
11015                }
11016                if (op == glslang::EOpDotPackedEXT || op == glslang::EOpDotPackedAccSatEXT) {
11017                    operands2.push_back({false, 0});
11018                }
11019                return builder.createOp(opCode, typeId, operands2);
11020            }
11021        }
11022        break;
11023    case glslang::EOpAtan:
11024        libCall = spv::GLSLstd450Atan2;
11025        break;
11026
11027    case glslang::EOpClamp:
11028        if (isFloat)
11029            libCall = nanMinMaxClamp ? spv::GLSLstd450NClamp : spv::GLSLstd450FClamp;
11030        else if (isUnsigned)
11031            libCall = spv::GLSLstd450UClamp;
11032        else
11033            libCall = spv::GLSLstd450SClamp;
11034        builder.promoteScalar(precision, operands.front(), operands[1]);
11035        builder.promoteScalar(precision, operands.front(), operands[2]);
11036        break;
11037    case glslang::EOpMix:
11038        if (! builder.isBoolType(builder.getScalarTypeId(builder.getTypeId(operands.back())))) {
11039            assert(isFloat);
11040            libCall = spv::GLSLstd450FMix;
11041        } else {
11042            opCode = spv::Op::OpSelect;
11043            std::swap(operands.front(), operands.back());
11044        }
11045        builder.promoteScalar(precision, operands.front(), operands.back());
11046        break;
11047    case glslang::EOpStep:
11048        libCall = spv::GLSLstd450Step;
11049        builder.promoteScalar(precision, operands.front(), operands.back());
11050        break;
11051    case glslang::EOpSmoothStep:
11052        libCall = spv::GLSLstd450SmoothStep;
11053        builder.promoteScalar(precision, operands[0], operands[2]);
11054        builder.promoteScalar(precision, operands[1], operands[2]);
11055        break;
11056
11057    case glslang::EOpDistance:
11058        libCall = spv::GLSLstd450Distance;
11059        break;
11060    case glslang::EOpCross:
11061        libCall = spv::GLSLstd450Cross;
11062        break;
11063    case glslang::EOpFaceForward:
11064        libCall = spv::GLSLstd450FaceForward;
11065        break;
11066    case glslang::EOpReflect:
11067        libCall = spv::GLSLstd450Reflect;
11068        break;
11069    case glslang::EOpRefract:
11070        libCall = spv::GLSLstd450Refract;
11071        break;
11072    case glslang::EOpBarrier:
11073        {
11074            // This is for the extended controlBarrier function, with four operands.
11075            // The unextended barrier() goes through createNoArgOperation.
11076            assert(operands.size() == 4);
11077            auto const executionScope = (spv::Scope)builder.getConstantScalar(operands[0]);
11078            auto const memoryScope = (spv::Scope)builder.getConstantScalar(operands[1]);
11079            auto const semantics = (spv::MemorySemanticsMask)(builder.getConstantScalar(operands[2]) | builder.getConstantScalar(operands[3]));
11080            builder.createControlBarrier(executionScope, memoryScope,
11081                semantics);
11082            if (anySet(semantics, spv::MemorySemanticsMask::MakeAvailableKHR |
11083                                  spv::MemorySemanticsMask::MakeVisibleKHR |
11084                                  spv::MemorySemanticsMask::OutputMemoryKHR |
11085                                  spv::MemorySemanticsMask::Volatile)) {
11086                builder.addCapability(spv::Capability::VulkanMemoryModelKHR);
11087            }
11088            if (glslangIntermediate->usingVulkanMemoryModel() && (executionScope == spv::Scope::Device ||
11089                memoryScope == spv::Scope::Device)) {
11090                builder.addCapability(spv::Capability::VulkanMemoryModelDeviceScopeKHR);
11091            }
11092            return 0;
11093        }
11094        break;
11095    case glslang::EOpControlBarrierArriveEXT:
11096    case glslang::EOpControlBarrierWaitEXT:
11097        {
11098            builder.addExtension(spv::E_SPV_EXT_split_barrier);
11099            builder.addCapability(spv::Capability::SplitBarrierEXT);
11100            // This is for the extended controlBarrierArrive/Wait functions, with four operands.
11101            // The unextended controlBarrierArrive/controlBarrierWait() go through createNoArgOperation.
11102            assert(operands.size() == 4);
11103            auto const executionScope = (spv::Scope)builder.getConstantScalar(operands[0]);
11104            auto const memoryScope = (spv::Scope)builder.getConstantScalar(operands[1]);
11105            auto const semantics = (spv::MemorySemanticsMask)(builder.getConstantScalar(operands[2]) | builder.getConstantScalar(operands[3]));
11106            builder.createSplitControlBarrier((op == glslang::EOpControlBarrierArriveEXT ? spv::Op::OpControlBarrierArriveEXT
11107                                               : spv::Op::OpControlBarrierWaitEXT),
11108                                              executionScope, memoryScope, semantics);
11109            if (anySet(semantics, spv::MemorySemanticsMask::OutputMemoryKHR)) {
11110                builder.addCapability(spv::Capability::VulkanMemoryModelKHR);
11111            }
11112            return 0;
11113        }
11114        break;
11115    case glslang::EOpMemoryBarrier:
11116        {
11117            // This is for the extended memoryBarrier function, with three operands.
11118            // The unextended memoryBarrier() goes through createNoArgOperation.
11119            assert(operands.size() == 3);
11120            auto const memoryScope = (spv::Scope)builder.getConstantScalar(operands[0]);
11121            auto const semantics = (spv::MemorySemanticsMask)(builder.getConstantScalar(operands[1]) | builder.getConstantScalar(operands[2]));
11122            builder.createMemoryBarrier(memoryScope, semantics);
11123            if (anySet(semantics, spv::MemorySemanticsMask::MakeAvailableKHR |
11124                                  spv::MemorySemanticsMask::MakeVisibleKHR |
11125                                  spv::MemorySemanticsMask::OutputMemoryKHR |
11126                                  spv::MemorySemanticsMask::Volatile)) {
11127                builder.addCapability(spv::Capability::VulkanMemoryModelKHR);
11128            }
11129            if (glslangIntermediate->usingVulkanMemoryModel() && memoryScope == spv::Scope::Device) {
11130                builder.addCapability(spv::Capability::VulkanMemoryModelDeviceScopeKHR);
11131            }
11132            return 0;
11133        }
11134        break;
11135
11136    case glslang::EOpInterpolateAtSample:
11137        if (typeProxy == glslang::EbtFloat16)
11138            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
11139        libCall = spv::GLSLstd450InterpolateAtSample;
11140        break;
11141    case glslang::EOpInterpolateAtOffset:
11142        if (typeProxy == glslang::EbtFloat16)
11143            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
11144        libCall = spv::GLSLstd450InterpolateAtOffset;
11145        break;
11146    case glslang::EOpAddCarry:
11147        opCode = spv::Op::OpIAddCarry;
11148        typeId = builder.makeStructResultType(typeId0, typeId0);
11149        consumedOperands = 2;
11150        break;
11151    case glslang::EOpSubBorrow:
11152        opCode = spv::Op::OpISubBorrow;
11153        typeId = builder.makeStructResultType(typeId0, typeId0);
11154        consumedOperands = 2;
11155        break;
11156    case glslang::EOpUMulExtended:
11157        opCode = spv::Op::OpUMulExtended;
11158        typeId = builder.makeStructResultType(typeId0, typeId0);
11159        consumedOperands = 2;
11160        break;
11161    case glslang::EOpIMulExtended:
11162        opCode = spv::Op::OpSMulExtended;
11163        typeId = builder.makeStructResultType(typeId0, typeId0);
11164        consumedOperands = 2;
11165        break;
11166    case glslang::EOpBitfieldExtract:
11167        if (isUnsigned)
11168            opCode = spv::Op::OpBitFieldUExtract;
11169        else
11170            opCode = spv::Op::OpBitFieldSExtract;
11171        break;
11172    case glslang::EOpBitfieldInsert:
11173        opCode = spv::Op::OpBitFieldInsert;
11174        break;
11175
11176    case glslang::EOpFma:
11177        libCall = spv::GLSLstd450Fma;
11178        break;
11179    case glslang::EOpFrexp:
11180        {
11181            libCall = spv::GLSLstd450FrexpStruct;
11182            assert(builder.isPointerType(typeId1));
11183            typeId1 = builder.getContainedTypeId(typeId1);
11184            int width = builder.getScalarTypeWidth(typeId1);
11185            if (width == 16)
11186                // Using 16-bit exp operand, enable extension SPV_AMD_gpu_shader_int16
11187                builder.addExtension(spv::E_SPV_AMD_gpu_shader_int16);
11188            if (builder.getNumComponents(operands[0]) == 1)
11189                frexpIntType = builder.makeIntegerType(width, true);
11190            else if (builder.isCooperativeVector(operands[0]))
11191                frexpIntType = builder.makeCooperativeVectorTypeNV(builder.makeIntegerType(width, true),
11192                                                                   builder.getCooperativeVectorNumComponents(builder.getTypeId(operands[0])));
11193            else
11194                frexpIntType = builder.makeVectorType(builder.makeIntegerType(width, true),
11195                                                      builder.getNumComponents(operands[0]));
11196            typeId = builder.makeStructResultType(typeId0, frexpIntType);
11197            consumedOperands = 1;
11198        }
11199        break;
11200    case glslang::EOpLdexp:
11201        libCall = spv::GLSLstd450Ldexp;
11202        break;
11203
11204    case glslang::EOpReadInvocation:
11205        return createInvocationsOperation(op, typeId, operands, typeProxy);
11206
11207    case glslang::EOpSubgroupBroadcast:
11208    case glslang::EOpSubgroupBallotBitExtract:
11209    case glslang::EOpSubgroupShuffle:
11210    case glslang::EOpSubgroupShuffleXor:
11211    case glslang::EOpSubgroupShuffleUp:
11212    case glslang::EOpSubgroupShuffleDown:
11213    case glslang::EOpSubgroupRotate:
11214    case glslang::EOpSubgroupClusteredRotate:
11215    case glslang::EOpSubgroupClusteredAdd:
11216    case glslang::EOpSubgroupClusteredMul:
11217    case glslang::EOpSubgroupClusteredMin:
11218    case glslang::EOpSubgroupClusteredMax:
11219    case glslang::EOpSubgroupClusteredAnd:
11220    case glslang::EOpSubgroupClusteredOr:
11221    case glslang::EOpSubgroupClusteredXor:
11222    case glslang::EOpSubgroupQuadBroadcast:
11223    case glslang::EOpSubgroupPartitionedAdd:
11224    case glslang::EOpSubgroupPartitionedMul:
11225    case glslang::EOpSubgroupPartitionedMin:
11226    case glslang::EOpSubgroupPartitionedMax:
11227    case glslang::EOpSubgroupPartitionedAnd:
11228    case glslang::EOpSubgroupPartitionedOr:
11229    case glslang::EOpSubgroupPartitionedXor:
11230    case glslang::EOpSubgroupPartitionedInclusiveAdd:
11231    case glslang::EOpSubgroupPartitionedInclusiveMul:
11232    case glslang::EOpSubgroupPartitionedInclusiveMin:
11233    case glslang::EOpSubgroupPartitionedInclusiveMax:
11234    case glslang::EOpSubgroupPartitionedInclusiveAnd:
11235    case glslang::EOpSubgroupPartitionedInclusiveOr:
11236    case glslang::EOpSubgroupPartitionedInclusiveXor:
11237    case glslang::EOpSubgroupPartitionedExclusiveAdd:
11238    case glslang::EOpSubgroupPartitionedExclusiveMul:
11239    case glslang::EOpSubgroupPartitionedExclusiveMin:
11240    case glslang::EOpSubgroupPartitionedExclusiveMax:
11241    case glslang::EOpSubgroupPartitionedExclusiveAnd:
11242    case glslang::EOpSubgroupPartitionedExclusiveOr:
11243    case glslang::EOpSubgroupPartitionedExclusiveXor:
11244        return createSubgroupOperation(op, typeId, operands, typeProxy);
11245
11246    case glslang::EOpSwizzleInvocations:
11247        extBuiltins = getExtBuiltins(spv::E_SPV_AMD_shader_ballot);
11248        libCall = spv::SwizzleInvocationsAMD;
11249        break;
11250    case glslang::EOpSwizzleInvocationsMasked:
11251        extBuiltins = getExtBuiltins(spv::E_SPV_AMD_shader_ballot);
11252        libCall = spv::SwizzleInvocationsMaskedAMD;
11253        break;
11254    case glslang::EOpWriteInvocation:
11255        extBuiltins = getExtBuiltins(spv::E_SPV_AMD_shader_ballot);
11256        libCall = spv::WriteInvocationAMD;
11257        break;
11258
11259    case glslang::EOpMin3:
11260        extBuiltins = getExtBuiltins(spv::E_SPV_AMD_shader_trinary_minmax);
11261        if (isFloat)
11262            libCall = spv::FMin3AMD;
11263        else {
11264            if (isUnsigned)
11265                libCall = spv::UMin3AMD;
11266            else
11267                libCall = spv::SMin3AMD;
11268        }
11269        break;
11270    case glslang::EOpMax3:
11271        extBuiltins = getExtBuiltins(spv::E_SPV_AMD_shader_trinary_minmax);
11272        if (isFloat)
11273            libCall = spv::FMax3AMD;
11274        else {
11275            if (isUnsigned)
11276                libCall = spv::UMax3AMD;
11277            else
11278                libCall = spv::SMax3AMD;
11279        }
11280        break;
11281    case glslang::EOpMid3:
11282        extBuiltins = getExtBuiltins(spv::E_SPV_AMD_shader_trinary_minmax);
11283        if (isFloat)
11284            libCall = spv::FMid3AMD;
11285        else {
11286            if (isUnsigned)
11287                libCall = spv::UMid3AMD;
11288            else
11289                libCall = spv::SMid3AMD;
11290        }
11291        break;
11292
11293    case glslang::EOpInterpolateAtVertex:
11294        if (typeProxy == glslang::EbtFloat16)
11295            builder.addExtension(spv::E_SPV_AMD_gpu_shader_half_float);
11296        extBuiltins = getExtBuiltins(spv::E_SPV_AMD_shader_explicit_vertex_parameter);
11297        libCall = spv::InterpolateAtVertexAMD;
11298        break;
11299
11300    case glslang::EOpReportIntersection:
11301        typeId = builder.makeBoolType();
11302        opCode = spv::Op::OpReportIntersectionKHR;
11303        break;
11304    case glslang::EOpTraceNV:
11305        builder.createNoResultOp(spv::Op::OpTraceNV, operands);
11306        return 0;
11307    case glslang::EOpTraceRayMotionNV:
11308        builder.addExtension(spv::E_SPV_NV_ray_tracing_motion_blur);
11309        builder.addCapability(spv::Capability::RayTracingMotionBlurNV);
11310        builder.createNoResultOp(spv::Op::OpTraceRayMotionNV, operands);
11311        return 0;
11312    case glslang::EOpTraceKHR:
11313        builder.createNoResultOp(spv::Op::OpTraceRayKHR, operands);
11314        return 0;
11315    case glslang::EOpExecuteCallableNV:
11316        builder.createNoResultOp(spv::Op::OpExecuteCallableNV, operands);
11317        return 0;
11318    case glslang::EOpExecuteCallableKHR:
11319        builder.createNoResultOp(spv::Op::OpExecuteCallableKHR, operands);
11320        return 0;
11321
11322    case glslang::EOpRayQueryInitialize:
11323        builder.createNoResultOp(spv::Op::OpRayQueryInitializeKHR, operands);
11324        return 0;
11325    case glslang::EOpRayQueryTerminate:
11326        builder.createNoResultOp(spv::Op::OpRayQueryTerminateKHR, operands);
11327        return 0;
11328    case glslang::EOpRayQueryGenerateIntersection:
11329        builder.createNoResultOp(spv::Op::OpRayQueryGenerateIntersectionKHR, operands);
11330        return 0;
11331    case glslang::EOpRayQueryConfirmIntersection:
11332        builder.createNoResultOp(spv::Op::OpRayQueryConfirmIntersectionKHR, operands);
11333        return 0;
11334    case glslang::EOpRayQueryProceed:
11335        typeId = builder.makeBoolType();
11336        opCode = spv::Op::OpRayQueryProceedKHR;
11337        break;
11338    case glslang::EOpRayQueryGetIntersectionType:
11339        typeId = builder.makeUintType(32);
11340        opCode = spv::Op::OpRayQueryGetIntersectionTypeKHR;
11341        break;
11342    case glslang::EOpRayQueryGetRayTMin:
11343        typeId = builder.makeFloatType(32);
11344        opCode = spv::Op::OpRayQueryGetRayTMinKHR;
11345        break;
11346    case glslang::EOpRayQueryGetRayFlags:
11347        typeId = builder.makeIntType(32);
11348        opCode = spv::Op::OpRayQueryGetRayFlagsKHR;
11349        break;
11350    case glslang::EOpRayQueryGetIntersectionT:
11351        typeId = builder.makeFloatType(32);
11352        opCode = spv::Op::OpRayQueryGetIntersectionTKHR;
11353        break;
11354    case glslang::EOpRayQueryGetIntersectionInstanceCustomIndex:
11355        typeId = builder.makeIntType(32);
11356        opCode = spv::Op::OpRayQueryGetIntersectionInstanceCustomIndexKHR;
11357        break;
11358    case glslang::EOpRayQueryGetIntersectionInstanceId:
11359        typeId = builder.makeIntType(32);
11360        opCode = spv::Op::OpRayQueryGetIntersectionInstanceIdKHR;
11361        break;
11362    case glslang::EOpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffset:
11363        typeId = builder.makeUintType(32);
11364        opCode = spv::Op::OpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffsetKHR;
11365        break;
11366    case glslang::EOpRayQueryGetIntersectionGeometryIndex:
11367        typeId = builder.makeIntType(32);
11368        opCode = spv::Op::OpRayQueryGetIntersectionGeometryIndexKHR;
11369        break;
11370    case glslang::EOpRayQueryGetIntersectionPrimitiveIndex:
11371        typeId = builder.makeIntType(32);
11372        opCode = spv::Op::OpRayQueryGetIntersectionPrimitiveIndexKHR;
11373        break;
11374    case glslang::EOpRayQueryGetIntersectionBarycentrics:
11375        typeId = builder.makeVectorType(builder.makeFloatType(32), 2);
11376        opCode = spv::Op::OpRayQueryGetIntersectionBarycentricsKHR;
11377        break;
11378    case glslang::EOpRayQueryGetIntersectionFrontFace:
11379        typeId = builder.makeBoolType();
11380        opCode = spv::Op::OpRayQueryGetIntersectionFrontFaceKHR;
11381        break;
11382    case glslang::EOpRayQueryGetIntersectionCandidateAABBOpaque:
11383        typeId = builder.makeBoolType();
11384        opCode = spv::Op::OpRayQueryGetIntersectionCandidateAABBOpaqueKHR;
11385        break;
11386    case glslang::EOpRayQueryGetIntersectionObjectRayDirection:
11387        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11388        opCode = spv::Op::OpRayQueryGetIntersectionObjectRayDirectionKHR;
11389        break;
11390    case glslang::EOpRayQueryGetIntersectionObjectRayOrigin:
11391        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11392        opCode = spv::Op::OpRayQueryGetIntersectionObjectRayOriginKHR;
11393        break;
11394    case glslang::EOpRayQueryGetWorldRayDirection:
11395        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11396        opCode = spv::Op::OpRayQueryGetWorldRayDirectionKHR;
11397        break;
11398    case glslang::EOpRayQueryGetWorldRayOrigin:
11399        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11400        opCode = spv::Op::OpRayQueryGetWorldRayOriginKHR;
11401        break;
11402    case glslang::EOpRayQueryGetIntersectionObjectToWorld:
11403        typeId = builder.makeMatrixType(builder.makeFloatType(32), 4, 3);
11404        opCode = spv::Op::OpRayQueryGetIntersectionObjectToWorldKHR;
11405        break;
11406    case glslang::EOpRayQueryGetIntersectionClusterIdNV:
11407        typeId = builder.makeIntegerType(32, 1);
11408        opCode = spv::Op::OpRayQueryGetClusterIdNV;
11409        break;
11410    case glslang::EOpRayQueryGetIntersectionWorldToObject:
11411        typeId = builder.makeMatrixType(builder.makeFloatType(32), 4, 3);
11412        opCode = spv::Op::OpRayQueryGetIntersectionWorldToObjectKHR;
11413        break;
11414    case glslang::EOpRayQueryGetIntersectionSpherePositionNV:
11415        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11416        opCode = spv::Op::OpRayQueryGetIntersectionSpherePositionNV;
11417        break;
11418    case glslang::EOpRayQueryGetIntersectionSphereRadiusNV:
11419        typeId = builder.makeFloatType(32);
11420        opCode = spv::Op::OpRayQueryGetIntersectionSphereRadiusNV;
11421        break;
11422    case glslang::EOpRayQueryGetIntersectionLSSHitValueNV:
11423        typeId = builder.makeFloatType(32);
11424        opCode = spv::Op::OpRayQueryGetIntersectionLSSHitValueNV;
11425        break;
11426    case glslang::EOpRayQueryIsSphereHitNV:
11427        typeId = builder.makeBoolType();
11428        opCode = spv::Op::OpRayQueryIsSphereHitNV;
11429        break;
11430    case glslang::EOpRayQueryIsLSSHitNV:
11431        typeId = builder.makeBoolType();
11432        opCode = spv::Op::OpRayQueryIsLSSHitNV;
11433        break;
11434    case glslang::EOpWritePackedPrimitiveIndices4x8NV:
11435        builder.createNoResultOp(spv::Op::OpWritePackedPrimitiveIndices4x8NV, operands);
11436        return 0;
11437    case glslang::EOpEmitMeshTasksEXT:
11438        if (taskPayloadID)
11439            operands.push_back(taskPayloadID);
11440        // As per SPV_EXT_mesh_shader make it a terminating instruction in the current block
11441        builder.makeStatementTerminator(spv::Op::OpEmitMeshTasksEXT, operands, "post-OpEmitMeshTasksEXT");
11442        return 0;
11443    case glslang::EOpSetMeshOutputsEXT:
11444        builder.createNoResultOp(spv::Op::OpSetMeshOutputsEXT, operands);
11445        return 0;
11446    case glslang::EOpCooperativeMatrixMulAddNV:
11447        opCode = spv::Op::OpCooperativeMatrixMulAddNV;
11448        break;
11449    case glslang::EOpHitObjectTraceRayNV:
11450        builder.createNoResultOp(spv::Op::OpHitObjectTraceRayNV, operands);
11451        return 0;
11452    case glslang::EOpHitObjectTraceRayEXT:
11453        builder.createNoResultOp(spv::Op::OpHitObjectTraceRayEXT, operands);
11454        return 0;
11455    case glslang::EOpHitObjectTraceRayMotionNV:
11456        builder.createNoResultOp(spv::Op::OpHitObjectTraceRayMotionNV, operands);
11457        return 0;
11458    case glslang::EOpHitObjectTraceRayMotionEXT:
11459        builder.createNoResultOp(spv::Op::OpHitObjectTraceRayMotionEXT, operands);
11460        return 0;
11461    case glslang::EOpHitObjectRecordHitNV:
11462        builder.createNoResultOp(spv::Op::OpHitObjectRecordHitNV, operands);
11463        return 0;
11464    case glslang::EOpHitObjectRecordHitMotionNV:
11465        builder.createNoResultOp(spv::Op::OpHitObjectRecordHitMotionNV, operands);
11466        return 0;
11467    case glslang::EOpHitObjectRecordHitWithIndexNV:
11468        builder.createNoResultOp(spv::Op::OpHitObjectRecordHitWithIndexNV, operands);
11469        return 0;
11470    case glslang::EOpHitObjectRecordHitWithIndexMotionNV:
11471        builder.createNoResultOp(spv::Op::OpHitObjectRecordHitWithIndexMotionNV, operands);
11472        return 0;
11473    case glslang::EOpHitObjectRecordMissNV:
11474        builder.createNoResultOp(spv::Op::OpHitObjectRecordMissNV, operands);
11475        return 0;
11476    case glslang::EOpHitObjectRecordMissEXT:
11477        builder.createNoResultOp(spv::Op::OpHitObjectRecordMissEXT, operands);
11478        return 0;
11479    case glslang::EOpHitObjectRecordMissMotionNV:
11480        builder.createNoResultOp(spv::Op::OpHitObjectRecordMissMotionNV, operands);
11481        return 0;
11482    case glslang::EOpHitObjectRecordMissMotionEXT:
11483        builder.createNoResultOp(spv::Op::OpHitObjectRecordMissMotionEXT, operands);
11484        return 0;
11485    case glslang::EOpHitObjectExecuteShaderNV:
11486        builder.createNoResultOp(spv::Op::OpHitObjectExecuteShaderNV, operands);
11487        return 0;
11488    case glslang::EOpHitObjectExecuteShaderEXT:
11489        builder.createNoResultOp(spv::Op::OpHitObjectExecuteShaderEXT, operands);
11490        return 0;
11491    case glslang::EOpHitObjectIsEmptyNV:
11492        typeId = builder.makeBoolType();
11493        opCode = spv::Op::OpHitObjectIsEmptyNV;
11494        break;
11495    case glslang::EOpHitObjectIsEmptyEXT:
11496        typeId = builder.makeBoolType();
11497        opCode = spv::Op::OpHitObjectIsEmptyEXT;
11498        break;
11499    case glslang::EOpHitObjectIsMissNV:
11500        typeId = builder.makeBoolType();
11501        opCode = spv::Op::OpHitObjectIsMissNV;
11502        break;
11503    case glslang::EOpHitObjectIsMissEXT:
11504        typeId = builder.makeBoolType();
11505        opCode = spv::Op::OpHitObjectIsMissEXT;
11506        break;
11507    case glslang::EOpHitObjectIsHitNV:
11508        typeId = builder.makeBoolType();
11509        opCode = spv::Op::OpHitObjectIsHitNV;
11510        break;
11511    case glslang::EOpHitObjectIsSphereHitNV:
11512        typeId = builder.makeBoolType();
11513        opCode = spv::Op::OpHitObjectIsSphereHitNV;
11514        break;
11515    case glslang::EOpHitObjectIsLSSHitNV:
11516        typeId = builder.makeBoolType();
11517        opCode = spv::Op::OpHitObjectIsLSSHitNV;
11518        break;
11519    case glslang::EOpHitObjectIsHitEXT:
11520        typeId = builder.makeBoolType();
11521        opCode = spv::Op::OpHitObjectIsHitEXT;
11522        break;
11523    case glslang::EOpHitObjectGetRayTMinNV:
11524        typeId = builder.makeFloatType(32);
11525        opCode = spv::Op::OpHitObjectGetRayTMinNV;
11526        break;
11527    case glslang::EOpHitObjectGetRayTMinEXT:
11528        typeId = builder.makeFloatType(32);
11529        opCode = spv::Op::OpHitObjectGetRayTMinEXT;
11530        break;
11531    case glslang::EOpHitObjectGetRayTMaxNV:
11532        typeId = builder.makeFloatType(32);
11533        opCode = spv::Op::OpHitObjectGetRayTMaxNV;
11534        break;
11535    case glslang::EOpHitObjectGetRayTMaxEXT:
11536        typeId = builder.makeFloatType(32);
11537        opCode = spv::Op::OpHitObjectGetRayTMaxEXT;
11538        break;
11539    case glslang::EOpHitObjectGetRayFlagsEXT:
11540        typeId = builder.makeIntegerType(32, 0);
11541        opCode = spv::Op::OpHitObjectGetRayFlagsEXT;
11542        break;
11543    case glslang::EOpHitObjectGetObjectRayOriginNV:
11544        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11545        opCode = spv::Op::OpHitObjectGetObjectRayOriginNV;
11546        break;
11547    case glslang::EOpHitObjectGetObjectRayOriginEXT:
11548        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11549        opCode = spv::Op::OpHitObjectGetObjectRayOriginEXT;
11550        break;
11551    case glslang::EOpHitObjectGetObjectRayDirectionNV:
11552        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11553        opCode = spv::Op::OpHitObjectGetObjectRayDirectionNV;
11554        break;
11555    case glslang::EOpHitObjectGetObjectRayDirectionEXT:
11556        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11557        opCode = spv::Op::OpHitObjectGetObjectRayDirectionEXT;
11558        break;
11559    case glslang::EOpHitObjectGetWorldRayOriginNV:
11560        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11561        opCode = spv::Op::OpHitObjectGetWorldRayOriginNV;
11562        break;
11563    case glslang::EOpHitObjectGetWorldRayOriginEXT:
11564        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11565        opCode = spv::Op::OpHitObjectGetWorldRayOriginEXT;
11566        break;
11567    case glslang::EOpHitObjectGetWorldRayDirectionNV:
11568        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11569        opCode = spv::Op::OpHitObjectGetWorldRayDirectionNV;
11570        break;
11571    case glslang::EOpHitObjectGetWorldRayDirectionEXT:
11572        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11573        opCode = spv::Op::OpHitObjectGetWorldRayDirectionEXT;
11574        break;
11575    case glslang::EOpHitObjectGetWorldToObjectNV:
11576        typeId = builder.makeMatrixType(builder.makeFloatType(32), 4, 3);
11577        opCode = spv::Op::OpHitObjectGetWorldToObjectNV;
11578        break;
11579    case glslang::EOpHitObjectGetWorldToObjectEXT:
11580        typeId = builder.makeMatrixType(builder.makeFloatType(32), 4, 3);
11581        opCode = spv::Op::OpHitObjectGetWorldToObjectEXT;
11582        break;
11583    case glslang::EOpHitObjectGetObjectToWorldNV:
11584        typeId = builder.makeMatrixType(builder.makeFloatType(32), 4, 3);
11585        opCode = spv::Op::OpHitObjectGetObjectToWorldNV;
11586        break;
11587    case glslang::EOpHitObjectGetObjectToWorldEXT:
11588        typeId = builder.makeMatrixType(builder.makeFloatType(32), 4, 3);
11589        opCode = spv::Op::OpHitObjectGetObjectToWorldEXT;
11590        break;
11591    case glslang::EOpHitObjectGetInstanceCustomIndexNV:
11592        typeId = builder.makeIntegerType(32, 1);
11593        opCode = spv::Op::OpHitObjectGetInstanceCustomIndexNV;
11594        break;
11595    case glslang::EOpHitObjectGetInstanceCustomIndexEXT:
11596        typeId = builder.makeIntegerType(32, 1);
11597        opCode = spv::Op::OpHitObjectGetInstanceCustomIndexEXT;
11598        break;
11599    case glslang::EOpHitObjectGetInstanceIdNV:
11600        typeId = builder.makeIntegerType(32, 1);
11601        opCode = spv::Op::OpHitObjectGetInstanceIdNV;
11602        break;
11603    case glslang::EOpHitObjectGetInstanceIdEXT:
11604        typeId = builder.makeIntegerType(32, 1);
11605        opCode = spv::Op::OpHitObjectGetInstanceIdEXT;
11606        break;
11607    case glslang::EOpHitObjectGetGeometryIndexNV:
11608        typeId = builder.makeIntegerType(32, 1);
11609        opCode = spv::Op::OpHitObjectGetGeometryIndexNV;
11610        break;
11611    case glslang::EOpHitObjectGetGeometryIndexEXT:
11612        typeId = builder.makeIntegerType(32, 1);
11613        opCode = spv::Op::OpHitObjectGetGeometryIndexEXT;
11614        break;
11615    case glslang::EOpHitObjectGetPrimitiveIndexNV:
11616        typeId = builder.makeIntegerType(32, 1);
11617        opCode = spv::Op::OpHitObjectGetPrimitiveIndexNV;
11618        break;
11619    case glslang::EOpHitObjectGetPrimitiveIndexEXT:
11620        typeId = builder.makeIntegerType(32, 1);
11621        opCode = spv::Op::OpHitObjectGetPrimitiveIndexEXT;
11622        break;
11623    case glslang::EOpHitObjectGetHitKindNV:
11624        typeId = builder.makeIntegerType(32, 0);
11625        opCode = spv::Op::OpHitObjectGetHitKindNV;
11626        break;
11627    case glslang::EOpHitObjectGetHitKindEXT:
11628        typeId = builder.makeIntegerType(32, 0);
11629        opCode = spv::Op::OpHitObjectGetHitKindEXT;
11630        break;
11631    case glslang::EOpHitObjectGetCurrentTimeNV:
11632        typeId = builder.makeFloatType(32);
11633        opCode = spv::Op::OpHitObjectGetCurrentTimeNV;
11634        break;
11635    case glslang::EOpHitObjectGetCurrentTimeEXT:
11636        typeId = builder.makeFloatType(32);
11637        opCode = spv::Op::OpHitObjectGetCurrentTimeEXT;
11638        break;
11639    case glslang::EOpHitObjectGetShaderBindingTableRecordIndexNV:
11640        typeId = builder.makeIntegerType(32, 0);
11641        opCode = spv::Op::OpHitObjectGetShaderBindingTableRecordIndexNV;
11642        return 0;
11643    case glslang::EOpHitObjectGetShaderBindingTableRecordIndexEXT:
11644        typeId = builder.makeIntegerType(32, 0);
11645        opCode = spv::Op::OpHitObjectGetShaderBindingTableRecordIndexEXT;
11646        return 0;
11647    case glslang::EOpHitObjectGetAttributesNV:
11648        builder.createNoResultOp(spv::Op::OpHitObjectGetAttributesNV, operands);
11649        return 0;
11650    case glslang::EOpHitObjectGetAttributesEXT:
11651        builder.createNoResultOp(spv::Op::OpHitObjectGetAttributesEXT, operands);
11652        return 0;
11653    case glslang::EOpHitObjectRecordFromQueryEXT:
11654        builder.createNoResultOp(spv::Op::OpHitObjectRecordFromQueryEXT, operands);
11655        return 0;
11656    case glslang::EOpHitObjectGetShaderRecordBufferHandleNV:
11657        typeId = builder.makeVectorType(builder.makeUintType(32), 2);
11658        opCode = spv::Op::OpHitObjectGetShaderRecordBufferHandleNV;
11659        break;
11660    case glslang::EOpHitObjectGetClusterIdNV:
11661        typeId = builder.makeIntegerType(32, 1);
11662        opCode = spv::Op::OpHitObjectGetClusterIdNV;
11663        break;
11664    case glslang::EOpHitObjectGetShaderRecordBufferHandleEXT:
11665        typeId = builder.makeVectorType(builder.makeUintType(32), 2);
11666        opCode = spv::Op::OpHitObjectGetShaderRecordBufferHandleEXT;
11667        break;
11668    case glslang::EOpHitObjectSetShaderBindingTableRecordIndexEXT:
11669        builder.createNoResultOp(spv::Op::OpHitObjectSetShaderBindingTableRecordIndexEXT, operands);
11670        return 0;
11671    case glslang::EOpReorderThreadNV: {
11672        if (operands.size() == 2) {
11673            builder.createNoResultOp(spv::Op::OpReorderThreadWithHintNV, operands);
11674        } else {
11675            builder.createNoResultOp(spv::Op::OpReorderThreadWithHitObjectNV, operands);
11676        }
11677        return 0;
11678    }
11679    case glslang::EOpReorderThreadEXT: {
11680        if (operands.size() == 2) {
11681            builder.createNoResultOp(spv::Op::OpReorderThreadWithHintEXT, operands);
11682        } else {
11683            builder.createNoResultOp(spv::Op::OpReorderThreadWithHitObjectEXT, operands);
11684        }
11685        return 0;
11686    }
11687
11688    case glslang::EOpHitObjectReorderExecuteEXT: {
11689        if (operands.size() == 2) {
11690            builder.createNoResultOp(spv::Op::OpHitObjectReorderExecuteShaderEXT, operands);
11691        } else {
11692            // GLSL intrinsic is
11693            // hitObjectReorderExecuteEXT(hitObjectEXT hitObject, uint hint, uint bits,int payload) while
11694            // SPIRV is hitObject id , payload id, optional hint id, optional bits id hence reorder operands
11695            builder.createNoResultOp(spv::Op::OpHitObjectReorderExecuteShaderEXT, {operands[0], operands[3], operands[1], operands[2]});
11696        }
11697        return 0;
11698    }
11699
11700    case glslang::EOpHitObjectTraceReorderExecuteEXT: {
11701        if (operands.size() == 12) {
11702            builder.createNoResultOp(spv::Op::OpHitObjectTraceReorderExecuteEXT, operands);
11703        } else {
11704            std::vector<spv::Id> argOperands;
11705            std::copy(operands.begin(), operands.begin() + 11, std::back_inserter(argOperands));
11706            argOperands.push_back(operands[13]);
11707            argOperands.push_back(operands[11]);
11708            argOperands.push_back(operands[12]);
11709            builder.createNoResultOp(spv::Op::OpHitObjectTraceReorderExecuteEXT, argOperands);
11710        }
11711        return 0;
11712    }
11713    case glslang::EOpHitObjectTraceMotionReorderExecuteEXT: {
11714        if (operands.size() == 13) {
11715            builder.createNoResultOp(spv::Op::OpHitObjectTraceMotionReorderExecuteEXT, operands);
11716        } else {
11717            std::vector<spv::Id> argOperands;
11718            std::copy(operands.begin(), operands.begin() + 12, std::back_inserter(argOperands));
11719            argOperands.push_back(operands[14]);
11720            argOperands.push_back(operands[12]);
11721            argOperands.push_back(operands[13]);
11722            builder.createNoResultOp(spv::Op::OpHitObjectTraceMotionReorderExecuteEXT, argOperands);
11723        }
11724        return 0;
11725    }
11726    case glslang::EOpImageSampleWeightedQCOM:
11727        typeId = builder.makeVectorType(builder.makeFloatType(32), 4);
11728        opCode = spv::Op::OpImageSampleWeightedQCOM;
11729        addImageProcessingQCOMDecoration(operands[2], spv::Decoration::WeightTextureQCOM);
11730        break;
11731    case glslang::EOpImageBoxFilterQCOM:
11732        typeId = builder.makeVectorType(builder.makeFloatType(32), 4);
11733        opCode = spv::Op::OpImageBoxFilterQCOM;
11734        break;
11735    case glslang::EOpImageBlockMatchSADQCOM:
11736        typeId = builder.makeVectorType(builder.makeFloatType(32), 4);
11737        opCode = spv::Op::OpImageBlockMatchSADQCOM;
11738        addImageProcessingQCOMDecoration(operands[0], spv::Decoration::BlockMatchTextureQCOM);
11739        addImageProcessingQCOMDecoration(operands[2], spv::Decoration::BlockMatchTextureQCOM);
11740        break;
11741    case glslang::EOpImageBlockMatchSSDQCOM:
11742        typeId = builder.makeVectorType(builder.makeFloatType(32), 4);
11743        opCode = spv::Op::OpImageBlockMatchSSDQCOM;
11744        addImageProcessingQCOMDecoration(operands[0], spv::Decoration::BlockMatchTextureQCOM);
11745        addImageProcessingQCOMDecoration(operands[2], spv::Decoration::BlockMatchTextureQCOM);
11746        break;
11747
11748    case glslang::EOpFetchMicroTriangleVertexBarycentricNV:
11749        typeId = builder.makeVectorType(builder.makeFloatType(32), 2);
11750        opCode = spv::Op::OpFetchMicroTriangleVertexBarycentricNV;
11751        break;
11752
11753    case glslang::EOpFetchMicroTriangleVertexPositionNV:
11754        typeId = builder.makeVectorType(builder.makeFloatType(32), 3);
11755        opCode = spv::Op::OpFetchMicroTriangleVertexPositionNV;
11756        break;
11757
11758    case glslang::EOpImageBlockMatchWindowSSDQCOM:
11759        typeId = builder.makeVectorType(builder.makeFloatType(32), 4);
11760        opCode = spv::Op::OpImageBlockMatchWindowSSDQCOM;
11761        addImageProcessing2QCOMDecoration(operands[0], false);
11762        addImageProcessing2QCOMDecoration(operands[2], false);
11763        break;
11764    case glslang::EOpImageBlockMatchWindowSADQCOM:
11765        typeId = builder.makeVectorType(builder.makeFloatType(32), 4);
11766        opCode = spv::Op::OpImageBlockMatchWindowSADQCOM;
11767        addImageProcessing2QCOMDecoration(operands[0], false);
11768        addImageProcessing2QCOMDecoration(operands[2], false);
11769        break;
11770    case glslang::EOpImageBlockMatchGatherSSDQCOM:
11771        typeId = builder.makeVectorType(builder.makeFloatType(32), 4);
11772        opCode = spv::Op::OpImageBlockMatchGatherSSDQCOM;
11773        addImageProcessing2QCOMDecoration(operands[0], true);
11774        addImageProcessing2QCOMDecoration(operands[2], true);
11775        break;
11776    case glslang::EOpImageBlockMatchGatherSADQCOM:
11777        typeId = builder.makeVectorType(builder.makeFloatType(32), 4);
11778        opCode = spv::Op::OpImageBlockMatchGatherSADQCOM;
11779        addImageProcessing2QCOMDecoration(operands[0], true);
11780        addImageProcessing2QCOMDecoration(operands[2], true);
11781        break;
11782    case glslang::EOpCreateTensorLayoutNV:
11783        return builder.createOp(spv::Op::OpCreateTensorLayoutNV, typeId, std::vector<spv::Id>{});
11784    case glslang::EOpCreateTensorViewNV:
11785        return builder.createOp(spv::Op::OpCreateTensorViewNV, typeId, std::vector<spv::Id>{});
11786    case glslang::EOpTensorLayoutSetBlockSizeNV:
11787        opCode = spv::Op::OpTensorLayoutSetBlockSizeNV;
11788        break;
11789    case glslang::EOpTensorLayoutSetDimensionNV:
11790        opCode = spv::Op::OpTensorLayoutSetDimensionNV;
11791        break;
11792    case glslang::EOpTensorLayoutSetStrideNV:
11793        opCode = spv::Op::OpTensorLayoutSetStrideNV;
11794        break;
11795    case glslang::EOpTensorLayoutSliceNV:
11796        opCode = spv::Op::OpTensorLayoutSliceNV;
11797        break;
11798    case glslang::EOpTensorLayoutSetClampValueNV:
11799        opCode = spv::Op::OpTensorLayoutSetClampValueNV;
11800        break;
11801    case glslang::EOpTensorViewSetDimensionNV:
11802        opCode = spv::Op::OpTensorViewSetDimensionNV;
11803        break;
11804    case glslang::EOpTensorViewSetStrideNV:
11805        opCode = spv::Op::OpTensorViewSetStrideNV;
11806        break;
11807    case glslang::EOpTensorViewSetClipNV:
11808        opCode = spv::Op::OpTensorViewSetClipNV;
11809        break;
11810    case glslang::EOpCooperativeMatrixGetCoordinateEXT:
11811        opCode = spv::Op::OpCooperativeMatrixGetCoordinateEXT;
11812        break;
11813    case glslang::EOpBitcastExtractE2M1:
11814    case glslang::EOpBitcastExtractE3M2:
11815    case glslang::EOpBitcastExtractE2M3:
11816        {
11817            spv::Id scalarTypeId = op == glslang::EOpBitcastExtractE2M1 ? builder.makeFloatE2M1Type() :
11818                                   op == glslang::EOpBitcastExtractE3M2 ? builder.makeFloatE3M2Type() : builder.makeFloatE2M3Type();
11819            opCode = spv::Op::OpBitcastExtractEXT;
11820            typeId = scalarTypeId;
11821            if (builder.isVectorType(builder.getTypeId(operands[0]))) {
11822                int vecSize = builder.getNumTypeComponents(builder.getTypeId(operands[0]));
11823                typeId = builder.makeVectorType(typeId, vecSize);
11824            }
11825            builder.addExtension(spv::E_SPV_EXT_ocp_microscaling_types);
11826            builder.addCapability(spv::Capability::BitcastExtractEXT);
11827            break;
11828        }
11829    default:
11830        return 0;
11831    }
11832
11833    spv::Id id = 0;
11834    if (libCall >= 0) {
11835        // Use an extended instruction from the standard library.
11836        // Construct the call arguments, without modifying the original operands vector.
11837        // We might need the remaining arguments, e.g. in the EOpFrexp case.
11838        std::vector<spv::Id> callArguments(operands.begin(), operands.begin() + consumedOperands);
11839        id = builder.createBuiltinCall(typeId, extBuiltins >= 0 ? extBuiltins : stdBuiltins, libCall, callArguments);
11840    } else if (opCode == spv::Op::OpDot && !isFloat) {
11841        // int dot(int, int)
11842        // NOTE: never called for scalar/vector1, this is turned into simple mul before this can be reached
11843        const int componentCount = builder.getNumComponents(operands[0]);
11844        spv::Id mulOp = builder.createBinOp(spv::Op::OpIMul, builder.getTypeId(operands[0]), operands[0], operands[1]);
11845        builder.setPrecision(mulOp, precision);
11846        id = builder.createCompositeExtract(mulOp, typeId, 0);
11847        for (int i = 1; i < componentCount; ++i) {
11848            builder.setPrecision(id, precision);
11849            id = builder.createBinOp(spv::Op::OpIAdd, typeId, id, builder.createCompositeExtract(mulOp, typeId, i));
11850        }
11851    } else {
11852        switch (consumedOperands) {
11853        case 0:
11854            // should all be handled by visitAggregate and createNoArgOperation
11855            assert(0);
11856            return 0;
11857        case 1:
11858            // should all be handled by createUnaryOperation
11859            assert(0);
11860            return 0;
11861        case 2:
11862            id = builder.createBinOp(opCode, typeId, operands[0], operands[1]);
11863            break;
11864        default:
11865            // anything 3 or over doesn't have l-value operands, so all should be consumed
11866            assert(consumedOperands == operands.size());
11867            id = builder.createOp(opCode, typeId, operands);
11868            break;
11869        }
11870    }
11871
11872    // Decode the return types that were structures
11873    switch (op) {
11874    case glslang::EOpAddCarry:
11875    case glslang::EOpSubBorrow:
11876        builder.createStore(builder.createCompositeExtract(id, typeId0, 1), operands[2]);
11877        id = builder.createCompositeExtract(id, typeId0, 0);
11878        break;
11879    case glslang::EOpUMulExtended:
11880    case glslang::EOpIMulExtended:
11881        builder.createStore(builder.createCompositeExtract(id, typeId0, 0), operands[3]);
11882        builder.createStore(builder.createCompositeExtract(id, typeId0, 1), operands[2]);
11883        break;
11884    case glslang::EOpModf:
11885        {
11886            assert(operands.size() == 2);
11887            builder.createStore(builder.createCompositeExtract(id, typeId0, 1), operands[1]);
11888            id = builder.createCompositeExtract(id, typeId0, 0);
11889        }
11890        break;
11891    case glslang::EOpFrexp:
11892        {
11893            assert(operands.size() == 2);
11894            if (builder.isFloatType(builder.getScalarTypeId(typeId1))) {
11895                // "exp" is floating-point type (from HLSL intrinsic)
11896                spv::Id member1 = builder.createCompositeExtract(id, frexpIntType, 1);
11897                member1 = builder.createUnaryOp(spv::Op::OpConvertSToF, typeId1, member1);
11898                builder.createStore(member1, operands[1]);
11899            } else
11900                // "exp" is integer type (from GLSL built-in function)
11901                builder.createStore(builder.createCompositeExtract(id, frexpIntType, 1), operands[1]);
11902            id = builder.createCompositeExtract(id, typeId0, 0);
11903        }
11904        break;
11905    default:
11906        break;
11907    }
11908
11909    return builder.setPrecision(id, precision);
11910}
11911
11912// Intrinsics with no arguments (or no return value, and no precision).
11913spv::Id TGlslangToSpvTraverser::createNoArgOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId)
11914{
11915    // GLSL memory barriers use queuefamily scope in new model, device scope in old model
11916    spv::Scope memoryBarrierScope = glslangIntermediate->usingVulkanMemoryModel() ?
11917        spv::Scope::QueueFamilyKHR : spv::Scope::Device;
11918
11919    switch (op) {
11920    case glslang::EOpBarrier:
11921        if (glslangIntermediate->getStage() == EShLangTessControl) {
11922            if (glslangIntermediate->usingVulkanMemoryModel()) {
11923                builder.createControlBarrier(spv::Scope::Workgroup, spv::Scope::Workgroup,
11924                                             spv::MemorySemanticsMask::OutputMemoryKHR |
11925                                             spv::MemorySemanticsMask::AcquireRelease);
11926                builder.addCapability(spv::Capability::VulkanMemoryModelKHR);
11927            } else {
11928                builder.createControlBarrier(spv::Scope::Workgroup, spv::Scope::Invocation, spv::MemorySemanticsMask::MaskNone);
11929            }
11930        } else {
11931            builder.createControlBarrier(spv::Scope::Workgroup, spv::Scope::Workgroup,
11932                                            spv::MemorySemanticsMask::WorkgroupMemory |
11933                                            spv::MemorySemanticsMask::AcquireRelease);
11934        }
11935        return 0;
11936    case glslang::EOpControlBarrierArriveEXT:
11937        builder.addExtension(spv::E_SPV_EXT_split_barrier);
11938        builder.addCapability(spv::Capability::SplitBarrierEXT);
11939        if (glslangIntermediate->usingVulkanMemoryModel()) {
11940            builder.addCapability(spv::Capability::VulkanMemoryModelKHR);
11941        }
11942        builder.createSplitControlBarrier(spv::Op::OpControlBarrierArriveEXT, spv::Scope::Workgroup, spv::Scope::Workgroup,
11943                                          spv::MemorySemanticsMask::WorkgroupMemory |
11944                                          spv::MemorySemanticsMask::Release);
11945        return 0;
11946    case glslang::EOpControlBarrierWaitEXT:
11947        builder.addExtension(spv::E_SPV_EXT_split_barrier);
11948        builder.addCapability(spv::Capability::SplitBarrierEXT);
11949        if (glslangIntermediate->usingVulkanMemoryModel()) {
11950            builder.addCapability(spv::Capability::VulkanMemoryModelKHR);
11951        }
11952        builder.createSplitControlBarrier(spv::Op::OpControlBarrierWaitEXT, spv::Scope::Workgroup, spv::Scope::Workgroup,
11953                                          spv::MemorySemanticsMask::WorkgroupMemory |
11954                                          spv::MemorySemanticsMask::Acquire);
11955        return 0;
11956    case glslang::EOpMemoryBarrier:
11957        builder.createMemoryBarrier(memoryBarrierScope, spv::MemorySemanticsAllMemory |
11958                                                        spv::MemorySemanticsMask::AcquireRelease);
11959        return 0;
11960    case glslang::EOpMemoryBarrierBuffer:
11961        builder.createMemoryBarrier(memoryBarrierScope, spv::MemorySemanticsMask::UniformMemory |
11962                                                        spv::MemorySemanticsMask::AcquireRelease);
11963        return 0;
11964    case glslang::EOpMemoryBarrierShared:
11965        builder.createMemoryBarrier(memoryBarrierScope, spv::MemorySemanticsMask::WorkgroupMemory |
11966                                                        spv::MemorySemanticsMask::AcquireRelease);
11967        return 0;
11968    case glslang::EOpGroupMemoryBarrier:
11969        builder.createMemoryBarrier(spv::Scope::Workgroup, spv::MemorySemanticsAllMemory |
11970                                                           spv::MemorySemanticsMask::AcquireRelease);
11971        return 0;
11972    case glslang::EOpMemoryBarrierAtomicCounter:
11973        builder.createMemoryBarrier(memoryBarrierScope, spv::MemorySemanticsMask::AtomicCounterMemory |
11974                                                        spv::MemorySemanticsMask::AcquireRelease);
11975        return 0;
11976    case glslang::EOpMemoryBarrierImage:
11977        builder.createMemoryBarrier(memoryBarrierScope, spv::MemorySemanticsMask::ImageMemory |
11978                                                        spv::MemorySemanticsMask::AcquireRelease);
11979        return 0;
11980    case glslang::EOpAllMemoryBarrierWithGroupSync:
11981        builder.createControlBarrier(spv::Scope::Workgroup, spv::Scope::Device,
11982                                        spv::MemorySemanticsAllMemory |
11983                                        spv::MemorySemanticsMask::AcquireRelease);
11984        return 0;
11985    case glslang::EOpDeviceMemoryBarrier:
11986        builder.createMemoryBarrier(spv::Scope::Device, spv::MemorySemanticsMask::UniformMemory |
11987                                                        spv::MemorySemanticsMask::ImageMemory |
11988                                                        spv::MemorySemanticsMask::AcquireRelease);
11989        return 0;
11990    case glslang::EOpDeviceMemoryBarrierWithGroupSync:
11991        builder.createControlBarrier(spv::Scope::Workgroup, spv::Scope::Device, spv::MemorySemanticsMask::UniformMemory |
11992                                                                                spv::MemorySemanticsMask::ImageMemory |
11993                                                                                spv::MemorySemanticsMask::AcquireRelease);
11994        return 0;
11995    case glslang::EOpWorkgroupMemoryBarrier:
11996        builder.createMemoryBarrier(spv::Scope::Workgroup, spv::MemorySemanticsMask::WorkgroupMemory |
11997                                                           spv::MemorySemanticsMask::AcquireRelease);
11998        return 0;
11999    case glslang::EOpWorkgroupMemoryBarrierWithGroupSync:
12000        builder.createControlBarrier(spv::Scope::Workgroup, spv::Scope::Workgroup,
12001                                        spv::MemorySemanticsMask::WorkgroupMemory |
12002                                        spv::MemorySemanticsMask::AcquireRelease);
12003        return 0;
12004    case glslang::EOpSubgroupBarrier:
12005        builder.createControlBarrier(spv::Scope::Subgroup, spv::Scope::Subgroup, spv::MemorySemanticsAllMemory |
12006                                                                                 spv::MemorySemanticsMask::AcquireRelease);
12007        return spv::NoResult;
12008    case glslang::EOpSubgroupMemoryBarrier:
12009        builder.createMemoryBarrier(spv::Scope::Subgroup, spv::MemorySemanticsAllMemory |
12010                                                          spv::MemorySemanticsMask::AcquireRelease);
12011        return spv::NoResult;
12012    case glslang::EOpSubgroupMemoryBarrierBuffer:
12013        builder.createMemoryBarrier(spv::Scope::Subgroup, spv::MemorySemanticsMask::UniformMemory |
12014                                                          spv::MemorySemanticsMask::AcquireRelease);
12015        return spv::NoResult;
12016    case glslang::EOpSubgroupMemoryBarrierImage:
12017        builder.createMemoryBarrier(spv::Scope::Subgroup, spv::MemorySemanticsMask::ImageMemory |
12018                                                          spv::MemorySemanticsMask::AcquireRelease);
12019        return spv::NoResult;
12020    case glslang::EOpSubgroupMemoryBarrierShared:
12021        builder.createMemoryBarrier(spv::Scope::Subgroup, spv::MemorySemanticsMask::WorkgroupMemory |
12022                                                          spv::MemorySemanticsMask::AcquireRelease);
12023        return spv::NoResult;
12024
12025    case glslang::EOpEmitVertex:
12026        builder.createNoResultOp(spv::Op::OpEmitVertex);
12027        return 0;
12028    case glslang::EOpEndPrimitive:
12029        builder.createNoResultOp(spv::Op::OpEndPrimitive);
12030        return 0;
12031
12032    case glslang::EOpSubgroupElect: {
12033        std::vector<spv::Id> operands;
12034        return createSubgroupOperation(op, typeId, operands, glslang::EbtVoid);
12035    }
12036    case glslang::EOpTime:
12037    {
12038        std::vector<spv::Id> args; // Dummy arguments
12039        spv::Id id = builder.createBuiltinCall(typeId, getExtBuiltins(spv::E_SPV_AMD_gcn_shader), spv::TimeAMD, args);
12040        return builder.setPrecision(id, precision);
12041    }
12042    case glslang::EOpIgnoreIntersectionNV:
12043        builder.createNoResultOp(spv::Op::OpIgnoreIntersectionNV);
12044        return 0;
12045    case glslang::EOpTerminateRayNV:
12046        builder.createNoResultOp(spv::Op::OpTerminateRayNV);
12047        return 0;
12048    case glslang::EOpRayQueryInitialize:
12049        builder.createNoResultOp(spv::Op::OpRayQueryInitializeKHR);
12050        return 0;
12051    case glslang::EOpRayQueryTerminate:
12052        builder.createNoResultOp(spv::Op::OpRayQueryTerminateKHR);
12053        return 0;
12054    case glslang::EOpRayQueryGenerateIntersection:
12055        builder.createNoResultOp(spv::Op::OpRayQueryGenerateIntersectionKHR);
12056        return 0;
12057    case glslang::EOpRayQueryConfirmIntersection:
12058        builder.createNoResultOp(spv::Op::OpRayQueryConfirmIntersectionKHR);
12059        return 0;
12060    case glslang::EOpBeginInvocationInterlock:
12061        builder.createNoResultOp(spv::Op::OpBeginInvocationInterlockEXT);
12062        return 0;
12063    case glslang::EOpEndInvocationInterlock:
12064        builder.createNoResultOp(spv::Op::OpEndInvocationInterlockEXT);
12065        return 0;
12066
12067    case glslang::EOpIsHelperInvocation:
12068    {
12069        std::vector<spv::Id> args; // Dummy arguments
12070        builder.addExtension(spv::E_SPV_EXT_demote_to_helper_invocation);
12071        builder.addCapability(spv::Capability::DemoteToHelperInvocationEXT);
12072        return builder.createOp(spv::Op::OpIsHelperInvocationEXT, typeId, args);
12073    }
12074
12075    case glslang::EOpReadClockSubgroupKHR: {
12076        std::vector<spv::Id> args;
12077        args.push_back(builder.makeUintConstant(spv::Scope::Subgroup));
12078        builder.addExtension(spv::E_SPV_KHR_shader_clock);
12079        builder.addCapability(spv::Capability::ShaderClockKHR);
12080        return builder.createOp(spv::Op::OpReadClockKHR, typeId, args);
12081    }
12082
12083    case glslang::EOpReadClockDeviceKHR: {
12084        std::vector<spv::Id> args;
12085        args.push_back(builder.makeUintConstant(spv::Scope::Device));
12086        builder.addExtension(spv::E_SPV_KHR_shader_clock);
12087        builder.addCapability(spv::Capability::ShaderClockKHR);
12088        return builder.createOp(spv::Op::OpReadClockKHR, typeId, args);
12089    }
12090    case glslang::EOpStencilAttachmentReadEXT:
12091    case glslang::EOpDepthAttachmentReadEXT:
12092    {
12093        builder.addExtension(spv::E_SPV_EXT_shader_tile_image);
12094
12095        spv::Decoration precision;
12096        spv::Op spv_op;
12097        if (op == glslang::EOpStencilAttachmentReadEXT)
12098        {
12099            precision = spv::Decoration::RelaxedPrecision;
12100            spv_op = spv::Op::OpStencilAttachmentReadEXT;
12101            builder.addCapability(spv::Capability::TileImageStencilReadAccessEXT);
12102        }
12103        else
12104        {
12105            precision = spv::NoPrecision;
12106            spv_op = spv::Op::OpDepthAttachmentReadEXT;
12107            builder.addCapability(spv::Capability::TileImageDepthReadAccessEXT);
12108        }
12109
12110        std::vector<spv::Id> args; // Dummy args
12111        spv::Id result = builder.createOp(spv_op, typeId, args);
12112        return builder.setPrecision(result, precision);
12113    }
12114    default:
12115        break;
12116    }
12117
12118    logger->missingFunctionality("unknown operation with no arguments");
12119
12120    return 0;
12121}
12122
12123spv::Id TGlslangToSpvTraverser::getSymbolId(const glslang::TIntermSymbol* symbol)
12124{
12125    auto iter = symbolValues.find(symbol->getId());
12126    spv::Id id;
12127    if (symbolValues.end() != iter) {
12128        id = iter->second;
12129        return id;
12130    }
12131
12132    // it was not found, create it
12133    spv::BuiltIn builtIn = TranslateBuiltInDecoration(symbol->getQualifier().builtIn, false);
12134    auto forcedType = getForcedType(symbol->getQualifier().builtIn, symbol->getType());
12135
12136    // There are pairs of symbols that map to the same SPIR-V built-in:
12137    // gl_ObjectToWorldEXT and gl_ObjectToWorld3x4EXT, and gl_WorldToObjectEXT
12138    // and gl_WorldToObject3x4EXT. SPIR-V forbids having two OpVariables
12139    // with the same BuiltIn in the same storage class, so we must re-use one.
12140    const bool mayNeedToReuseBuiltIn =
12141        builtIn == spv::BuiltIn::ObjectToWorldKHR ||
12142        builtIn == spv::BuiltIn::WorldToObjectKHR;
12143
12144    // EXT_descriptor_heap
12145    const bool needToRemapDescHeap =
12146        builtIn == spv::BuiltIn::ResourceHeapEXT || builtIn == spv::BuiltIn::SamplerHeapEXT;
12147
12148    if (mayNeedToReuseBuiltIn || needToRemapDescHeap) {
12149        auto iter = builtInVariableIds.find(uint32_t(builtIn));
12150        if (builtInVariableIds.end() != iter) {
12151            id = iter->second;
12152            symbolValues[symbol->getId()] = id;
12153            if (forcedType.second != spv::NoType)
12154                forceType[id] = forcedType.second;
12155            return id;
12156        }
12157    }
12158
12159    if (symbol->getBasicType() == glslang::EbtFunction) {
12160        return 0;
12161    }
12162
12163    id = createSpvVariable(symbol, forcedType.first);
12164
12165    if (mayNeedToReuseBuiltIn) {
12166        builtInVariableIds.insert({uint32_t(builtIn), id});
12167    }
12168
12169    symbolValues[symbol->getId()] = id;
12170    if (forcedType.second != spv::NoType)
12171        forceType[id] = forcedType.second;
12172
12173    if (symbol->getBasicType() != glslang::EbtBlock) {
12174        builder.addDecoration(id, TranslatePrecisionDecoration(symbol->getType()));
12175        builder.addDecoration(id, TranslateInterpolationDecoration(symbol->getType().getQualifier()));
12176        builder.addDecoration(id, TranslateAuxiliaryStorageDecoration(symbol->getType().getQualifier()));
12177        addMeshNVDecoration(id, /*member*/ -1, symbol->getType().getQualifier());
12178        if (symbol->getQualifier().hasComponent())
12179            builder.addDecoration(id, spv::Decoration::Component, symbol->getQualifier().layoutComponent);
12180        if (symbol->getQualifier().hasIndex())
12181            builder.addDecoration(id, spv::Decoration::Index, symbol->getQualifier().layoutIndex);
12182        if (symbol->getType().getQualifier().hasSpecConstantId()) {
12183            builder.addDecoration(id, spv::Decoration::SpecId, symbol->getType().getQualifier().layoutSpecConstantId);
12184            // GL_EXT_opacity_micromap_ray_query_mode: if gl_EnableOpacityMicromapEXT is materialized (i.e.
12185            // it is read somewhere), remember its spec constant so finishSpv() references this same one for
12186            // the OpacityMicromapIdKHR execution mode rather than synthesizing a duplicate SpecId.
12187            if (glslangIntermediate->getEnableOpacityMicromapSpecId() != glslang::TQualifier::layoutNotSet &&
12188                symbol->getName() == "gl_EnableOpacityMicromapEXT")
12189                opacityMicromapEnableConstId = id;
12190        }
12191        // atomic counters use this:
12192        if (symbol->getQualifier().hasOffset())
12193            builder.addDecoration(id, spv::Decoration::Offset, symbol->getQualifier().layoutOffset);
12194    }
12195
12196    if (symbol->getQualifier().hasLocation()) {
12197        if (!(glslangIntermediate->isRayTracingStage() &&
12198              (glslangIntermediate->IsRequestedExtension(glslang::E_GL_EXT_ray_tracing) ||
12199               glslangIntermediate->IsRequestedExtension(glslang::E_GL_NV_shader_invocation_reorder) ||
12200               glslangIntermediate->IsRequestedExtension(glslang::E_GL_EXT_shader_invocation_reorder))
12201              && (builder.getStorageClass(id) == spv::StorageClass::RayPayloadKHR ||
12202                  builder.getStorageClass(id) == spv::StorageClass::IncomingRayPayloadKHR ||
12203                  builder.getStorageClass(id) == spv::StorageClass::CallableDataKHR ||
12204                  builder.getStorageClass(id) == spv::StorageClass::IncomingCallableDataKHR ||
12205                  builder.getStorageClass(id) == spv::StorageClass::HitObjectAttributeEXT ||
12206                  builder.getStorageClass(id) == spv::StorageClass::HitObjectAttributeNV))) {
12207            // Location values are used to link TraceRayKHR/ExecuteCallableKHR/HitObjectGetAttributesNV
12208            // to corresponding variables but are not valid in SPIRV since they are supported only
12209            // for Input/Output Storage classes.
12210            builder.addDecoration(id, spv::Decoration::Location, symbol->getQualifier().layoutLocation);
12211        }
12212    }
12213
12214    builder.addDecoration(id, TranslateInvariantDecoration(symbol->getType().getQualifier()));
12215    if (symbol->getQualifier().hasStream() && glslangIntermediate->isMultiStream()) {
12216        builder.addCapability(spv::Capability::GeometryStreams);
12217        builder.addDecoration(id, spv::Decoration::Stream, symbol->getQualifier().layoutStream);
12218    }
12219    if (symbol->getQualifier().hasSet())
12220        builder.addDecoration(id, spv::Decoration::DescriptorSet, symbol->getQualifier().layoutSet);
12221    else if (IsDescriptorResource(symbol->getType())) {
12222        // default to 0
12223        builder.addDecoration(id, spv::Decoration::DescriptorSet, 0);
12224    }
12225    if (symbol->getQualifier().hasBinding())
12226        builder.addDecoration(id, spv::Decoration::Binding, symbol->getQualifier().layoutBinding);
12227    else if (IsDescriptorResource(symbol->getType())) {
12228        // default to 0
12229        builder.addDecoration(id, spv::Decoration::Binding, 0);
12230    }
12231    if (symbol->getQualifier().hasAttachment())
12232        builder.addDecoration(id, spv::Decoration::InputAttachmentIndex, symbol->getQualifier().layoutAttachment);
12233    if (glslangIntermediate->getXfbMode()) {
12234        builder.addCapability(spv::Capability::TransformFeedback);
12235        if (symbol->getQualifier().hasXfbBuffer()) {
12236            builder.addDecoration(id, spv::Decoration::XfbBuffer, symbol->getQualifier().layoutXfbBuffer);
12237            unsigned stride = glslangIntermediate->getXfbStride(symbol->getQualifier().layoutXfbBuffer);
12238            if (stride != glslang::TQualifier::layoutXfbStrideEnd)
12239                builder.addDecoration(id, spv::Decoration::XfbStride, stride);
12240        }
12241        if (symbol->getQualifier().hasXfbOffset())
12242            builder.addDecoration(id, spv::Decoration::Offset, symbol->getQualifier().layoutXfbOffset);
12243    }
12244
12245    // add built-in variable decoration
12246    if (builtIn != spv::BuiltIn::Max) {
12247        // WorkgroupSize deprecated in spirv1.6
12248        if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_6 ||
12249            builtIn != spv::BuiltIn::WorkgroupSize)
12250            builder.addDecoration(id, spv::Decoration::BuiltIn, (int)builtIn);
12251    }
12252
12253    // Add volatile decoration to HelperInvocation for spirv1.6 and beyond
12254    if (builtIn == spv::BuiltIn::HelperInvocation &&
12255        !glslangIntermediate->usingVulkanMemoryModel() &&
12256        glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6) {
12257        builder.addDecoration(id, spv::Decoration::Volatile);
12258    }
12259
12260    // Subgroup builtins which have input storage class are volatile for ray tracing stages.
12261    if (symbol->getType().isImage() || symbol->getQualifier().isPipeInput()) {
12262        std::vector<spv::Decoration> memory;
12263        TranslateMemoryDecoration(symbol->getType().getQualifier(), memory,
12264            glslangIntermediate->usingVulkanMemoryModel());
12265        for (unsigned int i = 0; i < memory.size(); ++i)
12266            builder.addDecoration(id, memory[i]);
12267    }
12268
12269    if (builtIn == spv::BuiltIn::SampleMask) {
12270          spv::Decoration decoration;
12271          // GL_NV_sample_mask_override_coverage extension
12272          if (glslangIntermediate->getLayoutOverrideCoverage())
12273              decoration = spv::Decoration::OverrideCoverageNV;
12274          else
12275              decoration = spv::Decoration::Max;
12276        builder.addDecoration(id, decoration);
12277        if (decoration != spv::Decoration::Max) {
12278            builder.addCapability(spv::Capability::SampleMaskOverrideCoverageNV);
12279            builder.addExtension(spv::E_SPV_NV_sample_mask_override_coverage);
12280        }
12281    }
12282    else if (builtIn == spv::BuiltIn::Layer) {
12283        // SPV_NV_viewport_array2 extension
12284        if (symbol->getQualifier().layoutViewportRelative) {
12285            builder.addDecoration(id, spv::Decoration::ViewportRelativeNV);
12286            builder.addCapability(spv::Capability::ShaderViewportMaskNV);
12287            builder.addExtension(spv::E_SPV_NV_viewport_array2);
12288        }
12289        if (symbol->getQualifier().layoutSecondaryViewportRelativeOffset != -2048) {
12290            builder.addDecoration(id, spv::Decoration::SecondaryViewportRelativeNV,
12291                                  symbol->getQualifier().layoutSecondaryViewportRelativeOffset);
12292            builder.addCapability(spv::Capability::ShaderStereoViewNV);
12293            builder.addExtension(spv::E_SPV_NV_stereo_view_rendering);
12294        }
12295    }
12296
12297    if (symbol->getQualifier().layoutPassthrough) {
12298        builder.addDecoration(id, spv::Decoration::PassthroughNV);
12299        builder.addCapability(spv::Capability::GeometryShaderPassthroughNV);
12300        builder.addExtension(spv::E_SPV_NV_geometry_shader_passthrough);
12301    }
12302    if (symbol->getQualifier().pervertexNV) {
12303        builder.addDecoration(id, spv::Decoration::PerVertexNV);
12304        builder.addCapability(spv::Capability::FragmentBarycentricNV);
12305        builder.addExtension(spv::E_SPV_NV_fragment_shader_barycentric);
12306    }
12307
12308    if (symbol->getQualifier().pervertexEXT) {
12309        builder.addDecoration(id, spv::Decoration::PerVertexKHR);
12310        builder.addCapability(spv::Capability::FragmentBarycentricKHR);
12311        builder.addExtension(spv::E_SPV_KHR_fragment_shader_barycentric);
12312    }
12313
12314    if (glslangIntermediate->getHlslFunctionality1() && symbol->getType().getQualifier().semanticName != nullptr) {
12315        builder.addExtension("SPV_GOOGLE_hlsl_functionality1");
12316        builder.addDecoration(id, spv::Decoration::HlslSemanticGOOGLE,
12317                              symbol->getType().getQualifier().semanticName);
12318    }
12319
12320    if (symbol->isReference()) {
12321        builder.addDecoration(id, symbol->getType().getQualifier().restrict ?
12322            spv::Decoration::RestrictPointerEXT : spv::Decoration::AliasedPointerEXT);
12323    }
12324
12325    // Add SPIR-V decorations (GL_EXT_spirv_intrinsics)
12326    if (symbol->getType().getQualifier().hasSpirvDecorate())
12327        applySpirvDecorate(symbol->getType(), id, {});
12328
12329    if (symbol->getQualifier().hasBank()) {
12330        builder.addExtension(spv::E_SPV_NV_push_constant_bank);
12331        builder.addCapability(spv::Capability::PushConstantBanksNV);
12332        builder.addDecoration(id, spv::Decoration::BankNV, symbol->getQualifier().layoutBank);
12333    }
12334
12335    if (symbol->getQualifier().hasMemberOffset()) {
12336        builder.addExtension(spv::E_SPV_NV_push_constant_bank);
12337        builder.addCapability(spv::Capability::PushConstantBanksNV);
12338        builder.addDecoration(id, spv::Decoration::MemberOffsetNV, symbol->getQualifier().layoutMemberOffset);
12339    }
12340
12341    return id;
12342}
12343
12344// add per-primitive, per-view. per-task decorations to a struct member (member >= 0) or an object
12345void TGlslangToSpvTraverser::addMeshNVDecoration(spv::Id id, int member, const glslang::TQualifier& qualifier)
12346{
12347    bool isMeshShaderExt = (glslangIntermediate->getRequestedExtensions().find(glslang::E_GL_EXT_mesh_shader) !=
12348                            glslangIntermediate->getRequestedExtensions().end());
12349
12350    if (member >= 0) {
12351        if (qualifier.perPrimitiveNV) {
12352            // Need to add capability/extension for fragment shader.
12353            // Mesh shader already adds this by default.
12354            if (glslangIntermediate->getStage() == EShLangFragment) {
12355                if(isMeshShaderExt) {
12356                    builder.addCapability(spv::Capability::MeshShadingEXT);
12357                    builder.addExtension(spv::E_SPV_EXT_mesh_shader);
12358                } else {
12359                    builder.addCapability(spv::Capability::MeshShadingNV);
12360                    builder.addExtension(spv::E_SPV_NV_mesh_shader);
12361                }
12362            }
12363            builder.addMemberDecoration(id, (unsigned)member, spv::Decoration::PerPrimitiveNV);
12364        }
12365        if (qualifier.perViewNV)
12366            builder.addMemberDecoration(id, (unsigned)member, spv::Decoration::PerViewNV);
12367        if (qualifier.perTaskNV)
12368            builder.addMemberDecoration(id, (unsigned)member, spv::Decoration::PerTaskNV);
12369    } else {
12370        if (qualifier.perPrimitiveNV) {
12371            // Need to add capability/extension for fragment shader.
12372            // Mesh shader already adds this by default.
12373            if (glslangIntermediate->getStage() == EShLangFragment) {
12374                if(isMeshShaderExt) {
12375                    builder.addCapability(spv::Capability::MeshShadingEXT);
12376                    builder.addExtension(spv::E_SPV_EXT_mesh_shader);
12377                } else {
12378                    builder.addCapability(spv::Capability::MeshShadingNV);
12379                    builder.addExtension(spv::E_SPV_NV_mesh_shader);
12380                }
12381            }
12382            builder.addDecoration(id, spv::Decoration::PerPrimitiveNV);
12383        }
12384        if (qualifier.perViewNV)
12385            builder.addDecoration(id, spv::Decoration::PerViewNV);
12386        if (qualifier.perTaskNV)
12387            builder.addDecoration(id, spv::Decoration::PerTaskNV);
12388    }
12389}
12390
12391bool TGlslangToSpvTraverser::hasQCOMImageProceessingDecoration(spv::Id id, spv::Decoration decor)
12392{
12393  std::vector<spv::Decoration> &decoVec = idToQCOMDecorations[id];
12394  for ( auto d : decoVec ) {
12395    if ( d == decor )
12396      return true;
12397  }
12398  return false;
12399}
12400
12401void TGlslangToSpvTraverser::addImageProcessingQCOMDecoration(spv::Id id, spv::Decoration decor)
12402{
12403  spv::Op opc = builder.getOpCode(id);
12404  if (opc == spv::Op::OpSampledImage) {
12405    id  = builder.getIdOperand(id, 0);
12406    opc = builder.getOpCode(id);
12407  }
12408
12409  if (opc == spv::Op::OpLoad) {
12410    spv::Id texid = builder.getIdOperand(id, 0);
12411    if (!hasQCOMImageProceessingDecoration(texid, decor)) {//
12412      builder.addDecoration(texid, decor);
12413      idToQCOMDecorations[texid].push_back(decor);
12414    }
12415  }
12416}
12417
12418void TGlslangToSpvTraverser::addImageProcessing2QCOMDecoration(spv::Id id, bool isForGather)
12419{
12420  if (isForGather) {
12421    return addImageProcessingQCOMDecoration(id, spv::Decoration::BlockMatchTextureQCOM);
12422  }
12423
12424  auto addDecor =
12425    [this](spv::Id id, spv::Decoration decor) {
12426      spv::Op tsopc = this->builder.getOpCode(id);
12427      if (tsopc == spv::Op::OpLoad) {
12428        spv::Id tsid = this->builder.getIdOperand(id, 0);
12429        if (this->glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) {
12430          assert(iOSet.count(tsid) > 0);
12431        }
12432        if (!hasQCOMImageProceessingDecoration(tsid, decor)) {
12433          this->builder.addDecoration(tsid, decor);
12434          idToQCOMDecorations[tsid].push_back(decor);
12435        }
12436      }
12437    };
12438
12439  spv::Op opc = builder.getOpCode(id);
12440  bool isInterfaceObject = (opc != spv::Op::OpSampledImage);
12441
12442  if (!isInterfaceObject) {
12443    addDecor(builder.getIdOperand(id, 0), spv::Decoration::BlockMatchTextureQCOM);
12444    addDecor(builder.getIdOperand(id, 1), spv::Decoration::BlockMatchSamplerQCOM);
12445  } else {
12446    addDecor(id, spv::Decoration::BlockMatchTextureQCOM);
12447    addDecor(id, spv::Decoration::BlockMatchSamplerQCOM);
12448  }
12449}
12450
12451// Make a full tree of instructions to build a SPIR-V specialization constant,
12452// or regular constant if possible.
12453//
12454// TBD: this is not yet done, nor verified to be the best design, it does do the leaf symbols though
12455//
12456// Recursively walk the nodes.  The nodes form a tree whose leaves are
12457// regular constants, which themselves are trees that createSpvConstant()
12458// recursively walks.  So, this function walks the "top" of the tree:
12459//  - emit specialization constant-building instructions for specConstant
12460//  - when running into a non-spec-constant, switch to createSpvConstant()
12461spv::Id TGlslangToSpvTraverser::createSpvConstant(const glslang::TIntermTyped& node)
12462{
12463    assert(node.getQualifier().isConstant());
12464
12465    // Handle front-end constants first (non-specialization constants).
12466    if (! node.getQualifier().specConstant) {
12467        // hand off to the non-spec-constant path
12468        assert(node.getAsConstantUnion() != nullptr || node.getAsSymbolNode() != nullptr);
12469        int nextConst = 0;
12470        return createSpvConstantFromConstUnionArray(node.getType(), node.getAsConstantUnion() ?
12471            node.getAsConstantUnion()->getConstArray() : node.getAsSymbolNode()->getConstArray(),
12472            nextConst, false);
12473    }
12474
12475    // We now know we have a specialization constant to build
12476
12477    // Extra capabilities may be needed.
12478    if (node.getType().contains8BitInt())
12479        builder.addCapability(spv::Capability::Int8);
12480    if (node.getType().contains16BitFloat())
12481        builder.addCapability(spv::Capability::Float16);
12482    if (node.getType().contains16BitInt())
12483        builder.addCapability(spv::Capability::Int16);
12484    if (node.getType().contains64BitInt())
12485        builder.addCapability(spv::Capability::Int64);
12486    if (node.getType().containsDouble())
12487        builder.addCapability(spv::Capability::Float64);
12488
12489    // gl_WorkGroupSize is a special case until the front-end handles hierarchical specialization constants,
12490    // even then, it's specialization ids are handled by special case syntax in GLSL: layout(local_size_x = ...
12491    if (node.getType().getQualifier().builtIn == glslang::EbvWorkGroupSize) {
12492        std::vector<spv::Id> dimConstId;
12493        for (int dim = 0; dim < 3; ++dim) {
12494            bool specConst = (glslangIntermediate->getLocalSizeSpecId(dim) != glslang::TQualifier::layoutNotSet);
12495            dimConstId.push_back(builder.makeUintConstant(glslangIntermediate->getLocalSize(dim), specConst));
12496            if (specConst) {
12497                builder.addDecoration(dimConstId.back(), spv::Decoration::SpecId,
12498                                      glslangIntermediate->getLocalSizeSpecId(dim));
12499            }
12500        }
12501        return builder.makeCompositeConstant(builder.makeVectorType(builder.makeUintType(32), 3), dimConstId, true);
12502    }
12503
12504    // An AST node labelled as specialization constant should be a symbol node.
12505    // Its initializer should either be a sub tree with constant nodes, or a constant union array.
12506    if (auto* sn = node.getAsSymbolNode()) {
12507        spv::Id result;
12508        if (auto* sub_tree = sn->getConstSubtree()) {
12509            // Traverse the constant constructor sub tree like generating normal run-time instructions.
12510            // During the AST traversal, if the node is marked as 'specConstant', SpecConstantOpModeGuard
12511            // will set the builder into spec constant op instruction generating mode.
12512            sub_tree->traverse(this);
12513            result = accessChainLoad(sub_tree->getType());
12514        } else if (auto* const_union_array = &sn->getConstArray()) {
12515            int nextConst = 0;
12516            result = createSpvConstantFromConstUnionArray(sn->getType(), *const_union_array, nextConst, true);
12517        } else {
12518            logger->missingFunctionality("Invalid initializer for spec constant.");
12519            return spv::NoResult;
12520        }
12521        builder.addName(result, sn->getName().c_str());
12522        return result;
12523    }
12524
12525    // Neither a front-end constant node, nor a specialization constant node with constant union array or
12526    // constant sub tree as initializer.
12527    logger->missingFunctionality("Neither a front-end constant nor a spec constant.");
12528    return spv::NoResult;
12529}
12530
12531// Use 'consts' as the flattened glslang source of scalar constants to recursively
12532// build the aggregate SPIR-V constant.
12533//
12534// If there are not enough elements present in 'consts', 0 will be substituted;
12535// an empty 'consts' can be used to create a fully zeroed SPIR-V constant.
12536//
12537spv::Id TGlslangToSpvTraverser::createSpvConstantFromConstUnionArray(const glslang::TType& glslangType,
12538    const glslang::TConstUnionArray& consts, int& nextConst, bool specConstant)
12539{
12540    // vector of constants for SPIR-V
12541    std::vector<spv::Id> spvConsts;
12542
12543    // Type is used for struct and array constants
12544    spv::Id typeId = convertGlslangToSpvType(glslangType);
12545
12546    if (glslangType.isArray()) {
12547        glslang::TType elementType(glslangType, 0);
12548        for (int i = 0; i < glslangType.getOuterArraySize(); ++i)
12549            spvConsts.push_back(createSpvConstantFromConstUnionArray(elementType, consts, nextConst, false));
12550    } else if (glslangType.isMatrix()) {
12551        glslang::TType vectorType(glslangType, 0);
12552        for (int col = 0; col < glslangType.getMatrixCols(); ++col)
12553            spvConsts.push_back(createSpvConstantFromConstUnionArray(vectorType, consts, nextConst, false));
12554    } else if (glslangType.isCoopMat()) {
12555        glslang::TType componentType(glslangType.getBasicType());
12556        spvConsts.push_back(createSpvConstantFromConstUnionArray(componentType, consts, nextConst, false));
12557    } else if (glslangType.isStruct()) {
12558        glslang::TVector<glslang::TTypeLoc>::const_iterator iter;
12559        for (iter = glslangType.getStruct()->begin(); iter != glslangType.getStruct()->end(); ++iter)
12560            spvConsts.push_back(createSpvConstantFromConstUnionArray(*iter->type, consts, nextConst, false));
12561    } else if (glslangType.getVectorSize() > 1 || glslangType.isCoopVecOrLongVector()) {
12562        unsigned int numComponents = glslangType.isCoopVecOrLongVector() ? glslangType.getTypeParameters()->arraySizes->getDimSize(0) : glslangType.getVectorSize();
12563        for (unsigned int i = 0; i < numComponents; ++i) {
12564            bool zero = nextConst >= consts.size();
12565            switch (glslangType.getBasicType()) {
12566            case glslang::EbtInt:
12567                spvConsts.push_back(builder.makeIntConstant(zero ? 0 : consts[nextConst].getIConst()));
12568                break;
12569            case glslang::EbtUint:
12570                spvConsts.push_back(builder.makeUintConstant(zero ? 0 : consts[nextConst].getUConst()));
12571                break;
12572            case glslang::EbtFloat:
12573                spvConsts.push_back(builder.makeFloatConstant(zero ? 0.0F : (float)consts[nextConst].getDConst()));
12574                break;
12575            case glslang::EbtBool:
12576                spvConsts.push_back(builder.makeBoolConstant(zero ? false : consts[nextConst].getBConst()));
12577                break;
12578            case glslang::EbtInt8:
12579                builder.addCapability(spv::Capability::Int8);
12580                spvConsts.push_back(builder.makeInt8Constant(zero ? 0 : consts[nextConst].getI8Const()));
12581                break;
12582            case glslang::EbtUint8:
12583                builder.addCapability(spv::Capability::Int8);
12584                spvConsts.push_back(builder.makeUint8Constant(zero ? 0 : consts[nextConst].getU8Const()));
12585                break;
12586            case glslang::EbtInt16:
12587                builder.addCapability(spv::Capability::Int16);
12588                spvConsts.push_back(builder.makeInt16Constant(zero ? 0 : consts[nextConst].getI16Const()));
12589                break;
12590            case glslang::EbtUint16:
12591                builder.addCapability(spv::Capability::Int16);
12592                spvConsts.push_back(builder.makeUint16Constant(zero ? 0 : consts[nextConst].getU16Const()));
12593                break;
12594            case glslang::EbtInt64:
12595                spvConsts.push_back(builder.makeInt64Constant(zero ? 0 : consts[nextConst].getI64Const()));
12596                break;
12597            case glslang::EbtUint64:
12598                spvConsts.push_back(builder.makeUint64Constant(zero ? 0 : consts[nextConst].getU64Const()));
12599                break;
12600            case glslang::EbtDouble:
12601                spvConsts.push_back(builder.makeDoubleConstant(zero ? 0.0 : consts[nextConst].getDConst()));
12602                break;
12603            case glslang::EbtFloat16:
12604                builder.addCapability(spv::Capability::Float16);
12605                spvConsts.push_back(builder.makeFloat16Constant(zero ? 0.0F : (float)consts[nextConst].getDConst()));
12606                break;
12607            case glslang::EbtBFloat16:
12608                spvConsts.push_back(builder.makeBFloat16Constant(zero ? 0.0F : (float)consts[nextConst].getDConst()));
12609                break;
12610            case glslang::EbtFloatE5M2:
12611                spvConsts.push_back(builder.makeFloatE5M2Constant(zero ? 0.0F : (float)consts[nextConst].getDConst()));
12612                break;
12613            case glslang::EbtFloatE4M3:
12614                spvConsts.push_back(builder.makeFloatE4M3Constant(zero ? 0.0F : (float)consts[nextConst].getDConst()));
12615                break;
12616            case glslang::EbtFloatE2M1:
12617                spvConsts.push_back(builder.makeFloatE2M1Constant(zero ? 0.0F : (float)consts[nextConst].getDConst()));
12618                break;
12619            case glslang::EbtFloatE3M2:
12620                spvConsts.push_back(builder.makeFloatE3M2Constant(zero ? 0.0F : (float)consts[nextConst].getDConst()));
12621                break;
12622            case glslang::EbtFloatE2M3:
12623                spvConsts.push_back(builder.makeFloatE2M3Constant(zero ? 0.0F : (float)consts[nextConst].getDConst()));
12624                break;
12625            case glslang::EbtFloatUE8M0:
12626                spvConsts.push_back(builder.makeFloatUE8M0Constant(zero ? 0.0F : (float)consts[nextConst].getDConst()));
12627                break;
12628            case glslang::EbtFloatMXINT8:
12629                spvConsts.push_back(builder.makeFloatMXINT8Constant(zero ? 0.0F : (float)consts[nextConst].getDConst()));
12630                break;
12631            default:
12632                assert(0);
12633                break;
12634            }
12635            ++nextConst;
12636        }
12637    } else {
12638        // we have a non-aggregate (scalar) constant
12639        bool zero = nextConst >= consts.size();
12640        spv::Id scalar = 0;
12641        switch (glslangType.getBasicType()) {
12642        case glslang::EbtInt:
12643            scalar = builder.makeIntConstant(zero ? 0 : consts[nextConst].getIConst(), specConstant);
12644            break;
12645        case glslang::EbtUint:
12646            scalar = builder.makeUintConstant(zero ? 0 : consts[nextConst].getUConst(), specConstant);
12647            break;
12648        case glslang::EbtFloat:
12649            scalar = builder.makeFloatConstant(zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
12650            break;
12651        case glslang::EbtBool:
12652            scalar = builder.makeBoolConstant(zero ? false : consts[nextConst].getBConst(), specConstant);
12653            break;
12654        case glslang::EbtInt8:
12655            builder.addCapability(spv::Capability::Int8);
12656            scalar = builder.makeInt8Constant(zero ? 0 : consts[nextConst].getI8Const(), specConstant);
12657            break;
12658        case glslang::EbtUint8:
12659            builder.addCapability(spv::Capability::Int8);
12660            scalar = builder.makeUint8Constant(zero ? 0 : consts[nextConst].getU8Const(), specConstant);
12661            break;
12662        case glslang::EbtInt16:
12663            builder.addCapability(spv::Capability::Int16);
12664            scalar = builder.makeInt16Constant(zero ? 0 : consts[nextConst].getI16Const(), specConstant);
12665            break;
12666        case glslang::EbtUint16:
12667            builder.addCapability(spv::Capability::Int16);
12668            scalar = builder.makeUint16Constant(zero ? 0 : consts[nextConst].getU16Const(), specConstant);
12669            break;
12670        case glslang::EbtInt64:
12671            scalar = builder.makeInt64Constant(zero ? 0 : consts[nextConst].getI64Const(), specConstant);
12672            break;
12673        case glslang::EbtUint64:
12674            scalar = builder.makeUint64Constant(zero ? 0 : consts[nextConst].getU64Const(), specConstant);
12675            break;
12676        case glslang::EbtDouble:
12677            scalar = builder.makeDoubleConstant(zero ? 0.0 : consts[nextConst].getDConst(), specConstant);
12678            break;
12679        case glslang::EbtFloat16:
12680            builder.addCapability(spv::Capability::Float16);
12681            scalar = builder.makeFloat16Constant(zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
12682            break;
12683        case glslang::EbtBFloat16:
12684            scalar = builder.makeBFloat16Constant(zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
12685            break;
12686        case glslang::EbtFloatE5M2:
12687            scalar = builder.makeFloatE5M2Constant(zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
12688            break;
12689        case glslang::EbtFloatE4M3:
12690            scalar = builder.makeFloatE4M3Constant(zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
12691            break;
12692        case glslang::EbtFloatE2M1:
12693            scalar = builder.makeFloatE2M1Constant(zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
12694            break;
12695        case glslang::EbtFloatE3M2:
12696            scalar = builder.makeFloatE3M2Constant(zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
12697            break;
12698        case glslang::EbtFloatE2M3:
12699            scalar = builder.makeFloatE2M3Constant(zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
12700            break;
12701        case glslang::EbtFloatUE8M0:
12702            scalar = builder.makeFloatUE8M0Constant(zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
12703            break;
12704        case glslang::EbtFloatMXINT8:
12705            scalar = builder.makeFloatMXINT8Constant(zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
12706            break;
12707        case glslang::EbtReference:
12708            scalar = builder.makeUint64Constant(zero ? 0 : consts[nextConst].getU64Const(), specConstant);
12709            scalar = builder.createUnaryOp(spv::Op::OpBitcast, typeId, scalar);
12710            break;
12711        case glslang::EbtString:
12712            scalar = builder.getStringId(consts[nextConst].getSConst()->c_str());
12713            break;
12714        default:
12715            assert(0);
12716            break;
12717        }
12718        ++nextConst;
12719        return scalar;
12720    }
12721
12722    return builder.makeCompositeConstant(typeId, spvConsts);
12723}
12724
12725// Return true if the node is a constant or symbol whose reading has no
12726// non-trivial observable cost or effect.
12727bool TGlslangToSpvTraverser::isTrivialLeaf(const glslang::TIntermTyped* node)
12728{
12729    // don't know what this is
12730    if (node == nullptr)
12731        return false;
12732
12733    // a constant is safe
12734    if (node->getAsConstantUnion() != nullptr)
12735        return true;
12736
12737    // not a symbol means non-trivial
12738    if (node->getAsSymbolNode() == nullptr)
12739        return false;
12740
12741    // a symbol, depends on what's being read
12742    switch (node->getType().getQualifier().storage) {
12743    case glslang::EvqTemporary:
12744    case glslang::EvqGlobal:
12745    case glslang::EvqIn:
12746    case glslang::EvqInOut:
12747    case glslang::EvqConst:
12748    case glslang::EvqConstReadOnly:
12749    case glslang::EvqUniform:
12750        return true;
12751    default:
12752        return false;
12753    }
12754}
12755
12756// A node is trivial if it is a single operation with no side effects.
12757// HLSL (and/or vectors) are always trivial, as it does not short circuit.
12758// Otherwise, error on the side of saying non-trivial.
12759// Return true if trivial.
12760bool TGlslangToSpvTraverser::isTrivial(const glslang::TIntermTyped* node)
12761{
12762    if (node == nullptr)
12763        return false;
12764
12765    // count non scalars as trivial, as well as anything coming from HLSL
12766    if (! node->getType().isScalarOrVec1() || glslangIntermediate->getSource() == glslang::EShSourceHlsl)
12767        return true;
12768
12769    // symbols and constants are trivial
12770    if (isTrivialLeaf(node))
12771        return true;
12772
12773    // otherwise, it needs to be a simple operation or one or two leaf nodes
12774
12775    // not a simple operation
12776    const glslang::TIntermBinary* binaryNode = node->getAsBinaryNode();
12777    const glslang::TIntermUnary* unaryNode = node->getAsUnaryNode();
12778    if (binaryNode == nullptr && unaryNode == nullptr)
12779        return false;
12780
12781    // not on leaf nodes
12782    if (binaryNode && (! isTrivialLeaf(binaryNode->getLeft()) || ! isTrivialLeaf(binaryNode->getRight())))
12783        return false;
12784
12785    if (unaryNode && ! isTrivialLeaf(unaryNode->getOperand())) {
12786        return false;
12787    }
12788
12789    if (IsOpNumericConv(node->getAsOperator()->getOp()) &&
12790        node->getType().getBasicType() == glslang::EbtBool) {
12791        return true;
12792    }
12793
12794    switch (node->getAsOperator()->getOp()) {
12795    case glslang::EOpLogicalNot:
12796    case glslang::EOpEqual:
12797    case glslang::EOpNotEqual:
12798    case glslang::EOpLessThan:
12799    case glslang::EOpGreaterThan:
12800    case glslang::EOpLessThanEqual:
12801    case glslang::EOpGreaterThanEqual:
12802    case glslang::EOpIndexDirect:
12803    case glslang::EOpIndexDirectStruct:
12804    case glslang::EOpLogicalXor:
12805    case glslang::EOpAny:
12806    case glslang::EOpAll:
12807        return true;
12808    default:
12809        return false;
12810    }
12811}
12812
12813// Emit short-circuiting code, where 'right' is never evaluated unless
12814// the left side is true (for &&) or false (for ||).
12815spv::Id TGlslangToSpvTraverser::createShortCircuit(glslang::TOperator op, glslang::TIntermTyped& left,
12816    glslang::TIntermTyped& right)
12817{
12818    spv::Id boolTypeId = builder.makeBoolType();
12819
12820    // emit left operand
12821    builder.clearAccessChain();
12822    left.traverse(this);
12823    spv::Id leftId = accessChainLoad(left.getType());
12824
12825    // Operands to accumulate OpPhi operands
12826    std::vector<spv::Id> phiOperands;
12827    phiOperands.reserve(4);
12828    // accumulate left operand's phi information
12829    phiOperands.push_back(leftId);
12830    phiOperands.push_back(builder.getBuildPoint()->getId());
12831
12832    // Make the two kinds of operation symmetric with a "!"
12833    //   || => emit "if (! left) result = right"
12834    //   && => emit "if (  left) result = right"
12835    //
12836    // TODO: this runtime "not" for || could be avoided by adding functionality
12837    // to 'builder' to have an "else" without an "then"
12838    if (op == glslang::EOpLogicalOr)
12839        leftId = builder.createUnaryOp(spv::Op::OpLogicalNot, boolTypeId, leftId);
12840
12841    // make an "if" based on the left value
12842    spv::Builder::If ifBuilder(leftId, spv::SelectionControlMask::MaskNone, builder);
12843
12844    // emit right operand as the "then" part of the "if"
12845    builder.clearAccessChain();
12846    right.traverse(this);
12847    spv::Id rightId = accessChainLoad(right.getType());
12848
12849    // accumulate left operand's phi information
12850    phiOperands.push_back(rightId);
12851    phiOperands.push_back(builder.getBuildPoint()->getId());
12852
12853    // finish the "if"
12854    ifBuilder.makeEndIf();
12855
12856    // phi together the two results
12857    return builder.createOp(spv::Op::OpPhi, boolTypeId, phiOperands);
12858}
12859
12860// Return type Id of the imported set of extended instructions corresponds to the name.
12861// Import this set if it has not been imported yet.
12862spv::Id TGlslangToSpvTraverser::getExtBuiltins(const char* name)
12863{
12864    if (extBuiltinMap.find(name) != extBuiltinMap.end())
12865        return extBuiltinMap[name];
12866    else {
12867        spv::Id extBuiltins = builder.import(name);
12868        extBuiltinMap[name] = extBuiltins;
12869        return extBuiltins;
12870    }
12871}
12872
12873void TGlslangToSpvTraverser::enableCoopMatConversions()
12874{
12875    if (glslangIntermediate->usingCoopMatMaint1()) {
12876        builder.addCapability(spv::Capability::CooperativeMatrixConversionsEXT);
12877        builder.addExtension(spv::E_SPV_EXT_cooperative_matrix_maintenance1);
12878    } else {
12879        builder.addCapability(spv::Capability::CooperativeMatrixConversionsNV);
12880        builder.addExtension(spv::E_SPV_NV_cooperative_matrix2);
12881    }
12882}
12883
12884} // end anonymous namespace
12885
12886namespace glslang {
12887
12888void GetSpirvVersion(std::string& version)
12889{
12890    const int bufSize = 100;
12891    char buf[bufSize];
12892    snprintf(buf, bufSize, "0x%08x, Revision %d", spv::Version, spv::Revision);
12893    version = buf;
12894}
12895
12896// For low-order part of the generator's magic number. Bump up
12897// when there is a change in the style (e.g., if SSA form changes,
12898// or a different instruction sequence to do something gets used).
12899int GetSpirvGeneratorVersion()
12900{
12901    // return 1; // start
12902    // return 2; // EOpAtomicCounterDecrement gets a post decrement, to map between GLSL -> SPIR-V
12903    // return 3; // change/correct barrier-instruction operands, to match memory model group decisions
12904    // return 4; // some deeper access chains: for dynamic vector component, and local Boolean component
12905    // return 5; // make OpArrayLength result type be an int with signedness of 0
12906    // return 6; // revert version 5 change, which makes a different (new) kind of incorrect code,
12907                 // versions 4 and 6 each generate OpArrayLength as it has long been done
12908    // return 7; // GLSL volatile keyword maps to both SPIR-V decorations Volatile and Coherent
12909    // return 8; // switch to new dead block eliminator; use OpUnreachable
12910    // return 9; // don't include opaque function parameters in OpEntryPoint global's operand list
12911    // return 10; // Generate OpFUnordNotEqual for != comparisons
12912    return 11; // Make OpEmitMeshTasksEXT a terminal instruction
12913}
12914
12915// Write SPIR-V out to a binary file
12916bool OutputSpvBin(const std::vector<unsigned int>& spirv, const char* baseName)
12917{
12918    std::ofstream out;
12919    out.open(baseName, std::ios::binary | std::ios::out);
12920    if (out.fail()) {
12921        printf("ERROR: Failed to open file: %s\n", baseName);
12922        return false;
12923    }
12924    for (int i = 0; i < (int)spirv.size(); ++i) {
12925        unsigned int word = spirv[i];
12926        out.write((const char*)&word, 4);
12927    }
12928    out.close();
12929    return true;
12930}
12931
12932// Write SPIR-V out to a text file with 32-bit hexadecimal words
12933bool OutputSpvHex(const std::vector<unsigned int>& spirv, const char* baseName, const char* varName)
12934{
12935    std::ofstream out;
12936    out.open(baseName, std::ios::binary | std::ios::out);
12937    if (out.fail()) {
12938        printf("ERROR: Failed to open file: %s\n", baseName);
12939        return false;
12940    }
12941    out << "\t// " <<
12942        GetSpirvGeneratorVersion() <<
12943        GLSLANG_VERSION_MAJOR << "." << GLSLANG_VERSION_MINOR << "." << GLSLANG_VERSION_PATCH <<
12944        GLSLANG_VERSION_FLAVOR << std::endl;
12945    if (varName != nullptr) {
12946        out << "\t #pragma once" << std::endl;
12947        out << "const uint32_t " << varName << "[] = {" << std::endl;
12948    }
12949    const int WORDS_PER_LINE = 8;
12950    for (int i = 0; i < (int)spirv.size(); i += WORDS_PER_LINE) {
12951        out << "\t";
12952        for (int j = 0; j < WORDS_PER_LINE && i + j < (int)spirv.size(); ++j) {
12953            const unsigned int word = spirv[i + j];
12954            out << "0x" << std::hex << std::setw(8) << std::setfill('0') << word;
12955            if (i + j + 1 < (int)spirv.size()) {
12956                out << ",";
12957            }
12958        }
12959        out << std::endl;
12960    }
12961    if (varName != nullptr) {
12962        out << "};";
12963        out << std::endl;
12964    }
12965    out.close();
12966    return true;
12967}
12968
12969//
12970// Set up the glslang traversal
12971//
12972void GlslangToSpv(const TIntermediate& intermediate, std::vector<unsigned int>& spirv, SpvOptions* options)
12973{
12974    spv::SpvBuildLogger logger;
12975    GlslangToSpv(intermediate, spirv, &logger, options);
12976}
12977
12978void GlslangToSpv(const TIntermediate& intermediate, std::vector<unsigned int>& spirv,
12979                  spv::SpvBuildLogger* logger, SpvOptions* options)
12980{
12981    TIntermNode* root = intermediate.getTreeRoot();
12982
12983    if (root == nullptr)
12984        return;
12985
12986    SpvOptions defaultOptions;
12987    if (options == nullptr)
12988        options = &defaultOptions;
12989
12990    GetThreadPoolAllocator().push();
12991
12992    TGlslangToSpvTraverser it(intermediate.getSpv().spv, &intermediate, logger, *options);
12993    root->traverse(&it);
12994    it.finishSpv(options->compileOnly);
12995    it.dumpSpv(spirv);
12996
12997#if ENABLE_OPT
12998    // If from HLSL, run spirv-opt to "legalize" the SPIR-V for Vulkan
12999    // eg. forward and remove memory writes of opaque types.
13000    bool prelegalization = intermediate.getSource() == EShSourceHlsl;
13001    if ((prelegalization || options->optimizeSize || options->optimizePerformance) && !options->disableOptimizer) {
13002        SpirvToolsTransform(intermediate, spirv, logger, options, prelegalization);
13003        prelegalization = false;
13004    }
13005    else if (options->stripDebugInfo) {
13006        // Strip debug info even if optimization is disabled.
13007        SpirvToolsStripDebugInfo(intermediate, spirv, logger);
13008    }
13009
13010    if (options->validate)
13011        SpirvToolsValidate(intermediate, spirv, logger, prelegalization);
13012
13013    if (options->disassemble)
13014        SpirvToolsDisassemble(std::cout, spirv);
13015
13016#endif
13017
13018    GetThreadPoolAllocator().pop();
13019}
13020
13021} // end namespace glslang