Mesh shader support (#2464)

* Add gdb generated files to .gitignore

* Switch to c++17

TODO: Ellie update coding style doc

* WIP mesh shaders

* Add MeshOutputType and mesh output decorations

* Lift array type layout creation out of _createTypeLayout

in preparation for sharing it elsewhere

* Initial pass at GLSL legalization for mesh shaders

* Create output types for builtin mesh outputs

This should be rendered as an out paramter block

* Handle writes to member fields in mesh shader output

* Per primitive output from mesh shaders

* Add mesh shader tests

* Redeclare mesh output builtins

* Remove unused instruction

* Emit explicit mesh output max max size

* Add unimplemented warning for array members in mesh output

* Implement mesh output splitting for GLSL in terms of getSubscriptVal

* Allow HLSL syntax for mesh output modifiers

* Improve error messages for mesh output

* Add test for HLSL style mesh output syntax

* Emit explicit mesh output indices max size

* HLSL generation support for mesh shaders

* Better errors for mesh shader misuse

* Neaten comments

* Regenerate vs2019 project files

* Fix build on vs2019

* Retreat on c++17

Will make the change in a separate PR

* slang-glslang binary dep 11.10.0 -> 11.12.0-32

* Fixes for msvc compiler

* Update msvc project

1 files changed, 58 insertions, 0 deletions

diff --git a/source/slang/slang-check-decl.cpp b/source/slang/slang-check-decl.cpp
index b33c33e7a..aae741770 100644
--- a/source/slang/slang-check-decl.cpp
+++ b/source/slang/slang-check-decl.cpp
@@ -47,6 +47,7 @@ namespace Slang
 
         void checkDerivativeMemberAttribute(VarDeclBase* varDecl, DerivativeMemberAttribute* attr);
         void checkExtensionExternVarAttribute(VarDeclBase* varDecl, ExtensionExternVarModifier* m);
+        void checkMeshOutputDecl(VarDeclBase* varDecl);
 
         void checkVarDeclCommon(VarDeclBase* varDecl);
 
@@ -1178,6 +1179,8 @@ namespace Slang
             validateArraySizeForVariable(varDecl);
         }
 
+        checkMeshOutputDecl(varDecl);
+
         // The NVAPI library allows user code to express extended operations
         // (not supported natively by D3D HLSL) by communicating with
         // a specially identified shader parameter called `g_NvidiaExt`.
@@ -5398,7 +5401,62 @@ namespace Slang
         {
             typeExpr = CheckUsableType(typeExpr);
             paramDecl->type = typeExpr;
+            checkMeshOutputDecl(paramDecl);
+        }
+    }
+
+    // This checks that the declaration is marked as "out" and changes the hlsl
+    // modifier based syntax into a proper type.
+    void SemanticsDeclHeaderVisitor::checkMeshOutputDecl(VarDeclBase* varDecl)
+    {
+        auto modifier = varDecl->findModifier<HLSLMeshShaderOutputModifier>();
+        auto meshOutputType = as<MeshOutputType>(varDecl->type.type);
+        bool isMeshOutput = modifier || meshOutputType;
+
+        if(!isMeshOutput)
+        {
+            return;
+        }
+        if(!varDecl->findModifier<OutModifier>())
+        {
+            getSink()->diagnose(varDecl, Diagnostics::meshOutputMustBeOut);
+        }
+
+        //
+        // If necessary, convert to our typed representation
+        //
+        if(!modifier)
+        {
+            return;
         }
+        if(meshOutputType)
+        {
+            getSink()->diagnose(modifier, Diagnostics::unnecessaryHLSLMeshOutputModifier);
+            varDecl->type.type = m_astBuilder->getErrorType();
+            return;
+        }
+        auto indexExpr = as<IndexExpr>(varDecl->type.exp);
+        if(!indexExpr)
+        {
+            getSink()->diagnose(varDecl, Diagnostics::meshOutputMustBeArray);
+            varDecl->type.type = m_astBuilder->getErrorType();
+            return;
+        }
+        if(indexExpr->indexExprs.getCount() != 1)
+        {
+            getSink()->diagnose(varDecl, Diagnostics::meshOutputArrayMustHaveSize);
+            varDecl->type.type = m_astBuilder->getErrorType();
+            return;
+        }
+        auto base = ExpectAType(indexExpr->baseExpression);
+        auto index = CheckIntegerConstantExpression(
+            indexExpr->indexExprs[0],
+            IntegerConstantExpressionCoercionType::AnyInteger,
+            nullptr,
+            getSink());
+
+        Type* d = m_astBuilder->getMeshOutputTypeFromModifier(modifier, base, index);
+        varDecl->type.type = d;
     }
 
     void SemanticsDeclBodyVisitor::visitParamDecl(ParamDecl* paramDecl)