Get IR working for `AdaptiveTessellationCS40/Render` test

I had expected this to be the first case where control-flow instructions were needed, but it turns out that we aren't testing that entry point right now.

The real work/fix here ended up being handling of the `row_major` layout qualifier on a matrix inside a `cbuffer`. The existing AST-based code was passing it through easily (although I don't believe it was handling the layout rules right).
Getting it working in the IR involved beefing up the type-layout behavior so that it can handle explicit layout qualifiers (at least for matrix layout) which should also improve the layout computation for non-square matrices with nonstandard layout in the AST-based path.

There are still some annoying things left to do:

- The `row_major` and `column_major` layout qualifiers in HLSL/GLSL mean different things (well, they mean the reverse of one another) so I need to validate that the GLSL case is working remotely correctly.

- The layout logic isn't handling other explicit-layout cases as supported by GLSL (but of course GLSL is a far lower priority than HLSL/Slang)

- There is currently no way to pass in an explicit matrix layout flag to Slang to control the default behavior.
  - Any client who was using Slang for HLSL pass-through and then applying a non-default flag on their HLSL->* compilation will get nasty unexpected behavior when the IR goes live - and they are already dealing with nasty behavior around non-square matrices not matching layout between Slang and the downstream.

- The logic that gleans layout modes from a variable declaration is currently only being applied for fields of structure types (which applies to `cbuffer` declarations as well), and not to global-scope uniform variables.

- We need test cases for all of this.

4 files changed, 277 insertions, 37 deletions

diff --git a/source/slang/emit.cpp b/source/slang/emit.cpp
index 40366c43c..209436da4 100644
--- a/source/slang/emit.cpp
+++ b/source/slang/emit.cpp
@@ -4416,6 +4416,16 @@ emitDeclImpl(decl, nullptr);
             emit("]");
             break;
 
+        case kIROp_Mul_Vector_Matrix:
+        case kIROp_Mul_Matrix_Vector:
+        case kIROp_Mul_Matrix_Matrix:
+            emit("mul(");
+            emitIROperand(context, inst->getArg(1));
+            emit(", ");
+            emitIROperand(context, inst->getArg(2));
+            emit(")");
+            break;
+
         default:
             emit("/* uhandled */");
             break;
@@ -4575,6 +4585,34 @@ emitDeclImpl(decl, nullptr);
         emit("};\n");
     }
 
+    void emitIRVarModifiers(
+        EmitContext*    context,
+        VarLayout*      layout)
+    {
+        if (!layout)
+            return;
+
+        // We need to handle the case where the variable has
+        // a matrix type, and has been given a non-standard
+        // layout attribute (for HLSL, `row_major` is the
+        // non-standard layout).
+        //
+        if (auto matrixTypeLayout = layout->typeLayout.As<MatrixTypeLayout>())
+        {
+            switch (matrixTypeLayout->mode)
+            {
+            case kMatrixLayoutMode_ColumnMajor:
+                emit("column_major ");
+                break;
+
+            case kMatrixLayoutMode_RowMajor:
+                emit("row_major ");
+                break;
+            }
+
+        }
+    }
+
     void emitIRParameterBlock(
         EmitContext*            context,
         IRVar*                  varDecl,
@@ -4624,7 +4662,7 @@ emitDeclImpl(decl, nullptr);
 
                     auto fieldLayout = structTypeLayout->fields[fieldIndex++];
 
-
+                    emitIRVarModifiers(context, fieldLayout);
 
                     auto fieldType = ff->getFieldType();
                     emitIRType(context, fieldType, getName(ff));
@@ -4662,6 +4700,11 @@ emitDeclImpl(decl, nullptr);
             break;
         }
 
+        // Need to emit appropriate modifiers here.
+
+        auto layout = getVarLayout(context, varDecl);
+        
+        emitIRVarModifiers(context, layout);
 
         emitIRType(context, varType, getName(varDecl));
 
diff --git a/source/slang/type-layout.cpp b/source/slang/type-layout.cpp
index 544071b84..00a625427 100644
--- a/source/slang/type-layout.cpp
+++ b/source/slang/type-layout.cpp
@@ -25,6 +25,14 @@ static size_t RoundUpToPowerOfTwo( size_t value )
     return result;
 }
 
+//
+
+MatrixLayoutMode LayoutRulesImpl::getDefaultMatrixLayoutMode() {
+    return family->getDefaultMatrixLayoutMode();
+}
+
+//
+
 struct DefaultLayoutRulesImpl : SimpleLayoutRulesImpl
 {
     // Get size and alignment for a single value of base type.
@@ -95,6 +103,15 @@ struct DefaultLayoutRulesImpl : SimpleLayoutRulesImpl
 
     SimpleLayoutInfo GetMatrixLayout(SimpleLayoutInfo elementInfo, size_t rowCount, size_t columnCount) override
     {
+        // The default behavior here is to lay out a matrix
+        // as an array of column vectors (that is column-major).
+        // That is because this is the default convention
+        // used by HLSL.
+        //
+        // In practice, the code that calls `GetMatrixLayout` will
+        // potentially transpose the row/column counts in order
+        // to get layouts with a different convention.
+        //
         return GetArrayLayout(
             GetVectorLayout(elementInfo, columnCount),
             rowCount);
@@ -412,6 +429,25 @@ struct GLSLLayoutRulesFamilyImpl : LayoutRulesFamilyImpl
     virtual LayoutRulesImpl* getVaryingOutputRules() override;
     virtual LayoutRulesImpl* getSpecializationConstantRules() override;
     virtual LayoutRulesImpl* getShaderStorageBufferRules() override;
+
+    virtual MatrixLayoutMode getDefaultMatrixLayoutMode() override
+    {
+        // The default matrix layout mode in GLSL is specified
+        // to be "column major" but what GLSL calls a "column"
+        // is actually what HLSL (and hence Slang) calls a row.
+        //
+        // That is, an HLSL `float3x4` has 3 rows and 4 columns,
+        // and indexing yields a `float4`.
+        //
+        // A GLSL `mat3x4` has 3 "columns" and 4 "rows", and
+        // indexing into it yields a `vec4`.
+        //
+        // The Slang compiler needs to be consistent about this mess,
+        // and so when the GLSL spec says that "column"-major is
+        // the default, we know that they actually mean what we
+        // call row-major.
+        return kMatrixLayoutMode_RowMajor;
+    }
 };
 
 struct HLSLLayoutRulesFamilyImpl : LayoutRulesFamilyImpl
@@ -423,6 +459,11 @@ struct HLSLLayoutRulesFamilyImpl : LayoutRulesFamilyImpl
     virtual LayoutRulesImpl* getVaryingOutputRules() override;
     virtual LayoutRulesImpl* getSpecializationConstantRules() override;
     virtual LayoutRulesImpl* getShaderStorageBufferRules() override;
+
+    virtual MatrixLayoutMode getDefaultMatrixLayoutMode() override
+    {
+        return kMatrixLayoutMode_ColumnMajor;
+    }
 };
 
 GLSLLayoutRulesFamilyImpl kGLSLLayoutRulesFamilyImpl;
@@ -678,14 +719,32 @@ static SimpleLayoutInfo getParameterBlockLayoutInfo(
     }
 }
 
+struct TypeLayoutContext
+{
+    // The layout rules to use (e.g., we compute
+    // layout differently in a `cbuffer` vs. the
+    // parameter list of a fragment shader).
+    LayoutRulesImpl*    rules;
+
+    // Whether to lay out matrices column-major
+    // or row-major.
+    MatrixLayoutMode    matrixLayoutMode;
+};
+
+RefPtr<TypeLayout> createTypeLayout(
+    TypeLayoutContext*  context,
+    Type*               type,
+    SimpleLayoutInfo    offset);
 
 RefPtr<ParameterBlockTypeLayout>
 createParameterBlockTypeLayout(
+    TypeLayoutContext*          context,
     RefPtr<ParameterBlockType>  parameterBlockType,
-    LayoutRulesImpl*            parameterBlockRules,
     SimpleLayoutInfo            parameterBlockInfo,
     RefPtr<TypeLayout>          elementTypeLayout)
 {
+    auto parameterBlockRules = context->rules;
+
     auto typeLayout = new ParameterBlockTypeLayout();
 
     typeLayout->type = parameterBlockType;
@@ -740,9 +799,29 @@ RefPtr<ParameterBlockTypeLayout>
 createParameterBlockTypeLayout(
     RefPtr<ParameterBlockType>  parameterBlockType,
     LayoutRulesImpl*            parameterBlockRules,
-    RefPtr<Type>      elementType,
+    SimpleLayoutInfo            parameterBlockInfo,
+    RefPtr<TypeLayout>          elementTypeLayout)
+{
+    TypeLayoutContext context;
+    context.rules = parameterBlockRules;
+    context.matrixLayoutMode = parameterBlockRules->getDefaultMatrixLayoutMode();
+
+    return createParameterBlockTypeLayout(
+        &context,
+        parameterBlockType,
+        parameterBlockInfo,
+        elementTypeLayout);
+}
+
+RefPtr<ParameterBlockTypeLayout>
+createParameterBlockTypeLayout(
+    TypeLayoutContext*          context,
+    RefPtr<ParameterBlockType>  parameterBlockType,
+    RefPtr<Type>                elementType,
     LayoutRulesImpl*            elementTypeRules)
 {
+    auto parameterBlockRules = context->rules;
+
     // First compute resource usage of the block itself.
     // For now we assume that the layout of the block can
     // always be described in a `SimpleLayoutInfo` (only
@@ -766,14 +845,16 @@ createParameterBlockTypeLayout(
     // the elements of the block use the same resource kind consumed
     // by the block itself.
 
-    auto elementTypeLayout = CreateTypeLayout(
+    TypeLayoutContext elementContext = *context;
+    elementContext.rules = elementTypeRules;
+    auto elementTypeLayout = createTypeLayout(
+        &elementContext,
         elementType,
-        elementTypeRules,
         info);
 
     return createParameterBlockTypeLayout(
+        context,
         parameterBlockType,
-        parameterBlockRules,
         info,
         elementTypeLayout);
 }
@@ -811,9 +892,11 @@ LayoutRulesImpl* getParameterBufferElementTypeLayoutRules(
 
 RefPtr<ParameterBlockTypeLayout>
 createParameterBlockTypeLayout(
-    RefPtr<ParameterBlockType>  parameterBlockType,
-    LayoutRulesImpl*            parameterBlockRules)
+    TypeLayoutContext*          context,
+    RefPtr<ParameterBlockType>  parameterBlockType)
 {
+    auto parameterBlockRules = context->rules;
+
     // Determine the layout rules to use for the contents of the block
     auto elementTypeRules = getParameterBufferElementTypeLayoutRules(
         parameterBlockType,
@@ -822,8 +905,8 @@ createParameterBlockTypeLayout(
     auto elementType = parameterBlockType->elementType;
 
     return createParameterBlockTypeLayout(
+        context,
         parameterBlockType,
-        parameterBlockRules,
         elementType,
         elementTypeRules);
 }
@@ -888,25 +971,50 @@ createStructuredBufferTypeLayout(
 }
 
 SimpleLayoutInfo GetLayoutImpl(
-    Type*     type,
-    LayoutRulesImpl*    rules,
+    TypeLayoutContext*  context,
+    Type*               type,
     RefPtr<TypeLayout>* outTypeLayout,
     SimpleLayoutInfo    offset);
 
 SimpleLayoutInfo GetLayoutImpl(
-    Type*     type,
-    LayoutRulesImpl*    rules,
+    TypeLayoutContext*  context,
+    Type*               type,
     RefPtr<TypeLayout>* outTypeLayout)
 {
-    return GetLayoutImpl(type, rules, outTypeLayout, SimpleLayoutInfo());
+    return GetLayoutImpl(context, type, outTypeLayout, SimpleLayoutInfo());
 }
 
 SimpleLayoutInfo GetLayoutImpl(
-    Type*     type,
-    LayoutRulesImpl*    rules,
+    TypeLayoutContext*  context,
+    Type*               type,
+    RefPtr<TypeLayout>* outTypeLayout,
+    Decl*               declForModifiers)
+{
+    TypeLayoutContext subContext = *context;
+
+    if (declForModifiers)
+    {
+        if (declForModifiers->HasModifier<RowMajorLayoutModifier>())
+            subContext.matrixLayoutMode = kMatrixLayoutMode_RowMajor;
+
+        if (declForModifiers->HasModifier<ColumnMajorLayoutModifier>())
+            subContext.matrixLayoutMode = kMatrixLayoutMode_ColumnMajor;
+
+        // TODO: really need to look for other modifiers that affect
+        // layout, such as GLSL `std140`.
+    }
+
+    return GetLayoutImpl(&subContext, type, outTypeLayout, SimpleLayoutInfo());
+}
+
+SimpleLayoutInfo GetLayoutImpl(
+    TypeLayoutContext*  context,
+    Type*               type,
     RefPtr<TypeLayout>* outTypeLayout,
     SimpleLayoutInfo    offset)
 {
+    auto rules = context->rules;
+
     if (auto parameterBlockType = type->As<ParameterBlockType>())
     {
         // If the user is just interested in uniform layout info,
@@ -930,8 +1038,8 @@ SimpleLayoutInfo GetLayoutImpl(
         if (outTypeLayout)
         {
             *outTypeLayout = createParameterBlockTypeLayout(
-                parameterBlockType,
-                rules);
+                context,
+                parameterBlockType);
         }
 
         return info;
@@ -1075,21 +1183,51 @@ SimpleLayoutInfo GetLayoutImpl(
     }
     else if(auto matType = type->As<MatrixExpressionType>())
     {
-        return GetSimpleLayoutImpl(
-            rules->GetMatrixLayout(
-                GetLayout(matType->getElementType(), rules),
-                (size_t) GetIntVal(matType->getRowCount()),
-                (size_t) GetIntVal(matType->getColumnCount())),
-            type,
-            rules,
-            outTypeLayout);
+        // The `GetMatrixLayout` implementation in the layout rules
+        // currently defaults to assuming column-major layout,
+        // so if we want row-major layout we achieve it here by
+        // transposing the row/column counts.
+        //
+        // TODO: If it is really a universal convention that matrices
+        // are laid out just like arrays of vectors, when we can
+        // probably eliminate the `virtual` `GetLayout` method entirely,
+        // and have the code here be responsible for the layout choice.
+        //
+        size_t rowCount = (size_t) GetIntVal(matType->getRowCount());
+        size_t colCount = (size_t) GetIntVal(matType->getColumnCount());
+        if (context->matrixLayoutMode == kMatrixLayoutMode_ColumnMajor)
+        {
+            size_t tmp = rowCount;
+            rowCount = colCount;
+            colCount = tmp;
+        }
+
+        auto info = rules->GetMatrixLayout(
+            GetLayout(matType->getElementType(), rules),
+            rowCount,
+            colCount);
+
+        if (outTypeLayout)
+        {
+            RefPtr<MatrixTypeLayout> typeLayout = new MatrixTypeLayout();
+            *outTypeLayout = typeLayout;
+
+            typeLayout->type = type;
+            typeLayout->rules = rules;
+            typeLayout->uniformAlignment = info.alignment;
+            typeLayout->mode = context->matrixLayoutMode;
+
+            typeLayout->addResourceUsage(info.kind, info.size);
+        }
+
+        return info;
     }
     else if (auto arrayType = type->As<ArrayExpressionType>())
     {
         RefPtr<TypeLayout> elementTypeLayout;
         auto elementInfo = GetLayoutImpl(
+            context,
             arrayType->BaseType.Ptr(),
-            rules,
             outTypeLayout ? &elementTypeLayout : nullptr);
 
         // For layout purposes, we treat an unsized array as an array of zero elements.
@@ -1171,9 +1309,10 @@ SimpleLayoutInfo GetLayoutImpl(
             {
                 RefPtr<TypeLayout> fieldTypeLayout;
                 UniformLayoutInfo fieldInfo = GetLayoutImpl(
+                    context,
                     GetType(field).Ptr(),
-                    rules,
-                    outTypeLayout ? &fieldTypeLayout : nullptr).getUniformLayout();
+                    outTypeLayout ? &fieldTypeLayout : nullptr,
+                    field.getDecl()).getUniformLayout();
 
                 // Note: we don't add any zero-size fields
                 // when computing structure layout, just
@@ -1272,13 +1411,43 @@ SimpleLayoutInfo GetLayoutImpl(
 
 SimpleLayoutInfo GetLayout(Type* inType, LayoutRulesImpl* rules)
 {
-    return GetLayoutImpl(inType, rules, nullptr);
+    TypeLayoutContext context;
+    context.rules = rules;
+    context.matrixLayoutMode = rules->getDefaultMatrixLayoutMode();
+
+    return GetLayoutImpl(&context, inType, nullptr);
 }
 
-RefPtr<TypeLayout> CreateTypeLayout(Type* type, LayoutRulesImpl* rules, SimpleLayoutInfo offset)
+RefPtr<TypeLayout> createTypeLayout(
+    TypeLayoutContext*  context,
+    Type*               type,
+    SimpleLayoutInfo    offset)
 {
     RefPtr<TypeLayout> typeLayout;
-    GetLayoutImpl(type, rules, &typeLayout, offset);
+    GetLayoutImpl(context, type, &typeLayout, offset);
+    return typeLayout;
+}
+
+RefPtr<TypeLayout> createTypeLayout(
+    TypeLayoutContext*  context,
+    Type*               type)
+{
+    RefPtr<TypeLayout> typeLayout;
+    GetLayoutImpl(context, type, &typeLayout, SimpleLayoutInfo());
+    return typeLayout;
+}
+
+RefPtr<TypeLayout> CreateTypeLayout(
+    Type* type,
+    LayoutRulesImpl* rules,
+    SimpleLayoutInfo offset)
+{
+    TypeLayoutContext context;
+    context.rules = rules;
+    context.matrixLayoutMode = rules->getDefaultMatrixLayoutMode();
+
+    RefPtr<TypeLayout> typeLayout;
+    GetLayoutImpl(&context, type, &typeLayout, offset);
     return typeLayout;
 }
 
diff --git a/source/slang/type-layout.h b/source/slang/type-layout.h
index 8a034b5a6..971088aac 100644
--- a/source/slang/type-layout.h
+++ b/source/slang/type-layout.h
@@ -313,6 +313,22 @@ public:
     size_t              uniformStride;
 };
 
+// When storing the layout for a matrix-type
+// value, we need to know whether it has been
+// laid ot with row-major or column-major
+// storage.
+//
+enum MatrixLayoutMode
+{
+    kMatrixLayoutMode_RowMajor,
+    kMatrixLayoutMode_ColumnMajor,
+};
+class MatrixTypeLayout : public TypeLayout
+{
+public:
+    MatrixLayoutMode    mode;
+};
+
 // Specific case of type layout for a struct
 class StructTypeLayout : public TypeLayout
 {
@@ -515,6 +531,8 @@ struct LayoutRulesImpl
     //
 
     LayoutRulesFamilyImpl* getLayoutRulesFamily() { return family; }
+
+    MatrixLayoutMode getDefaultMatrixLayoutMode();
 };
 
 struct LayoutRulesFamilyImpl
@@ -526,6 +544,7 @@ struct LayoutRulesFamilyImpl
     virtual LayoutRulesImpl* getVaryingOutputRules()        = 0;
     virtual LayoutRulesImpl* getSpecializationConstantRules()   = 0;
     virtual LayoutRulesImpl* getShaderStorageBufferRules()      = 0;
+    virtual MatrixLayoutMode getDefaultMatrixLayoutMode() = 0;
 };
 
 LayoutRulesImpl* GetLayoutRulesImpl(LayoutRule rule);
@@ -541,21 +560,30 @@ RefPtr<TypeLayout> CreateTypeLayout(Type* type, LayoutRulesImpl* rules, SimpleLa
 
 //
 
+struct TypeLayoutContext;
+
 // Create a type layout for a parameter block type.
 RefPtr<ParameterBlockTypeLayout>
 createParameterBlockTypeLayout(
-    RefPtr<ParameterBlockType>  parameterBlockType,
-    LayoutRulesImpl*            rules);
+    TypeLayoutContext*          context,
+    RefPtr<ParameterBlockType>  parameterBlockType);
 
 RefPtr<ParameterBlockTypeLayout>
 createParameterBlockTypeLayout(
+    TypeLayoutContext*          context,
     RefPtr<ParameterBlockType>  parameterBlockType,
-    LayoutRulesImpl*            parameterBlockRules,
-    RefPtr<Type>      elementType,
+    RefPtr<Type>                elementType,
     LayoutRulesImpl*            elementTypeRules);
 
 RefPtr<ParameterBlockTypeLayout>
 createParameterBlockTypeLayout(
+    TypeLayoutContext*          context,
+    RefPtr<ParameterBlockType>  parameterBlockType,
+    SimpleLayoutInfo            parameterBlockInfo,
+    RefPtr<TypeLayout>          elementTypeLayout);
+
+RefPtr<ParameterBlockTypeLayout>
+createParameterBlockTypeLayout(
     RefPtr<ParameterBlockType>  parameterBlockType,
     LayoutRulesImpl*            parameterBlockRules,
     SimpleLayoutInfo            parameterBlockInfo,
diff --git a/tests/hlsl/dxsdk/AdaptiveTessellationCS40/Render.hlsl b/tests/hlsl/dxsdk/AdaptiveTessellationCS40/Render.hlsl
index c106c46e7..2fc3536c1 100644
--- a/tests/hlsl/dxsdk/AdaptiveTessellationCS40/Render.hlsl
+++ b/tests/hlsl/dxsdk/AdaptiveTessellationCS40/Render.hlsl
@@ -1,4 +1,4 @@
-//TEST(smoke):COMPARE_HLSL: -profile vs_4_0 -entry RenderBaseVS -profile ps_4_0 -entry RenderPS -target dxbc-assembly
+//TEST(smoke):COMPARE_HLSL: -use-ir -profile vs_4_0 -entry RenderBaseVS -profile ps_4_0 -entry RenderPS -target dxbc-assembly
 //--------------------------------------------------------------------------------------
 // File: Render.hlsl
 //