Fix IR layout of 3-element vectors in cbuffers for -fvk-use-dx-layout (#7282)

* Better handling for 16-byte boundary of d3d cbuffer

Fixes #6921

D3D cbuffers have slightly different packing rules that allow packing
vectors into a 16-byte slot at element alignments, except when
a field would cross a 16-byte boundary. In that case, we need to
realign the field to the next 16-byte boundary.

In particular, this impacts vec3s, which are not a power of two in
size and thus require slightly different alignment logic, compared to
std430 and std140. (Example: a float and float3 should fit together in
that order in a single slot.)

Adds test cases.

Adds documentation page for GLSL target

2 files changed, 86 insertions, 41 deletions

diff --git a/source/slang/slang-ir-layout.cpp b/source/slang/slang-ir-layout.cpp
index 7b3aa4340..795f47f55 100644
--- a/source/slang/slang-ir-layout.cpp
+++ b/source/slang/slang-ir-layout.cpp
@@ -151,6 +151,8 @@ static Result _calcSizeAndAlignment(
             auto structType = cast<IRStructType>(type);
             IRSizeAndAlignment structLayout;
             IRIntegerValue offset = 0;
+            IRIntegerValue lastFieldAlignment = 0;
+            IRType* lastFieldType = NULL;
             bool seenFinalUnsizedArrayField = false;
             for (auto field : structType->getFields())
             {
@@ -159,18 +161,26 @@ static Result _calcSizeAndAlignment(
                 // subsequent offsets
                 SLANG_ASSERT(!seenFinalUnsizedArrayField);
 
-                if (auto offsetDecor =
-                        field->getKey()->findDecoration<IRVkStructOffsetDecoration>())
-                {
-                    offset = offsetDecor->getOffset()->getValue();
-                }
-
                 IRSizeAndAlignment fieldTypeLayout;
                 SLANG_RETURN_ON_FAIL(
                     getSizeAndAlignment(optionSet, rules, field->getFieldType(), &fieldTypeLayout));
                 seenFinalUnsizedArrayField =
                     fieldTypeLayout.size == IRSizeAndAlignment::kIndeterminateSize;
 
+                if (auto offsetDecor =
+                        field->getKey()->findDecoration<IRVkStructOffsetDecoration>())
+                {
+                    offset = offsetDecor->getOffset()->getValue();
+                }
+                else
+                {
+                    offset = rules->adjustOffset(
+                        offset,
+                        fieldTypeLayout.size,
+                        lastFieldType,
+                        lastFieldAlignment);
+                }
+
                 structLayout.size = align(offset, fieldTypeLayout.alignment);
                 structLayout.alignment =
                     std::max(structLayout.alignment, fieldTypeLayout.alignment);
@@ -195,14 +205,8 @@ static Result _calcSizeAndAlignment(
                 if (!seenFinalUnsizedArrayField)
                     structLayout.size += fieldTypeLayout.size;
                 offset = structLayout.size;
-                if (as<IRMatrixType>(field->getFieldType()) ||
-                    as<IRArrayTypeBase>(field->getFieldType()) ||
-                    as<IRStructType>(field->getFieldType()))
-                {
-                    offset = rules->adjustOffsetForNextAggregateMember(
-                        offset,
-                        fieldTypeLayout.alignment);
-                }
+                lastFieldType = field->getFieldType();
+                lastFieldAlignment = fieldTypeLayout.alignment;
             }
             *outSizeAndAlignment = rules->alignCompositeElement(structLayout);
             return SLANG_OK;
@@ -498,17 +502,23 @@ Result getOffset(
 struct NaturalLayoutRules : IRTypeLayoutRules
 {
     NaturalLayoutRules() { ruleName = IRTypeLayoutRuleName::Natural; }
-    virtual IRIntegerValue adjustOffsetForNextAggregateMember(
-        IRIntegerValue currentSize,
-        IRIntegerValue lastElementAlignment)
+    virtual IRIntegerValue adjustOffset(
+        IRIntegerValue offset,
+        IRIntegerValue elementSize,
+        IRType* lastFieldType,
+        IRIntegerValue lastFieldAlignment)
     {
-        SLANG_UNUSED(lastElementAlignment);
-        return currentSize;
+        SLANG_UNUSED(elementSize);
+        SLANG_UNUSED(lastFieldType);
+        SLANG_UNUSED(lastFieldAlignment);
+        return offset;
     }
+
     virtual IRSizeAndAlignment alignCompositeElement(IRSizeAndAlignment elementSize)
     {
         return elementSize;
     }
+
     virtual IRSizeAndAlignment getVectorSizeAndAlignment(
         IRSizeAndAlignment element,
         IRIntegerValue count)
@@ -528,22 +538,30 @@ struct ConstantBufferLayoutRules : IRTypeLayoutRules
         return IRSizeAndAlignment(currentSize.size, 16);
     }
 
-    virtual IRIntegerValue adjustOffsetForNextAggregateMember(
-        IRIntegerValue currentSize,
-        IRIntegerValue lastElementAlignment)
+    virtual IRIntegerValue adjustOffset(
+        IRIntegerValue offset,
+        IRIntegerValue elementSize,
+        IRType* lastFieldType,
+        IRIntegerValue lastFieldAlignment)
     {
-        SLANG_UNUSED(lastElementAlignment);
-        return currentSize;
+        SLANG_UNUSED(lastFieldType);
+        SLANG_UNUSED(lastFieldAlignment);
+
+        // If the element would cross a 16-byte boundary, align to the next boundary
+        auto currentChunk = offset / 16;
+        auto endChunk = (offset + elementSize - 1) / 16;
+        if (currentChunk != endChunk)
+        {
+            return align(offset, 16);
+        }
+        return offset;
     }
 
     virtual IRSizeAndAlignment getVectorSizeAndAlignment(
         IRSizeAndAlignment element,
         IRIntegerValue count)
     {
-        IRIntegerValue countForAlignment = count;
-        return IRSizeAndAlignment(
-            (int)(element.size * count),
-            (int)(element.size * countForAlignment));
+        return IRSizeAndAlignment(element.size * count, element.alignment);
     }
 };
 
@@ -551,15 +569,24 @@ struct Std430LayoutRules : IRTypeLayoutRules
 {
     Std430LayoutRules() { ruleName = IRTypeLayoutRuleName::Std430; }
 
-    virtual IRSizeAndAlignment alignCompositeElement(IRSizeAndAlignment elementSize)
+    virtual IRIntegerValue adjustOffset(
+        IRIntegerValue offset,
+        IRIntegerValue elementSize,
+        IRType* lastFieldType,
+        IRIntegerValue lastFieldAlignment)
     {
-        return elementSize;
+        SLANG_UNUSED(elementSize);
+        if (as<IRMatrixType>(lastFieldType) || as<IRArrayTypeBase>(lastFieldType) ||
+            as<IRStructType>(lastFieldType))
+        {
+            return align(offset, (int)lastFieldAlignment);
+        }
+        return offset;
     }
-    virtual IRIntegerValue adjustOffsetForNextAggregateMember(
-        IRIntegerValue currentSize,
-        IRIntegerValue lastElementAlignment)
+
+    virtual IRSizeAndAlignment alignCompositeElement(IRSizeAndAlignment elementSize)
     {
-        return align(currentSize, (int)lastElementAlignment);
+        return elementSize;
     }
 
     virtual IRSizeAndAlignment getVectorSizeAndAlignment(
@@ -579,18 +606,28 @@ struct Std140LayoutRules : IRTypeLayoutRules
 {
     Std140LayoutRules() { ruleName = IRTypeLayoutRuleName::Std140; }
 
-    virtual IRIntegerValue adjustOffsetForNextAggregateMember(
-        IRIntegerValue currentSize,
-        IRIntegerValue lastElementAlignment)
+    virtual IRIntegerValue adjustOffset(
+        IRIntegerValue offset,
+        IRIntegerValue elementSize,
+        IRType* lastFieldType,
+        IRIntegerValue lastFieldAlignment)
     {
-        return align(currentSize, (int)lastElementAlignment);
+        SLANG_UNUSED(elementSize);
+        if (as<IRMatrixType>(lastFieldType) || as<IRArrayTypeBase>(lastFieldType) ||
+            as<IRStructType>(lastFieldType))
+        {
+            return align(offset, (int)lastFieldAlignment);
+        }
+        return offset;
     }
+
     virtual IRSizeAndAlignment alignCompositeElement(IRSizeAndAlignment elementSize)
     {
         elementSize.alignment = (int)align(elementSize.alignment, 16);
         elementSize.size = align(elementSize.size, elementSize.alignment);
         return elementSize;
     }
+
     virtual IRSizeAndAlignment getVectorSizeAndAlignment(
         IRSizeAndAlignment element,
         IRIntegerValue count)
diff --git a/source/slang/slang-ir-layout.h b/source/slang/slang-ir-layout.h
index db115a936..1918f31b6 100644
--- a/source/slang/slang-ir-layout.h
+++ b/source/slang/slang-ir-layout.h
@@ -65,9 +65,17 @@ public:
     virtual IRSizeAndAlignment getVectorSizeAndAlignment(
         IRSizeAndAlignment element,
         IRIntegerValue count) = 0;
-    virtual IRIntegerValue adjustOffsetForNextAggregateMember(
-        IRIntegerValue currentSize,
-        IRIntegerValue lastElementAlignment) = 0;
+
+    /// Adjust the offset of an element. Handles two cases; alignmment when
+    /// the previous field was a composite, and the D3D constant buffer case
+    /// where an element is aligned to the next 16-byte boundary if it doesn't
+    /// fit entirely within the current one.
+    virtual IRIntegerValue adjustOffset(
+        IRIntegerValue offset,
+        IRIntegerValue elementSize,
+        IRType* lastFieldType,
+        IRIntegerValue lastFieldAlignment) = 0;
+
     static IRTypeLayoutRules* getStd430();
     static IRTypeLayoutRules* getStd140();
     static IRTypeLayoutRules* getNatural();