Legalize array size of SV_TessFactor and SV_InsideTessFactor (#6409)

* Legalize array size of SV_TessFactor and SV_InsideTessFactor

When targeting SPIR-V or GLSL, the type for SV_TessFactor must be
float[4]; note that it is not a vector but an array.
Similarly the type of SV_InsideTessFactor has to be an array of
float[2].

When the user shader declare them as arrays smaller than the required
size, Slang will legalize them to the required sizes.

Note that it is not the user mistake to declare floar[3] when the hull
mode is in "tri", as an example. The unused components are expected to
be ignored.

Note also that HLSL allows the type to be float[2|3|4] whereas GLSL and
SPIR-V requires it to be always float[4].

* Change from "quad" domain mode to "tri"

* Handle error case

---------

Co-authored-by: Yong He <yonghe@outlook.com>

4 files changed, 180 insertions, 0 deletions

diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h
index 03f52a701..789eac88f 100644
--- a/source/slang/slang-diagnostic-defs.h
+++ b/source/slang/slang-diagnostic-defs.h
@@ -606,6 +606,11 @@ DIAGNOSTIC(30019, Error, typeMismatch, "expected an expression of type '$0', got
 DIAGNOSTIC(30021, Error, noApplicationFunction, "$0: no overload takes arguments ($1)")
 DIAGNOSTIC(30022, Error, invalidTypeCast, "invalid type cast between \"$0\" and \"$1\".")
 DIAGNOSTIC(30023, Error, typeHasNoPublicMemberOfName, "\"$0\" does not have public member \"$1\".")
+DIAGNOSTIC(
+    30024,
+    Error,
+    cannotConvertArrayOfSmallerToLargerSize,
+    "Cannot convert array of size $0 to array of size $1 as this would truncate data")
 DIAGNOSTIC(30025, Error, invalidArraySize, "array size must be larger than zero.")
 DIAGNOSTIC(
     30026,
diff --git a/source/slang/slang-ir-glsl-legalize.cpp b/source/slang/slang-ir-glsl-legalize.cpp
index c11ccc66d..11d5399cf 100644
--- a/source/slang/slang-ir-glsl-legalize.cpp
+++ b/source/slang/slang-ir-glsl-legalize.cpp
@@ -1528,6 +1528,43 @@ ScalarizedVal createSimpleGLSLGlobalVarying(
 
                 val = ScalarizedVal::typeAdapter(typeAdapter);
             }
+
+            if (auto requiredArrayType = as<IRArrayTypeBase>(systemValueInfo->requiredType))
+            {
+                // Find first array declarator and handle size mismatch
+                for (auto dd = declarator; dd; dd = dd->next)
+                {
+                    if (dd->flavor != GlobalVaryingDeclarator::Flavor::array)
+                        continue;
+
+                    // Compare the array size
+                    auto declaredArraySize = dd->elementCount;
+                    auto requiredArraySize = requiredArrayType->getElementCount();
+                    if (declaredArraySize == requiredArraySize)
+                        break;
+
+                    auto toSize = getIntVal(requiredArraySize);
+                    auto fromSize = getIntVal(declaredArraySize);
+                    if (toSize < fromSize)
+                    {
+                        context->getSink()->diagnose(
+                            inVarLayout,
+                            Diagnostics::cannotConvertArrayOfSmallerToLargerSize,
+                            fromSize,
+                            toSize);
+                    }
+
+                    // Array sizes differ, need type adapter
+                    RefPtr<ScalarizedTypeAdapterValImpl> typeAdapter =
+                        new ScalarizedTypeAdapterValImpl;
+                    typeAdapter->actualType = systemValueInfo->requiredType;
+                    typeAdapter->pretendType = builder->getArrayType(inType, declaredArraySize);
+                    typeAdapter->val = val;
+
+                    val = ScalarizedVal::typeAdapter(typeAdapter);
+                    break;
+                }
+            }
         }
     }
     else
@@ -1971,6 +2008,42 @@ ScalarizedVal adaptType(IRBuilder* builder, IRInst* val, IRType* toType, IRType*
                 val,
                 builder->getIntValue(builder->getIntType(), 0));
         }
+        else if (auto toArray = as<IRArrayTypeBase>(toType))
+        {
+            // If array sizes differ, we need to reshape the array
+            if (fromArray->getElementCount() != toArray->getElementCount())
+            {
+                List<IRInst*> elements;
+
+                // Get array sizes once
+                auto fromSize = getIntVal(fromArray->getElementCount());
+                auto toSize = getIntVal(toArray->getElementCount());
+                SLANG_ASSERT(fromSize <= toSize);
+
+                // Extract elements one at a time up to the source array size
+                for (Index i = 0; i < fromSize; i++)
+                {
+                    auto element = builder->emitElementExtract(
+                        fromArray->getElementType(),
+                        val,
+                        builder->getIntValue(builder->getIntType(), i));
+                    elements.add(element);
+                }
+
+                if (fromSize < toSize)
+                {
+                    // Fill remaining elements with default value up to target size
+                    auto elementType = toArray->getElementType();
+                    auto defaultValue = builder->emitDefaultConstruct(elementType);
+                    for (Index i = fromSize; i < toSize; i++)
+                    {
+                        elements.add(defaultValue);
+                    }
+                }
+
+                val = builder->emitMakeArray(toType, elements.getCount(), elements.getBuffer());
+            }
+        }
     }
     // TODO: actually consider what needs to go on here...
     return ScalarizedVal::value(builder->emitCast(toType, val));
diff --git a/tests/diagnostics/tessellation-legalize-array-size-too-big.slang b/tests/diagnostics/tessellation-legalize-array-size-too-big.slang
new file mode 100644
index 000000000..6153d505e
--- /dev/null
+++ b/tests/diagnostics/tessellation-legalize-array-size-too-big.slang
@@ -0,0 +1,47 @@
+//TEST:SIMPLE(filecheck=CHK):-target glsl -entry hullMain -stage hull -allow-glsl
+//TEST:SIMPLE(filecheck=CHK):-target spirv-asm -entry hullMain -stage hull -allow-glsl
+
+// Tessllation outsie factor must be an array of size 4 or less
+// Tessllation inside factor must be an array of size 2 or less
+
+struct HsOut
+{
+    float2 pos;
+    float2 hm;
+};
+
+struct HscOut
+{
+    //CHK: error 30024: Cannot convert array of size 5 to array of size 4 as this would truncate data
+    float EdgeTessFactor[5] : SV_TessFactor;
+
+    //CHK: error 30024: Cannot convert array of size 3 to array of size 2 as this would truncate data
+    float InsideTessFactor[3] : SV_InsideTessFactor;
+};
+
+[domain("tri")]
+[partitioning("integer")]
+[outputtopology("triangle_ccw")]
+[outputcontrolpoints(4)]
+[patchconstantfunc("constants")]
+HsOut hullMain()
+{
+    HsOut o;
+    o.pos = 1;
+    o.hm = 2;
+    return o;
+}
+
+HscOut constants()
+{
+    HscOut o;
+    o.EdgeTessFactor[0] = 1;
+    o.EdgeTessFactor[1] = 2;
+    o.EdgeTessFactor[2] = 3;
+    o.EdgeTessFactor[3] = 4;
+    o.EdgeTessFactor[4] = 5;
+    o.InsideTessFactor[0] = 0.5;
+    o.InsideTessFactor[1] = 0.3;
+    o.InsideTessFactor[2] = 0.2;
+    return o;
+}
diff --git a/tests/spirv/tess-factor-legalize-array-size.slang b/tests/spirv/tess-factor-legalize-array-size.slang
new file mode 100644
index 000000000..70d020deb
--- /dev/null
+++ b/tests/spirv/tess-factor-legalize-array-size.slang
@@ -0,0 +1,55 @@
+//TEST:SIMPLE(filecheck=GLSL):-target glsl -entry hullMain -stage hull -allow-glsl
+//TEST:SIMPLE(filecheck=SPIRV):-target spirv -entry hullMain -stage hull -allow-glsl
+
+// When targeting SPIRV, the tessellation factors should be an array with size 4 or 2.
+
+// GLSL: gl_TessLevelOuter[3] = 0.0;
+// GLSL: gl_TessLevelInner[1] = 0.0;
+
+// SPIRV: OpDecorate %gl_TessLevelOuter BuiltIn TessLevelOuter
+// SPIRV: OpDecorate %gl_TessLevelInner BuiltIn TessLevelInner
+
+// SPIRV-DAG: OpStore %gl_TessLevelOuter %[[OuterOpCompositeConstruct:[1-9][0-9]*]]
+// SPIRV-DAG: %[[OuterOpCompositeConstruct]] = OpCompositeConstruct %[[OuterOpTypeArray:[a-zA-Z_0-9]*]]
+// SPIRV-DAG: %[[OuterOpTypeArray]] = OpTypeArray %{{[^ ]*}} %[[OuterOpConstant:[a-zA-Z_0-9]*]]
+// SPIRV-DAG: %[[OuterOpConstant]] = OpConstant %{{[^ ]*}} 4
+
+// SPIRV-DAG: OpStore %gl_TessLevelInner %[[InnerOpCompositeConstruct:[1-9][0-9]*]]
+// SPIRV-DAG: %[[InnerOpCompositeConstruct]] = OpCompositeConstruct %[[InnerOpTypeArray:[a-zA-Z_0-9]*]]
+// SPIRV-DAG: %[[InnerOpTypeArray]] = OpTypeArray %{{[^ ]*}} %[[InnerOpConstant:[a-zA-Z_0-9]*]]
+// SPIRV-DAG: %[[InnerOpConstant]] = OpConstant %{{[^ ]*}} 2
+
+struct HsOut
+{
+    float2 pos;
+    float2 hm;
+};
+
+struct HscOut
+{
+    float EdgeTessFactor[3] : SV_TessFactor; // Should be legalized to float[4]
+    float InsideTessFactor[1] : SV_InsideTessFactor; // Should be legalized to float[2]
+};
+
+[domain("tri")]
+[partitioning("integer")]
+[outputtopology("triangle_ccw")]
+[outputcontrolpoints(4)]
+[patchconstantfunc("constants")]
+HsOut hullMain()
+{
+    HsOut o;
+    o.pos = 1;
+    o.hm = 2;
+    return o;
+}
+
+HscOut constants()
+{
+    HscOut o;
+    o.EdgeTessFactor[0] = 1;
+    o.EdgeTessFactor[1] = 2;
+    o.EdgeTessFactor[2] = 3;
+    o.InsideTessFactor[0] = 0.5;
+    return o;
+}