Legalize array size of SV_TessFactor and SV_InsideTessFactor (#6409)

* Legalize array size of SV_TessFactor and SV_InsideTessFactor

When targeting SPIR-V or GLSL, the type for SV_TessFactor must be
float[4]; note that it is not a vector but an array.
Similarly the type of SV_InsideTessFactor has to be an array of
float[2].

When the user shader declare them as arrays smaller than the required
size, Slang will legalize them to the required sizes.

Note that it is not the user mistake to declare floar[3] when the hull
mode is in "tri", as an example. The unused components are expected to
be ignored.

Note also that HLSL allows the type to be float[2|3|4] whereas GLSL and
SPIR-V requires it to be always float[4].

* Change from "quad" domain mode to "tri"

* Handle error case

---------

Co-authored-by: Yong He <yonghe@outlook.com>

1 files changed, 47 insertions, 0 deletions

diff --git a/tests/diagnostics/tessellation-legalize-array-size-too-big.slang b/tests/diagnostics/tessellation-legalize-array-size-too-big.slang
new file mode 100644
index 000000000..6153d505e
--- /dev/null
+++ b/tests/diagnostics/tessellation-legalize-array-size-too-big.slang
@@ -0,0 +1,47 @@
+//TEST:SIMPLE(filecheck=CHK):-target glsl -entry hullMain -stage hull -allow-glsl
+//TEST:SIMPLE(filecheck=CHK):-target spirv-asm -entry hullMain -stage hull -allow-glsl
+
+// Tessllation outsie factor must be an array of size 4 or less
+// Tessllation inside factor must be an array of size 2 or less
+
+struct HsOut
+{
+    float2 pos;
+    float2 hm;
+};
+
+struct HscOut
+{
+    //CHK: error 30024: Cannot convert array of size 5 to array of size 4 as this would truncate data
+    float EdgeTessFactor[5] : SV_TessFactor;
+
+    //CHK: error 30024: Cannot convert array of size 3 to array of size 2 as this would truncate data
+    float InsideTessFactor[3] : SV_InsideTessFactor;
+};
+
+[domain("tri")]
+[partitioning("integer")]
+[outputtopology("triangle_ccw")]
+[outputcontrolpoints(4)]
+[patchconstantfunc("constants")]
+HsOut hullMain()
+{
+    HsOut o;
+    o.pos = 1;
+    o.hm = 2;
+    return o;
+}
+
+HscOut constants()
+{
+    HscOut o;
+    o.EdgeTessFactor[0] = 1;
+    o.EdgeTessFactor[1] = 2;
+    o.EdgeTessFactor[2] = 3;
+    o.EdgeTessFactor[3] = 4;
+    o.EdgeTessFactor[4] = 5;
+    o.InsideTessFactor[0] = 0.5;
+    o.InsideTessFactor[1] = 0.3;
+    o.InsideTessFactor[2] = 0.2;
+    return o;
+}