Legalize array size of SV_TessFactor and SV_InsideTessFactor (#6409)

* Legalize array size of SV_TessFactor and SV_InsideTessFactor

When targeting SPIR-V or GLSL, the type for SV_TessFactor must be
float[4]; note that it is not a vector but an array.
Similarly the type of SV_InsideTessFactor has to be an array of
float[2].

When the user shader declare them as arrays smaller than the required
size, Slang will legalize them to the required sizes.

Note that it is not the user mistake to declare floar[3] when the hull
mode is in "tri", as an example. The unused components are expected to
be ignored.

Note also that HLSL allows the type to be float[2|3|4] whereas GLSL and
SPIR-V requires it to be always float[4].

* Change from "quad" domain mode to "tri"

* Handle error case

---------

Co-authored-by: Yong He <yonghe@outlook.com>

1 files changed, 55 insertions, 0 deletions

diff --git a/tests/spirv/tess-factor-legalize-array-size.slang b/tests/spirv/tess-factor-legalize-array-size.slang
new file mode 100644
index 000000000..70d020deb
--- /dev/null
+++ b/tests/spirv/tess-factor-legalize-array-size.slang
@@ -0,0 +1,55 @@
+//TEST:SIMPLE(filecheck=GLSL):-target glsl -entry hullMain -stage hull -allow-glsl
+//TEST:SIMPLE(filecheck=SPIRV):-target spirv -entry hullMain -stage hull -allow-glsl
+
+// When targeting SPIRV, the tessellation factors should be an array with size 4 or 2.
+
+// GLSL: gl_TessLevelOuter[3] = 0.0;
+// GLSL: gl_TessLevelInner[1] = 0.0;
+
+// SPIRV: OpDecorate %gl_TessLevelOuter BuiltIn TessLevelOuter
+// SPIRV: OpDecorate %gl_TessLevelInner BuiltIn TessLevelInner
+
+// SPIRV-DAG: OpStore %gl_TessLevelOuter %[[OuterOpCompositeConstruct:[1-9][0-9]*]]
+// SPIRV-DAG: %[[OuterOpCompositeConstruct]] = OpCompositeConstruct %[[OuterOpTypeArray:[a-zA-Z_0-9]*]]
+// SPIRV-DAG: %[[OuterOpTypeArray]] = OpTypeArray %{{[^ ]*}} %[[OuterOpConstant:[a-zA-Z_0-9]*]]
+// SPIRV-DAG: %[[OuterOpConstant]] = OpConstant %{{[^ ]*}} 4
+
+// SPIRV-DAG: OpStore %gl_TessLevelInner %[[InnerOpCompositeConstruct:[1-9][0-9]*]]
+// SPIRV-DAG: %[[InnerOpCompositeConstruct]] = OpCompositeConstruct %[[InnerOpTypeArray:[a-zA-Z_0-9]*]]
+// SPIRV-DAG: %[[InnerOpTypeArray]] = OpTypeArray %{{[^ ]*}} %[[InnerOpConstant:[a-zA-Z_0-9]*]]
+// SPIRV-DAG: %[[InnerOpConstant]] = OpConstant %{{[^ ]*}} 2
+
+struct HsOut
+{
+    float2 pos;
+    float2 hm;
+};
+
+struct HscOut
+{
+    float EdgeTessFactor[3] : SV_TessFactor; // Should be legalized to float[4]
+    float InsideTessFactor[1] : SV_InsideTessFactor; // Should be legalized to float[2]
+};
+
+[domain("tri")]
+[partitioning("integer")]
+[outputtopology("triangle_ccw")]
+[outputcontrolpoints(4)]
+[patchconstantfunc("constants")]
+HsOut hullMain()
+{
+    HsOut o;
+    o.pos = 1;
+    o.hm = 2;
+    return o;
+}
+
+HscOut constants()
+{
+    HscOut o;
+    o.EdgeTessFactor[0] = 1;
+    o.EdgeTessFactor[1] = 2;
+    o.EdgeTessFactor[2] = 3;
+    o.InsideTessFactor[0] = 0.5;
+    return o;
+}