Fix SPIRV for mesh shaders, checks for invalid target code&recursion. (#3788)

* Fix #3780.

* Fixers #3781.

* Add test for #3781.

* Diagnose error on unsupported builtin intrinsic types.

* Add check for recursion.

* Fix.

* Fix.

* Fix recursion detection.

* Fix.

* Fix.

* Fix recursion logic.

* More fix.

6 files changed, 118 insertions, 3 deletions

diff --git a/tests/bugs/gh-3780.slang b/tests/bugs/gh-3780.slang
new file mode 100644
index 000000000..62a9542d7
--- /dev/null
+++ b/tests/bugs/gh-3780.slang
@@ -0,0 +1,58 @@
+//TEST:SIMPLE(filecheck=CHECK): -target spirv -emit-spirv-directly
+
+// CHECK: OpEntryPoint
+
+const static float2 positions[3] = {
+  float2(0.0, -0.5),
+  float2(0.5, 0.5),
+  float2(-0.5, 0.5)
+};
+
+const static float3 colors[3] = {
+  float3(1.0, 1.0, 0.0),
+  float3(0.0, 1.0, 1.0),
+  float3(1.0, 0.0, 1.0)
+};
+
+struct Vertex
+{
+    float4 pos : SV_Position;
+    float3 color;
+};
+
+const static uint MAX_VERTS = 3;
+const static uint MAX_PRIMS = 1;
+
+[outputtopology("triangle")]
+[numthreads(3, 1, 1)]
+[shader("mesh")]
+void entry_mesh(
+    in uint tig : SV_GroupIndex,
+    out vertices Vertex verts[MAX_VERTS],
+    out indices uint3 triangles[MAX_PRIMS])
+{
+    const uint numVertices = 3;
+    const uint numPrimitives = 1;
+    SetMeshOutputCounts(numVertices, numPrimitives);
+
+    if(tig < numVertices) {
+        verts[tig] = {float4(positions[tig], 0, 1), colors[tig]};
+    }
+
+    if(tig < numPrimitives) {
+        triangles[tig] = uint3(0,1,2);
+    }
+}
+
+struct FragmentOut
+{
+    [[vk::location(0)]] float3 color;
+};
+
+[shader("fragment")]
+FragmentOut entry_fragment(in Vertex vertex)
+{
+    FragmentOut frag_out;
+    frag_out.color = float3(1,1,1);
+    return frag_out;
+}
\ No newline at end of file
diff --git a/tests/bugs/gh-3781.slang b/tests/bugs/gh-3781.slang
new file mode 100644
index 000000000..89b2957b9
--- /dev/null
+++ b/tests/bugs/gh-3781.slang
@@ -0,0 +1,22 @@
+//TEST:SIMPLE(filecheck=CHECK): -target spirv -emit-spirv-directly
+
+// CHECK: OpEntryPoint
+
+struct Vertex
+{
+  float4 pos : SV_Position;
+  [[vk::location(0)]] float3 color;
+};
+
+const static uint MAX_VERTS = 3;
+const static uint MAX_PRIMS = 1;
+
+[outputtopology("triangle")]
+[numthreads(3, 1, 1)]
+[shader("mesh")]
+void entry_mesh(
+    in uint tig : SV_GroupIndex,
+    out vertices Vertex verts[MAX_VERTS],
+    out indices uint3 triangles[MAX_PRIMS])
+{
+}
\ No newline at end of file
diff --git a/tests/bugs/gh-3783.slang b/tests/bugs/gh-3783.slang
new file mode 100644
index 000000000..9e343b22f
--- /dev/null
+++ b/tests/bugs/gh-3783.slang
@@ -0,0 +1,17 @@
+//TEST:SIMPLE(filecheck=CHECK):-target spirv -emit-spirv-directly
+
+// CHECK: error 55200:
+
+RWStructuredBuffer<float> outputBuffer;
+
+float testFunc(vector<vector<float, 3>, 2> v1)
+{
+    return v1.x.x;
+}
+
+[numthreads(1,1,1)]
+void main(uint v : SV_DispatchThreadID)
+{
+    vector<vector<float, 3>, 2> v1 = { { 1, 2, 3 }, { 4, 5, v } };
+    outputBuffer[0] = testFunc(v1);
+}
\ No newline at end of file
diff --git a/tests/cross-compile/array-of-buffers.slang b/tests/cross-compile/array-of-buffers.slang
index 31738ccfc..bf2574f29 100644
--- a/tests/cross-compile/array-of-buffers.slang
+++ b/tests/cross-compile/array-of-buffers.slang
@@ -1,7 +1,10 @@
 // array-of-buffers.slang
 
-//TEST:CROSS_COMPILE:-target spirv-assembly -entry main -stage fragment
-//TEST:CROSS_COMPILE:-target dxil-assembly  -entry main -stage fragment -profile sm_6_0
+//TEST:SIMPLE(filecheck=SPIRV):-target spirv-assembly -entry main -stage fragment
+//TEST:SIMPLE(filecheck=DXIL):-target dxil-assembly  -entry main -stage fragment -profile sm_6_0
+
+// SPIRV: OpEntryPoint
+// DXIL: define void @main()
 
 // This test ensures that we cross-compile arrays of structured/constant
 // buffers into appropriate GLSL, where these are not first-class types.
diff --git a/tests/diagnostics/recursion.slang b/tests/diagnostics/recursion.slang
new file mode 100644
index 000000000..b5746820e
--- /dev/null
+++ b/tests/diagnostics/recursion.slang
@@ -0,0 +1,15 @@
+//TEST:SIMPLE(filecheck=CHECK): -target spirv -emit-spirv-directly
+
+// CHECK: error 55201:
+
+float test<let x : int>()
+{
+    return test<x>();
+}
+
+RWStructuredBuffer<float> outputBuffer;
+[numthreads(1,1,1)]
+void main()
+{
+    outputBuffer[0] = test<1>();
+}
\ No newline at end of file
diff --git a/tests/spirv/scalar-buffer-packing.slang b/tests/spirv/scalar-buffer-packing.slang
index bc42cfa39..1224ae664 100644
--- a/tests/spirv/scalar-buffer-packing.slang
+++ b/tests/spirv/scalar-buffer-packing.slang
@@ -51,6 +51,6 @@ void computeMain()
 
 // SPIRV: OpEntryPoint GLCompute
 
-// SPIRV-DAG: %[[STRUCTNAME:[A-Za-z0-9_]+]]  = OpTypeStruct %int %_Array_natural_int32
+// SPIRV-DAG: %[[STRUCTNAME:[A-Za-z0-9_]+]]  = OpTypeStruct %int %_Array_natural_vector_int_3_2
 
 // SPIRV-DAG: OpMemberDecorate %[[STRUCTNAME:[A-Za-z0-9_]+]] 1 Offset 4