Fix SPIRV for mesh shaders, checks for invalid target code&recursion. (#3788)

* Fix #3780.

* Fixers #3781.

* Add test for #3781.

* Diagnose error on unsupported builtin intrinsic types.

* Add check for recursion.

* Fix.

* Fix.

* Fix recursion detection.

* Fix.

* Fix.

* Fix recursion logic.

* More fix.

3 files changed, 97 insertions, 0 deletions

diff --git a/tests/bugs/gh-3780.slang b/tests/bugs/gh-3780.slang
new file mode 100644
index 000000000..62a9542d7
--- /dev/null
+++ b/tests/bugs/gh-3780.slang
@@ -0,0 +1,58 @@
+//TEST:SIMPLE(filecheck=CHECK): -target spirv -emit-spirv-directly
+
+// CHECK: OpEntryPoint
+
+const static float2 positions[3] = {
+  float2(0.0, -0.5),
+  float2(0.5, 0.5),
+  float2(-0.5, 0.5)
+};
+
+const static float3 colors[3] = {
+  float3(1.0, 1.0, 0.0),
+  float3(0.0, 1.0, 1.0),
+  float3(1.0, 0.0, 1.0)
+};
+
+struct Vertex
+{
+    float4 pos : SV_Position;
+    float3 color;
+};
+
+const static uint MAX_VERTS = 3;
+const static uint MAX_PRIMS = 1;
+
+[outputtopology("triangle")]
+[numthreads(3, 1, 1)]
+[shader("mesh")]
+void entry_mesh(
+    in uint tig : SV_GroupIndex,
+    out vertices Vertex verts[MAX_VERTS],
+    out indices uint3 triangles[MAX_PRIMS])
+{
+    const uint numVertices = 3;
+    const uint numPrimitives = 1;
+    SetMeshOutputCounts(numVertices, numPrimitives);
+
+    if(tig < numVertices) {
+        verts[tig] = {float4(positions[tig], 0, 1), colors[tig]};
+    }
+
+    if(tig < numPrimitives) {
+        triangles[tig] = uint3(0,1,2);
+    }
+}
+
+struct FragmentOut
+{
+    [[vk::location(0)]] float3 color;
+};
+
+[shader("fragment")]
+FragmentOut entry_fragment(in Vertex vertex)
+{
+    FragmentOut frag_out;
+    frag_out.color = float3(1,1,1);
+    return frag_out;
+}
\ No newline at end of file
diff --git a/tests/bugs/gh-3781.slang b/tests/bugs/gh-3781.slang
new file mode 100644
index 000000000..89b2957b9
--- /dev/null
+++ b/tests/bugs/gh-3781.slang
@@ -0,0 +1,22 @@
+//TEST:SIMPLE(filecheck=CHECK): -target spirv -emit-spirv-directly
+
+// CHECK: OpEntryPoint
+
+struct Vertex
+{
+  float4 pos : SV_Position;
+  [[vk::location(0)]] float3 color;
+};
+
+const static uint MAX_VERTS = 3;
+const static uint MAX_PRIMS = 1;
+
+[outputtopology("triangle")]
+[numthreads(3, 1, 1)]
+[shader("mesh")]
+void entry_mesh(
+    in uint tig : SV_GroupIndex,
+    out vertices Vertex verts[MAX_VERTS],
+    out indices uint3 triangles[MAX_PRIMS])
+{
+}
\ No newline at end of file
diff --git a/tests/bugs/gh-3783.slang b/tests/bugs/gh-3783.slang
new file mode 100644
index 000000000..9e343b22f
--- /dev/null
+++ b/tests/bugs/gh-3783.slang
@@ -0,0 +1,17 @@
+//TEST:SIMPLE(filecheck=CHECK):-target spirv -emit-spirv-directly
+
+// CHECK: error 55200:
+
+RWStructuredBuffer<float> outputBuffer;
+
+float testFunc(vector<vector<float, 3>, 2> v1)
+{
+    return v1.x.x;
+}
+
+[numthreads(1,1,1)]
+void main(uint v : SV_DispatchThreadID)
+{
+    vector<vector<float, 3>, 2> v1 = { { 1, 2, 3 }, { 4, 5, v } };
+    outputBuffer[0] = testFunc(v1);
+}
\ No newline at end of file