Legalise out parameters for vertex shaders on metal (#6943)

* Handle pointer types when getting type cast style

Closes https://github.com/shader-slang/slang/issues/6025

* Move vertex shader out parameters to return type for Metal

Closes https://github.com/shader-slang/slang/issues/6025

* More asserts

* Make struct instead of tuple

* More layout preservation

* Handle same function result

* more layout

* remove layout

* a

* more debug code

* more debug code

* a

* layout working

* refactored

* more tests

* more tests

* fuse loops

* remove unused comments

* Correct filecheck usage

* debug code

* correct name and order of filecheck vars

* simplify

* Address review comments

fix warning

* simplify handling of simple vertex shaders

7 files changed, 95 insertions, 18 deletions

diff --git a/tests/metal/simple-vertex-position-implicit-semantic.slang b/tests/metal/simple-vertex-position-implicit-semantic.slang
new file mode 100644
index 000000000..6e51e0bea
--- /dev/null
+++ b/tests/metal/simple-vertex-position-implicit-semantic.slang
@@ -0,0 +1,11 @@
+//TEST:SIMPLE(filecheck=METAL): -target metal -stage vertex -entry vertexMain
+//TEST:SIMPLE(filecheck=METALLIB): -target metallib -stage vertex -entry vertexMain
+
+//METAL: [position]]
+//METALLIB: @vertexMain
+
+// Vertex Shader which writes to position in a returned value 
+float4 vertexMain() // : SV_Position
+{
+    return float4(1,2,3,4);
+}
diff --git a/tests/metal/simple-vertex-position-out-struct.slang b/tests/metal/simple-vertex-position-out-struct.slang
new file mode 100644
index 000000000..87b618560
--- /dev/null
+++ b/tests/metal/simple-vertex-position-out-struct.slang
@@ -0,0 +1,14 @@
+//TEST:SIMPLE(filecheck=METAL): -target metal -stage vertex -entry vertexMain
+//TEST:SIMPLE(filecheck=METALLIB): -target metallib -stage vertex -entry vertexMain
+
+//METAL: [position]]
+//METALLIB: @vertexMain
+struct VertexOut
+{
+    float4 position : SV_Position; 
+}
+// Vertex Shader which writes to position in a returned struct (as metal expects)
+void vertexMain(out VertexOut o)
+{
+    o.position = float4(1,2,3,4);
+}
diff --git a/tests/metal/simple-vertex-position-return-struct.slang b/tests/metal/simple-vertex-position-return-struct.slang
new file mode 100644
index 000000000..cc54ee5f1
--- /dev/null
+++ b/tests/metal/simple-vertex-position-return-struct.slang
@@ -0,0 +1,16 @@
+//TEST:SIMPLE(filecheck=METAL): -target metal -stage vertex -entry vertexMain
+//TEST:SIMPLE(filecheck=METALLIB): -target metallib -stage vertex -entry vertexMain
+
+//METAL: [position]]
+//METALLIB: @vertexMain
+struct VertexOut
+{
+    float4 position : SV_Position; 
+}
+// Vertex Shader which writes to position in a returned struct (as metal expects)
+VertexOut vertexMain()
+{
+    VertexOut o;
+    o.position = float4(1,2,3,4);
+    return o;
+}
diff --git a/tests/metal/simple-vertex-position-return-value-and-out.slang b/tests/metal/simple-vertex-position-return-value-and-out.slang
new file mode 100644
index 000000000..f150eef20
--- /dev/null
+++ b/tests/metal/simple-vertex-position-return-value-and-out.slang
@@ -0,0 +1,13 @@
+//TEST:SIMPLE(filecheck=METAL): -target metal -stage vertex -entry vertexMain
+//TEST:SIMPLE(filecheck=METALLIB): -target metallib -stage vertex -entry vertexMain
+
+//METAL: [position]]
+//METAL: [user(AA)]]
+//METALLIB: @vertexMain
+
+// Vertex Shader which writes to position in a returned value (as metal expects), and as an out parameter which it doesn't
+float4 vertexMain(out uint aa : AA) : SV_Position
+{
+    aa = 0;
+    return float4(1,2,3,4);
+}
diff --git a/tests/metal/simple-vertex-position-return-value.slang b/tests/metal/simple-vertex-position-return-value.slang
new file mode 100644
index 000000000..9fe824774
--- /dev/null
+++ b/tests/metal/simple-vertex-position-return-value.slang
@@ -0,0 +1,11 @@
+//TEST:SIMPLE(filecheck=METAL): -target metal -stage vertex -entry vertexMain
+//TEST:SIMPLE(filecheck=METALLIB): -target metallib -stage vertex -entry vertexMain
+
+//METAL: [position]]
+//METALLIB: @vertexMain
+
+// Vertex Shader which writes to position in a returned value (as metal expects)
+float4 vertexMain() : SV_Position
+{
+    return float4(1,2,3,4);
+}
diff --git a/tests/metal/simple-vertex-position.slang b/tests/metal/simple-vertex-position.slang
new file mode 100644
index 000000000..04c157aa6
--- /dev/null
+++ b/tests/metal/simple-vertex-position.slang
@@ -0,0 +1,11 @@
+//TEST:SIMPLE(filecheck=METAL): -target metal -stage vertex -entry vertexMain
+//TEST:SIMPLE(filecheck=METALLIB): -target metallib -stage vertex -entry vertexMain
+
+//METAL: [position]]
+//METALLIB: @vertexMain
+
+// Vertex Shader which writes to position
+void vertexMain(out float4 position : SV_Position, in uint vertexID : SV_VertexID)
+{
+    position = float4(0.6, 0.1, 0.6, 0.33);
+}
diff --git a/tests/metal/stage-in.slang b/tests/metal/stage-in.slang
index d494a76e0..9a50d163b 100644
--- a/tests/metal/stage-in.slang
+++ b/tests/metal/stage-in.slang
@@ -1,23 +1,8 @@
 //TEST:SIMPLE(filecheck=CHECK): -target metal
 //TEST:SIMPLE(filecheck=CHECK-ASM): -target metallib
 
-// CHECK:     struct VOut{{.*}}
-// CHECK-NEXT:{
-// CHECK-NEXT:    float4 position{{.*}} {{\[\[}}position{{\]\]}};
-// CHECK-NEXT:    float4 vertexColor{{.*}} {{\[\[}}user(_SLANG_ATTR){{\]\]}};
-// CHECK-NEXT:    float2 vertexUV{{.*}} {{\[\[}}user(_SLANG_ATTR_1){{\]\]}};
-// CHECK-NEXT:    float3 vertexNormal{{.*}} {{\[\[}}user(NORMAL){{\]\]}};
-// CHECK-NEXT:};
-
-// CHECK:     struct vertexInput{{.*}}
-// CHECK-NEXT:{
-// CHECK-NEXT:    float4 position{{.*}} {{\[\[}}attribute(0){{\]\]}};
-// CHECK-NEXT:    float4 color{{.*}} {{\[\[}}attribute(1){{\]\]}};
-// CHECK-NEXT:};
-
-// CHECK: {{\[\[}}vertex{{\]\]}} VOut{{.*}} main_vertex(vertexInput{{.*}}{{\[\[}}stage_in{{\]\]}}, uint vid{{.*}}{{\[\[}}vertex_id{{\]\]}}, uint instanceID{{.*}} {{\[\[}}instance_id{{\]\]}})
 
-// CHECK:     struct pixelOutput{{.*}}
+// CHECK:     struct [[pixelOutput:pixelOutput_[0-9]+]]
 // CHECK-NEXT:{
 // CHECK-NEXT:    float4 output{{.*}}{{\[\[}}color(0){{\]\]}};
 // CHECK-NEXT:};
@@ -29,7 +14,7 @@
 // CHECK-NEXT:    float3 vertexNormal{{.*}} {{\[\[}}user(NORMAL){{\]\]}};
 // CHECK-NEXT:};
 
-// CHECK: {{\[\[}}fragment{{\]\]}} pixelOutput{{.*}} main_fragment(pixelInput{{.*}} {{\[\[}}stage_in{{\]\]}}, float4 position{{.*}} {{\[\[}}position{{\]\]}})
+// CHECK: {{\[\[}}fragment{{\]\]}} [[pixelOutput]] main_fragment(pixelInput{{.*}} {{\[\[}}stage_in{{\]\]}}, float4 position{{.*}} {{\[\[}}position{{\]\]}})
 
 // CHECK:     struct FragOut{{.*}}
 // CHECK-NEXT:{
@@ -37,6 +22,22 @@
 // CHECK-NEXT:    float depth{{.*}} {{\[\[}}depth(any){{\]\]}};
 // CHECK-NEXT:};
 
+// CHECK:     struct [[vertexOutput:main_vertex_Result_[0-9]+]]
+// CHECK-NEXT:{
+// CHECK-NEXT:    float4 position{{.*}} {{\[\[}}position{{\]\]}};
+// CHECK-NEXT:    float4 vertexColor{{.*}} {{\[\[}}user(_SLANG_ATTR){{\]\]}};
+// CHECK-NEXT:    float2 vertexUV{{.*}} {{\[\[}}user(_SLANG_ATTR_1){{\]\]}};
+// CHECK-NEXT:    float3 vertexNormal{{.*}} {{\[\[}}user(NORMAL){{\]\]}};
+// CHECK-NEXT:};
+
+// CHECK:     struct vertexInput{{.*}}
+// CHECK-NEXT:{
+// CHECK-NEXT:    float4 position{{.*}} {{\[\[}}attribute(0){{\]\]}};
+// CHECK-NEXT:    float4 color{{.*}} {{\[\[}}attribute(1){{\]\]}};
+// CHECK-NEXT:};
+
+// CHECK: {{\[\[}}vertex{{\]\]}} [[vertexOutput]] main_vertex(vertexInput{{.*}}{{\[\[}}stage_in{{\]\]}}, uint vid{{.*}}{{\[\[}}vertex_id{{\]\]}}, uint instanceID{{.*}} {{\[\[}}instance_id{{\]\]}})
+
 // CHECK-ASM: define {{.*}} @main_vertex
 // CHECK-ASM: define {{.*}} @main_fragment
 // CHECK-ASM: define {{.*}} @main_fragment1
@@ -86,4 +87,4 @@ FragOut main_fragment1(VOut fragmentIn)
     fragOut.color = fragmentIn.vertexColor;
     fragOut.depth = 0.5;
     return fragOut;
-}
\ No newline at end of file
+}