Fixes to Metal Input parameters and Output value input/output semantics (#4536)

* initial change to test with CI for CPU/CUDA errors

* Fixes to Metal Input parameters and Output values

Note:
1. Flattening a struct is the process of making a struct have 0 struct/class members.

Changes:
1. Separated `legalizeSystemValueParameters`. This was done to make it easier to run `legalizeSystemValue` 1 system-value at a time to simplify logic. This change is optional and can be undone if not preferred.
2. Wrap everything inside a Metal legalization context. This was done since it simplifies a lot of logic and will be required for #4375
3. Created `convertSystemValueSemanticNameToEnum` and expanded the existing System-Value Enum system. This allows (sometimes) faster comparisons and helps prepare code for porting into `slang-ir-legalize-varying-params.cpp` (#4375)
4. Added a more dynamic `legalizeSystemValue` system so more than 2 types can be targeted for legalization. This is required to legalize `output`. There is still no preference for any converted type, the first valid type will be converted to.
5. Flatten all input(`flattenInputParameters`)/output(part of `wrapReturnValueInStruct`) structs and assign semantics accordingly.
6. Semantics when legalized have no specific logic other than to: 1. avoid overlapping semantics 2. Prefer assigning explicit semantics specified by a user.
7. Fixed some issue with incorrect output semantics if not a fragment stage (when there are not any assigned semantics)

* change metallib test to the correct metal test

* comment code & cleanup -- Did not address all review

Added comments for clarity + cleaned up some odd areas which were messy

* Add comment to `fixFieldSemanticsOfFlatStruct`

I found `fixFieldSemanticsOfFlatStruct` to still be confusing at a cursory glance. Added comments to make the function be more understandable.

* white space

* Address review comments

1. Fix semantic propegation.
2. Fix how we map struct fields of the flat struct to struct. This is specifically important for if reusing the same struct twice since struct member info is not unique per struct instance used.

* Fix semantic legalization by adding TreeMap

Add TreeMap to allow proper sorted-object data iteration.

* Fix some compile issues

* try to fix gcc compile error

* compile error

* fix logic bug in treeMap iterator next-semantic setter

* fix vsproject filters

* filter file syntax error

* remove need of a context to make copies stable

* Rename treemap to the more appropriate name of "treeset", adjust code comments accordingly.

* remove custom type `TreeSet` and use `std::set`

* remove TreeMap fully

---------

Co-authored-by: Yong He <yonghe@outlook.com>

12 files changed, 267 insertions, 6 deletions

diff --git a/tests/compute/compile-time-loop.slang b/tests/compute/compile-time-loop.slang
index f69708e0c..9035bde2a 100644
--- a/tests/compute/compile-time-loop.slang
+++ b/tests/compute/compile-time-loop.slang
@@ -1,5 +1,5 @@
 //TEST(compute):COMPARE_RENDER_COMPUTE: -shaderobj
-//DISABLE_TEST(compute):COMPARE_COMPUTE:-slang -shaderobj -mtl
+//TEST(compute):COMPARE_RENDER_COMPUTE: -mtl -shaderobj
 
 //TEST_INPUT: Texture2D(size=4, content = one):name t
 //TEST_INPUT: Sampler:name s
diff --git a/tests/compute/constexpr.slang b/tests/compute/constexpr.slang
index 9aa5c1d56..9c7c9d131 100644
--- a/tests/compute/constexpr.slang
+++ b/tests/compute/constexpr.slang
@@ -1,7 +1,7 @@
 // constexpr.slang
 //TEST(compute):COMPARE_COMPUTE_EX:-slang -gcompute -shaderobj
 //DISABLED://TEST(compute, vulkan):COMPARE_COMPUTE_EX:-vk -gcompute -shaderobj
-//DISABLE_TEST(compute):COMPARE_COMPUTE:-slang -shaderobj -mtl
+//TEST(compute):COMPARE_COMPUTE_EX:-mtl -gcompute -shaderobj
 
 //TEST_INPUT: Texture2D(size=4, content = one):name tex
 //TEST_INPUT: Sampler:name samp
diff --git a/tests/compute/discard-stmt.slang b/tests/compute/discard-stmt.slang
index 90a81c0ff..fa00c9ec3 100644
--- a/tests/compute/discard-stmt.slang
+++ b/tests/compute/discard-stmt.slang
@@ -1,5 +1,5 @@
 //TEST(compute):COMPARE_RENDER_COMPUTE: -shaderobj
-//DISABLE_TEST(compute):COMPARE_COMPUTE:-slang -shaderobj -mtl
+//TEST(compute):COMPARE_RENDER_COMPUTE: -mtl -shaderobj
 //TEST_INPUT: Texture2D(size=4, content = one):name tex
 //TEST_INPUT: Sampler:name samp
 //TEST_INPUT: ubuffer(data=[0 0], stride=4):out,name outputBuffer
diff --git a/tests/compute/texture-sampling.slang b/tests/compute/texture-sampling.slang
index 89041dafb..0e319680a 100644
--- a/tests/compute/texture-sampling.slang
+++ b/tests/compute/texture-sampling.slang
@@ -1,6 +1,6 @@
 //TEST(compute):COMPARE_RENDER_COMPUTE: -shaderobj -output-using-type
 //TEST(compute):COMPARE_RENDER_COMPUTE: -shaderobj -output-using-type -vk
-//DISABLE_TEST(compute):COMPARE_COMPUTE:-slang -shaderobj -mtl
+//TEST(compute):COMPARE_RENDER_COMPUTE: -shaderobj -output-using-type -mtl
 
 
 //TEST_INPUT: Texture1D(size=4, content = one):name=t1D
@@ -104,7 +104,7 @@ FragmentStageOutput fragmentMain(FragmentStageInput input)
     val += tCubeArray.Sample(samplerState, float4(uv, 0.5, 0.0));
     val += tCube.Sample(samplerState, float3(uv, 0.5));
 
-    val += t2D.Load(int3(0), int2(0));
+    val += t2D.Load(int3(0));
     val += t2dArray.Load(int4(0));
 
     val += t3D[int3(0)];
diff --git a/tests/metal/atomic-intrinsics.slang b/tests/metal/atomic-intrinsics.slang
index 3533ea2aa..5d47db913 100644
--- a/tests/metal/atomic-intrinsics.slang
+++ b/tests/metal/atomic-intrinsics.slang
@@ -2,6 +2,7 @@
 //TEST:SIMPLE(filecheck=LIB):-target metallib -entry computeMain -stage compute -DMETAL
 //TEST(compute):COMPARE_COMPUTE_EX(filecheck-buffer=CHK):-slang -compute -dx12 -profile cs_6_0 -use-dxil -shaderobj -output-using-type
 //TEST(compute, vulkan):COMPARE_COMPUTE_EX(filecheck-buffer=CHK):-vk -emit-spirv-directly -compute -shaderobj -output-using-type
+//TEST(compute, vulkan):COMPARE_COMPUTE_EX(filecheck-buffer=CHK):-vk -emit-spirv-via-glsl -compute -shaderobj -output-using-type
 
 //DISABLE_TEST(compute):COMPARE_COMPUTE_EX:-cpu -compute -shaderobj -output-using-type
 //DISABLE_TEST(compute):COMPARE_COMPUTE_EX:-cuda -compute -shaderobj -output-using-type
diff --git a/tests/metal/nested-struct-fragment-input.slang b/tests/metal/nested-struct-fragment-input.slang
new file mode 100644
index 000000000..727b5b1e5
--- /dev/null
+++ b/tests/metal/nested-struct-fragment-input.slang
@@ -0,0 +1,68 @@
+//TEST:SIMPLE(filecheck=METAL): -target metal -stage fragment -entry fragmentMain
+//TEST:SIMPLE(filecheck=METALLIB): -target metallib -stage fragment -entry fragmentMain
+
+// METAL: COARSEVERTEX_7
+// METAL: COARSEVERTEX_6
+
+// Ensure each attribute which may vary only appears once.
+// Ensure 1, 2, 3, 4 all appear
+
+// METAL-DAG: [[ATTR1:COARSEVERTEX_(1|2|3|4)]]
+
+// METAL-NOT: [[ATTR1]]
+// METAL-DAG: [[ATTR2:COARSEVERTEX_(1|2|3|4)]]
+
+// METAL: COARSEVERTEX{{(_0|())}}
+
+// METAL-NOT: [[ATTR2]]
+// METAL-DAG: [[ATTR3:COARSEVERTEX_(1|2|3|4)]]
+
+// METAL-NOT: [[ATTR3]]
+// METAL-DAG: [[ATTR4:COARSEVERTEX_(1|2|3|4)]]
+
+// METALLIB: @fragmentMain
+
+RWStructuredBuffer<float> outputBuffer;
+
+struct BottomFragment1
+{
+    float p1;
+};
+struct BottomFragment2
+{
+    float p1;
+};
+
+struct MiddleFragment1
+{
+    float p1;
+    BottomFragment1 p2;
+    BottomFragment2 p3;
+};
+struct TopFragment
+{
+    float p1 : CoarseVertex7;
+    MiddleFragment1 p2 : CoarseVertex6;
+    MiddleFragment1 p3 : CoarseVertex0;
+};
+
+struct FragmentStageInput
+{
+	TopFragment	coarseVertex : CoarseVertex;
+};
+
+float4 fragmentMain(FragmentStageInput input)
+{
+    // METAL-DAG: {{.*}}->p1{{.*}}=
+
+    // METAL-DAG: {{.*}}->p2{{.*}}->p1{{.*}}=
+    // METAL-DAG: {{.*}}->p2{{.*}}->p2{{.*}}->p1{{.*}}=
+    // METAL-DAG: {{.*}}->p2{{.*}}->p3{{.*}}->p1{{.*}}=
+
+    // METAL-DAG: {{.*}}->p3{{.*}}->p1{{.*}}=
+    // METAL-DAG: {{.*}}->p3{{.*}}->p2{{.*}}->p1{{.*}}=
+    // METAL-DAG: {{.*}}->p3{{.*}}->p3{{.*}}->p1{{.*}}=
+
+    outputBuffer[0] = input.coarseVertex.p1 + input.coarseVertex.p2.p1 + +input.coarseVertex.p3.p1;
+    return float4(0, 0, 0, 0);
+}
diff --git a/tests/metal/nested-struct-fragment-output.slang b/tests/metal/nested-struct-fragment-output.slang
new file mode 100644
index 000000000..1d002c124
--- /dev/null
+++ b/tests/metal/nested-struct-fragment-output.slang
@@ -0,0 +1,74 @@
+//TEST:SIMPLE(filecheck=METAL): -target metal -stage fragment -entry fragmentMain
+//TEST:SIMPLE(filecheck=METALLIB): -target metallib -stage fragment -entry fragmentMain
+
+//METAL-DAG: color(0)
+//METAL-DAG: color(1)
+//METAL-DAG: color(2)
+//METAL-DAG: color(3)
+//METAL-DAG: color(4)
+//METAL-DAG: color(5)
+//METAL-DAG: color(6)
+//METAL-NOT: color(7)
+
+//METALLIB: @fragmentMain
+
+RWStructuredBuffer<float> outputBuffer;
+
+struct BottomFragment1
+{
+    float p1;
+};
+struct BottomFragment2
+{
+    float p1;
+};
+
+struct MiddleFragment1
+{
+    float p1;
+    BottomFragment1 p2;
+    BottomFragment2 p3;
+};
+struct TopFragment
+{
+	float p1;
+    MiddleFragment1 p2;
+    MiddleFragment1 p3;
+};
+
+struct FragmentStageInput
+{
+	float4	coarseVertex	: CoarseVertex;
+};
+
+struct FragmentStageOutput
+{
+	TopFragment fragment	: SV_Target;
+};
+
+FragmentStageOutput fragmentMain(FragmentStageInput input)
+{
+    FragmentStageOutput output;
+    output.fragment.p1 = 1;
+
+    output.fragment.p2.p1 = 3;
+    output.fragment.p2.p2.p1 = 4;
+    output.fragment.p2.p3.p1 = 5;
+
+    output.fragment.p3.p1 = 8;
+    output.fragment.p3.p2.p1 = 9;
+    output.fragment.p3.p3.p1 = 10;
+
+    // METAL-DAG: ={{.*}}.p1
+
+    // METAL-DAG: ={{.*}}.p2{{.*}}.p1
+    // METAL-DAG: ={{.*}}.p2{{.*}}.p2{{.*}}.p1
+    // METAL-DAG: ={{.*}}.p2{{.*}}.p3{{.*}}.p1
+
+    // METAL-DAG: ={{.*}}.p3{{.*}}.p1
+    // METAL-DAG: ={{.*}}.p3{{.*}}.p2{{.*}}.p1
+    // METAL-DAG: ={{.*}}.p3{{.*}}.p3{{.*}}.p1
+
+	outputBuffer[0] = 1;
+	return output;
+}
diff --git a/tests/metal/nested-struct-multi-entry-point-vertex.slang b/tests/metal/nested-struct-multi-entry-point-vertex.slang
new file mode 100644
index 000000000..779b66704
--- /dev/null
+++ b/tests/metal/nested-struct-multi-entry-point-vertex.slang
@@ -0,0 +1,45 @@
+//TEST:SIMPLE(filecheck=METAL1): -target metal -stage vertex -entry vertexMain1
+//TEST:SIMPLE(filecheck=METALLIB1): -target metallib -stage vertex -entry vertexMain1
+//TEST:SIMPLE(filecheck=METAL2): -target metal -stage vertex -entry vertexMain2
+//TEST:SIMPLE(filecheck=METALLIB2): -target metallib -stage vertex -entry vertexMain2
+
+//METALLIB1: @vertexMain1
+//METAL1-DAG: attribute(0)
+//METAL1-DAG: attribute(1)
+//METAL1-NOT: attribute(2)
+
+//METALLIB2: @vertexMain2
+//METAL2-DAG: attribute(0)
+//METAL2-DAG: attribute(1)
+//METAL2-DAG: attribute(2)
+
+struct SharedStruct
+{
+    float4 position;
+    float4 color;
+};
+
+struct VertexStageInput
+{
+	SharedStruct assembledVertex	: CoarseVertex;
+};
+
+float4 vertexMain1(VertexStageInput vertex)
+{
+	return vertex.assembledVertex.position;
+}
+
+struct sharedStructWrapper
+{
+    float2 uv;
+    SharedStruct sharedData;
+};
+struct VertexStageInput2
+{
+	sharedStructWrapper assembledVertex	: CoarseVertex;
+};
+
+float4 vertexMain2(VertexStageInput2 vertex)
+{
+	return vertex.assembledVertex.sharedData.position;
+}
\ No newline at end of file
diff --git a/tests/metal/no-struct-vertex-output.slang b/tests/metal/no-struct-vertex-output.slang
new file mode 100644
index 000000000..f4988b685
--- /dev/null
+++ b/tests/metal/no-struct-vertex-output.slang
@@ -0,0 +1,12 @@
+//TEST:SIMPLE(filecheck=METAL): -target metallib -stage vertex -entry vertexMain
+//TEST:SIMPLE(filecheck=METALLIB): -target metallib -stage vertex -entry vertexMain
+
+//METAL-DAG: position
+//METALLIB: @vertexMain
+
+// Vertex Shader
+
+float4 vertexMain()
+{
+	return float4(1,1,1,1);
+}
\ No newline at end of file
diff --git a/tests/metal/stage-in-2.slang b/tests/metal/stage-in-2.slang
index 2b1e61306..5bdf3f3eb 100644
--- a/tests/metal/stage-in-2.slang
+++ b/tests/metal/stage-in-2.slang
@@ -8,7 +8,7 @@
 
 // CHECK: struct pixelInput
 // CHECK-NEXT: {
-// CHECK-NEXT: CoarseVertex{{.*}} coarseVertex{{.*}} {{\[\[}}user(COARSEVERTEX){{\]\]}};
+// CHECK-NEXT: {{\[\[}}user(COARSEVERTEX){{\]\]}};
 
 // Uniform data to be passed from application -> shader.
 cbuffer Uniforms
diff --git a/tests/metal/sv_target-complex-1.slang b/tests/metal/sv_target-complex-1.slang
new file mode 100644
index 000000000..a830ff3d2
--- /dev/null
+++ b/tests/metal/sv_target-complex-1.slang
@@ -0,0 +1,34 @@
+//TEST:SIMPLE(filecheck=CHECK): -target metal
+//TEST:SIMPLE(filecheck=CHECK-ASM): -target metallib
+
+struct NestedReturn
+{
+    float4 debug;
+};
+struct NestedReturn2
+{
+    float4 debugAux1;
+    float4 debugAux2;
+};
+
+// Semantics are supposed to ignore uppercase/lowercase differences
+struct Output
+{
+    float4 Diffuse : SV_TarGet0;
+    NestedReturn debug1 : SV_Target1;
+    float4 Material : SV_TArgeT2;
+    NestedReturn2 debug2 : SV_TaRget3;
+}
+
+// CHECK-ASM: define {{.*}} @fragmentMain
+// CHECK: color(0)
+// CHECK-DAG: color(1)
+// CHECK-DAG: color(2)
+// CHECK-DAG: color(3)
+// CHECK-DAG: color(4)
+
+[shader("fragment")]
+Output fragmentMain()
+{
+    return { float4(1), {float4(2)}, float4(3) };
+}
\ No newline at end of file
diff --git a/tests/metal/sv_target-complex-2.slang b/tests/metal/sv_target-complex-2.slang
new file mode 100644
index 000000000..9cc59cc5f
--- /dev/null
+++ b/tests/metal/sv_target-complex-2.slang
@@ -0,0 +1,27 @@
+//TEST:SIMPLE(filecheck=CHECK): -target metal
+//TEST:SIMPLE(filecheck=CHECK-ASM): -target metallib
+
+struct NestedReturn
+{
+    float4 debug1;
+    float4 debug2;
+};
+
+struct Output
+{
+    float4 Diffuse : SV_Target0;
+    NestedReturn val : SV_Target1;
+    float4 Material : SV_Target2;
+}
+
+// CHECK-ASM: define {{.*}} @fragmentMain
+// CHECK: color(0)
+// CHECK: color(1)
+// CHECK-DAG: color(3)
+// CHECK-DAG: color(2)
+
+[shader("fragment")]
+Output fragmentMain()
+{
+    return { float4(1), {float4(2), float4(2)}, float4(3) };
+}
\ No newline at end of file