Fix compiler crash when enum is used as vertex output data (#7915)

* Initial plan

* Initial investigation and plan for enum vertex output fix

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Fix compiler crash when enum is used as vertex output data

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Fix compiler crash when enum is used as vertex output data

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Address reviewer feedback: use SLANG_ASSERT and improve CHECK directives

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

5 files changed, 131 insertions, 0 deletions

diff --git a/source/slang/slang-parameter-binding.cpp b/source/slang/slang-parameter-binding.cpp
index 06d2b1f34..89e633857 100644
--- a/source/slang/slang-parameter-binding.cpp
+++ b/source/slang/slang-parameter-binding.cpp
@@ -2554,6 +2554,16 @@ static RefPtr<TypeLayout> processEntryPointVaryingParameter(
                     globalGenericParamDecl.getDecl());
             }
         }
+        else if (auto enumDeclRef = declRef.as<EnumDecl>())
+        {
+            // We handle an enumeration type as its tag type for varying parameters.
+            // This allows enums to be used in vertex output/input similar to their
+            // underlying integer types.
+            //
+            auto tagType = enumDeclRef.getDecl()->tagType;
+            SLANG_ASSERT(tagType);
+            return processEntryPointVaryingParameter(context, tagType, state, varLayout);
+        }
         else if (auto associatedTypeParam = declRef.as<AssocTypeDecl>())
         {
             RefPtr<TypeLayout> assocTypeLayout = new TypeLayout();
diff --git a/tests/bugs/enum-different-types.slang b/tests/bugs/enum-different-types.slang
new file mode 100644
index 000000000..93deffdd8
--- /dev/null
+++ b/tests/bugs/enum-different-types.slang
@@ -0,0 +1,39 @@
+//TEST:SIMPLE(filecheck=CHECK):-target hlsl -stage vertex -entry main
+
+// Test enum without explicit tag type (should default to int)
+public enum SimpleEnum {
+    value1,
+    value2,
+    value3
+}
+
+// Test enum with different tag types
+public enum ByteEnum : uint8_t {
+    a, b, c
+}
+
+public enum IntEnum : int {
+    x = -1, y = 0, z = 1
+}
+
+struct VertexInput {
+    float3 position : POSITION_ATTR;
+};
+
+public struct VertexOutput {
+    float4 position : SV_Position;
+    SimpleEnum simple : TEXCOORD0;
+    IntEnum intVal : TEXCOORD1;
+}
+
+[shader("vertex")]
+VertexOutput main(in VertexInput input) {
+    VertexOutput output;
+    output.position = float4(0);
+    output.simple = SimpleEnum.value1;
+    output.intVal = IntEnum.x;
+    return output;
+}
+
+//CHECK: simple_0 : TEXCOORD0
+//CHECK: intVal_0 : TEXCOORD1
\ No newline at end of file
diff --git a/tests/bugs/enum-fragment-input.slang b/tests/bugs/enum-fragment-input.slang
new file mode 100644
index 000000000..e03514467
--- /dev/null
+++ b/tests/bugs/enum-fragment-input.slang
@@ -0,0 +1,25 @@
+//TEST:SIMPLE(filecheck=CHECK):-target hlsl -stage fragment -entry main
+
+public enum VertexLayout : uint {
+    position = 0,
+    color = 1,
+    uv = 2
+}
+
+struct FragmentInput {
+    float4 position : SV_Position;
+    VertexLayout layout : TEXCOORD0;
+};
+
+struct FragmentOutput {
+    float4 color : SV_Target;
+};
+
+[shader("fragment")]
+FragmentOutput main(in FragmentInput input) {
+    FragmentOutput output;
+    output.color = float4(float(input.layout), 0, 0, 1);
+    return output;
+}
+
+//CHECK: main
\ No newline at end of file
diff --git a/tests/bugs/enum-vertex-output-explicit.slang b/tests/bugs/enum-vertex-output-explicit.slang
new file mode 100644
index 000000000..f80007a21
--- /dev/null
+++ b/tests/bugs/enum-vertex-output-explicit.slang
@@ -0,0 +1,28 @@
+//TEST:SIMPLE(filecheck=CHECK):-target hlsl -stage vertex -entry main
+
+public enum VertexLayout : uint {
+    position = 0,
+    color = 1,
+    uv = 2
+}
+
+struct VertexInput {
+    float3 position : POSITION_ATTR;
+    uint color      : COLOR_ATTR;
+    float2 uv       : UV_ATTR;
+};
+
+public struct VertexOutput {
+    float4 position     : SV_Position;
+    VertexLayout layout : TEXCOORD0;
+}
+
+[shader("vertex")]
+VertexOutput main(in VertexInput input) {
+    VertexOutput output;
+    output.position = float4(0);
+    output.layout = VertexLayout.position;
+    return output;
+}
+
+//CHECK: main
\ No newline at end of file
diff --git a/tests/bugs/enum-vertex-output.slang b/tests/bugs/enum-vertex-output.slang
new file mode 100644
index 000000000..ad04f1163
--- /dev/null
+++ b/tests/bugs/enum-vertex-output.slang
@@ -0,0 +1,29 @@
+//TEST:SIMPLE(filecheck=CHECK):-target hlsl -stage vertex -entry main
+
+[Flags]
+public enum VertexLayout : uint {
+    position,
+    color,
+    uv
+}
+
+struct VertexInput {
+    float3 position : POSITION_ATTR;
+    uint color      : COLOR_ATTR;
+    float2 uv       : UV_ATTR;
+};
+
+public struct VertexOutput {
+    float4 position     : SV_Position;
+    VertexLayout layout : TEXCOORD0;
+}
+
+[shader("vertex")]
+VertexOutput main(in VertexInput input) {
+    VertexOutput output;
+    output.position = float4(0);
+    output.layout = VertexLayout.position;
+    return output;
+}
+
+//CHECK: layout_0 : TEXCOORD0
\ No newline at end of file