Fix internal error when a generic in an extension is unused (#7665)

* Initial plan

* Add diagnostic for unreferenced generic parameters in extensions

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Use GenericTypeParamDeclBase and eliminate intermediate list

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Update test to use filecheck format for better verification

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Update extension validation to use getMembersOfType for better constraint collection

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Add temporary workaround for inheritance constraints but issue persists

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Fix inheritance constraint validation by moving to SemanticsDeclConformancesVisitor

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Fix inheritance constraint validation by removing inheritance declaration checks

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Move extension generic parameter validation back to SemanticsDeclBasesVisitor

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Fix test.

* format code (#7671)

Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>

* Fix and update.

* Refine fix.

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>
Co-authored-by: Yong He <yonghe@outlook.com>
Co-authored-by: slangbot <ellieh+slangbot@nvidia.com>
Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>

5 files changed, 87 insertions, 3 deletions

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index 9afc79389..180478fef 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -1,8 +1,14 @@
 ## Configure CMake and Build
 
-To configure cmake, run `cmake --preset default --fresh`.
-To build, run `cmake --workflow --preset debug` or `cmake --workflow --preset release`.
-Build instructions can be found in docs/building.md
+Slang is already built in debug configuration, so you should be able to run targets
+like `slangc`, `slang-test`, `slangi` etc. right away.
+
+If you made some changes and need to rebuild Slang, follow these steps:
+
+1. Configure cmake with `cmake --preset default`.
+2. Run `cmake --workflow --preset debug` to build.
+
+Detailed build instructions can be found in docs/building.md
 
 ## Formatting
 
diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml
index fe47a7778..64dd326c7 100644
--- a/.github/workflows/copilot-setup-steps.yml
+++ b/.github/workflows/copilot-setup-steps.yml
@@ -30,3 +30,5 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y libx11-dev
+          cmake --preset default --fresh
+          cmake --workflow --preset debug
diff --git a/source/slang/slang-check-decl.cpp b/source/slang/slang-check-decl.cpp
index 5de83e24d..847f296bb 100644
--- a/source/slang/slang-check-decl.cpp
+++ b/source/slang/slang-check-decl.cpp
@@ -446,6 +446,7 @@ struct SemanticsDeclBasesVisitor : public SemanticsDeclVisitorBase,
     /// Validate that the target type of an extension `decl` is valid.
     void _validateExtensionDeclTargetType(ExtensionDecl* decl);
     void _validateExtensionDeclMembers(ExtensionDecl* decl);
+    void _validateExtensionDeclGenericParams(ExtensionDecl* decl);
 
     void visitExtensionDecl(ExtensionDecl* decl);
 };
@@ -10620,6 +10621,9 @@ bool getExtensionTargetDeclList(
 
 void SemanticsDeclBasesVisitor::_validateExtensionDeclTargetType(ExtensionDecl* decl)
 {
+    // Validate generic parameters first
+    _validateExtensionDeclGenericParams(decl);
+
     if (auto targetDeclRefType = as<DeclRefType>(decl->targetType))
     {
         // Attach our extension to that type as a candidate...
@@ -10695,6 +10699,47 @@ void SemanticsDeclBasesVisitor::_validateExtensionDeclMembers(ExtensionDecl* dec
     }
 }
 
+void SemanticsDeclBasesVisitor::_validateExtensionDeclGenericParams(ExtensionDecl* decl)
+{
+    // Check if any generic parameter on the extension is not referenced by the target type
+    // or by constraints in the generic declaration
+    if (auto genericDecl = as<GenericDecl>(decl->parentDecl))
+    {
+        ensureDecl(genericDecl, DeclCheckState::ReadyForReference);
+
+        // Collect all declarations referenced by the target type
+        HashSet<Decl*> referencedDecls;
+        collectReferencedDecls(decl->targetType.type, referencedDecls);
+
+        // Also collect declarations referenced by generic constraints
+        for (auto constraint :
+             getMembersOfType<GenericTypeConstraintDecl>(getASTBuilder(), genericDecl))
+        {
+            collectReferencedDecls(constraint.getDecl()->sup.type, referencedDecls);
+        }
+
+        // Note: We intentionally do NOT check inheritance declarations in the extension.
+        // Being referenced only in inheritance declarations is not sufficient for the
+        // type system to solve for generic parameters when applying the extension.
+
+        // Check each generic parameter directly
+        for (auto member : genericDecl->getDirectMemberDecls())
+        {
+            if (as<GenericTypeParamDeclBase>(member) || as<GenericValueParamDecl>(member))
+            {
+                if (!referencedDecls.contains(member))
+                {
+                    getSink()->diagnose(
+                        member,
+                        Diagnostics::unreferencedGenericParamInExtension,
+                        member->getName(),
+                        decl->targetType);
+                }
+            }
+        }
+    }
+}
+
 void SemanticsDeclBasesVisitor::visitExtensionDecl(ExtensionDecl* decl)
 {
     // We check the target type expression and members, and then validate
diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h
index 249bc4f88..b3d90f27c 100644
--- a/source/slang/slang-diagnostic-defs.h
+++ b/source/slang/slang-diagnostic-defs.h
@@ -1790,6 +1790,11 @@ DIAGNOSTIC(
     Error,
     overrideModifierNotOverridingBaseDecl,
     "'$0' marked as 'override' is not overriding any base declarations.")
+DIAGNOSTIC(
+    30855,
+    Error,
+    unreferencedGenericParamInExtension,
+    "generic parameter '$0' is not referenced by extension target type '$1'.")
 
 // 309xx: subscripts
 DIAGNOSTIC(
diff --git a/tests/diagnostics/extension-unreferenced-generic-param.slang b/tests/diagnostics/extension-unreferenced-generic-param.slang
new file mode 100644
index 000000000..80aba4adb
--- /dev/null
+++ b/tests/diagnostics/extension-unreferenced-generic-param.slang
@@ -0,0 +1,26 @@
+//TEST:SIMPLE(filecheck=CHECK): -target spirv
+// Test unreferenced generic parameters in extensions
+
+// Error case 1: Simple unreferenced generic parameter
+// CHECK: ([[# @LINE+1]]): error 30855:
+extension<T> int
+{
+    void foo() {}
+}
+
+// Error case 2: Multiple generic parameters, some unused
+// CHECK: ([[# @LINE+1]]): error 30855:
+extension<T, U> vector<T, 3>
+{
+    void bar() {}
+}
+
+// Error case 3: Parameters only used in inheritance declaration should be errors
+interface IFooGeneric<S> {}
+struct MyType {}
+
+// CHECK: ([[# @LINE+1]]): error 30855:
+extension<S> MyType : IFooGeneric<S>
+{
+    void shouldFail() {}
+}