Fix internal error when a generic in an extension is unused (#7665)

* Initial plan

* Add diagnostic for unreferenced generic parameters in extensions

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Use GenericTypeParamDeclBase and eliminate intermediate list

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Update test to use filecheck format for better verification

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Update extension validation to use getMembersOfType for better constraint collection

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Add temporary workaround for inheritance constraints but issue persists

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Fix inheritance constraint validation by moving to SemanticsDeclConformancesVisitor

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Fix inheritance constraint validation by removing inheritance declaration checks

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Move extension generic parameter validation back to SemanticsDeclBasesVisitor

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Fix test.

* format code (#7671)

Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>

* Fix and update.

* Refine fix.

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>
Co-authored-by: Yong He <yonghe@outlook.com>
Co-authored-by: slangbot <ellieh+slangbot@nvidia.com>
Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>

2 files changed, 50 insertions, 0 deletions

diff --git a/source/slang/slang-check-decl.cpp b/source/slang/slang-check-decl.cpp
index 5de83e24d..847f296bb 100644
--- a/source/slang/slang-check-decl.cpp
+++ b/source/slang/slang-check-decl.cpp
@@ -446,6 +446,7 @@ struct SemanticsDeclBasesVisitor : public SemanticsDeclVisitorBase,
     /// Validate that the target type of an extension `decl` is valid.
     void _validateExtensionDeclTargetType(ExtensionDecl* decl);
     void _validateExtensionDeclMembers(ExtensionDecl* decl);
+    void _validateExtensionDeclGenericParams(ExtensionDecl* decl);
 
     void visitExtensionDecl(ExtensionDecl* decl);
 };
@@ -10620,6 +10621,9 @@ bool getExtensionTargetDeclList(
 
 void SemanticsDeclBasesVisitor::_validateExtensionDeclTargetType(ExtensionDecl* decl)
 {
+    // Validate generic parameters first
+    _validateExtensionDeclGenericParams(decl);
+
     if (auto targetDeclRefType = as<DeclRefType>(decl->targetType))
     {
         // Attach our extension to that type as a candidate...
@@ -10695,6 +10699,47 @@ void SemanticsDeclBasesVisitor::_validateExtensionDeclMembers(ExtensionDecl* dec
     }
 }
 
+void SemanticsDeclBasesVisitor::_validateExtensionDeclGenericParams(ExtensionDecl* decl)
+{
+    // Check if any generic parameter on the extension is not referenced by the target type
+    // or by constraints in the generic declaration
+    if (auto genericDecl = as<GenericDecl>(decl->parentDecl))
+    {
+        ensureDecl(genericDecl, DeclCheckState::ReadyForReference);
+
+        // Collect all declarations referenced by the target type
+        HashSet<Decl*> referencedDecls;
+        collectReferencedDecls(decl->targetType.type, referencedDecls);
+
+        // Also collect declarations referenced by generic constraints
+        for (auto constraint :
+             getMembersOfType<GenericTypeConstraintDecl>(getASTBuilder(), genericDecl))
+        {
+            collectReferencedDecls(constraint.getDecl()->sup.type, referencedDecls);
+        }
+
+        // Note: We intentionally do NOT check inheritance declarations in the extension.
+        // Being referenced only in inheritance declarations is not sufficient for the
+        // type system to solve for generic parameters when applying the extension.
+
+        // Check each generic parameter directly
+        for (auto member : genericDecl->getDirectMemberDecls())
+        {
+            if (as<GenericTypeParamDeclBase>(member) || as<GenericValueParamDecl>(member))
+            {
+                if (!referencedDecls.contains(member))
+                {
+                    getSink()->diagnose(
+                        member,
+                        Diagnostics::unreferencedGenericParamInExtension,
+                        member->getName(),
+                        decl->targetType);
+                }
+            }
+        }
+    }
+}
+
 void SemanticsDeclBasesVisitor::visitExtensionDecl(ExtensionDecl* decl)
 {
     // We check the target type expression and members, and then validate
diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h
index 249bc4f88..b3d90f27c 100644
--- a/source/slang/slang-diagnostic-defs.h
+++ b/source/slang/slang-diagnostic-defs.h
@@ -1790,6 +1790,11 @@ DIAGNOSTIC(
     Error,
     overrideModifierNotOverridingBaseDecl,
     "'$0' marked as 'override' is not overriding any base declarations.")
+DIAGNOSTIC(
+    30855,
+    Error,
+    unreferencedGenericParamInExtension,
+    "generic parameter '$0' is not referenced by extension target type '$1'.")
 
 // 309xx: subscripts
 DIAGNOSTIC(