fix overload in extension issue (#7999)

close #7931.

For a generic callable, we have two passes overload resolution, in first pass, we will resolve
the generic by only checking the generic parameters, while in the second pass, we will resolve
the function signature to resolve the overload.

But in our candidate comparison logic, we pick a preferred generic even two generics are equally
good. However, we should not make this decision in the first pass, because we don't know about
the function arguments in this pass yet. So we just return OverloadEpxr2 in this case, and let the
function overload resolution to break the tie.

2 files changed, 64 insertions, 18 deletions

diff --git a/source/slang/slang-check-overload.cpp b/source/slang/slang-check-overload.cpp
index 9e5560b3a..72de3fd61 100644
--- a/source/slang/slang-check-overload.cpp
+++ b/source/slang/slang-check-overload.cpp
@@ -1368,18 +1368,6 @@ int SemanticsVisitor::CompareLookupResultItems(
     bool leftIsExtern = left.declRef.getDecl()->hasModifier<ExternModifier>();
     bool rigthIsExtern = right.declRef.getDecl()->hasModifier<ExternModifier>();
 
-    // If both left and right are extern, then they are equal.
-    // If only one of them is extern, then the other one is preferred.
-    // If neither is extern, then we continue with the rest of the checks.
-    if (leftIsExtern)
-    {
-        return (rigthIsExtern ? 0 : 1);
-    }
-    if (rigthIsExtern)
-    {
-        return (leftIsExtern ? -1 : 0);
-    }
-
     // Prefer declarations that are not in free-form generic extensions, i.e.
     // `extension<T:IFoo> T { /* declaration here should have lower precedence. */ }
     if (auto leftExt = as<ExtensionDecl>(leftDeclRefParent.getDecl()))
@@ -1395,11 +1383,6 @@ int SemanticsVisitor::CompareLookupResultItems(
             rightIsFreeFormExtension = true;
     }
 
-    // If one of the candidates is a free-form extension, it is always worse than
-    // a non-free-form extension.
-    if (leftIsFreeFormExtension != rightIsFreeFormExtension)
-        return int(leftIsFreeFormExtension) - int(rightIsFreeFormExtension);
-
     // It is possible for lookup to return both an interface requirement
     // and the concrete function that satisfies that requirement.
     // We always want to favor a concrete method over an interface
@@ -1416,7 +1399,42 @@ int SemanticsVisitor::CompareLookupResultItems(
     bool leftIsInterfaceRequirement = isInterfaceRequirement(left.declRef.getDecl());
     bool rightIsInterfaceRequirement = isInterfaceRequirement(right.declRef.getDecl());
     if (leftIsInterfaceRequirement != rightIsInterfaceRequirement)
-        return int(leftIsInterfaceRequirement) - int(rightIsInterfaceRequirement);
+    {
+        // Normally we should always choose the non-Interface candidate, but if one
+        // of the candidate is a free-form extension, this rule doesn't apply, and we
+        // will let free-form extension rule to decide which one is better later.
+        if (!leftIsFreeFormExtension && !rightIsFreeFormExtension)
+        {
+            return (int)(leftIsInterfaceRequirement) - int(rightIsInterfaceRequirement);
+        }
+    }
+
+    // If both candidates are generic functions, we cannot decide which one is better if
+    // above two rules cannot resolve them.
+    auto genericsLeft = as<GenericDecl>(left.declRef.getDecl());
+    auto genericsRight = as<GenericDecl>(right.declRef.getDecl());
+    if ((genericsLeft && as<CallableDecl>(genericsLeft->inner)) ||
+        (genericsRight && as<CallableDecl>(genericsRight->inner)))
+    {
+        return 0;
+    }
+
+    // If both left and right are extern, then they are equal.
+    // If only one of them is extern, then the other one is preferred.
+    // If neither is extern, then we continue with the rest of the checks.
+    if (leftIsExtern)
+    {
+        return (rigthIsExtern ? 0 : 1);
+    }
+    if (rigthIsExtern)
+    {
+        return (leftIsExtern ? -1 : 0);
+    }
+
+    // If one of the candidates is a free-form extension, it is always worse than
+    // a non-free-form extension.
+    if (leftIsFreeFormExtension != rightIsFreeFormExtension)
+        return int(leftIsFreeFormExtension) - int(rightIsFreeFormExtension);
 
     // Prefer non-extension declarations over extension declarations.
     if (leftIsExtension != rightIsExtension)
diff --git a/tests/language-feature/extensions/extension-override-3.slang b/tests/language-feature/extensions/extension-override-3.slang
new file mode 100644
index 000000000..2a023bca3
--- /dev/null
+++ b/tests/language-feature/extensions/extension-override-3.slang
@@ -0,0 +1,28 @@
+//TEST:SIMPLE(filecheck=CHECK):-target spirv
+
+// This test verifies that a generic method defined in an extension can be found.
+
+// CHECK: OpEntryPoint
+struct Test {
+    void genericMethod<int u>(float2 x) {}
+    __subscript<T:__BuiltinIntegerType>(T idx1, T idx2) -> int
+    {
+        get { return 0; }
+    }
+}
+
+extension Test {
+    void genericMethod<int u>() {}
+    __subscript<T:__BuiltinIntegerType>(T idx1) -> int
+    {
+        get { return 1; }
+    }
+}
+
+[shader("compute")]
+void main()
+{
+    Test t;
+    t.genericMethod<3>();    // Should work - calls extension method
+    int a = t[0];            // Should work - calls extension method
+}