Fix several bugs with `specializeWithArgTypes()` (#5365)

* Fix several bugs with `specializeWithArgTypes()`

* Make all types L-values for the purposes of reflection API resolution

1 files changed, 70 insertions, 10 deletions

diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index 313e94439..7949fd9f7 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -1369,16 +1369,68 @@ DeclRef<GenericDecl> getGenericParentDeclRef(
     // Create substituted parent decl ref.
     auto decl = declRef.getDecl();
 
-    while (!as<GenericDecl>(decl))
+    while (decl && !as<GenericDecl>(decl))
     {
         decl = decl->parentDecl;
     }
 
+    if (!decl)
+    {
+        // No generic parent
+        return DeclRef<GenericDecl>();
+    }
+
     auto genericDecl = as<GenericDecl>(decl);
     auto genericDeclRef = createDefaultSubstitutionsIfNeeded(astBuilder, visitor, DeclRef(genericDecl)).as<GenericDecl>();
     return substituteDeclRef(SubstitutionSet(declRef), astBuilder, genericDeclRef).as<GenericDecl>();
 }
 
+bool Linkage::isSpecialized(DeclRef<Decl> declRef)
+{
+    // For now, we only support two 'states': fully applied or not at all.
+    // If we add support for partial specialization, we will need to update this logic.
+    // 
+    // If it's not specialized, then declRef will be the one with default substitutions.
+    //
+    SemanticsVisitor visitor(getSemanticsForReflection());
+
+    auto decl = declRef.getDecl();
+    while (decl && !as<GenericDecl>(decl))
+    {
+        decl = decl->parentDecl;
+    }
+
+    if(!decl)
+        return true; // no generics => always specialized
+    
+    auto defaultArgs = getDefaultSubstitutionArgs(getASTBuilder(), &visitor, as<GenericDecl>(decl));
+    auto currentArgs = SubstitutionSet(declRef).findGenericAppDeclRef(as<GenericDecl>(decl))->getArgs();
+    
+    if (defaultArgs.getCount() != currentArgs.getCount()) // should really never happen.
+        return true;
+
+    for (Index i = 0; i < defaultArgs.getCount(); ++i)
+    {
+        if (defaultArgs[i] != currentArgs[i])
+            return true;
+    }
+
+    return false;
+}
+
+bool isFuncGeneric(DeclRef<Decl> declRef)
+{
+    if (auto funcDecl = as<FuncDecl>(declRef.getDecl()))
+    {
+        if (funcDecl->parentDecl && as<GenericDecl>(funcDecl->parentDecl))
+        {
+            return true;
+        }
+    }
+    
+    return false;
+}
+
 DeclRef<Decl> Linkage::specializeWithArgTypes(
     Expr*               funcExpr,
     List<Type*>         argTypes,
@@ -1387,16 +1439,22 @@ DeclRef<Decl> Linkage::specializeWithArgTypes(
     SemanticsVisitor visitor(getSemanticsForReflection());
     visitor = visitor.withSink(sink);
 
-    ASTBuilder* astBuilder = getASTBuilder();
+    SLANG_AST_BUILDER_RAII(getASTBuilder());
     
     if (auto declRefFuncExpr = as<DeclRefExpr>(funcExpr))
     {
-        auto genericDeclRefExpr = astBuilder->create<DeclRefExpr>();
-        genericDeclRefExpr->declRef = getGenericParentDeclRef(
-            getASTBuilder(),
-            &visitor,
-            declRefFuncExpr->declRef);
-        funcExpr = genericDeclRefExpr;
+        if (isFuncGeneric(declRefFuncExpr->declRef) && !isSpecialized(declRefFuncExpr->declRef))
+        {
+            if (auto genericDeclRef = getGenericParentDeclRef(
+                getCurrentASTBuilder(),
+                &visitor,
+                declRefFuncExpr->declRef))
+            {
+                auto genericDeclRefExpr = getCurrentASTBuilder()->create<DeclRefExpr>();
+                genericDeclRefExpr->declRef = genericDeclRef;
+                funcExpr = genericDeclRefExpr;
+            }
+        }
     }
 
     List<Expr*> argExprs;
@@ -1407,17 +1465,19 @@ DeclRef<Decl> Linkage::specializeWithArgTypes(
         // Create an 'empty' expr with the given type. Ideally, the expression itself should not matter
         // only its checked type.
         //
-        auto argExpr = astBuilder->create<VarExpr>();
+        auto argExpr = getCurrentASTBuilder()->create<VarExpr>();
         argExpr->type = argType;
+        argExpr->type.isLeftValue = true;
         argExprs.add(argExpr);
     }
 
     // Construct invoke expr.
-    auto invokeExpr = astBuilder->create<InvokeExpr>();
+    auto invokeExpr = getCurrentASTBuilder()->create<InvokeExpr>();
     invokeExpr->functionExpr = funcExpr;
     invokeExpr->arguments = argExprs;
 
     auto checkedInvokeExpr = visitor.CheckInvokeExprWithCheckedOperands(invokeExpr);
+
     return as<DeclRefExpr>(as<InvokeExpr>(checkedInvokeExpr)->functionExpr)->declRef;
 }