Allow partial specialization of existential arguments. (#6487)

* Allow partial specialization of existential arguments.

* Fix.

* Add test case for improved diagnostics.

* Fix compile error.

* Fix tests.

* Fix.

* Fix test.

* Fix compile issue.

* Fix typo.

* Address comment.

1 files changed, 22 insertions, 8 deletions

diff --git a/source/slang/slang-legalize-types.cpp b/source/slang/slang-legalize-types.cpp
index 7d107bbce..e26475522 100644
--- a/source/slang/slang-legalize-types.cpp
+++ b/source/slang/slang-legalize-types.cpp
@@ -198,31 +198,40 @@ bool isResourceType(IRType* type)
     return false;
 }
 
-bool isOpaqueType(IRType* type, List<IRType*>& opaqueTypes)
+
+bool isOpaqueTypeImpl(IRType* type, HashSet<IRType*>& visited, IRType** outLeafOpaqueHandleType)
 {
+    if (visited.contains(type))
+    {
+        if (outLeafOpaqueHandleType)
+            *outLeafOpaqueHandleType = type;
+        return true;
+    }
+
     if (isResourceType(type))
     {
-        opaqueTypes.add(type);
+        if (outLeafOpaqueHandleType)
+            *outLeafOpaqueHandleType = type;
         return true;
     }
 
     if (auto structType = as<IRStructType>(type))
     {
+        visited.add(type);
         for (auto field : structType->getFields())
         {
-            if (isOpaqueType(field->getFieldType(), opaqueTypes))
+            if (isOpaqueTypeImpl(field->getFieldType(), visited, outLeafOpaqueHandleType))
             {
-                opaqueTypes.add(type);
                 return true;
             }
         }
+        visited.remove(type);
     }
 
     if (auto arrayType = as<IRArrayTypeBase>(type))
     {
-        if (isOpaqueType(arrayType->getElementType(), opaqueTypes))
+        if (isOpaqueTypeImpl(arrayType->getElementType(), visited, outLeafOpaqueHandleType))
         {
-            opaqueTypes.add(type);
             return true;
         }
     }
@@ -233,9 +242,8 @@ bool isOpaqueType(IRType* type, List<IRType*>& opaqueTypes)
         {
             if (auto elementType = as<IRType>(tupleType->getOperand(i)))
             {
-                if (isOpaqueType(elementType, opaqueTypes))
+                if (isOpaqueTypeImpl(elementType, visited, outLeafOpaqueHandleType))
                 {
-                    opaqueTypes.add(type);
                     return true;
                 }
             }
@@ -245,6 +253,12 @@ bool isOpaqueType(IRType* type, List<IRType*>& opaqueTypes)
     return false;
 }
 
+bool isOpaqueType(IRType* type, IRType** outLeafOpaqueHandleType)
+{
+    HashSet<IRType*> visited;
+    return isOpaqueTypeImpl(type, visited, outLeafOpaqueHandleType);
+}
+
 SourceLoc findBestSourceLocFromUses(IRInst* inst)
 {
     for (auto use = inst->firstUse; use; use = use->nextUse)