Fix stack overflow in lookupWitness lowering pass. (#2798)

Co-authored-by: Yong He <yhe@nvidia.com>

1 files changed, 16 insertions, 4 deletions

diff --git a/source/slang/slang-ir-lower-witness-lookup.cpp b/source/slang/slang-ir-lower-witness-lookup.cpp
index 1a1900cd2..d3e96eb21 100644
--- a/source/slang/slang-ir-lower-witness-lookup.cpp
+++ b/source/slang/slang-ir-lower-witness-lookup.cpp
@@ -39,12 +39,24 @@ struct WitnessLookupLoweringContext
     {
         if (!type)
             return false;
-        if (type->getOp() == kIROp_AssociatedType)
-            return true;
-        for (UInt i = 0; i < type->getOperandCount(); i++)
+        
+        HashSet<IRInst*> processedSet;
+        List<IRInst*> workList;
+        workList.add(type);
+        processedSet.add(type);
+        for (Index i = 0; i < workList.getCount(); i++)
         {
-            if (hasAssocType(type->getOperand(i)))
+            auto inst = workList[i];
+            if (inst->getOp() == kIROp_AssociatedType)
                 return true;
+            
+            for (UInt j = 0; j < inst->getOperandCount(); j++)
+            {
+                if (!inst->getOperand(j))
+                    continue;
+                if (processedSet.Add(inst->getOperand(j)))
+                    workList.add(inst->getOperand(j));
+            }
         }
         return false;
     }