Update `model-viewer` example and fixing compiler bugs. (#1795)

11 files changed, 263 insertions, 31 deletions

diff --git a/source/core/core.natvis b/source/core/core.natvis
index 1087b4e6a..08446db8d 100644
--- a/source/core/core.natvis
+++ b/source/core/core.natvis
@@ -89,6 +89,18 @@
     </Expand>
 </Type>
 
+    <Type Name="Slang::OrderedHashSet&lt;*&gt;">
+        <DisplayString>{{ size={dict._count} }}</DisplayString>
+        <Expand>
+            <LinkedListItems>
+                <Size>dict._count</Size>
+                <HeadPointer>dict.kvPairs.head</HeadPointer>
+                <NextPointer>next</NextPointer>
+                <ValueNode>Value</ValueNode>
+            </LinkedListItems>
+        </Expand>
+    </Type>
+    
 <Type Name="Slang::RefPtr&lt;*&gt;">
     <SmartPointer Usage="Minimal">pointer</SmartPointer>
     <DisplayString Condition="pointer == 0">empty</DisplayString>
diff --git a/source/slang/slang-api.cpp b/source/slang/slang-api.cpp
index bb0e2e174..8347b8c30 100644
--- a/source/slang/slang-api.cpp
+++ b/source/slang/slang-api.cpp
@@ -83,6 +83,12 @@ SLANG_API SlangResult slang_createGlobalSession(
     slang::IGlobalSession** outGlobalSession)
 {
     Slang::ComPtr<slang::IGlobalSession> globalSession;
+
+#ifdef SLANG_ENABLE_IR_BREAK_ALLOC
+    // Set inst debug alloc counter to 0 so IRInsts for stdlib always starts from a large value.
+    Slang::_debugGetIRAllocCounter() = 0x80000000;
+#endif
+
     SLANG_RETURN_ON_FAIL(slang_createGlobalSessionWithoutStdLib(apiVersion, globalSession.writeRef()));
 
     // If we have the embedded stdlib, load from that, else compile it
@@ -106,6 +112,12 @@ SLANG_API SlangResult slang_createGlobalSession(
     }
 
     *outGlobalSession = globalSession.detach();
+
+#ifdef SLANG_ENABLE_IR_BREAK_ALLOC
+    // Reset inst debug alloc counter to 0 so IRInsts for user code always starts from 0.
+    Slang::_debugGetIRAllocCounter() = 0;
+#endif
+
     return SLANG_OK;
 }
 
diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp
index 44cfebf13..19a5fddf8 100755
--- a/source/slang/slang-compiler.cpp
+++ b/source/slang/slang-compiler.cpp
@@ -2309,6 +2309,9 @@ SlangResult dissassembleDXILUsingDXC(
             sink,
             m_program);
 
+        backEndRequest->shouldDumpIR =
+            (m_targetReq->getTargetFlags() & SLANG_TARGET_FLAG_DUMP_IR) != 0;
+
         return _createWholeProgramResult(
             backEndRequest,
             nullptr);
@@ -2339,6 +2342,9 @@ SlangResult dissassembleDXILUsingDXC(
             sink,
             m_program);
 
+        backEndRequest->shouldDumpIR =
+            (m_targetReq->getTargetFlags() & SLANG_TARGET_FLAG_DUMP_IR) != 0;
+
         return _createEntryPointResult(
             entryPointIndex,
             backEndRequest,
diff --git a/source/slang/slang-emit.cpp b/source/slang/slang-emit.cpp
index 01b682a39..af870d02b 100644
--- a/source/slang/slang-emit.cpp
+++ b/source/slang/slang-emit.cpp
@@ -138,11 +138,14 @@ static void dumpIRIfEnabled(
     if(compileRequest->shouldDumpIR)
     {
         DiagnosticSinkWriter writer(compileRequest->getSink());
-
+        //FILE* f = nullptr;
+        //fopen_s(&f, (String("dump-") + label + ".txt").getBuffer(), "wt");
+        //FileWriter writer(f, 0);
         IRDumpOptions options;
         options.sourceManager = compileRequest->getSourceManager();
 
         dumpIR(irModule, options, label, &writer);
+        //fclose(f);
     }
 }
 
@@ -309,8 +312,10 @@ Result linkAndOptimizeIR(
     // perform specialization of functions based on parameter
     // values that need to be compile-time constants.
     //
+    dumpIRIfEnabled(compileRequest, irModule, "BEFORE-SPECIALIZE");
     if (!compileRequest->disableSpecialization)
         specializeModule(irModule);
+    dumpIRIfEnabled(compileRequest, irModule, "AFTER-SPECIALIZE");
 
     eliminateDeadCode(irModule);
 
@@ -319,7 +324,7 @@ Result linkAndOptimizeIR(
     // function pointers.
     dumpIRIfEnabled(compileRequest, irModule, "BEFORE-LOWER-GENERICS");
     lowerGenerics(targetRequest, irModule, sink);
-    dumpIRIfEnabled(compileRequest, irModule, "LOWER-GENERICS");
+    dumpIRIfEnabled(compileRequest, irModule, "AFTER-LOWER-GENERICS");
 
     if (sink->getErrorCount() != 0)
         return SLANG_FAIL;
diff --git a/source/slang/slang-ir-dce.cpp b/source/slang/slang-ir-dce.cpp
index db01929a2..285e5100c 100644
--- a/source/slang/slang-ir-dce.cpp
+++ b/source/slang/slang-ir-dce.cpp
@@ -120,6 +120,15 @@ struct DeadCodeEliminationContext
             UInt operandCount = inst->getOperandCount();
             for( UInt ii = 0; ii < operandCount; ++ii )
             {
+                switch (inst->getOp())
+                {
+                case kIROp_BoundInterfaceType:
+                    if (inst->getOperand(ii)->getOp() == kIROp_WitnessTable)
+                        continue;
+                    break;
+                default:
+                    break;
+                }
                 markInstAsLive(inst->getOperand(ii));
             }
 
diff --git a/source/slang/slang-ir-lower-generic-function.cpp b/source/slang/slang-ir-lower-generic-function.cpp
index 6e9754744..8cd292e49 100644
--- a/source/slang/slang-ir-lower-generic-function.cpp
+++ b/source/slang/slang-ir-lower-generic-function.cpp
@@ -26,6 +26,7 @@ namespace Slang
                 return genericValue;
             auto genericParent = as<IRGeneric>(genericValue);
             SLANG_ASSERT(genericParent);
+            SLANG_ASSERT(genericParent->getDataType());
             auto func = as<IRFunc>(findGenericReturnVal(genericParent));
             if (!func)
             {
@@ -37,7 +38,8 @@ namespace Slang
                 return genericValue;
             }
             SLANG_ASSERT(func);
-            if (!func->isDefinition())
+            // Do not lower intrinsic functions.
+            if (!func->isDefinition() || func->findDecoration<IRTargetIntrinsicDecoration>())
             {
                 sharedContext->loweredGenericFunctions[genericValue] = genericValue;
                 return genericValue;
@@ -47,7 +49,10 @@ namespace Slang
             builder.sharedBuilder = &sharedContext->sharedBuilderStorage;
             builder.setInsertBefore(genericParent);
             auto loweredFunc = cast<IRFunc>(cloneInstAndOperands(&cloneEnv, &builder, func));
-            loweredFunc->setFullType(lowerGenericFuncType(&builder, cast<IRGeneric>(genericParent->getFullType())));
+            auto loweredGenericType =
+                lowerGenericFuncType(&builder, cast<IRGeneric>(genericParent->getFullType()));
+            SLANG_ASSERT(loweredGenericType);
+            loweredFunc->setFullType(loweredGenericType);
             List<IRInst*> clonedParams;
 
             for (auto genericChild : genericParent->getFirstBlock()->getChildren())
diff --git a/source/slang/slang-ir.cpp b/source/slang/slang-ir.cpp
index a983188b7..1f68623bb 100644
--- a/source/slang/slang-ir.cpp
+++ b/source/slang/slang-ir.cpp
@@ -1587,6 +1587,25 @@ namespace Slang
         value->sourceLoc = sourceLocInfo->sourceLoc;
     }
 
+#if SLANG_ENABLE_IR_BREAK_ALLOC
+    SLANG_API uint32_t _slangIRAllocBreak = 0xFFFFFFFF;
+    uint32_t& _debugGetIRAllocCounter()
+    {
+        static uint32_t counter = 0;
+        return counter;
+    }
+    uint32_t _debugGetAndIncreaseInstCounter()
+    {
+        if (_slangIRAllocBreak != 0xFFFFFFFF && _debugGetIRAllocCounter() == _slangIRAllocBreak)
+        {
+#if _WIN32 && defined(_MSC_VER)
+            __debugbreak();
+#endif
+        }
+        return _debugGetIRAllocCounter()++;
+    }
+#endif
+
     // Create an IR instruction/value and initialize it.
     //
     // In this case `argCount` and `args` represent the
@@ -1622,6 +1641,10 @@ namespace Slang
         // TODO: Do we need to run ctor after zeroing?
         new(inst)T();
 
+#if SLANG_ENABLE_IR_BREAK_ALLOC
+        inst->_debugUID = _debugGetAndIncreaseInstCounter();
+#endif
+
         inst->operandCount = (uint32_t)(fixedArgCount + varArgCount);
 
         inst->m_op = op;
@@ -1677,6 +1700,10 @@ namespace Slang
         // TODO: Do we need to run ctor after zeroing?
         new (inst) IRInst;
 
+#if SLANG_ENABLE_IR_BREAK_ALLOC
+        inst->_debugUID = _debugGetAndIncreaseInstCounter();
+#endif
+
         inst->m_op = op;
         if (type)
         {
@@ -2213,6 +2240,9 @@ namespace Slang
         SLANG_UNUSED(endCursor);
 
         new(inst) IRInst();
+#if SLANG_ENABLE_IR_BREAK_ALLOC
+        inst->_debugUID = _debugGetAndIncreaseInstCounter();
+#endif
         inst->m_op = op;
         inst->typeUse.usedValue = type;
         inst->operandCount = (uint32_t) operandCount;
@@ -2300,6 +2330,9 @@ namespace Slang
         SLANG_UNUSED(endCursor);
 
         new(inst) IRInst();
+#if SLANG_ENABLE_IR_BREAK_ALLOC
+        inst->_debugUID = _debugGetAndIncreaseInstCounter();
+#endif
         inst->m_op = op;
         inst->typeUse.usedValue = type;
         inst->operandCount = (uint32_t)operandCount;
@@ -3155,7 +3188,7 @@ namespace Slang
                 //
                 // We want to emit `makeExistential(getValueFromBoundInterface(value) : C, witnessTable)`.
                 //
-                auto concreteType = cast<IRType>(slotArgs[0]);
+                auto concreteType = (IRType*)(slotArgs[0]);
                 auto witnessTable = slotArgs[1];
                 if (slotArgs[0]->getOp() == kIROp_DynamicType)
                     return value;
@@ -4501,6 +4534,18 @@ namespace Slang
         return name;
     }
 
+    static void dumpDebugID(IRDumpContext* context, IRInst* inst)
+    {
+#if SLANG_ENABLE_IR_BREAK_ALLOC
+        dump(context, "[#");
+        dump(context, String(inst->_debugUID));
+        dump(context, "]");
+#else
+        SLANG_UNUSED(context);
+        SLANG_UNUSED(inst);
+#endif
+    }
+
     static void dumpID(
         IRDumpContext* context,
         IRInst*        inst)
@@ -4520,9 +4565,8 @@ namespace Slang
         {
             dump(context, "_");
         }
+        dumpDebugID(context, inst);
     }
-
-
     
     struct StringEncoder
     {
@@ -4973,7 +5017,7 @@ namespace Slang
         }
 
         dump(context, opInfo.name);
-
+        dumpDebugID(context, inst);
         dumpInstOperandList(context, inst);
     }
 
diff --git a/source/slang/slang-ir.h b/source/slang/slang-ir.h
index b27542424..9eb03c269 100644
--- a/source/slang/slang-ir.h
+++ b/source/slang/slang-ir.h
@@ -432,6 +432,12 @@ struct IRInst
 
     void removeAndDeallocateAllDecorationsAndChildren();
 
+#ifdef SLANG_ENABLE_IR_BREAK_ALLOC
+    // Unique allocation ID for this instruction since start of current process.
+    // Used to aid debugging only.
+    uint32_t _debugUID;
+#endif
+
     // The type of the result value of this instruction,
     // or `null` to indicate that the instruction has
     // no value.
@@ -603,7 +609,14 @@ struct IRType : IRInst
 {
     IRType* getCanonicalType() { return this; }
 
-    IR_PARENT_ISA(Type)
+    // Hack: specialize can also be a type. We should consider using a
+    // separate `specializeType` op code for types so we can use the normal
+    // `IR_PARENT_ISA` macro here.
+    static bool isaImpl(IROp opIn)
+    {
+        const int op = (kIROpMeta_OpMask & opIn);
+        return (op >= kIROp_FirstType && op <= kIROp_LastType) || op == kIROp_Specialize;
+    }
 };
 
 IRType* unwrapArray(IRType* type);
@@ -1550,6 +1563,10 @@ bool isBuiltin(IRInst* inst);
     // Get the enclosuing function of an instruction.
 IRFunc* getParentFunc(IRInst* inst);
 
+#if SLANG_ENABLE_IR_BREAK_ALLOC
+uint32_t& _debugGetIRAllocCounter();
+#endif
+
 }
 
 #endif
diff --git a/source/slang/slang-lower-to-ir.cpp b/source/slang/slang-lower-to-ir.cpp
index 3c9256178..de5a07c0d 100644
--- a/source/slang/slang-lower-to-ir.cpp
+++ b/source/slang/slang-lower-to-ir.cpp
@@ -15,6 +15,7 @@
 #include "slang-ir-strip.h"
 #include "slang-ir-validate.h"
 #include "slang-ir-string-hash.h"
+#include "slang-ir-clone.h"
 
 #include "slang-mangle.h"
 #include "slang-type-layout.h"
@@ -6098,23 +6099,32 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
             auto entry = subBuilder->createInterfaceRequirementEntry(
                 getInterfaceRequirementKey(requirementDecl),
                 nullptr);
-            IRInst* requirementVal = ensureDecl(subContext, requirementDecl).val;
-            if (requirementVal)
+            if (auto inheritance = as<InheritanceDecl>(requirementDecl))
             {
-                switch (requirementVal->getOp())
-                {
-                case kIROp_Func:
-                case kIROp_Generic:
+                auto irBaseType = lowerType(context, inheritance->base.type);
+                auto irWitnessTableType = subBuilder->getWitnessTableType(irBaseType);
+                entry->setRequirementVal(irWitnessTableType);
+            }
+            else
+            {
+                IRInst* requirementVal = ensureDecl(subContext, requirementDecl).val;
+                if (requirementVal)
                 {
-                    // Remove lowered `IRFunc`s since we only care about
-                    // function types.
-                    auto reqType = requirementVal->getFullType();
-                    entry->setRequirementVal(reqType);
-                    break;
-                }
-                default:
-                    entry->setRequirementVal(requirementVal);
-                    break;
+                    switch (requirementVal->getOp())
+                    {
+                    case kIROp_Func:
+                    case kIROp_Generic:
+                        {
+                            // Remove lowered `IRFunc`s since we only care about
+                            // function types.
+                            auto reqType = requirementVal->getFullType();
+                            entry->setRequirementVal(reqType);
+                            break;
+                        }
+                    default:
+                        entry->setRequirementVal(requirementVal);
+                        break;
+                    }
                 }
             }
             irInterface->setOperand(entryIndex, entry);
@@ -6598,6 +6608,34 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
         return nullptr;
     }
 
+    static bool isChildOf(IRInst* child, IRInst* parent)
+    {
+        while (child && child->getParent() != parent)
+            child = child->getParent();
+        return child != nullptr;
+    }
+    static void markInstsToClone(HashSet<IRInst*>& valuesToClone, IRInst* parentBlock, IRInst* value)
+    {
+        if (!isChildOf(value, parentBlock))
+            return;
+        if (valuesToClone.Add(value))
+        {
+            for (UInt i = 0; i < value->getOperandCount(); i++)
+            {
+                auto operand = value->getOperand(i);
+                markInstsToClone(valuesToClone, parentBlock, operand);
+            }
+        }
+        for (auto child : value->getChildren())
+            markInstsToClone(valuesToClone, parentBlock, child);
+        auto parent = parentBlock->getParent();
+        while (parent && parent != parentBlock)
+        {
+            valuesToClone.Add(parent);
+            parent = parent->getParent();
+        }
+    }
+
     // If any generic declarations have been created by `emitOuterGenerics`,
     // then finish them off by emitting `return` instructions for the
     // values that they should produce.
@@ -6612,9 +6650,94 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
         IRGeneric*  parentGeneric)
     {
         IRInst* v = val;
+
+        IRInst* returnType = v->getFullType();
+
         while (parentGeneric)
         {
+            // Create a universal type in `outterBlock` that will be used
+            // as the type of this generic inst. The return value of the
+            // generic inst will have a specialized type.
+            // For example, if we have a generic function
+            // g0 = generic<T> { return f: T->int }
+            // The type for `g0` should be:
+            // g0Type = generic<T1> { return IRFuncType{T1->int} }
+            // with `g0Type`, we can rewrite `g0` into:
+            // ```
+            //    g0 : g0Type = generic<T>
+            //    {
+            //       ftype = specialize(g0Type, T);
+            //       return f : ftype;
+            //    }
+            // ```
+            IRBuilder typeBuilder;
+            typeBuilder.sharedBuilder = subBuilder->sharedBuilder;
+            IRCloneEnv cloneEnv = {};
+            if (returnType)
+            {
+                HashSet<IRInst*> valuesToClone;
+                markInstsToClone(valuesToClone, parentGeneric->getFirstBlock(), returnType);
+                if (valuesToClone.Count() == 0)
+                {
+                    // If returnType is independent of generic parameters, set
+                    // the generic inst's type to just `returnType`.
+                    parentGeneric->setFullType((IRType*)returnType);
+                }
+                else
+                {
+                    // In the general case, we need to construct a separate
+                    // generic value for the return type, and set the generic's type
+                    // to the newly construct generic value.
+                    typeBuilder.setInsertBefore(parentGeneric);
+                    auto typeGeneric = typeBuilder.emitGeneric();
+                    typeBuilder.setInsertInto(typeGeneric);
+                    typeBuilder.emitBlock();
+                    
+                    for (auto child : parentGeneric->getFirstBlock()->getChildren())
+                    {
+                        if (valuesToClone.Contains(child))
+                        {
+                            cloneInst(&cloneEnv, &typeBuilder, child);
+                        }
+                    }
+                    IRInst* clonedReturnType = nullptr;
+                    cloneEnv.mapOldValToNew.TryGetValue(returnType, clonedReturnType);
+                    SLANG_ASSERT(clonedReturnType);
+                    typeBuilder.emitReturn(clonedReturnType);
+                    parentGeneric->setFullType((IRType*)typeGeneric);
+                    returnType = typeGeneric;
+                }
+            }
+
             subBuilder->setInsertInto(parentGeneric->getFirstBlock());
+#if 0
+            // TODO: we cannot enable this right now as it breaks too many existing code
+            // that is assuming a generic function type is `IRFuncType` rather than `IRSpecialize`.
+            if (v->getFullType() != returnType)
+            {
+                // We need to rewrite the type of the return value as
+                // `specialize(returnType, ...)`.
+                SLANG_ASSERT(returnType->getOp() == kIROp_Generic);
+                auto oldType = v->getFullType();
+                SLANG_ASSERT(isChildOf(oldType, parentGeneric->getFirstBlock()));
+
+                List<IRInst*> specializeArgs;
+                for (auto param : parentGeneric->getParams())
+                {
+                    IRInst* arg = nullptr;
+                    if (cloneEnv.mapOldValToNew.TryGetValue(param, arg))
+                    {
+                        specializeArgs.add(arg);
+                    }
+                }
+                auto specializedType = subBuilder->emitSpecializeInst(
+                    subBuilder->getTypeKind(),
+                    returnType,
+                    (UInt)specializeArgs.getCount(),
+                    specializeArgs.getBuffer());
+                oldType->replaceUsesWith(specializedType);
+            }
+#endif
             subBuilder->emitReturn(v);
             parentGeneric->moveToEnd();
 
@@ -7261,11 +7384,6 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
         // If this function is defined inside an interface, add a reference to the IRFunc from
         // the interface's type definition.
         auto finalVal = finishOuterGenerics(subBuilder, irFunc, outerGeneric);
-        if (auto genericVal = as<IRGeneric>(finalVal))
-        {
-            auto funcType = lowerFuncType(decl);
-            genericVal->setFullType((IRType*)funcType);
-        }
 
         return LoweredValInfo::simple(finalVal);
     }
diff --git a/source/slang/slang-syntax.cpp b/source/slang/slang-syntax.cpp
index accbdb27e..d5f2c13db 100644
--- a/source/slang/slang-syntax.cpp
+++ b/source/slang/slang-syntax.cpp
@@ -30,7 +30,10 @@ void printDiagnosticArg(StringBuilder& sb, Val* val)
 
 void printDiagnosticArg(StringBuilder& sb, TypeExp const& type)
 {
-    type.type->toText(sb);
+    if (type.type)
+        type.type->toText(sb);
+    else
+        sb << "<null>";
 }
 
 void printDiagnosticArg(StringBuilder& sb, QualType const& type)
diff --git a/source/slang/slang.natvis b/source/slang/slang.natvis
index cb272f974..ea010c187 100644
--- a/source/slang/slang.natvis
+++ b/source/slang/slang.natvis
@@ -72,9 +72,10 @@
     </Expand>
   </Type>
   <Type Name="Slang::IRInst">
-    <DisplayString>{{{m_op}}}</DisplayString>
+    <DisplayString>{{{m_op} #{_debugUID}}}</DisplayString>
     <Expand>
       <Item Name="[op]">m_op</Item>
+      <Item Name="[UID]">_debugUID</Item>
       <Item Name="[type]">typeUse.usedValue</Item>
       <CustomListItems MaxItemsPerView="3">
 		  <Variable Name="child" InitialValue="m_decorationsAndChildren.first"/>