Fix crash when compiling specialized generic entrypoint containing a static const decl. (#8392)

Closes #8184.

We fixed three issues with this regression test:
1. After generating IR for a `SpecializeComponentType`, we should also
strip the frontend
decorations from the IR so there is no HighLevelDeclDecoration that will
go into the backend.
2. When lowering a static const inside a generic function, we should not
give the static const
a linkage, because it won't such constant will not appear in global
scope. Trying to give it a
linkage decoration will lead to the parent generic (for the function) to
have two duplicate
Export/Import decorations with different mangle names, and confuses the
linker.
3. Make sure internal exceptions does not leak through
    `IComponentType::getEntryPointCode`/`getTargetCode`.

3 files changed, 88 insertions, 49 deletions

diff --git a/source/slang/slang-linkable.cpp b/source/slang/slang-linkable.cpp
index 0eef62742..7a307cbc3 100644
--- a/source/slang/slang-linkable.cpp
+++ b/source/slang/slang-linkable.cpp
@@ -657,57 +657,75 @@ IArtifact* ComponentType::getTargetArtifact(Int targetIndex, slang::IBlob** outD
     {
         return artifact.get();
     }
-
-    // If the user hasn't specified any entry points, then we should
-    // discover all entrypoints that are defined in linked modules, and
-    // include all of them in the compile.
-    //
-    if (getEntryPointCount() == 0)
+    try
     {
-        List<Module*> modules;
-        this->enumerateModules([&](Module* module) { modules.add(module); });
-        List<RefPtr<ComponentType>> components;
-        components.add(this);
-        bool entryPointsDiscovered = false;
-        for (auto module : modules)
+        // If the user hasn't specified any entry points, then we should
+        // discover all entrypoints that are defined in linked modules, and
+        // include all of them in the compile.
+        //
+        if (getEntryPointCount() == 0)
         {
-            for (auto entryPoint : module->getEntryPoints())
+            List<Module*> modules;
+            this->enumerateModules([&](Module* module) { modules.add(module); });
+            List<RefPtr<ComponentType>> components;
+            components.add(this);
+            bool entryPointsDiscovered = false;
+            for (auto module : modules)
             {
-                components.add(entryPoint);
-                entryPointsDiscovered = true;
+                for (auto entryPoint : module->getEntryPoints())
+                {
+                    components.add(entryPoint);
+                    entryPointsDiscovered = true;
+                }
             }
-        }
 
-        // If any entry points were discovered, then we should emit the program with entrypoints
-        // linked.
-        if (entryPointsDiscovered)
-        {
-            RefPtr<CompositeComponentType> composite =
-                new CompositeComponentType(linkage, components);
-            ComPtr<IComponentType> linkedComponentType;
-            SLANG_RETURN_NULL_ON_FAIL(
-                composite->link(linkedComponentType.writeRef(), outDiagnostics));
-            auto targetArtifact = static_cast<ComponentType*>(linkedComponentType.get())
-                                      ->getTargetArtifact(targetIndex, outDiagnostics);
-            if (targetArtifact)
+            // If any entry points were discovered, then we should emit the program with entrypoints
+            // linked.
+            if (entryPointsDiscovered)
             {
-                m_targetArtifacts[targetIndex] = targetArtifact;
+                RefPtr<CompositeComponentType> composite =
+                    new CompositeComponentType(linkage, components);
+                ComPtr<IComponentType> linkedComponentType;
+                SLANG_RETURN_NULL_ON_FAIL(
+                    composite->link(linkedComponentType.writeRef(), outDiagnostics));
+                auto targetArtifact = static_cast<ComponentType*>(linkedComponentType.get())
+                                          ->getTargetArtifact(targetIndex, outDiagnostics);
+                if (targetArtifact)
+                {
+                    m_targetArtifacts[targetIndex] = targetArtifact;
+                }
+                return targetArtifact;
             }
-            return targetArtifact;
         }
-    }
 
-    auto target = linkage->targets[targetIndex];
-    auto targetProgram = getTargetProgram(target);
+        auto target = linkage->targets[targetIndex];
+        auto targetProgram = getTargetProgram(target);
 
-    DiagnosticSink sink(linkage->getSourceManager(), Lexer::sourceLocationLexer);
-    applySettingsToDiagnosticSink(&sink, &sink, linkage->m_optionSet);
-    applySettingsToDiagnosticSink(&sink, &sink, m_optionSet);
+        DiagnosticSink sink(linkage->getSourceManager(), Lexer::sourceLocationLexer);
+        applySettingsToDiagnosticSink(&sink, &sink, linkage->m_optionSet);
+        applySettingsToDiagnosticSink(&sink, &sink, m_optionSet);
 
-    IArtifact* targetArtifact = targetProgram->getOrCreateWholeProgramResult(&sink);
-    sink.getBlobIfNeeded(outDiagnostics);
-    m_targetArtifacts[targetIndex] = ComPtr<IArtifact>(targetArtifact);
-    return targetArtifact;
+        IArtifact* targetArtifact = targetProgram->getOrCreateWholeProgramResult(&sink);
+        sink.getBlobIfNeeded(outDiagnostics);
+        m_targetArtifacts[targetIndex] = ComPtr<IArtifact>(targetArtifact);
+        return targetArtifact;
+    }
+    catch (const Exception& e)
+    {
+        if (outDiagnostics && !*outDiagnostics)
+        {
+            DiagnosticSink sink(linkage->getSourceManager(), Lexer::sourceLocationLexer);
+            applySettingsToDiagnosticSink(&sink, &sink, linkage->m_optionSet);
+            applySettingsToDiagnosticSink(&sink, &sink, m_optionSet);
+            sink.diagnose(
+                SourceLoc(),
+                Diagnostics::compilationAbortedDueToException,
+                typeid(e).name(),
+                e.Message);
+            sink.getBlobIfNeeded(outDiagnostics);
+        }
+        return nullptr;
+    }
 }
 
 SLANG_NO_THROW SlangResult SLANG_MCALL
diff --git a/source/slang/slang-lower-to-ir.cpp b/source/slang/slang-lower-to-ir.cpp
index 4df778ee6..3fbf0bda9 100644
--- a/source/slang/slang-lower-to-ir.cpp
+++ b/source/slang/slang-lower-to-ir.cpp
@@ -8896,6 +8896,8 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
 
         IRGeneric* outerGeneric = nullptr;
 
+        bool needLinkage = true;
+
         // If we are static, then we need to insert the declaration before the parent.
         // This tries to match the behavior of previous `lowerFunctionStaticConstVarDecl`
         // functionality
@@ -8906,6 +8908,11 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
             // be the global scope, but it might be an outer
             // generic if we are lowering a generic function.
             subBuilder->setInsertBefore(subBuilder->getFunc());
+
+            // static values inside a function does not need a linkage.
+            // trying to insert a linkage decoration to a static constant defined
+            // inside a generic function can lead to errorneous IR.
+            needLinkage = false;
         }
         else if (!isFunctionVarDecl(decl))
         {
@@ -8960,8 +8967,8 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
         // All of the attributes/decorations we can attach
         // belong on the IR constant node.
         //
-
-        addLinkageDecoration(context, irConstant, decl);
+        if (needLinkage)
+            addLinkageDecoration(context, irConstant, decl);
 
         addNameHint(context, irConstant, decl);
         addVarDecorations(context, irConstant, decl);
@@ -12034,6 +12041,8 @@ static void lowerProgramEntryPointToIR(
             existentialSlotArgs.getCount(),
             existentialSlotArgs.getBuffer());
     }
+
+    stripFrontEndOnlyInstructions(builder->getModule(), IRStripOptions());
 }
 
 /// Ensure that `decl` and all relevant declarations under it get emitted.
diff --git a/source/slang/slang-target-program.cpp b/source/slang/slang-target-program.cpp
index ffb859b55..d7e7a73bb 100644
--- a/source/slang/slang-target-program.cpp
+++ b/source/slang/slang-target-program.cpp
@@ -98,16 +98,28 @@ IArtifact* TargetProgram::getOrCreateEntryPointResult(Int entryPointIndex, Diagn
     if (IArtifact* artifact = m_entryPointResults[entryPointIndex])
         return artifact;
 
-    // If we haven't yet computed a layout for this target
-    // program, we need to make sure that is done before
-    // code generation.
-    //
-    if (!getOrCreateIRModuleForLayout(sink))
+    try
+    {
+        // If we haven't yet computed a layout for this target
+        // program, we need to make sure that is done before
+        // code generation.
+        //
+        if (!getOrCreateIRModuleForLayout(sink))
+        {
+            return nullptr;
+        }
+
+        return _createEntryPointResult(entryPointIndex, sink);
+    }
+    catch (const Exception& e)
     {
+        sink->diagnose(
+            SourceLoc(),
+            Diagnostics::compilationAbortedDueToException,
+            typeid(e).name(),
+            e.Message);
         return nullptr;
     }
-
-    return _createEntryPointResult(entryPointIndex, sink);
 }
 
 } // namespace Slang