Actual global support (#2262)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Use TerminatedUnownedStringSlice for literals in output C++.

* Remove Escape/Unescape functions used in slang-token-reader.cpp
Add target type of 'host-cpp' etc to map to the target types.

* Fix some corner cases around string encoding.

* Added unit test for string escaping.
Fixed some assorted escaping bugs.

* Updated test output.

* Added decode test.

* Stop using hex output, to get around 'greedy' aspect. Use octal instead.

* Added HostHostCallable
Small changes to use ArtifactDesc/Info instead of large switches.

* Fix C++ emit to handle arbitrary function export.

* Add options handling for callable without an output being specified.

* Can compile with COM interface. Added example using com interface.

* Use the IR Ptr type instead of hack in C++ emit for interfaces.

* Fix issue with outputting the COM call when ptr is used.

* Fix crash issue on compilation failure.

* Add support for __global.

* Added `ActualGlobalRate`
Added special handling around globals and COM interfaces.
Tested out in cpu-com-example.

* Fix typo in NodeBase.

* Support for accessing globals by name working.

* Check that actual global initialization is working.

* Refactor the com replacement such that it doesn't need a cache or do anything special with GlobalVar.

* Remove context.
Only create replacement if needed.

* Split out COM host-callable into a unit-test.

* host-callable com testing on C++and llvm.

* Comment around the COM ptr replacement.

* Disable com test on vs 32 bit.
Fix C++ prelude

* Disable 32 bit targets testing com host-callable.

* Use JSON parsing to locate VS version.

* Need platform detection in C++prelude.

* Fix com host callable test for LLVM.

* Work around for not being able to include "targetConditionals.h"

15 files changed, 174 insertions, 72 deletions

diff --git a/source/slang/slang-ast-base.h b/source/slang/slang-ast-base.h
index c74edb938..3126aab71 100644
--- a/source/slang/slang-ast-base.h
+++ b/source/slang/slang-ast-base.h
@@ -33,6 +33,9 @@ class NodeBase
         /// correctly constructed (through ASTBuilder) NodeBase derived class. 
         /// The actual type is set when constructed on the ASTBuilder. 
     ASTNodeType astNodeType = ASTNodeType(-1);
+
+    // Handy when debugging, shouldn't be checked in though!
+    // virtual ~NodeBase() {}
 };
 
 // Casting of NodeBase
diff --git a/source/slang/slang-ast-modifier.h b/source/slang/slang-ast-modifier.h
index 4522b7148..012c74377 100644
--- a/source/slang/slang-ast-modifier.h
+++ b/source/slang/slang-ast-modifier.h
@@ -31,6 +31,10 @@ class ConstExprModifier : public Modifier { SLANG_AST_CLASS(ConstExprModifier)};
 class GloballyCoherentModifier : public Modifier { SLANG_AST_CLASS(GloballyCoherentModifier)};
 class ExternCppModifier : public Modifier { SLANG_AST_CLASS(ExternCppModifier)};
 
+// An 'ActualGlobal' is a global that is output as a normal global in CPU code. 
+// Globals in HLSL/Slang are constant state passed into kernel execution 
+class ActualGlobalModifier : public Modifier { SLANG_AST_CLASS(ActualGlobalModifier)};
+
     /// A modifier that indicates an `InheritanceDecl` should be ignored during name lookup (and related checks).
 class IgnoreForLookupModifier : public Modifier { SLANG_AST_CLASS(IgnoreForLookupModifier) };
 
diff --git a/source/slang/slang-check-decl.cpp b/source/slang/slang-check-decl.cpp
index a2311b186..bb762c1c6 100644
--- a/source/slang/slang-check-decl.cpp
+++ b/source/slang/slang-check-decl.cpp
@@ -267,6 +267,9 @@ namespace Slang
         /// Is `decl` a global shader parameter declaration?
     bool isGlobalShaderParameter(VarDeclBase* decl)
     {
+        // If it's an *actual* global it is not a global shader parameter
+        if (decl->hasModifier<ActualGlobalModifier>()) { return false; }
+        
         // A global shader parameter must be declared at global or namespace
         // scope, so that it has a single definition across the module.
         //
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index ff1f660a1..148b0205b 100755
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -2887,7 +2887,7 @@ namespace Slang
         void addTransition(CodeGenTarget source, CodeGenTarget target, PassThroughMode compiler)
         {
             SLANG_ASSERT(source != target);
-            m_map.Add(Pair{ source, target }, compiler);
+            m_map.Set(Pair{ source, target }, compiler);
         }
         bool hasTransition(CodeGenTarget source, CodeGenTarget target) const
         {
diff --git a/source/slang/slang-emit-cpp.cpp b/source/slang/slang-emit-cpp.cpp
index ddc8b24ed..c23135c70 100644
--- a/source/slang/slang-emit-cpp.cpp
+++ b/source/slang/slang-emit-cpp.cpp
@@ -2547,7 +2547,50 @@ void CPPSourceEmitter::emitPreModuleImpl()
     }
 }
 
-/* virtual */void CPPSourceEmitter::emitFuncDecorationsImpl(IRFunc* func)
+
+void CPPSourceEmitter::emitGlobalInstImpl(IRInst* inst)
+{
+    if (as<IRGlobalVar>(inst) && inst->findDecoration<IRExternCppDecoration>())
+    {
+        // JS:
+        // Turns out just doing extern "C" means something different on a variable
+        // So we need to wrap in extern "C" { }
+        m_writer->emit("extern \"C\" {\n");
+        Super::emitGlobalInstImpl(inst);
+        m_writer->emit("\n}\n");
+    }
+    else
+    {
+        Super::emitGlobalInstImpl(inst);
+    }
+}
+
+static bool _isExported(IRInst* inst)
+{
+    for (auto decoration : inst->getDecorations())
+    {
+        const auto op = decoration->getOp();
+        if (op == kIROp_PublicDecoration ||
+            op == kIROp_HLSLExportDecoration)
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
+void CPPSourceEmitter::emitVarDecorationsImpl(IRInst* inst)
+{
+    if (as<IRGlobalVar>(inst) && _isExported(inst))
+    {
+        m_writer->emit("SLANG_PRELUDE_SHARED_LIB_EXPORT\n");
+    }
+
+    Super::emitVarDecorationsImpl(inst);
+}
+
+
+void CPPSourceEmitter::_maybeEmitExportLike(IRInst* inst)
 {
     // Specially handle export, as we don't want to emit it multiple times
     if (getTargetReq()->isWholeProgramRequest())
@@ -2556,7 +2599,7 @@ void CPPSourceEmitter::emitPreModuleImpl()
         bool isExported = false;
 
         // If public/export made it externally visible
-        for (auto decoration : func->getDecorations())
+        for (auto decoration : inst->getDecorations())
         {
             const auto op = decoration->getOp();
             if (op == kIROp_ExternCppDecoration)
@@ -2581,6 +2624,11 @@ void CPPSourceEmitter::emitPreModuleImpl()
             m_writer->emit("extern \"C\"\n");
         }
     }
+}
+
+/* virtual */void CPPSourceEmitter::emitFuncDecorationsImpl(IRFunc* func)
+{
+    _maybeEmitExportLike(func);
 
     // Use the default for others
     Super::emitFuncDecorationsImpl(func);
diff --git a/source/slang/slang-emit-cpp.h b/source/slang/slang-emit-cpp.h
index f5ba35933..6199c33f2 100644
--- a/source/slang/slang-emit-cpp.h
+++ b/source/slang/slang-emit-cpp.h
@@ -75,7 +75,9 @@ protected:
     virtual void emitIntrinsicCallExprImpl(IRCall* inst, IRTargetIntrinsicDecoration* targetIntrinsic, EmitOpInfo const& inOuterPrec) SLANG_OVERRIDE;
     virtual void emitLoopControlDecorationImpl(IRLoopControlDecoration* decl) SLANG_OVERRIDE;
     virtual void emitFuncDecorationsImpl(IRFunc* func) SLANG_OVERRIDE;
-   
+    virtual void emitVarDecorationsImpl(IRInst* var) SLANG_OVERRIDE;
+    virtual void emitGlobalInstImpl(IRInst* inst) SLANG_OVERRIDE;
+
     virtual const UnownedStringSlice* getVectorElementNames(BaseType elemType, Index elemCount);
     
     // Replaceable for classes derived from CPPSourceEmitter
@@ -130,6 +132,9 @@ protected:
         // of all the witness table objects in `pendingWitnessTableDefinitions`.
     void _emitWitnessTableDefinitions();
 
+        /// Maybe emits 'export' (such that visible outside binary/dll) and `extern "C"` naming
+    void _maybeEmitExportLike(IRInst* inst);
+
     HLSLIntrinsic* _addIntrinsic(HLSLIntrinsic::Op op, IRType* returnType, IRType*const* argTypes, Index argTypeCount);
 
     static bool _isVariable(IROp op);
@@ -137,7 +142,6 @@ protected:
     Dictionary<IRType*, StringSlicePool::Handle> m_typeNameMap;
     Dictionary<const HLSLIntrinsic*, StringSlicePool::Handle> m_intrinsicNameMap;
 
-
     IRTypeSet m_typeSet;
     RefPtr<HLSLIntrinsicOpLookup> m_opLookup;
     HLSLIntrinsicSet m_intrinsicSet;
diff --git a/source/slang/slang-ir-com-interface.cpp b/source/slang/slang-ir-com-interface.cpp
index 009d2314d..899596209 100644
--- a/source/slang/slang-ir-com-interface.cpp
+++ b/source/slang/slang-ir-com-interface.cpp
@@ -7,92 +7,105 @@
 namespace Slang
 {
 
-struct ComInterfaceLoweringContext
+static bool _canReplace(IRUse* use)
 {
-    IRModule* module;
-    DiagnosticSink* diagnosticSink;
-
-    ArtifactStyle artifactStyle;
-
-    SharedIRBuilder sharedBuilder;
-
-    void replaceTypeUses(IRInst* inst, IRInst* newValue)
+    switch (use->getUser()->getOp())
     {
-        List<IRUse*> uses;
-        for (auto use = inst->firstUse; use; use = use->nextUse)
+        case kIROp_WitnessTableIDType:
+        case kIROp_WitnessTableType:
+        case kIROp_RTTIPointerType:
+        case kIROp_RTTIHandleType:
         {
-            uses.add(use);
+            // Don't replace
+            return false;
         }
-        for (auto use : uses)
+        case kIROp_ThisType:
         {
-            switch (use->getUser()->getOp())
-            {
-            case kIROp_WitnessTableIDType:
-            case kIROp_WitnessTableType:
-            case kIROp_ThisType:
-            case kIROp_RTTIPointerType:
-            case kIROp_RTTIHandleType:
-            case kIROp_ComPtrType:
-            case kIROp_PtrType:
-                continue;
-            default:
-                break;
-            }
-            use->set(newValue);
+            // Appears replacable.
+            break;
         }
+        case kIROp_ComPtrType:
+        case kIROp_PtrType:
+        {
+            // We can have ** and ComPtr<T>*.
+            // If it's a pointer type it could be because it is a global.
+            break;
+        }
+        default: break;
     }
+    return true;
+}
 
-    IRType* processInterfaceType(IRInterfaceType* type)
+void lowerComInterfaces(IRModule* module, ArtifactStyle artifactStyle, DiagnosticSink* sink)
+{
+    SLANG_UNUSED(sink);
+
+    // Find all of the COM interfaces
+    List<IRInterfaceType*> comInterfaces;
+    for (auto child : module->getGlobalInsts())
     {
-        if (!type->findDecoration<IRComInterfaceDecoration>())
-            return nullptr;
-     
+        auto intf = as<IRInterfaceType>(child);
+        if (intf && intf->findDecoration<IRComInterfaceDecoration>())
+        {
+            comInterfaces.add(intf);
+        }
+    }
+
+    // For all interfaces found replace uses
+    {
+        SharedIRBuilder sharedBuilder;
+        sharedBuilder.init(module);
+
         IRBuilder builder(sharedBuilder);
         builder.setInsertInto(module->getModuleInst());
 
-        IRType* result = (artifactStyle == ArtifactStyle::Kernel) ?
-            static_cast<IRType*>(builder.getPtrType(type)) :
-            static_cast<IRType*>(builder.getComPtrType(type));
+        List<IRUse*> uses;
 
-        replaceTypeUses(type, result);
-        return result;
-    }
+        for (auto comIntf : comInterfaces)
+        {
+            uses.clear();
 
-    void processThisType(IRThisType* type)
-    {
-        auto comPtrType = processInterfaceType(as<IRInterfaceType>(type->getConstraintType()));
-        if (!comPtrType)
-            return;
-        replaceTypeUses(type, comPtrType);
-    }
+            // Find all of the uses *before* doing any replacement
+            // Otherwise we end up replacing the replacement leading 
+            // to it pointing to itself.
+            for (auto use = comIntf->firstUse; use; use = use->nextUse)
+            {
+                // Only store off uses where replacement can be made
+                if (_canReplace(use))
+                {
+                    uses.add(use);
+                }
+            }
 
-    void processModule()
-    {
-        for (auto child : module->getGlobalInsts())
-        {
-            switch (child->getOp())
+            // If there are no uses that can be replaced, then we don't need
+            // to create a replacement result
+            if (uses.getCount() <= 0)
             {
-            case kIROp_InterfaceType:
-                processInterfaceType(as<IRInterfaceType>(child));
-                break;
-            case kIROp_ThisType:
-                processThisType(as<IRThisType>(child));
-                break;
-            default:
-                break;
+                continue;
+            }
+
+            // NOTE! The following code relies on the fact that the builder
+            // *doesn't* dedup in general, and in particular doesn't ptr types. 
+            // This allows the creation a 'new'  pointer type, and subsequent replacment all old uses, 
+            // leading to a `IInterface*` becoming `IInterface**`. 
+            // 
+
+            // TODO(JS): This is a temporary fix, in that whether kernel or not 
+            // shouldn't control the ptr type in general
+            // It's necessary here though because Kernel doesn't have ComPtr<>
+            // so has to be a raw pointer
+            IRType* result = (artifactStyle == ArtifactStyle::Host) ?
+                static_cast<IRType*>(builder.getComPtrType(comIntf)) :
+                static_cast<IRType*>(builder.getPtrType(comIntf));
+
+            // Go through replacing all of the replacable uses
+            for (auto use : uses)
+            {
+                // Do the replacement
+                use->set(result);
             }
         }
     }
-};
-
-void lowerComInterfaces(IRModule* module, ArtifactStyle artifactStyle, DiagnosticSink* sink)
-{
-    ComInterfaceLoweringContext context;
-    context.module = module;
-    context.diagnosticSink = sink;
-    context.artifactStyle = artifactStyle;
-    context.sharedBuilder.init(module);
-    return context.processModule();
 }
 
 }
diff --git a/source/slang/slang-ir-explicit-global-context.cpp b/source/slang/slang-ir-explicit-global-context.cpp
index 6ab0d68f2..6e88c4cd7 100644
--- a/source/slang/slang-ir-explicit-global-context.cpp
+++ b/source/slang/slang-ir-explicit-global-context.cpp
@@ -50,6 +50,12 @@ struct IntroduceExplicitGlobalContextPass
                     //
                     auto globalVar = cast<IRGlobalVar>(inst);
 
+                    // Actual globals don't need to be moved to the context
+                    if (as<IRActualGlobalRate>(globalVar->getRate()))
+                    {
+                        continue;
+                    }
+
                     // One important exception is that CUDA *does* support
                     // global variables with the `__shared__` qualifer, with
                     // semantics that exactly match HLSL/Slang `groupshared`.
diff --git a/source/slang/slang-ir-explicit-global-init.cpp b/source/slang/slang-ir-explicit-global-init.cpp
index 07397902e..94c065514 100644
--- a/source/slang/slang-ir-explicit-global-init.cpp
+++ b/source/slang/slang-ir-explicit-global-init.cpp
@@ -87,6 +87,12 @@ struct MoveGlobalVarInitializationToEntryPointsPass
             if(!globalVar)
                 continue;
 
+            // If it's an `Actual Global` we don't want to move initialization
+            if (as<IRActualGlobalRate>(globalVar->getRate()))
+            {
+                continue;
+            }
+        
             auto firstBlock = globalVar->getFirstBlock();
             if(!firstBlock)
                 continue;
diff --git a/source/slang/slang-ir-inst-defs.h b/source/slang/slang-ir-inst-defs.h
index b77621720..6547d949e 100644
--- a/source/slang/slang-ir-inst-defs.h
+++ b/source/slang/slang-ir-inst-defs.h
@@ -79,6 +79,7 @@ INST(Nop, nop, 0, 0)
     /* Rate */
         INST(ConstExprRate, ConstExpr, 0, 0)
         INST(GroupSharedRate, GroupShared, 0, 0)
+        INST(ActualGlobalRate, ActualGlobalRate, 0, 0)
     INST_RANGE(Rate, ConstExprRate, GroupSharedRate)
 
     INST(RateQualifiedType, RateQualified, 2, 0)
diff --git a/source/slang/slang-ir-insts.h b/source/slang/slang-ir-insts.h
index 9c1569ca8..5e8e11f84 100644
--- a/source/slang/slang-ir-insts.h
+++ b/source/slang/slang-ir-insts.h
@@ -2235,6 +2235,7 @@ public:
 
     IRConstExprRate* getConstExprRate();
     IRGroupSharedRate* getGroupSharedRate();
+    IRActualGlobalRate* getActualGlobalRate();
 
     IRRateQualifiedType* getRateQualifiedType(
         IRRate* rate,
diff --git a/source/slang/slang-ir.cpp b/source/slang/slang-ir.cpp
index ee84cce73..562d0ea1a 100644
--- a/source/slang/slang-ir.cpp
+++ b/source/slang/slang-ir.cpp
@@ -2795,6 +2795,10 @@ namespace Slang
     {
         return (IRGroupSharedRate*)getType(kIROp_GroupSharedRate);
     }
+    IRActualGlobalRate* IRBuilder::getActualGlobalRate()
+    {
+        return (IRActualGlobalRate*)getType(kIROp_ActualGlobalRate);
+    }
 
     IRRateQualifiedType* IRBuilder::getRateQualifiedType(
         IRRate* rate,
diff --git a/source/slang/slang-ir.h b/source/slang/slang-ir.h
index 7a9a1ebee..403376dca 100644
--- a/source/slang/slang-ir.h
+++ b/source/slang/slang-ir.h
@@ -1226,6 +1226,7 @@ SIMPLE_IR_TYPE(UnsizedArrayType, ArrayTypeBase)
 SIMPLE_IR_PARENT_TYPE(Rate, Type)
 SIMPLE_IR_TYPE(ConstExprRate, Rate)
 SIMPLE_IR_TYPE(GroupSharedRate, Rate)
+SIMPLE_IR_TYPE(ActualGlobalRate, Rate)
 
 struct IRRateQualifiedType : IRType
 {
diff --git a/source/slang/slang-lower-to-ir.cpp b/source/slang/slang-lower-to-ir.cpp
index 5ffb1bf33..d175b69dd 100644
--- a/source/slang/slang-lower-to-ir.cpp
+++ b/source/slang/slang-lower-to-ir.cpp
@@ -2034,6 +2034,12 @@ void maybeSetRate(
             builder->getGroupSharedRate(),
             inst->getFullType()));
     }
+    else if (decl->hasModifier<ActualGlobalModifier>())
+    {
+        inst->setFullType(builder->getRateQualifiedType(
+            builder->getActualGlobalRate(),
+            inst->getFullType()));
+    }
 }
 
 static String getNameForNameHint(
diff --git a/source/slang/slang-parser.cpp b/source/slang/slang-parser.cpp
index 1c32499ef..b2179c1af 100644
--- a/source/slang/slang-parser.cpp
+++ b/source/slang/slang-parser.cpp
@@ -6339,6 +6339,8 @@ namespace Slang
         _makeParseModifier("instance",      InstanceModifier::kReflectClassInfo),
         _makeParseModifier("__builtin",     BuiltinModifier::kReflectClassInfo),
 
+        _makeParseModifier("__global",      ActualGlobalModifier::kReflectClassInfo),
+
         _makeParseModifier("inline",        InlineModifier::kReflectClassInfo),
         _makeParseModifier("public",        PublicModifier::kReflectClassInfo),
         _makeParseModifier("require",       RequireModifier::kReflectClassInfo),