Make Optional<PointerType> lower to PointerType instead of a struct. (#2373)

18 files changed, 171 insertions, 90 deletions

diff --git a/source/slang/slang-check-type.cpp b/source/slang/slang-check-type.cpp
index aa2c69126..c59e1b308 100644
--- a/source/slang/slang-check-type.cpp
+++ b/source/slang/slang-check-type.cpp
@@ -187,15 +187,27 @@ namespace Slang
         return DeclRefType::create(m_astBuilder, innerDeclRef);
     }
 
+    bool isManagedType(Type* type)
+    {
+        if (auto declRefValueType = as<DeclRefType>(type))
+        {
+            if (as<ClassDecl>(declRefValueType->declRef.getDecl()))
+                return true;
+            if (as<InterfaceDecl>(declRefValueType->declRef.getDecl()))
+                return true;
+        }
+        return false;
+    }
+
     bool SemanticsVisitor::CoerceToProperTypeImpl(
         TypeExp const&  typeExp,
         Type**   outProperType,
         DiagnosticSink* diagSink)
     {
+        Type* result = nullptr;
         Type* type = typeExp.type;
         auto originalExpr = typeExp.exp;
         auto expr = originalExpr;
-
         if(!type && expr)
         {
             expr = maybeResolveOverloadedExpr(expr, LookupMask::type, diagSink);
@@ -289,18 +301,26 @@ namespace Slang
                     // ignore non-parameter members
                 }
             }
-            if (outProperType)
-            {
-                *outProperType = InstantiateGenericType(genericDeclRef, args);
-            }
-            return true;
+            result = InstantiateGenericType(genericDeclRef, args);
         }
             
         // default case: we expect this to already be a proper type
-        if (outProperType)
+        if (!result)
+        {
+            result = type;
+        }
+
+        // Check for invalid types.
+        // We don't allow pointers to managed types.
+        if (auto ptrType = as<PtrType>(result))
         {
-            *outProperType = type;
+            if (isManagedType(ptrType->getValueType()))
+            {
+                getSink()->diagnose(typeExp.exp, Diagnostics::cannotDefinePtrTypeToManagedResource);
+            }
         }
+
+        *outProperType = result;
         return true;
     }
 
diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h
index e44a880a2..811359d71 100644
--- a/source/slang/slang-diagnostic-defs.h
+++ b/source/slang/slang-diagnostic-defs.h
@@ -338,6 +338,10 @@ DIAGNOSTIC(32003, Error, unexpectedEnumTagExpr,     "unexpected form for 'enum'
 // 303xx: interfaces and associated types
 DIAGNOSTIC(30300, Error, assocTypeInInterfaceOnly, "'associatedtype' can only be defined in an 'interface'.")
 DIAGNOSTIC(30301, Error, globalGenParamInGlobalScopeOnly, "'type_param' can only be defined global scope.")
+
+// Interop
+DIAGNOSTIC(30400, Error, cannotDefinePtrTypeToManagedResource, "pointer to a managed resource is invalid, use `NativeRef<T>` instead")
+
 // TODO: need to assign numbers to all these extra diagnostics...
 DIAGNOSTIC(39999, Fatal, cyclicReference, "cyclic reference '$0'.")
 DIAGNOSTIC(39999, Error, localVariableUsedBeforeDeclared, "local variable '$0' is being used before its declaration.")
diff --git a/source/slang/slang-emit.cpp b/source/slang/slang-emit.cpp
index 3ec03d0e7..11e0812d9 100644
--- a/source/slang/slang-emit.cpp
+++ b/source/slang/slang-emit.cpp
@@ -212,22 +212,6 @@ Result linkAndOptimizeIR(
     // un-specialized IR.
     dumpIRIfEnabled(codeGenContext, irModule);
 
-    switch (target)
-    {
-        case CodeGenTarget::CPPSource:
-        case CodeGenTarget::HostCPPSource:
-        {
-            lowerComInterfaces(irModule, artifactDesc.style, sink);
-            generateDllImportFuncs(codeGenContext->getTargetReq(), irModule, sink);
-            generateDllExportFuncs(irModule, sink);
-            break;
-        }
-        default: break;
-    }
-
-    // Lower `Result<T,E>` types into ordinary struct types.
-    lowerResultType(irModule, sink);
-
     // Replace any global constants with their values.
     //
     replaceGlobalConstants(irModule);
@@ -323,6 +307,24 @@ Result linkAndOptimizeIR(
         break;
     }
 
+    lowerOptionalType(irModule, sink);
+    simplifyIR(irModule);
+
+    switch (target)
+    {
+    case CodeGenTarget::CPPSource:
+    case CodeGenTarget::HostCPPSource:
+    {
+        lowerComInterfaces(irModule, artifactDesc.style, sink);
+        generateDllImportFuncs(codeGenContext->getTargetReq(), irModule, sink);
+        generateDllExportFuncs(irModule, sink);
+        break;
+    }
+    default: break;
+    }
+
+    // Lower `Result<T,E>` types into ordinary struct types.
+    lowerResultType(irModule, sink);
 
     // Desguar any union types, since these will be illegal on
     // various targets.
@@ -393,8 +395,6 @@ Result linkAndOptimizeIR(
     //
     simplifyIR(irModule);
 
-    lowerOptionalType(irModule, sink);
-
 #if 0
     dumpIRIfEnabled(codeGenContext, irModule, "AFTER DCE");
 #endif
diff --git a/source/slang/slang-ir-generics-lowering-context.cpp b/source/slang/slang-ir-generics-lowering-context.cpp
index 48178340d..0ed7d75d7 100644
--- a/source/slang/slang-ir-generics-lowering-context.cpp
+++ b/source/slang/slang-ir-generics-lowering-context.cpp
@@ -3,6 +3,7 @@
 #include "slang-ir-generics-lowering-context.h"
 
 #include "slang-ir-layout.h"
+#include "slang-ir-util.h"
 
 namespace Slang
 {
@@ -36,24 +37,6 @@ namespace Slang
         return false;
     }
 
-    bool isComInterfaceType(IRType* type)
-    {
-        if (!type) return false;
-        if (type->findDecoration<IRComInterfaceDecoration>() || 
-            type->getOp() == kIROp_ComPtrType)
-        {
-            return true;
-        }
-
-        if (auto ptrType = as<IRNativePtrType>(type))
-        {
-            auto valueType = ptrType->getValueType();
-            return valueType->findDecoration<IRComInterfaceDecoration>() != nullptr;
-        }
-
-        return false;
-    }
-
     bool isTypeValue(IRInst* typeInst)
     {
         if (typeInst)
diff --git a/source/slang/slang-ir-generics-lowering-context.h b/source/slang/slang-ir-generics-lowering-context.h
index e373e2bec..85ba2443d 100644
--- a/source/slang/slang-ir-generics-lowering-context.h
+++ b/source/slang/slang-ir-generics-lowering-context.h
@@ -104,8 +104,6 @@ namespace Slang
 
     bool isPolymorphicType(IRInst* typeInst);
 
-    bool isComInterfaceType(IRType* type);
-
     // Returns true if typeInst represents a type and should be lowered into
     // Ptr(RTTIType).
     bool isTypeValue(IRInst* typeInst);
diff --git a/source/slang/slang-ir-insts.h b/source/slang/slang-ir-insts.h
index 7f4e991b0..61011634c 100644
--- a/source/slang/slang-ir-insts.h
+++ b/source/slang/slang-ir-insts.h
@@ -2927,6 +2927,8 @@ public:
         IRType* type,
         IRInst* val);
 
+    IRInst* emitCastPtrToBool(IRInst* val);
+
     IRGlobalConstant* emitGlobalConstant(
         IRType* type);
 
diff --git a/source/slang/slang-ir-lower-existential.cpp b/source/slang/slang-ir-lower-existential.cpp
index cfdacc7ac..2d533b424 100644
--- a/source/slang/slang-ir-lower-existential.cpp
+++ b/source/slang/slang-ir-lower-existential.cpp
@@ -4,6 +4,7 @@
 #include "slang-ir-generics-lowering-context.h"
 #include "slang-ir.h"
 #include "slang-ir-insts.h"
+#include "slang-ir-util.h"
 
 namespace Slang
 {
diff --git a/source/slang/slang-ir-lower-generic-call.cpp b/source/slang/slang-ir-lower-generic-call.cpp
index 2a97bc28a..574db5dab 100644
--- a/source/slang/slang-ir-lower-generic-call.cpp
+++ b/source/slang/slang-ir-lower-generic-call.cpp
@@ -1,6 +1,7 @@
 // slang-ir-lower-generic-call.cpp
 #include "slang-ir-lower-generic-call.h"
 #include "slang-ir-generics-lowering-context.h"
+#include "slang-ir-util.h"
 
 namespace Slang
 {
diff --git a/source/slang/slang-ir-lower-generic-function.cpp b/source/slang/slang-ir-lower-generic-function.cpp
index 1bd785e8e..e990447bf 100644
--- a/source/slang/slang-ir-lower-generic-function.cpp
+++ b/source/slang/slang-ir-lower-generic-function.cpp
@@ -5,6 +5,7 @@
 #include "slang-ir.h"
 #include "slang-ir-clone.h"
 #include "slang-ir-insts.h"
+#include "slang-ir-util.h"
 
 namespace Slang
 {
diff --git a/source/slang/slang-ir-lower-generics.cpp b/source/slang/slang-ir-lower-generics.cpp
index 6b8150605..f94ce57d3 100644
--- a/source/slang/slang-ir-lower-generics.cpp
+++ b/source/slang/slang-ir-lower-generics.cpp
@@ -14,6 +14,7 @@
 #include "slang-ir-specialize-dynamic-associatedtype-lookup.h"
 #include "slang-ir-witness-table-wrapper.h"
 #include "slang-ir-ssa-simplification.h"
+#include "slang-ir-util.h"
 
 
 namespace Slang
diff --git a/source/slang/slang-ir-lower-optional-type.cpp b/source/slang/slang-ir-lower-optional-type.cpp
index 79be4e042..96f8353bb 100644
--- a/source/slang/slang-ir-lower-optional-type.cpp
+++ b/source/slang/slang-ir-lower-optional-type.cpp
@@ -3,6 +3,7 @@
 #include "slang-ir-lower-optional-type.h"
 #include "slang-ir.h"
 #include "slang-ir-insts.h"
+#include "slang-ir-util.h"
 
 namespace Slang
 {
@@ -35,6 +36,23 @@ namespace Slang
                 return type;
         }
 
+        bool typeHasNullValue(IRInst* type)
+        {
+            switch (type->getOp())
+            {
+            case kIROp_ComPtrType:
+            case kIROp_NativePtrType:
+            case kIROp_NativeStringType:
+            case kIROp_PtrType:
+            case kIROp_ClassType:
+                return true;
+            case kIROp_InterfaceType:
+                return isComInterfaceType((IRType*)type);
+            default:
+                return false;
+            }
+        }
+
         LoweredOptionalTypeInfo* getLoweredOptionalType(IRBuilder* builder, IRInst* type)
         {
             if (auto loweredInfo = loweredOptionalTypes.TryGetValue(type))
@@ -48,25 +66,30 @@ namespace Slang
                 return nullptr;
 
             RefPtr<LoweredOptionalTypeInfo> info = new LoweredOptionalTypeInfo();
-            info->optionalType = (IRType*)type;
             auto optionalType = cast<IROptionalType>(type);
             auto valueType = optionalType->getValueType();
+            info->optionalType = (IRType*)type;
             info->valueType = valueType;
-
-            auto structType = builder->createStructType();
-            info->loweredType = structType;
-            builder->addNameHintDecoration(structType, UnownedStringSlice("OptionalType"));
-
-            info->valueType = valueType;
-            auto valueKey = builder->createStructKey();
-            builder->addNameHintDecoration(valueKey, UnownedStringSlice("value"));
-            info->valueField = builder->createStructField(structType, valueKey, (IRType*)valueType);
-
-            auto boolType = builder->getBoolType();
-            auto hasValueKey = builder->createStructKey();
-            builder->addNameHintDecoration(hasValueKey, UnownedStringSlice("hasValue"));
-            info->hasValueField = builder->createStructField(structType, hasValueKey, (IRType*)boolType);
-
+            if (typeHasNullValue(valueType))
+            {
+                info->loweredType = valueType;
+            }
+            else
+            {
+                auto structType = builder->createStructType();
+                info->loweredType = structType;
+                builder->addNameHintDecoration(structType, UnownedStringSlice("OptionalType"));
+
+                info->valueType = valueType;
+                auto valueKey = builder->createStructKey();
+                builder->addNameHintDecoration(valueKey, UnownedStringSlice("value"));
+                info->valueField = builder->createStructField(structType, valueKey, (IRType*)valueType);
+
+                auto boolType = builder->getBoolType();
+                auto hasValueKey = builder->createStructKey();
+                builder->addNameHintDecoration(hasValueKey, UnownedStringSlice("hasValue"));
+                info->hasValueField = builder->createStructField(structType, hasValueKey, (IRType*)boolType);
+            }
             mapLoweredTypeToOptionalTypeInfo[info->loweredType] = info;
             loweredOptionalTypes[type] = info;
             return info.Ptr();
@@ -123,12 +146,19 @@ namespace Slang
         {
             auto loweredOptionalTypeInfo = getLoweredOptionalType(builder, optionalInst->getDataType());
             SLANG_ASSERT(loweredOptionalTypeInfo);
-
-            auto value = builder->emitFieldExtract(
-                builder->getBoolType(),
-                optionalInst,
-                loweredOptionalTypeInfo->hasValueField->getKey());
-            return value;
+            IRInst* result = nullptr;
+            if (loweredOptionalTypeInfo->loweredType != loweredOptionalTypeInfo->valueType)
+            {
+                result = builder->emitFieldExtract(
+                    builder->getBoolType(),
+                    optionalInst,
+                    loweredOptionalTypeInfo->hasValueField->getKey());
+            }
+            else
+            {
+                result = builder->emitCastPtrToBool(optionalInst);
+            }
+            return result;
         }
 
         void processGetOptionalHasValue(IROptionalHasValue* inst)
@@ -151,13 +181,20 @@ namespace Slang
 
             auto base = inst->getOptionalOperand();
             auto loweredOptionalTypeInfo = getLoweredOptionalType(builder, base->getDataType());
-            SLANG_ASSERT(loweredOptionalTypeInfo);
-            SLANG_ASSERT(loweredOptionalTypeInfo->valueField);
-            auto getElement = builder->emitFieldExtract(
-                loweredOptionalTypeInfo->valueType,
-                base,
-                loweredOptionalTypeInfo->valueField->getKey());
-            inst->replaceUsesWith(getElement);
+            if (loweredOptionalTypeInfo->loweredType != loweredOptionalTypeInfo->valueType)
+            {
+                SLANG_ASSERT(loweredOptionalTypeInfo);
+                SLANG_ASSERT(loweredOptionalTypeInfo->valueField);
+                auto getElement = builder->emitFieldExtract(
+                    loweredOptionalTypeInfo->valueType,
+                    base,
+                    loweredOptionalTypeInfo->valueField->getKey());
+                inst->replaceUsesWith(getElement);
+            }
+            else
+            {
+                inst->replaceUsesWith(base);
+            }
             inst->removeAndDeallocate();
         }
 
diff --git a/source/slang/slang-ir-specialize-dynamic-associatedtype-lookup.cpp b/source/slang/slang-ir-specialize-dynamic-associatedtype-lookup.cpp
index dbcc7a54c..441d32d24 100644
--- a/source/slang/slang-ir-specialize-dynamic-associatedtype-lookup.cpp
+++ b/source/slang/slang-ir-specialize-dynamic-associatedtype-lookup.cpp
@@ -3,6 +3,7 @@
 #include "slang-ir-generics-lowering-context.h"
 #include "slang-ir-insts.h"
 #include "slang-ir.h"
+#include "slang-ir-util.h"
 
 namespace Slang
 {
diff --git a/source/slang/slang-ir-util.cpp b/source/slang/slang-ir-util.cpp
index d91252fcc..7775fdb91 100644
--- a/source/slang/slang-ir-util.cpp
+++ b/source/slang/slang-ir-util.cpp
@@ -1,4 +1,5 @@
 #include "slang-ir-util.h"
+#include "slang-ir-insts.h"
 
 namespace Slang
 {
@@ -43,4 +44,24 @@ bool isPtrToArrayType(IRInst* type)
     return isPointerOfType(type, kIROp_ArrayType) || isPointerOfType(type, kIROp_UnsizedArrayType);
 }
 
+
+bool isComInterfaceType(IRType* type)
+{
+    if (!type) return false;
+    if (type->findDecoration<IRComInterfaceDecoration>() ||
+        type->getOp() == kIROp_ComPtrType)
+    {
+        return true;
+    }
+
+    if (auto ptrType = as<IRNativePtrType>(type))
+    {
+        auto valueType = ptrType->getValueType();
+        return valueType->findDecoration<IRComInterfaceDecoration>() != nullptr;
+    }
+
+    return false;
+}
+
+
 }
diff --git a/source/slang/slang-ir-util.h b/source/slang/slang-ir-util.h
index 5a09638d4..2300c929d 100644
--- a/source/slang/slang-ir-util.h
+++ b/source/slang/slang-ir-util.h
@@ -22,6 +22,8 @@ bool isPointerOfType(IRInst* ptrType, IROp opCode);
 // Builds a dictionary that maps from requirement key to requirement value for `interfaceType`.
 Dictionary<IRInst*, IRInst*> buildInterfaceRequirementDict(IRInterfaceType* interfaceType);
 
+bool isComInterfaceType(IRType* type);
+
 }
 
 #endif
diff --git a/source/slang/slang-ir-witness-table-wrapper.cpp b/source/slang/slang-ir-witness-table-wrapper.cpp
index 5d1a9360e..1d84eee19 100644
--- a/source/slang/slang-ir-witness-table-wrapper.cpp
+++ b/source/slang/slang-ir-witness-table-wrapper.cpp
@@ -5,6 +5,7 @@
 #include "slang-ir.h"
 #include "slang-ir-clone.h"
 #include "slang-ir-insts.h"
+#include "slang-ir-util.h"
 
 namespace Slang
 {
diff --git a/source/slang/slang-ir.cpp b/source/slang/slang-ir.cpp
index fd7cbe408..20bf6060d 100644
--- a/source/slang/slang-ir.cpp
+++ b/source/slang/slang-ir.cpp
@@ -4369,6 +4369,17 @@ namespace Slang
         return inst;
     }
 
+    IRInst* IRBuilder::emitCastPtrToBool(IRInst* val)
+    {
+        auto inst = createInst<IRInst>(
+            this,
+            kIROp_CastPtrToBool,
+            getBoolType(),
+            val);
+        addInst(inst);
+        return inst;
+    }
+
     IRGlobalConstant* IRBuilder::emitGlobalConstant(
         IRType* type)
     {
diff --git a/tests/cpu-program/gfx-smoke.slang b/tests/cpu-program/gfx-smoke.slang
index e04848f5d..fda6d5454 100644
--- a/tests/cpu-program/gfx-smoke.slang
+++ b/tests/cpu-program/gfx-smoke.slang
@@ -8,8 +8,8 @@ public __extern_cpp int main()
 {
     gfx.DeviceDesc deviceDesc = {};
     deviceDesc.deviceType = gfx.DeviceType.CPU;
-    gfx.IDevice obj;
-    if (gfx.succeeded(gfx.gfxCreateDevice(&deviceDesc, obj)))
+    Optional<gfx.IDevice> obj;
+    if (gfx.succeeded(gfx.gfxCreateDevice(&deviceDesc, obj)) && obj.hasValue)
         writeln("succ");
     else
         writeln("fail");
diff --git a/tools/gfx/gfx.slang b/tools/gfx/gfx.slang
index 7c6aefe79..02e31fad2 100644
--- a/tools/gfx/gfx.slang
+++ b/tools/gfx/gfx.slang
@@ -808,7 +808,7 @@ struct AccelerationStructureBuildInputs
 struct AccelerationStructureCreateDesc
 {
     AccelerationStructureKind kind;
-    IBufferResource *buffer;
+    NativeRef<IBufferResource> buffer;
     Offset offset;
     Size size;
 };
@@ -816,8 +816,8 @@ struct AccelerationStructureCreateDesc
 struct AccelerationStructureBuildDesc
 {
     AccelerationStructureBuildInputs inputs;
-    IAccelerationStructure *source;
-    IAccelerationStructure *dest;
+    NativeRef<IAccelerationStructure> source;
+    NativeRef<IAccelerationStructure> dest;
     DeviceAddress scratchData;
 };
 
@@ -889,7 +889,7 @@ interface IShaderObject
     Size getSize();
 
     /// Use the provided constant buffer instead of the internally created one.
-    Result setConstantBufferOverride(IBufferResource *constantBuffer);
+    Result setConstantBufferOverride(IBufferResource constantBuffer);
 };
 
 enum class StencilOp : uint8_t
@@ -1358,7 +1358,7 @@ interface IResourceCommandEncoder : ICommandEncoder
         ResourceState dstState,
         SubresourceRange dstSubresource,
         int3 dstOffset,
-        ITextureResource *src,
+        NativeRef<ITextureResource> src,
         ResourceState srcState,
         SubresourceRange srcSubresource,
         int3 srcOffset,
@@ -1422,7 +1422,7 @@ interface IRenderCommandEncoder : IResourceCommandEncoder
     Result bindPipeline(IPipelineState state, out IShaderObject outRootShaderObject);
 
     // Sets the current pipeline state along with a pre-created mutable root shader object.
-    Result bindPipelineWithRootObject(IPipelineState state, IShaderObject *rootObject);
+    Result bindPipelineWithRootObject(IPipelineState state, NativeRef<IShaderObject> rootObject);
 
     void setViewports(GfxCount count, Viewport *viewports);
     void setScissorRects(GfxCount count, ScissorRect *scissors);
@@ -1504,8 +1504,8 @@ interface IRayTracingCommandEncoder : IResourceCommandEncoder
                  GfxCount propertyQueryCount,
                  AccelerationStructureQueryDesc *queryDescs);
     void copyAccelerationStructure(
-        IAccelerationStructure *dest,
-        IAccelerationStructure *src,
+        NativeRef<IAccelerationStructure> dest,
+        NativeRef<IAccelerationStructure> src,
         AccelerationStructureCopyMode mode);
     void queryAccelerationStructureProperties(
         GfxCount accelerationStructureCount,
@@ -1523,7 +1523,7 @@ interface IRayTracingCommandEncoder : IResourceCommandEncoder
     /// `rayGenShaderIndex` specifies the index into the shader table that identifies the ray generation shader.
     void dispatchRays(
         GfxIndex rayGenShaderIndex,
-        IShaderTable *shaderTable,
+        NativeRef<IShaderTable> shaderTable,
         GfxCount width,
         GfxCount height,
         GfxCount depth);
@@ -1565,9 +1565,6 @@ struct CommandQueueDesc
 [COM("14e2bed0-0ad0-4dc8-b341-06-3f-e7-2d-bf-0e")]
 interface ICommandQueue
 {
-    // For D3D12, this is the pointer to the queue. For Vulkan, this is the queue itself.
-    typedef uint64_t NativeHandle;
-
     const CommandQueueDesc* getDesc();
 
     void executeCommandBuffers(
@@ -1936,7 +1933,7 @@ SLANG_GFX_IMPORT bool gfxIsTypelessFormat(Format format);
 SLANG_GFX_IMPORT Result gfxGetFormatInfo(Format format, FormatInfo *outInfo);
 
 /// Given a type returns a function that can construct it, or nullptr if there isn't one
-SLANG_GFX_IMPORT Result gfxCreateDevice(const DeviceDesc* desc, out IDevice outDevice);
+SLANG_GFX_IMPORT Result gfxCreateDevice(const DeviceDesc* desc, out Optional<IDevice> outDevice);
 
 /// Sets a callback for receiving debug messages.
 /// The layer does not hold a strong reference to the callback object.