Fix linux gfx issue (#2131)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Fix linux build issue with renderer-shared.h

* Move to .cpp file.

* Move some templates to stop definition order issue on linux.

* Move to header because it's templated.

* Fix warning about GeometryInstanceFlags::Enum flags : 8; being too small by making the enum backed by 8 bits.

* Check if vkDestroySampler function is available.

3 files changed, 181 insertions, 159 deletions

diff --git a/slang-gfx.h b/slang-gfx.h
index 809d588a1..5d59e2fda 100644
--- a/slang-gfx.h
+++ b/slang-gfx.h
@@ -909,7 +909,7 @@ public:
     {
         // The enum values are kept consistent with D3D12_RAYTRACING_INSTANCE_FLAGS
         // and VkGeometryInstanceFlagBitsKHR.
-        enum Enum : uint32_t
+        enum Enum : uint8_t
         {
             None = 0,
             TriangleFacingCullDisable = 0x00000001,
diff --git a/tools/gfx/renderer-shared.h b/tools/gfx/renderer-shared.h
index b27134306..aaf69a015 100644
--- a/tools/gfx/renderer-shared.h
+++ b/tools/gfx/renderer-shared.h
@@ -597,38 +597,7 @@ public:
         return SLANG_OK;
     }
 
-    void setSpecializationArgsForContainerElement(ExtendedShaderObjectTypeList& specializationArgs)
-    {
-        // Compute specialization args for the structured buffer object.
-        // If we haven't filled anything to `m_structuredBufferSpecializationArgs` yet,
-        // use `specializationArgs` directly.
-        if (m_structuredBufferSpecializationArgs.getCount() == 0)
-        {
-            m_structuredBufferSpecializationArgs = Slang::_Move(specializationArgs);
-        }
-        else
-        {
-            // If `m_structuredBufferSpecializationArgs` already contains some arguments, we
-            // need to check if they are the same as `specializationArgs`, and replace
-            // anything that is different with `__Dynamic` because we cannot specialize the
-            // buffer type if the element types are not the same.
-            SLANG_ASSERT(
-                m_structuredBufferSpecializationArgs.getCount() == specializationArgs.getCount());
-            auto device = getRenderer();
-            for (Slang::Index i = 0; i < m_structuredBufferSpecializationArgs.getCount(); i++)
-            {
-                if (m_structuredBufferSpecializationArgs[i].componentID !=
-                    specializationArgs[i].componentID)
-                {
-                    auto dynamicType = device->slangContext.session->getDynamicType();
-                    m_structuredBufferSpecializationArgs.componentIDs[i] =
-                        device->shaderCache.getComponentId(dynamicType);
-                    m_structuredBufferSpecializationArgs.components[i] =
-                        slang::SpecializationArg::fromType(dynamicType);
-                }
-            }
-        }
-    }
+    void setSpecializationArgsForContainerElement(ExtendedShaderObjectTypeList& specializationArgs);
 
     virtual SLANG_NO_THROW Result SLANG_MCALL
         setObject(ShaderOffset const& offset, IShaderObject* object) SLANG_OVERRIDE
@@ -795,26 +764,7 @@ public:
     Result getExtendedShaderTypeListFromSpecializationArgs(
         ExtendedShaderObjectTypeList& list,
         const slang::SpecializationArg* args,
-        uint32_t count)
-    {
-        auto device = getRenderer();
-        for (uint32_t i = 0; i < count; i++)
-        {
-            gfx::ExtendedShaderObjectType extendedType;
-            switch (args[i].kind)
-            {
-            case slang::SpecializationArg::Kind::Type:
-                extendedType.slangType = args[i].type;
-                extendedType.componentID = device->shaderCache.getComponentId(args[i].type);
-                break;
-            default:
-                SLANG_ASSERT(false && "Unexpected specialization argument kind.");
-                return SLANG_FAIL;
-            }
-            list.add(extendedType);
-        }
-        return SLANG_OK;
-    }
+        uint32_t count);
 
     virtual SLANG_NO_THROW Result SLANG_MCALL setSpecializationArgs(
         ShaderOffset const& offset,
@@ -860,111 +810,7 @@ public:
 
     // Appends all types that are used to specialize the element type of this shader object in
     // `args` list.
-    virtual Result collectSpecializationArgs(ExtendedShaderObjectTypeList& args) override
-    {
-        if (m_layout->getContainerType() != ShaderObjectContainerType::None)
-        {
-            args.addRange(m_structuredBufferSpecializationArgs);
-            return SLANG_OK;
-        }
-
-        auto device = getRenderer();
-        auto& subObjectRanges = getLayout()->getSubObjectRanges();
-        // The following logic is built on the assumption that all fields that involve
-        // existential types (and therefore require specialization) will results in a sub-object
-        // range in the type layout. This allows us to simply scan the sub-object ranges to find
-        // out all specialization arguments.
-        Slang::Index subObjectRangeCount = subObjectRanges.getCount();
-
-        for (Slang::Index subObjectRangeIndex = 0; subObjectRangeIndex < subObjectRangeCount;
-             subObjectRangeIndex++)
-        {
-            auto const& subObjectRange = subObjectRanges[subObjectRangeIndex];
-            auto const& bindingRange =
-                getLayout()->getBindingRange(subObjectRange.bindingRangeIndex);
-
-            Slang::Index oldArgsCount = args.getCount();
-
-            Slang::Index count = bindingRange.count;
-
-            for (Slang::Index subObjectIndexInRange = 0; subObjectIndexInRange < count;
-                 subObjectIndexInRange++)
-            {
-                ExtendedShaderObjectTypeList typeArgs;
-                Slang::Index objectIndex = bindingRange.subObjectIndex + subObjectIndexInRange;
-                auto subObject = m_objects[objectIndex];
-
-                if (!subObject)
-                    continue;
-
-                if (objectIndex < m_userProvidedSpecializationArgs.getCount() &&
-                    m_userProvidedSpecializationArgs[objectIndex])
-                {
-                    args.addRange(*m_userProvidedSpecializationArgs[objectIndex]);
-                    continue;
-                }
-
-                switch (bindingRange.bindingType)
-                {
-                case slang::BindingType::ExistentialValue:
-                    {
-                        // A binding type of `ExistentialValue` means the sub-object represents a
-                        // interface-typed field. In this case the specialization argument for this
-                        // field is the actual specialized type of the bound shader object. If the
-                        // shader object's type is an ordinary type without existential fields, then
-                        // the type argument will simply be the ordinary type. But if the sub
-                        // object's type is itself a specialized type, we need to make sure to use
-                        // that type as the specialization argument.
-
-                        ExtendedShaderObjectType specializedSubObjType;
-                        SLANG_RETURN_ON_FAIL(
-                            subObject->getSpecializedShaderObjectType(&specializedSubObjType));
-                        typeArgs.add(specializedSubObjType);
-                        break;
-                    }
-                case slang::BindingType::ParameterBlock:
-                case slang::BindingType::ConstantBuffer:
-                case slang::BindingType::RawBuffer:
-                case slang::BindingType::MutableRawBuffer:
-                    // Currently we only handle the case where the field's type is
-                    // `ParameterBlock<SomeStruct>` or `ConstantBuffer<SomeStruct>`, where
-                    // `SomeStruct` is a struct type (not directly an interface type). In this case,
-                    // we just recursively collect the specialization arguments from the bound sub
-                    // object.
-                    SLANG_RETURN_ON_FAIL(subObject->collectSpecializationArgs(typeArgs));
-                    // TODO: we need to handle the case where the field is of the form
-                    // `ParameterBlock<IFoo>`. We should treat this case the same way as the
-                    // `ExistentialValue` case here, but currently we lack a mechanism to
-                    // distinguish the two scenarios.
-                    break;
-                }
-
-                auto addedTypeArgCountForCurrentRange = args.getCount() - oldArgsCount;
-                if (addedTypeArgCountForCurrentRange == 0)
-                {
-                    args.addRange(typeArgs);
-                }
-                else
-                {
-                    // If type arguments for each elements in the array is different, use
-                    // `__Dynamic` type for the differing argument to disable specialization.
-                    SLANG_ASSERT(addedTypeArgCountForCurrentRange == typeArgs.getCount());
-                    for (Slang::Index i = 0; i < addedTypeArgCountForCurrentRange; i++)
-                    {
-                        if (args[i + oldArgsCount].componentID != typeArgs[i].componentID)
-                        {
-                            auto dynamicType = device->slangContext.session->getDynamicType();
-                            args.componentIDs[i + oldArgsCount] =
-                                device->shaderCache.getComponentId(dynamicType);
-                            args.components[i + oldArgsCount] =
-                                slang::SpecializationArg::fromType(dynamicType);
-                        }
-                    }
-                }
-            }
-        }
-        return SLANG_OK;
-    }
+    virtual Result collectSpecializationArgs(ExtendedShaderObjectTypeList& args) override;
 };
 
 class ShaderProgramBase : public IShaderProgram, public Slang::ComObject
@@ -1417,4 +1263,177 @@ inline IDebugCallback* getDebugCallback()
         return _getNullDebugCallback();
     }
 }
+
+
+// Implementations that have to come after RendererBase
+
+//--------------------------------------------------------------------------------
+template<typename TShaderObjectImpl, typename TShaderObjectLayoutImpl, typename TShaderObjectData>
+void ShaderObjectBaseImpl<TShaderObjectImpl, TShaderObjectLayoutImpl, TShaderObjectData>::setSpecializationArgsForContainerElement(ExtendedShaderObjectTypeList& specializationArgs)
+{
+    // Compute specialization args for the structured buffer object.
+    // If we haven't filled anything to `m_structuredBufferSpecializationArgs` yet,
+    // use `specializationArgs` directly.
+    if (m_structuredBufferSpecializationArgs.getCount() == 0)
+    {
+        m_structuredBufferSpecializationArgs = Slang::_Move(specializationArgs);
+    }
+    else
+    {
+        // If `m_structuredBufferSpecializationArgs` already contains some arguments, we
+        // need to check if they are the same as `specializationArgs`, and replace
+        // anything that is different with `__Dynamic` because we cannot specialize the
+        // buffer type if the element types are not the same.
+        SLANG_ASSERT(
+            m_structuredBufferSpecializationArgs.getCount() == specializationArgs.getCount());
+        auto device = getRenderer();
+        for (Slang::Index i = 0; i < m_structuredBufferSpecializationArgs.getCount(); i++)
+        {
+            if (m_structuredBufferSpecializationArgs[i].componentID !=
+                specializationArgs[i].componentID)
+            {
+                auto dynamicType = device->slangContext.session->getDynamicType();
+                m_structuredBufferSpecializationArgs.componentIDs[i] =
+                    device->shaderCache.getComponentId(dynamicType);
+                m_structuredBufferSpecializationArgs.components[i] =
+                    slang::SpecializationArg::fromType(dynamicType);
+            }
+        }
+    }
+}
+
+//--------------------------------------------------------------------------------
+template<typename TShaderObjectImpl, typename TShaderObjectLayoutImpl, typename TShaderObjectData>
+Result ShaderObjectBaseImpl<TShaderObjectImpl, TShaderObjectLayoutImpl, TShaderObjectData>::getExtendedShaderTypeListFromSpecializationArgs(
+    ExtendedShaderObjectTypeList& list,
+    const slang::SpecializationArg* args,
+    uint32_t count)
+{
+    auto device = getRenderer();
+    for (uint32_t i = 0; i < count; i++)
+    {
+        gfx::ExtendedShaderObjectType extendedType;
+        switch (args[i].kind)
+        {
+            case slang::SpecializationArg::Kind::Type:
+                extendedType.slangType = args[i].type;
+                extendedType.componentID = device->shaderCache.getComponentId(args[i].type);
+                break;
+            default:
+                SLANG_ASSERT(false && "Unexpected specialization argument kind.");
+                return SLANG_FAIL;
+        }
+        list.add(extendedType);
+    }
+    return SLANG_OK;
+}
+
+//--------------------------------------------------------------------------------
+template<typename TShaderObjectImpl, typename TShaderObjectLayoutImpl, typename TShaderObjectData>
+Result ShaderObjectBaseImpl<TShaderObjectImpl, TShaderObjectLayoutImpl, TShaderObjectData>::collectSpecializationArgs(ExtendedShaderObjectTypeList& args)
+{
+    if (m_layout->getContainerType() != ShaderObjectContainerType::None)
+    {
+        args.addRange(m_structuredBufferSpecializationArgs);
+        return SLANG_OK;
+    }
+
+    auto device = getRenderer();
+    auto& subObjectRanges = getLayout()->getSubObjectRanges();
+    // The following logic is built on the assumption that all fields that involve
+    // existential types (and therefore require specialization) will results in a sub-object
+    // range in the type layout. This allows us to simply scan the sub-object ranges to find
+    // out all specialization arguments.
+    Slang::Index subObjectRangeCount = subObjectRanges.getCount();
+
+    for (Slang::Index subObjectRangeIndex = 0; subObjectRangeIndex < subObjectRangeCount;
+        subObjectRangeIndex++)
+    {
+        auto const& subObjectRange = subObjectRanges[subObjectRangeIndex];
+        auto const& bindingRange =
+            getLayout()->getBindingRange(subObjectRange.bindingRangeIndex);
+
+        Slang::Index oldArgsCount = args.getCount();
+
+        Slang::Index count = bindingRange.count;
+
+        for (Slang::Index subObjectIndexInRange = 0; subObjectIndexInRange < count;
+            subObjectIndexInRange++)
+        {
+            ExtendedShaderObjectTypeList typeArgs;
+            Slang::Index objectIndex = bindingRange.subObjectIndex + subObjectIndexInRange;
+            auto subObject = m_objects[objectIndex];
+
+            if (!subObject)
+                continue;
+
+            if (objectIndex < m_userProvidedSpecializationArgs.getCount() &&
+                m_userProvidedSpecializationArgs[objectIndex])
+            {
+                args.addRange(*m_userProvidedSpecializationArgs[objectIndex]);
+                continue;
+            }
+
+            switch (bindingRange.bindingType)
+            {
+                case slang::BindingType::ExistentialValue:
+                {
+                    // A binding type of `ExistentialValue` means the sub-object represents a
+                    // interface-typed field. In this case the specialization argument for this
+                    // field is the actual specialized type of the bound shader object. If the
+                    // shader object's type is an ordinary type without existential fields, then
+                    // the type argument will simply be the ordinary type. But if the sub
+                    // object's type is itself a specialized type, we need to make sure to use
+                    // that type as the specialization argument.
+
+                    ExtendedShaderObjectType specializedSubObjType;
+                    SLANG_RETURN_ON_FAIL(
+                        subObject->getSpecializedShaderObjectType(&specializedSubObjType));
+                    typeArgs.add(specializedSubObjType);
+                    break;
+                }
+                case slang::BindingType::ParameterBlock:
+                case slang::BindingType::ConstantBuffer:
+                case slang::BindingType::RawBuffer:
+                case slang::BindingType::MutableRawBuffer:
+                    // Currently we only handle the case where the field's type is
+                    // `ParameterBlock<SomeStruct>` or `ConstantBuffer<SomeStruct>`, where
+                    // `SomeStruct` is a struct type (not directly an interface type). In this case,
+                    // we just recursively collect the specialization arguments from the bound sub
+                    // object.
+                    SLANG_RETURN_ON_FAIL(subObject->collectSpecializationArgs(typeArgs));
+                    // TODO: we need to handle the case where the field is of the form
+                    // `ParameterBlock<IFoo>`. We should treat this case the same way as the
+                    // `ExistentialValue` case here, but currently we lack a mechanism to
+                    // distinguish the two scenarios.
+                    break;
+            }
+
+            auto addedTypeArgCountForCurrentRange = args.getCount() - oldArgsCount;
+            if (addedTypeArgCountForCurrentRange == 0)
+            {
+                args.addRange(typeArgs);
+            }
+            else
+            {
+                // If type arguments for each elements in the array is different, use
+                // `__Dynamic` type for the differing argument to disable specialization.
+                SLANG_ASSERT(addedTypeArgCountForCurrentRange == typeArgs.getCount());
+                for (Slang::Index i = 0; i < addedTypeArgCountForCurrentRange; i++)
+                {
+                    if (args[i + oldArgsCount].componentID != typeArgs[i].componentID)
+                    {
+                        auto dynamicType = device->slangContext.session->getDynamicType();
+                        args.componentIDs[i + oldArgsCount] =
+                            device->shaderCache.getComponentId(dynamicType);
+                        args.components[i + oldArgsCount] =
+                            slang::SpecializationArg::fromType(dynamicType);
+                    }
+                }
+            }
+        }
+    }
+    return SLANG_OK;
+}
+
 }
diff --git a/tools/gfx/vulkan/render-vk.cpp b/tools/gfx/vulkan/render-vk.cpp
index eb4044639..3d77fc50b 100644
--- a/tools/gfx/vulkan/render-vk.cpp
+++ b/tools/gfx/vulkan/render-vk.cpp
@@ -6395,7 +6395,10 @@ VKDevice::~VKDevice()
     shaderCache.free();
     m_deviceObjectsWithPotentialBackReferences.clearAndDeallocate();
 
-    m_api.vkDestroySampler(m_device, m_defaultSampler, nullptr);
+    if (m_api.vkDestroySampler)
+    {
+        m_api.vkDestroySampler(m_device, m_defaultSampler, nullptr);
+    }
 
     m_deviceQueue.destroy();