Enable swiftshader in linux CI builds (#1909)

5 files changed, 121 insertions, 89 deletions

diff --git a/tools/gfx/vulkan/render-vk.cpp b/tools/gfx/vulkan/render-vk.cpp
index 703c0665f..64263887d 100644
--- a/tools/gfx/vulkan/render-vk.cpp
+++ b/tools/gfx/vulkan/render-vk.cpp
@@ -5200,6 +5200,8 @@ VkPipelineShaderStageCreateInfo VKDevice::compileEntryPoint(
 
 Result VKDevice::initVulkanInstanceAndDevice(bool useValidationLayer)
 {
+    m_features.clear();
+
     m_queueAllocCount = 0;
 
     VkInstance instance = VK_NULL_HANDLE;
@@ -5210,95 +5212,91 @@ Result VKDevice::initVulkanInstanceAndDevice(bool useValidationLayer)
     applicationInfo.engineVersion = 1;
     applicationInfo.applicationVersion = 1;
 
-    for (int tryUseSurfaceExtensions = 1; tryUseSurfaceExtensions >= 0; tryUseSurfaceExtensions--)
-    {
-        Array<const char*, 4> instanceExtensions;
+    Array<const char*, 4> instanceExtensions;
 
-        instanceExtensions.add(VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME);
-        if (tryUseSurfaceExtensions)
-        {
-            instanceExtensions.add(VK_KHR_SURFACE_EXTENSION_NAME);
+    instanceExtensions.add(VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME);
+
+    // Software (swiftshader) implementation currently does not support surface extension,
+    // so only use it with a hardware implementation.
+    if (!m_api.m_module->isSoftware())
+    {
+        instanceExtensions.add(VK_KHR_SURFACE_EXTENSION_NAME);
 #if SLANG_WINDOWS_FAMILY
-            instanceExtensions.add(VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
+        instanceExtensions.add(VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
 #elif defined(SLANG_ENABLE_XLIB)
-            instanceExtensions.add(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
+        instanceExtensions.add(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
 #endif
 #if ENABLE_VALIDATION_LAYER
-            instanceExtensions.add(VK_EXT_DEBUG_REPORT_EXTENSION_NAME);
+        instanceExtensions.add(VK_EXT_DEBUG_REPORT_EXTENSION_NAME);
 #endif
-        }
+    }
 
-        VkInstanceCreateInfo instanceCreateInfo = {VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO};
-        instanceCreateInfo.pApplicationInfo = &applicationInfo;
-        instanceCreateInfo.enabledExtensionCount = (uint32_t)instanceExtensions.getCount();
-        instanceCreateInfo.ppEnabledExtensionNames = &instanceExtensions[0];
+    VkInstanceCreateInfo instanceCreateInfo = {VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO};
+    instanceCreateInfo.pApplicationInfo = &applicationInfo;
+    instanceCreateInfo.enabledExtensionCount = (uint32_t)instanceExtensions.getCount();
+    instanceCreateInfo.ppEnabledExtensionNames = &instanceExtensions[0];
 
-        if (useValidationLayer)
+    if (useValidationLayer)
+    {
+        // Depending on driver version, validation layer may or may not exist.
+        // Newer drivers comes with "VK_LAYER_KHRONOS_validation", while older
+        // drivers provide only the deprecated
+        // "VK_LAYER_LUNARG_standard_validation" layer.
+        // We will check what layers are available, and use the newer
+        // "VK_LAYER_KHRONOS_validation" layer when possible.
+        uint32_t layerCount;
+        m_api.vkEnumerateInstanceLayerProperties(&layerCount, nullptr);
+
+        List<VkLayerProperties> availableLayers;
+        availableLayers.setCount(layerCount);
+        m_api.vkEnumerateInstanceLayerProperties(&layerCount, availableLayers.getBuffer());
+
+        const char* layerNames[] = {nullptr};
+        for (auto& layer : availableLayers)
+        {
+            if (strncmp(
+                    layer.layerName,
+                    "VK_LAYER_KHRONOS_validation",
+                    sizeof("VK_LAYER_KHRONOS_validation")) == 0)
+            {
+                layerNames[0] = "VK_LAYER_KHRONOS_validation";
+                break;
+            }
+        }
+        // On older drivers, only "VK_LAYER_LUNARG_standard_validation" exists,
+        // so we try to use it if we can't find "VK_LAYER_KHRONOS_validation".
+        if (!layerNames[0])
         {
-            // Depending on driver version, validation layer may or may not exist.
-            // Newer drivers comes with "VK_LAYER_KHRONOS_validation", while older
-            // drivers provide only the deprecated
-            // "VK_LAYER_LUNARG_standard_validation" layer.
-            // We will check what layers are available, and use the newer
-            // "VK_LAYER_KHRONOS_validation" layer when possible.
-            uint32_t layerCount;
-            m_api.vkEnumerateInstanceLayerProperties(&layerCount, nullptr);
-
-            List<VkLayerProperties> availableLayers;
-            availableLayers.setCount(layerCount);
-            m_api.vkEnumerateInstanceLayerProperties(&layerCount, availableLayers.getBuffer());
-
-            const char* layerNames[] = {nullptr};
             for (auto& layer : availableLayers)
             {
                 if (strncmp(
                         layer.layerName,
-                        "VK_LAYER_KHRONOS_validation",
-                        sizeof("VK_LAYER_KHRONOS_validation")) == 0)
+                        "VK_LAYER_LUNARG_standard_validation",
+                        sizeof("VK_LAYER_LUNARG_standard_validation")) == 0)
                 {
-                    layerNames[0] = "VK_LAYER_KHRONOS_validation";
+                    layerNames[0] = "VK_LAYER_LUNARG_standard_validation";
                     break;
                 }
             }
-            // On older drivers, only "VK_LAYER_LUNARG_standard_validation" exists,
-            // so we try to use it if we can't find "VK_LAYER_KHRONOS_validation".
-            if (!layerNames[0])
-            {
-                for (auto& layer : availableLayers)
-                {
-                    if (strncmp(
-                            layer.layerName,
-                            "VK_LAYER_LUNARG_standard_validation",
-                            sizeof("VK_LAYER_LUNARG_standard_validation")) == 0)
-                    {
-                        layerNames[0] = "VK_LAYER_LUNARG_standard_validation";
-                        break;
-                    }
-                }
-            }
-            if (layerNames[0])
-            {
-                instanceCreateInfo.enabledLayerCount = SLANG_COUNT_OF(layerNames);
-                instanceCreateInfo.ppEnabledLayerNames = layerNames;
-            }
         }
-        uint32_t apiVersionsToTry[] = {VK_API_VERSION_1_2, VK_API_VERSION_1_1, VK_API_VERSION_1_0};
-        for (auto apiVersion : apiVersionsToTry)
+        if (layerNames[0])
         {
-            applicationInfo.apiVersion = apiVersion;
-            if (m_api.vkCreateInstance(&instanceCreateInfo, nullptr, &instance) == VK_SUCCESS)
-            {
-                break;
-            }
+            instanceCreateInfo.enabledLayerCount = SLANG_COUNT_OF(layerNames);
+            instanceCreateInfo.ppEnabledLayerNames = layerNames;
         }
-        if (instance)
+    }
+    uint32_t apiVersionsToTry[] = {VK_API_VERSION_1_2, VK_API_VERSION_1_1, VK_API_VERSION_1_0};
+    for (auto apiVersion : apiVersionsToTry)
+    {
+        applicationInfo.apiVersion = apiVersion;
+        if (m_api.vkCreateInstance(&instanceCreateInfo, nullptr, &instance) == VK_SUCCESS)
+        {
             break;
+        }
     }
     if (!instance)
         return SLANG_FAIL;
-
     SLANG_RETURN_ON_FAIL(m_api.initInstanceProcs(instance));
-
     if (useValidationLayer && m_api.vkCreateDebugReportCallbackEXT)
     {
         VkDebugReportFlagsEXT debugFlags = VK_DEBUG_REPORT_ERROR_BIT_EXT | VK_DEBUG_REPORT_WARNING_BIT_EXT;
@@ -5589,7 +5587,6 @@ Result VKDevice::initVulkanInstanceAndDevice(bool useValidationLayer)
 
     deviceCreateInfo.enabledExtensionCount = uint32_t(deviceExtensions.getCount());
     deviceCreateInfo.ppEnabledExtensionNames = deviceExtensions.getBuffer();
- 
     if (m_api.vkCreateDevice(m_api.m_physicalDevice, &deviceCreateInfo, nullptr, &m_device) != VK_SUCCESS)
         return SLANG_FAIL;
     SLANG_RETURN_ON_FAIL(m_api.initDeviceProcs(m_device));
@@ -5612,12 +5609,20 @@ SlangResult VKDevice::initialize(const Desc& desc)
     m_desc = desc;
 
     SLANG_RETURN_ON_FAIL(RendererBase::initialize(desc));
+    SlangResult initDeviceResult = SLANG_OK;
+    for (int forceSoftware = 0; forceSoftware <= 1; forceSoftware++)
+    {
+        if (m_module.init(forceSoftware != 0) != SLANG_OK)
+            continue;
+        if (m_api.initGlobalProcs(m_module) != SLANG_OK)
+            continue;
+        descriptorSetAllocator.m_api = &m_api;
+        initDeviceResult = initVulkanInstanceAndDevice(ENABLE_VALIDATION_LAYER != 0);
+        if (initDeviceResult == SLANG_OK)
+            break;
+    }
+    SLANG_RETURN_ON_FAIL(initDeviceResult);
 
-    SLANG_RETURN_ON_FAIL(m_module.init());
-    SLANG_RETURN_ON_FAIL(m_api.initGlobalProcs(m_module));
-    descriptorSetAllocator.m_api = &m_api;
-
-    SLANG_RETURN_ON_FAIL(initVulkanInstanceAndDevice(ENABLE_VALIDATION_LAYER != 0));
     {
         VkQueue queue;
         m_api.vkGetDeviceQueue(m_device, m_queueFamilyIndex, 0, &queue);
diff --git a/tools/gfx/vulkan/vk-module.cpp b/tools/gfx/vulkan/vk-module.cpp
index 985df1a7b..cff75569e 100644
--- a/tools/gfx/vulkan/vk-module.cpp
+++ b/tools/gfx/vulkan/vk-module.cpp
@@ -11,45 +11,38 @@
 #   include <dlfcn.h>
 #endif
 
+#include "../renderer-shared.h"
+
 namespace gfx {
 using namespace Slang;
 
 // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! VulkanModule !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-Slang::Result VulkanModule::init()
+Slang::Result VulkanModule::init(bool useSoftwareImpl)
 {
     if (isInitialized())
     {
         destroy();
-        return SLANG_OK;
     }
 
     const char* dynamicLibraryName = "Unknown";
+    m_isSoftware = useSoftwareImpl;
 
 #if SLANG_WINDOWS_FAMILY
-    dynamicLibraryName = "vulkan-1.dll";
+    dynamicLibraryName = useSoftwareImpl ? "vk_swiftshader.dll" : "vulkan-1.dll";
     HMODULE module = ::LoadLibraryA(dynamicLibraryName);
     m_module = (void*)module;
 #else
-    dynamicLibraryName = "libvulkan.so.1";
-    m_module = dlopen(dynamicLibraryName, RTLD_NOW);
-#endif
-
-    if (!m_module)
+    dynamicLibraryName = useSoftwareImpl ? "libvk_swiftshader.so" : "libvulkan.so.1";
+    if (useSoftwareImpl)
     {
-        dynamicLibraryName = "vk_swiftshader";
-#if SLANG_WINDOWS_FAMILY
-        HMODULE swiftShaderModule = ::LoadLibraryA(dynamicLibraryName);
-        m_module = (void*)swiftShaderModule;
-#else
-        m_module = dlopen(dynamicLibraryName, RTLD_NOW);
-#endif
-        m_isSoftware = true;
+        dlopen("libpthread.so.0", RTLD_NOW | RTLD_GLOBAL);
     }
+    m_module = dlopen(dynamicLibraryName, RTLD_NOW);
+#endif
 
     if (!m_module)
     {
-        fprintf(stderr, "error: failed load '%s'\n", dynamicLibraryName);
         return SLANG_FAIL;
     }
 
diff --git a/tools/gfx/vulkan/vk-module.h b/tools/gfx/vulkan/vk-module.h
index eb1b3d570..f07f02b72 100644
--- a/tools/gfx/vulkan/vk-module.h
+++ b/tools/gfx/vulkan/vk-module.h
@@ -29,7 +29,7 @@ struct VulkanModule
     bool isSoftware() const { return m_isSoftware; }
 
         /// Initialize
-    Slang::Result init();
+    Slang::Result init(bool useSoftwareImpl);
         /// Destroy
     void destroy();
 
diff --git a/tools/render-test/render-test-main.cpp b/tools/render-test/render-test-main.cpp
index d71628fd0..96cc94aa3 100644
--- a/tools/render-test/render-test-main.cpp
+++ b/tools/render-test/render-test-main.cpp
@@ -22,7 +22,6 @@
 #include "window.h"
 
 #include "../../source/core/slang-test-tool-util.h"
-
 #define ENABLE_RENDERDOC_INTEGRATION 0
 
 #if ENABLE_RENDERDOC_INTEGRATION
@@ -1120,6 +1119,23 @@ static void renderDocBeginFrame(){}
 static void renderDocEndFrame(){}
 #endif
 
+class StdWritersDebugCallback : public gfx::IDebugCallback
+{
+public:
+    Slang::StdWriters* writers;
+    virtual SLANG_NO_THROW void SLANG_MCALL handleMessage(
+        gfx::DebugMessageType type,
+        gfx::DebugMessageSource source,
+        const char* message) override
+    {
+        SLANG_UNUSED(source);
+        if (type == gfx::DebugMessageType::Error)
+        {
+            writers->getOut().print("%s\n", message);
+        }
+    }
+};
+
 static SlangResult _innerMain(Slang::StdWriters* stdWriters, SlangSession* session, int argcIn, const char*const* argvIn)
 {
     using namespace renderer_test;
@@ -1245,6 +1261,9 @@ static SlangResult _innerMain(Slang::StdWriters* stdWriters, SlangSession* sessi
 #ifdef _DEBUG
     gfxEnableDebugLayer();
 #endif
+    StdWritersDebugCallback debugCallback;
+    debugCallback.writers = stdWriters;
+    gfxSetDebugCallback(&debugCallback);
 
     // Use the profile name set on options if set
     input.profile = options.profileName.getLength() ? options.profileName : input.profile;
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index 5e0d1142d..f477faf57 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -918,6 +918,15 @@ static RenderApiFlags _getAvailableRenderApiFlags(TestContext* context)
                     && TestToolUtil::getReturnCodeFromInt(exeRes.resultCode) == ToolReturnCode::Success)
                 {
                     availableRenderApiFlags |= RenderApiFlags(1) << int(apiType);
+                    StdWriters::getOut().print(
+                        "Check %s: Supported\n", RenderApiUtil::getApiName(apiType).begin());
+                }
+                else
+                {
+                    StdWriters::getOut().print(
+                        "Check %s: Not Supported\n", RenderApiUtil::getApiName(apiType).begin());
+                    StdWriters::getOut().print(
+                        "%s\n%s\n", exeRes.standardError.getBuffer(), exeRes.standardOutput.getBuffer());
                 }
             }
         }
@@ -3440,22 +3449,28 @@ SlangResult innerMain(int argc, char** argv)
             }
         }
 
+        StdWriters::getOut().print("Supported backends:");
         if (context.availableBackendFlags & PassThroughFlag::Fxc)
         {
+            StdWriters::getOut().print(" fxc");
             fxcCategory = categorySet.add("fxc", fullTestCategory);
         }
         if (context.availableBackendFlags & PassThroughFlag::Glslang)
         {
+            StdWriters::getOut().print(" glslang");
             glslangCategory = categorySet.add("glslang", fullTestCategory);
         }
         if (context.availableBackendFlags & PassThroughFlag::Dxc)
         {
+            StdWriters::getOut().print(" dxc");
             dxcCategory = categorySet.add("dxc", fullTestCategory);
         }
         if (context.availableBackendFlags & PassThroughFlag::NVRTC)
         {
+            StdWriters::getOut().print(" nvrtc");
             nvrtcCategory = categorySet.add("nvrtc", fullTestCategory);
         }
+        StdWriters::getOut().print("\n");
     }
 
     // Working out what renderApis is worked on on demand through