Dx11 & Dx12 device startup (#861)

* Added CombinationUtil to produce combinations of flags
Used in Dx11 device creation making it fall back to release driver if debug driver is not found

* Made dx12 renderer startup similar to dx11 - testing multiple configs.

* Small improvements in naming.

* * Moved functionality to gfx from core
* Use FlagCombiner to simplify construction, and can be iterated over, without need for array
* Share DeviceCheckFlags

* Improve comments.

* Re-add the comment about combinations tested to set up dx11 device.

* More comment improvements.

1 files changed, 96 insertions, 82 deletions

diff --git a/tools/gfx/render-d3d12.cpp b/tools/gfx/render-d3d12.cpp
index e8af974fa..ec9695cb1 100644
--- a/tools/gfx/render-d3d12.cpp
+++ b/tools/gfx/render-d3d12.cpp
@@ -26,6 +26,7 @@
 #include <d3dcompiler.h>
 
 #include "../../slang-com-ptr.h"
+#include "flag-combiner.h"
 
 #include "resource-d3d12.h"
 #include "descriptor-heap-d3d12.h"
@@ -101,6 +102,7 @@ public:
     ~D3D12Renderer();
 
 protected:
+    
     static const Int kMaxNumRenderFrames = 4;
     static const Int kMaxNumRenderTargets = 3;
 
@@ -477,8 +479,8 @@ protected:
 //    Result _calcBindParameters(BindParameters& params);
 //    RenderState* findRenderState(PipelineType pipelineType);
 
-    PFN_D3D12_SERIALIZE_ROOT_SIGNATURE m_D3D12SerializeRootSignature = nullptr;
-
+    Result _createAdaptor(DeviceCheckFlags deviceCheckFlags, ComPtr<IDXGIFactory4>& outDxgiFactory, ComPtr<IDXGIAdapter>& outAdapter);
+    
     D3D12CircularResourceHeap m_circularResourceHeap;
 
     int m_commandListOpenCount = 0;            ///< If >0 the command list should be open
@@ -560,6 +562,14 @@ protected:
     int32_t m_depthStencilUsageFlags = 0;    ///< D3DUtil::UsageFlag combination for depth stencil
     int32_t m_targetUsageFlags = 0;            ///< D3DUtil::UsageFlag combination for target
 
+    // Dll entry points
+    typedef HRESULT(WINAPI *PFN_DXGI_CREATE_FACTORY_2)(UINT Flags, REFIID riid, _COM_Outptr_ void **ppFactory);
+
+    PFN_D3D12_GET_DEBUG_INTERFACE m_D3D12GetDebugInterface = nullptr;
+    PFN_DXGI_CREATE_FACTORY_2 m_CreateDXGIFactory2 = nullptr;
+    PFN_D3D12_CREATE_DEVICE m_D3D12CreateDevice = nullptr;
+    PFN_D3D12_SERIALIZE_ROOT_SIGNATURE m_D3D12SerializeRootSignature = nullptr;
+
     HWND m_hwnd = nullptr;
 };
 
@@ -1278,82 +1288,24 @@ Result D3D12Renderer::_bindRenderState(PipelineStateImpl* pipelineStateImpl, ID3
 
 // !!!!!!!!!!!!!!!!!!!!!!!!!!!! Renderer interface !!!!!!!!!!!!!!!!!!!!!!!!!!
 
-Result D3D12Renderer::initialize(const Desc& desc, void* inWindowHandle)
-{
-    m_hwnd = (HWND)inWindowHandle;
-    // Rather than statically link against D3D, we load it dynamically.
 
-    HMODULE d3dModule = LoadLibraryA("d3d12.dll");
-    if (!d3dModule)
-    {
-        fprintf(stderr, "error: failed load 'd3d12.dll'\n");
-        return SLANG_FAIL;
-    }
-
-    HMODULE dxgiModule = LoadLibraryA("Dxgi.dll");
-    if (!dxgiModule)
-    {
-        fprintf(stderr, "error: failed load 'dxgi.dll'\n");
-        return SLANG_FAIL;
-    }
-
-
-#define LOAD_D3D_PROC(TYPE, NAME) \
-        TYPE NAME##_ = (TYPE) loadProc(d3dModule, #NAME);
-#define LOAD_DXGI_PROC(TYPE, NAME) \
-        TYPE NAME##_ = (TYPE) loadProc(dxgiModule, #NAME);
-
-    UINT dxgiFactoryFlags = 0;
-
-#if ENABLE_DEBUG_LAYER
-    {
-        LOAD_D3D_PROC(PFN_D3D12_GET_DEBUG_INTERFACE, D3D12GetDebugInterface);
-        if (D3D12GetDebugInterface_)
-        {
-            if (SUCCEEDED(D3D12GetDebugInterface_(IID_PPV_ARGS(m_dxDebug.writeRef()))))
-            {
-                m_dxDebug->EnableDebugLayer();
-                dxgiFactoryFlags |= DXGI_CREATE_FACTORY_DEBUG;
-            }
-        }
-    }
-#endif
-
-    m_D3D12SerializeRootSignature = (PFN_D3D12_SERIALIZE_ROOT_SIGNATURE)loadProc(d3dModule, "D3D12SerializeRootSignature");
-    if (!m_D3D12SerializeRootSignature)
-    {
-        return SLANG_FAIL;
-    }
+Result D3D12Renderer::_createAdaptor(DeviceCheckFlags deviceCheckFlags, ComPtr<IDXGIFactory4>& outDxgiFactory, ComPtr<IDXGIAdapter>& outAdapter)
+{
+    const UINT dxgiFactoryFlags = (deviceCheckFlags & DeviceCheckFlag::UseDebug) ? DXGI_CREATE_FACTORY_DEBUG  : 0;
 
     // Try and create DXGIFactory
     ComPtr<IDXGIFactory4> dxgiFactory;
-    {
-        typedef HRESULT(WINAPI *PFN_DXGI_CREATE_FACTORY_2)(UINT Flags, REFIID riid, _COM_Outptr_ void **ppFactory);
-        LOAD_DXGI_PROC(PFN_DXGI_CREATE_FACTORY_2, CreateDXGIFactory2);
-        if (!CreateDXGIFactory2_)
-        {
-            return SLANG_FAIL;
-        }
-        SLANG_RETURN_ON_FAIL(CreateDXGIFactory2_(dxgiFactoryFlags, IID_PPV_ARGS(dxgiFactory.writeRef())));
-    }
-
-    D3D_FEATURE_LEVEL featureLevel = D3D_FEATURE_LEVEL_11_0;
+    SLANG_RETURN_ON_FAIL(m_CreateDXGIFactory2(dxgiFactoryFlags, IID_PPV_ARGS(dxgiFactory.writeRef())));
+    
+    D3D_FEATURE_LEVEL featureLevel = DeviceCheckFlag::UseFullFeatureLevel ? D3D_FEATURE_LEVEL_11_1 : D3D_FEATURE_LEVEL_11_0;
 
     // Search for an adapter that meets our requirements
     ComPtr<IDXGIAdapter> adapter;
 
-    LOAD_D3D_PROC(PFN_D3D12_CREATE_DEVICE, D3D12CreateDevice);
-    if (!D3D12CreateDevice_)
-    {
-        return SLANG_FAIL;
-    }
-
-    const bool useWarp = false;
-
-    if (useWarp)
+    if ((deviceCheckFlags & DeviceCheckFlag::UseHardwareDevice) == 0)
     {
         SLANG_RETURN_ON_FAIL(dxgiFactory->EnumWarpAdapter(IID_PPV_ARGS(adapter.writeRef())));
-        SLANG_RETURN_ON_FAIL(D3D12CreateDevice_(adapter, featureLevel, IID_PPV_ARGS(m_device.writeRef())));
+        SLANG_RETURN_ON_FAIL(m_D3D12CreateDevice(adapter, featureLevel, IID_PPV_ARGS(m_device.writeRef())));
     }
     else
     {
@@ -1371,7 +1323,6 @@ Result D3D12Renderer::initialize(const Desc& desc, void* inWindowHandle)
 
             if (desc.Flags & DXGI_ADAPTER_FLAG_SOFTWARE)
             {
-                
                 // TODO: may want to allow software driver as fallback
             }
             else
@@ -1379,7 +1330,7 @@ Result D3D12Renderer::initialize(const Desc& desc, void* inWindowHandle)
                 continue;
             }
 
-            if (SUCCEEDED(D3D12CreateDevice_(candidateAdapter, featureLevel, IID_PPV_ARGS(m_device.writeRef()))))
+            if (SUCCEEDED(m_D3D12CreateDevice(candidateAdapter, featureLevel, IID_PPV_ARGS(m_device.writeRef()))))
             {
                 // We found one!
                 adapter = candidateAdapter;
@@ -1388,32 +1339,95 @@ Result D3D12Renderer::initialize(const Desc& desc, void* inWindowHandle)
         }
     }
 
-    if (!adapter)
+    if (m_dxDebug && (deviceCheckFlags & DeviceCheckFlag::UseDebug))
     {
-        // Couldn't find an adapter
+        m_dxDebug->EnableDebugLayer();
+    }
+
+    outDxgiFactory = dxgiFactory;
+    outAdapter = adapter;
+    return SLANG_OK;
+}
+
+Result D3D12Renderer::initialize(const Desc& desc, void* inWindowHandle)
+{
+    m_hwnd = (HWND)inWindowHandle;
+    // Rather than statically link against D3D, we load it dynamically.
+
+    HMODULE d3dModule = LoadLibraryA("d3d12.dll");
+    if (!d3dModule)
+    {
+        fprintf(stderr, "error: failed load 'd3d12.dll'\n");
         return SLANG_FAIL;
     }
 
-    // set up debug layer
-#ifndef NDEBUG
+    HMODULE dxgiModule = LoadLibraryA("Dxgi.dll");
+    if (!dxgiModule)
     {
+        fprintf(stderr, "error: failed load 'dxgi.dll'\n");
+        return SLANG_FAIL;
+    }
 
-        LOAD_D3D_PROC(PFN_D3D12_GET_DEBUG_INTERFACE, D3D12GetDebugInterface);
-        if (!D3D12GetDebugInterface_)
+    // Get all the dll entry points
+    m_D3D12SerializeRootSignature = (PFN_D3D12_SERIALIZE_ROOT_SIGNATURE)loadProc(d3dModule, "D3D12SerializeRootSignature");
+    if (!m_D3D12SerializeRootSignature)
+    {
+        return SLANG_FAIL;
+    }
+
+#if ENABLE_DEBUG_LAYER
+    m_D3D12GetDebugInterface = (PFN_D3D12_GET_DEBUG_INTERFACE)loadProc(d3dModule, "D3D12GetDebugInterface");
+    if (m_D3D12GetDebugInterface)
+    {
+        if (SUCCEEDED(m_D3D12GetDebugInterface(IID_PPV_ARGS(m_dxDebug.writeRef()))))
         {
-            return SLANG_FAIL;
+            m_dxDebug->EnableDebugLayer();
         }
+    }
+#endif
+
+    m_CreateDXGIFactory2 = (PFN_DXGI_CREATE_FACTORY_2)loadProc(dxgiModule, "CreateDXGIFactory2");
+    if (!m_CreateDXGIFactory2)
+    {
+        return SLANG_FAIL;
+    }
+    m_D3D12CreateDevice = (PFN_D3D12_CREATE_DEVICE)loadProc(d3dModule, "D3D12CreateDevice");
+    if (!m_D3D12CreateDevice)
+    {
+        return SLANG_FAIL;
+    }
+
+    FlagCombiner combiner;
+    combiner.add(DeviceCheckFlag::UseFullFeatureLevel, ChangeType::OnOff);      ///< First try fully featured, then degrade features
+    combiner.add(DeviceCheckFlag::UseHardwareDevice, ChangeType::OnOff);        ///< First try hardware, then reference
+
+    // TODO: we should probably provide a command-line option
+    // to override UseDebug of default rather than leave it
+    // up to each back-end to specify.
 
-        ComPtr<ID3D12Debug> debug;
+#if ENABLE_DEBUG_LAYER
+    combiner.add(DeviceCheckFlag::UseDebug, ChangeType::OnOff);                 ///< First try debug then non debug
+#else
+    combiner.add(DeviceCheckFlag::UseDebug, ChangeType::Off);                   ///< Don't bother with debug
+#endif
 
-        if (!SUCCEEDED(D3D12GetDebugInterface_(IID_PPV_ARGS(debug.writeRef()))))
+    ComPtr<IDXGIFactory4> dxgiFactory;
+    ComPtr<IDXGIAdapter> adapter;
+    const int numCombinations = combiner.getNumCombinations();
+    for (int i = 0; i < numCombinations; ++i)
+    {
+        adapter.setNull();
+        if (SLANG_SUCCEEDED(_createAdaptor(combiner.getCombination(i), dxgiFactory, adapter)))
         {
-            return SLANG_FAIL;
+            break;
         }
+    }
 
-        debug->EnableDebugLayer();
+    if (!adapter)
+    {
+        // Couldn't find an adapter
+        return SLANG_FAIL;
     }
-#endif
 
     m_numRenderFrames = 3;
     m_numRenderTargets = 2;