Dx11 & Dx12 device startup (#861)

* Added CombinationUtil to produce combinations of flags
Used in Dx11 device creation making it fall back to release driver if debug driver is not found

* Made dx12 renderer startup similar to dx11 - testing multiple configs.

* Small improvements in naming.

* * Moved functionality to gfx from core
* Use FlagCombiner to simplify construction, and can be iterated over, without need for array
* Share DeviceCheckFlags

* Improve comments.

* Re-add the comment about combinations tested to set up dx11 device.

* More comment improvements.

7 files changed, 249 insertions, 107 deletions

diff --git a/source/slang/slang.vcxproj b/source/slang/slang.vcxproj
index 577097c93..ff37c5e09 100644
--- a/source/slang/slang.vcxproj
+++ b/source/slang/slang.vcxproj
@@ -318,4 +318,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project> 
\ No newline at end of file
+</Project>
\ No newline at end of file
diff --git a/tools/gfx/flag-combiner.cpp b/tools/gfx/flag-combiner.cpp
new file mode 100644
index 000000000..04617368a
--- /dev/null
+++ b/tools/gfx/flag-combiner.cpp
@@ -0,0 +1,55 @@
+#include "flag-combiner.h"
+
+namespace gfx {
+using namespace Slang;
+
+void FlagCombiner::add(uint32_t flags, ChangeType type)
+{
+    // The flag/s must be set
+    SLANG_ASSERT(flags);
+    SLANG_ASSERT((flags & m_usedFlags) == 0);
+    // Mark the flags used
+    m_usedFlags |= flags;
+
+    if (type == ChangeType::On || type == ChangeType::OnOff)
+    {
+        m_invertBits |= flags;
+    }
+    if (type == ChangeType::OnOff || type == ChangeType::OffOn)
+    {
+        m_changingBits[m_numChangingBits++] = flags;
+    }
+}
+
+void FlagCombiner::calcCombinations(List<uint32_t>& outCombinations) const
+{
+    const int numCombinations = getNumCombinations();
+    outCombinations.SetSize(numCombinations);
+    uint32_t* dstCombinations = outCombinations.Buffer();
+    for (int i = 0; i < numCombinations; ++i)
+    {
+        dstCombinations[i] = getCombination(i);
+    }
+}
+
+uint32_t FlagCombiner::getCombination(int index) const
+{
+    SLANG_ASSERT(index >= 0 && index < getNumCombinations());
+
+    uint32_t combination = 0;
+    uint32_t bit = 1;
+    for (int i = 0; i < m_numChangingBits; ++i, bit += bit)
+    {
+        combination |= ((bit & index) ? m_changingBits[i] : 0);
+    }
+    return combination ^ m_invertBits;
+}
+
+void FlagCombiner::reset()
+{
+    m_numChangingBits = 0;
+    m_usedFlags = 0;
+    m_invertBits = 0;
+}
+
+} // namespace gfx
diff --git a/tools/gfx/flag-combiner.h b/tools/gfx/flag-combiner.h
new file mode 100644
index 000000000..90fffbbf4
--- /dev/null
+++ b/tools/gfx/flag-combiner.h
@@ -0,0 +1,62 @@
+#ifndef GFX_FLAG_COMBINER_H
+#define GFX_FLAG_COMBINER_H
+
+#include "../../source/core/list.h"
+
+namespace gfx {
+
+/* A default set of flags that can be used for checking devices */
+typedef uint32_t DeviceCheckFlags;
+struct DeviceCheckFlag
+{
+    enum Enum : DeviceCheckFlags
+    {
+        UseFullFeatureLevel = 0x1,      //< If set will use full feature level (on dx this is  D3D_FEATURE_LEVEL_11_1 else will try D3D_FEATURE_LEVEL_11_0)
+        UseHardwareDevice = 0x2,        //< If set will try a hardware device
+        UseDebug = 0x4,                 //< If set will enable use of debug
+    };
+};
+
+/* Controls how and the order flags are changed, on the FlagCombiner */
+enum class ChangeType
+{
+    On,                 ///< Always on
+    Off,                ///< Always off
+    OnOff,              ///< Initially on then off
+    OffOn,              ///< Initially off then on
+};
+
+/* Calculates all the combinations of flags as controlled by the change types.
+The earlier a flag/changeType is added increases the frequency the flag will change. This can be used to control the
+'importance' of a change, earlier added items will iterate through their combinations first.
+*/
+class FlagCombiner
+{
+public:
+        /// Add a flag and how it changes over the combinations
+        /// NOTE! That the order flags are added controls the order they change when combinations are calculated - earlier added
+        /// flags will change with the highest frequency
+    void add(uint32_t flags, ChangeType changeType);
+        /// Calculate all of the combinations and place in an array
+    void calcCombinations(Slang::List<uint32_t>& outCombinations) const;
+
+        /// Reset back to initial state
+    void reset(); 
+
+        /// Get the total amount of combinations
+    int getNumCombinations() const { return 1 << m_numChangingBits; }
+
+        /// Get the combination at i
+    uint32_t getCombination(int i) const;
+
+protected:
+    uint32_t m_changingBits[32];
+    int m_numChangingBits = 0;
+
+    uint32_t m_usedFlags = 0;               
+    uint32_t m_invertBits = 0;
+};
+
+} // namespace gfx
+
+#endif // GFX_FLAG_COMBINER_H
diff --git a/tools/gfx/gfx.vcxproj b/tools/gfx/gfx.vcxproj
index 6de379c73..059a99618 100644
--- a/tools/gfx/gfx.vcxproj
+++ b/tools/gfx/gfx.vcxproj
@@ -174,6 +174,7 @@
     <ClInclude Include="circular-resource-heap-d3d12.h" />
     <ClInclude Include="d3d-util.h" />
     <ClInclude Include="descriptor-heap-d3d12.h" />
+    <ClInclude Include="flag-combiner.h" />
     <ClInclude Include="gui.h" />
     <ClInclude Include="model.h" />
     <ClInclude Include="render-d3d11.h" />
@@ -195,6 +196,7 @@
     <ClCompile Include="circular-resource-heap-d3d12.cpp" />
     <ClCompile Include="d3d-util.cpp" />
     <ClCompile Include="descriptor-heap-d3d12.cpp" />
+    <ClCompile Include="flag-combiner.cpp" />
     <ClCompile Include="gui.cpp" />
     <ClCompile Include="model.cpp" />
     <ClCompile Include="render-d3d11.cpp" />
diff --git a/tools/gfx/gfx.vcxproj.filters b/tools/gfx/gfx.vcxproj.filters
index 8164d45b0..95b5f31d2 100644
--- a/tools/gfx/gfx.vcxproj.filters
+++ b/tools/gfx/gfx.vcxproj.filters
@@ -18,6 +18,9 @@
     <ClInclude Include="descriptor-heap-d3d12.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="flag-combiner.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="gui.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -77,6 +80,9 @@
     <ClCompile Include="descriptor-heap-d3d12.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="flag-combiner.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="gui.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
diff --git a/tools/gfx/render-d3d11.cpp b/tools/gfx/render-d3d11.cpp
index e049bad38..054786f81 100644
--- a/tools/gfx/render-d3d11.cpp
+++ b/tools/gfx/render-d3d11.cpp
@@ -15,6 +15,7 @@
 //#include <slang.h>
 
 #include "../../slang-com-ptr.h"
+#include "flag-combiner.h"
 
 // We will be rendering with Direct3D 11, so we need to include
 // the Windows and D3D11 headers
@@ -412,17 +413,6 @@ SlangResult D3D11Renderer::initialize(const Desc& desc, void* inWindowHandle)
         return SLANG_FAIL;
     }
 
-    UINT deviceFlags = 0;
-
-#ifdef _DEBUG
-    // We will enable the D3D debug more for debug builds.
-    //
-    // TODO: we should probably provide a command-line option
-    // to override this kind of default rather than leave it
-    // up to each back-end to specify.
-    deviceFlags |= D3D11_CREATE_DEVICE_DEBUG;
-#endif
-
     // Our swap chain uses RGBA8 with sRGB, with double buffering.
     DXGI_SWAP_CHAIN_DESC swapChainDesc = { 0 };
     swapChainDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
@@ -454,22 +444,35 @@ SlangResult D3D11Renderer::initialize(const Desc& desc, void* inWindowHandle)
     D3D_FEATURE_LEVEL featureLevel = D3D_FEATURE_LEVEL_9_1;
     const int totalNumFeatureLevels = SLANG_COUNT_OF(featureLevels);
 
-    // On a machine that does not have an up-to-date version of D3D installed,
-    // the `D3D11CreateDeviceAndSwapChain` call will fail with `E_INVALIDARG`
-    // if you ask for featuer level 11_1. The workaround is to call
-    // `D3D11CreateDeviceAndSwapChain` up to twice: the first time with 11_1
-    // at the start of the list of requested feature levels, and the second
-    // time without it.
-    // 
-    // We first try create using hardware and then software (reference) driver
-    // Test until we get a successful result
-
     {
+        // On a machine that does not have an up-to-date version of D3D installed,
+        // the `D3D11CreateDeviceAndSwapChain` call will fail with `E_INVALIDARG`
+        // if you ask for feature level 11_1 (DeviceCheckFlag::UseFullFeatureLevel).
+        // The workaround is to call `D3D11CreateDeviceAndSwapChain` the first time
+        // with 11_1 and then back off to 11_0 if that fails.
+
+        FlagCombiner combiner;
+        combiner.add(DeviceCheckFlag::UseFullFeatureLevel, ChangeType::OnOff);      ///< First try fully featured, then degrade features
+        combiner.add(DeviceCheckFlag::UseHardwareDevice, ChangeType::OnOff);        ///< First try hardware, then reference
+
+        // TODO: we should probably provide a command-line option
+        // to override UseDebug of default rather than leave it
+        // up to each back-end to specify.
+
+#if _DEBUG
+        combiner.add(DeviceCheckFlag::UseDebug, ChangeType::OnOff);                 ///< First try debug then non debug
+#else
+        combiner.add(DeviceCheckFlag::UseDebug, ChangeType::Off);                   ///< Don't bother with debug
+#endif
+
+        const int numCombinations = combiner.getNumCombinations();
         Result res = SLANG_FAIL;
-        for (int ii = 0; ii < 4; ++ii)
+        for (int i = 0; i < numCombinations; ++i)
         {
-            const D3D_DRIVER_TYPE driverType = (ii & 2) ? D3D_DRIVER_TYPE_REFERENCE : D3D_DRIVER_TYPE_HARDWARE;
-            const int startFeatureIndex = (ii & 1);
+            const auto deviceCheckFlags = combiner.getCombination(i);
+            const D3D_DRIVER_TYPE driverType = (deviceCheckFlags & DeviceCheckFlag::UseHardwareDevice) ? D3D_DRIVER_TYPE_HARDWARE : D3D_DRIVER_TYPE_REFERENCE;
+            const int startFeatureIndex = (deviceCheckFlags & DeviceCheckFlag::UseFullFeatureLevel) ? 0 : 1; 
+            const UINT deviceFlags = (deviceCheckFlags & DeviceCheckFlag::UseDebug) ? D3D11_CREATE_DEVICE_DEBUG : 0;
 
             res = D3D11CreateDeviceAndSwapChain_(
                 nullptr,                    // adapter (use default)
diff --git a/tools/gfx/render-d3d12.cpp b/tools/gfx/render-d3d12.cpp
index e8af974fa..ec9695cb1 100644
--- a/tools/gfx/render-d3d12.cpp
+++ b/tools/gfx/render-d3d12.cpp
@@ -26,6 +26,7 @@
 #include <d3dcompiler.h>
 
 #include "../../slang-com-ptr.h"
+#include "flag-combiner.h"
 
 #include "resource-d3d12.h"
 #include "descriptor-heap-d3d12.h"
@@ -101,6 +102,7 @@ public:
     ~D3D12Renderer();
 
 protected:
+    
     static const Int kMaxNumRenderFrames = 4;
     static const Int kMaxNumRenderTargets = 3;
 
@@ -477,8 +479,8 @@ protected:
 //    Result _calcBindParameters(BindParameters& params);
 //    RenderState* findRenderState(PipelineType pipelineType);
 
-    PFN_D3D12_SERIALIZE_ROOT_SIGNATURE m_D3D12SerializeRootSignature = nullptr;
-
+    Result _createAdaptor(DeviceCheckFlags deviceCheckFlags, ComPtr<IDXGIFactory4>& outDxgiFactory, ComPtr<IDXGIAdapter>& outAdapter);
+    
     D3D12CircularResourceHeap m_circularResourceHeap;
 
     int m_commandListOpenCount = 0;            ///< If >0 the command list should be open
@@ -560,6 +562,14 @@ protected:
     int32_t m_depthStencilUsageFlags = 0;    ///< D3DUtil::UsageFlag combination for depth stencil
     int32_t m_targetUsageFlags = 0;            ///< D3DUtil::UsageFlag combination for target
 
+    // Dll entry points
+    typedef HRESULT(WINAPI *PFN_DXGI_CREATE_FACTORY_2)(UINT Flags, REFIID riid, _COM_Outptr_ void **ppFactory);
+
+    PFN_D3D12_GET_DEBUG_INTERFACE m_D3D12GetDebugInterface = nullptr;
+    PFN_DXGI_CREATE_FACTORY_2 m_CreateDXGIFactory2 = nullptr;
+    PFN_D3D12_CREATE_DEVICE m_D3D12CreateDevice = nullptr;
+    PFN_D3D12_SERIALIZE_ROOT_SIGNATURE m_D3D12SerializeRootSignature = nullptr;
+
     HWND m_hwnd = nullptr;
 };
 
@@ -1278,82 +1288,24 @@ Result D3D12Renderer::_bindRenderState(PipelineStateImpl* pipelineStateImpl, ID3
 
 // !!!!!!!!!!!!!!!!!!!!!!!!!!!! Renderer interface !!!!!!!!!!!!!!!!!!!!!!!!!!
 
-Result D3D12Renderer::initialize(const Desc& desc, void* inWindowHandle)
-{
-    m_hwnd = (HWND)inWindowHandle;
-    // Rather than statically link against D3D, we load it dynamically.
 
-    HMODULE d3dModule = LoadLibraryA("d3d12.dll");
-    if (!d3dModule)
-    {
-        fprintf(stderr, "error: failed load 'd3d12.dll'\n");
-        return SLANG_FAIL;
-    }
-
-    HMODULE dxgiModule = LoadLibraryA("Dxgi.dll");
-    if (!dxgiModule)
-    {
-        fprintf(stderr, "error: failed load 'dxgi.dll'\n");
-        return SLANG_FAIL;
-    }
-
-
-#define LOAD_D3D_PROC(TYPE, NAME) \
-        TYPE NAME##_ = (TYPE) loadProc(d3dModule, #NAME);
-#define LOAD_DXGI_PROC(TYPE, NAME) \
-        TYPE NAME##_ = (TYPE) loadProc(dxgiModule, #NAME);
-
-    UINT dxgiFactoryFlags = 0;
-
-#if ENABLE_DEBUG_LAYER
-    {
-        LOAD_D3D_PROC(PFN_D3D12_GET_DEBUG_INTERFACE, D3D12GetDebugInterface);
-        if (D3D12GetDebugInterface_)
-        {
-            if (SUCCEEDED(D3D12GetDebugInterface_(IID_PPV_ARGS(m_dxDebug.writeRef()))))
-            {
-                m_dxDebug->EnableDebugLayer();
-                dxgiFactoryFlags |= DXGI_CREATE_FACTORY_DEBUG;
-            }
-        }
-    }
-#endif
-
-    m_D3D12SerializeRootSignature = (PFN_D3D12_SERIALIZE_ROOT_SIGNATURE)loadProc(d3dModule, "D3D12SerializeRootSignature");
-    if (!m_D3D12SerializeRootSignature)
-    {
-        return SLANG_FAIL;
-    }
+Result D3D12Renderer::_createAdaptor(DeviceCheckFlags deviceCheckFlags, ComPtr<IDXGIFactory4>& outDxgiFactory, ComPtr<IDXGIAdapter>& outAdapter)
+{
+    const UINT dxgiFactoryFlags = (deviceCheckFlags & DeviceCheckFlag::UseDebug) ? DXGI_CREATE_FACTORY_DEBUG  : 0;
 
     // Try and create DXGIFactory
     ComPtr<IDXGIFactory4> dxgiFactory;
-    {
-        typedef HRESULT(WINAPI *PFN_DXGI_CREATE_FACTORY_2)(UINT Flags, REFIID riid, _COM_Outptr_ void **ppFactory);
-        LOAD_DXGI_PROC(PFN_DXGI_CREATE_FACTORY_2, CreateDXGIFactory2);
-        if (!CreateDXGIFactory2_)
-        {
-            return SLANG_FAIL;
-        }
-        SLANG_RETURN_ON_FAIL(CreateDXGIFactory2_(dxgiFactoryFlags, IID_PPV_ARGS(dxgiFactory.writeRef())));
-    }
-
-    D3D_FEATURE_LEVEL featureLevel = D3D_FEATURE_LEVEL_11_0;
+    SLANG_RETURN_ON_FAIL(m_CreateDXGIFactory2(dxgiFactoryFlags, IID_PPV_ARGS(dxgiFactory.writeRef())));
+    
+    D3D_FEATURE_LEVEL featureLevel = DeviceCheckFlag::UseFullFeatureLevel ? D3D_FEATURE_LEVEL_11_1 : D3D_FEATURE_LEVEL_11_0;
 
     // Search for an adapter that meets our requirements
     ComPtr<IDXGIAdapter> adapter;
 
-    LOAD_D3D_PROC(PFN_D3D12_CREATE_DEVICE, D3D12CreateDevice);
-    if (!D3D12CreateDevice_)
-    {
-        return SLANG_FAIL;
-    }
-
-    const bool useWarp = false;
-
-    if (useWarp)
+    if ((deviceCheckFlags & DeviceCheckFlag::UseHardwareDevice) == 0)
     {
         SLANG_RETURN_ON_FAIL(dxgiFactory->EnumWarpAdapter(IID_PPV_ARGS(adapter.writeRef())));
-        SLANG_RETURN_ON_FAIL(D3D12CreateDevice_(adapter, featureLevel, IID_PPV_ARGS(m_device.writeRef())));
+        SLANG_RETURN_ON_FAIL(m_D3D12CreateDevice(adapter, featureLevel, IID_PPV_ARGS(m_device.writeRef())));
     }
     else
     {
@@ -1371,7 +1323,6 @@ Result D3D12Renderer::initialize(const Desc& desc, void* inWindowHandle)
 
             if (desc.Flags & DXGI_ADAPTER_FLAG_SOFTWARE)
             {
-                
                 // TODO: may want to allow software driver as fallback
             }
             else
@@ -1379,7 +1330,7 @@ Result D3D12Renderer::initialize(const Desc& desc, void* inWindowHandle)
                 continue;
             }
 
-            if (SUCCEEDED(D3D12CreateDevice_(candidateAdapter, featureLevel, IID_PPV_ARGS(m_device.writeRef()))))
+            if (SUCCEEDED(m_D3D12CreateDevice(candidateAdapter, featureLevel, IID_PPV_ARGS(m_device.writeRef()))))
             {
                 // We found one!
                 adapter = candidateAdapter;
@@ -1388,32 +1339,95 @@ Result D3D12Renderer::initialize(const Desc& desc, void* inWindowHandle)
         }
     }
 
-    if (!adapter)
+    if (m_dxDebug && (deviceCheckFlags & DeviceCheckFlag::UseDebug))
     {
-        // Couldn't find an adapter
+        m_dxDebug->EnableDebugLayer();
+    }
+
+    outDxgiFactory = dxgiFactory;
+    outAdapter = adapter;
+    return SLANG_OK;
+}
+
+Result D3D12Renderer::initialize(const Desc& desc, void* inWindowHandle)
+{
+    m_hwnd = (HWND)inWindowHandle;
+    // Rather than statically link against D3D, we load it dynamically.
+
+    HMODULE d3dModule = LoadLibraryA("d3d12.dll");
+    if (!d3dModule)
+    {
+        fprintf(stderr, "error: failed load 'd3d12.dll'\n");
         return SLANG_FAIL;
     }
 
-    // set up debug layer
-#ifndef NDEBUG
+    HMODULE dxgiModule = LoadLibraryA("Dxgi.dll");
+    if (!dxgiModule)
     {
+        fprintf(stderr, "error: failed load 'dxgi.dll'\n");
+        return SLANG_FAIL;
+    }
 
-        LOAD_D3D_PROC(PFN_D3D12_GET_DEBUG_INTERFACE, D3D12GetDebugInterface);
-        if (!D3D12GetDebugInterface_)
+    // Get all the dll entry points
+    m_D3D12SerializeRootSignature = (PFN_D3D12_SERIALIZE_ROOT_SIGNATURE)loadProc(d3dModule, "D3D12SerializeRootSignature");
+    if (!m_D3D12SerializeRootSignature)
+    {
+        return SLANG_FAIL;
+    }
+
+#if ENABLE_DEBUG_LAYER
+    m_D3D12GetDebugInterface = (PFN_D3D12_GET_DEBUG_INTERFACE)loadProc(d3dModule, "D3D12GetDebugInterface");
+    if (m_D3D12GetDebugInterface)
+    {
+        if (SUCCEEDED(m_D3D12GetDebugInterface(IID_PPV_ARGS(m_dxDebug.writeRef()))))
         {
-            return SLANG_FAIL;
+            m_dxDebug->EnableDebugLayer();
         }
+    }
+#endif
+
+    m_CreateDXGIFactory2 = (PFN_DXGI_CREATE_FACTORY_2)loadProc(dxgiModule, "CreateDXGIFactory2");
+    if (!m_CreateDXGIFactory2)
+    {
+        return SLANG_FAIL;
+    }
+    m_D3D12CreateDevice = (PFN_D3D12_CREATE_DEVICE)loadProc(d3dModule, "D3D12CreateDevice");
+    if (!m_D3D12CreateDevice)
+    {
+        return SLANG_FAIL;
+    }
+
+    FlagCombiner combiner;
+    combiner.add(DeviceCheckFlag::UseFullFeatureLevel, ChangeType::OnOff);      ///< First try fully featured, then degrade features
+    combiner.add(DeviceCheckFlag::UseHardwareDevice, ChangeType::OnOff);        ///< First try hardware, then reference
+
+    // TODO: we should probably provide a command-line option
+    // to override UseDebug of default rather than leave it
+    // up to each back-end to specify.
 
-        ComPtr<ID3D12Debug> debug;
+#if ENABLE_DEBUG_LAYER
+    combiner.add(DeviceCheckFlag::UseDebug, ChangeType::OnOff);                 ///< First try debug then non debug
+#else
+    combiner.add(DeviceCheckFlag::UseDebug, ChangeType::Off);                   ///< Don't bother with debug
+#endif
 
-        if (!SUCCEEDED(D3D12GetDebugInterface_(IID_PPV_ARGS(debug.writeRef()))))
+    ComPtr<IDXGIFactory4> dxgiFactory;
+    ComPtr<IDXGIAdapter> adapter;
+    const int numCombinations = combiner.getNumCombinations();
+    for (int i = 0; i < numCombinations; ++i)
+    {
+        adapter.setNull();
+        if (SLANG_SUCCEEDED(_createAdaptor(combiner.getCombination(i), dxgiFactory, adapter)))
         {
-            return SLANG_FAIL;
+            break;
         }
+    }
 
-        debug->EnableDebugLayer();
+    if (!adapter)
+    {
+        // Couldn't find an adapter
+        return SLANG_FAIL;
     }
-#endif
 
     m_numRenderFrames = 3;
     m_numRenderTargets = 2;