From 3a2f191d9edd7c61d49cae1e979fa7633b2aaf8b Mon Sep 17 00:00:00 2001
From: Tim Foley <tfoley@nvidia.com>
Date: Tue, 15 Aug 2017 08:52:13 -0700
Subject: Handle possibility of bad types in varying input/output signature.

Fixes #160

If the front-end runs into a type it doesn't understand in the parameter list of an entry point, it will create an `ErrorType` for that parameter, but then the parameter binding/layout rules will fail to create a `TypeLayout` for the prameter (and return `NULL`).
There were some places where the code was expecting that operation to succeed unconditionally, and so would crash when there was a bad type.

The specific case in the bug report was when the return type of a shader entry point was bad:

    // `vec4` is not an HLSL type
    vec4 main(...) { ... }

Note that the specific case in the buf report only manifests in "rewriter" mode (when the Slang compiler isn't allowed to issue error messages from the front-end), but the same basic thing would happen if the varying parameter/output had used a type that is invalid for varying input/output:

    Texture2D main(...) { ... }

I'm not 100% happy with just adding more `NULL` checks for this, because there is no easy way to tell if they are exhaustive.
A better solution in the longer term might be to construct a kind of `ErrorTypeLayout` to represent cases where we wanted a type layout, but none could be constructed.
---
 source/slang/parameter-binding.cpp | 32 +++++++++++++++++++-------------
 tests/rewriter/gh-160.hlsl         | 13 +++++++++++++
 tests/rewriter/gh-160.slang        |  3 +++
 3 files changed, 35 insertions(+), 13 deletions(-)
 create mode 100644 tests/rewriter/gh-160.hlsl
 create mode 100644 tests/rewriter/gh-160.slang

diff --git a/source/slang/parameter-binding.cpp b/source/slang/parameter-binding.cpp
index 877597640..757662101 100644
--- a/source/slang/parameter-binding.cpp
+++ b/source/slang/parameter-binding.cpp
@@ -1253,15 +1253,18 @@ static RefPtr<TypeLayout> processEntryPointParameter(
                     state,
                     fieldVarLayout);
 
-                fieldVarLayout->typeLayout = fieldTypeLayout;
-
-                for (auto rr : fieldTypeLayout->resourceInfos)
+                if(fieldTypeLayout)
                 {
-                    SLANG_RELEASE_ASSERT(rr.count != 0);
+                    fieldVarLayout->typeLayout = fieldTypeLayout;
+
+                    for (auto rr : fieldTypeLayout->resourceInfos)
+                    {
+                        SLANG_RELEASE_ASSERT(rr.count != 0);
 
-                    auto structRes = structLayout->findOrAddResourceInfo(rr.kind);
-                    fieldVarLayout->findOrAddResourceInfo(rr.kind)->index = structRes->count;
-                    structRes->count += rr.count;
+                        auto structRes = structLayout->findOrAddResourceInfo(rr.kind);
+                        fieldVarLayout->findOrAddResourceInfo(rr.kind)->index = structRes->count;
+                        structRes->count += rr.count;
+                    }
                 }
 
                 structLayout->fields.Add(fieldVarLayout);
@@ -1404,13 +1407,16 @@ static void collectEntryPointParameters(
             state,
             resultLayout);
 
-        resultLayout->typeLayout = resultTypeLayout;
-
-        for (auto rr : resultTypeLayout->resourceInfos)
+        if( resultTypeLayout )
         {
-            auto entryPointRes = entryPointLayout->findOrAddResourceInfo(rr.kind);
-            resultLayout->findOrAddResourceInfo(rr.kind)->index = entryPointRes->count;
-            entryPointRes->count += rr.count;
+            resultLayout->typeLayout = resultTypeLayout;
+
+            for (auto rr : resultTypeLayout->resourceInfos)
+            {
+                auto entryPointRes = entryPointLayout->findOrAddResourceInfo(rr.kind);
+                resultLayout->findOrAddResourceInfo(rr.kind)->index = entryPointRes->count;
+                entryPointRes->count += rr.count;
+            }
         }
 
         entryPointLayout->resultLayout = resultLayout;
diff --git a/tests/rewriter/gh-160.hlsl b/tests/rewriter/gh-160.hlsl
new file mode 100644
index 000000000..46ef6a92c
--- /dev/null
+++ b/tests/rewriter/gh-160.hlsl
@@ -0,0 +1,13 @@
+//TEST:COMPARE_HLSL: -no-checking -target dxbc-assembly -profile ps_4_0 -entry main
+
+#ifdef __SLANG__
+__import gh_160;
+#endif
+
+vec4 main(VS_OUT vOut) : SV_TARGET
+{
+	float3 color = float3(1,0,0);
+
+    vec4 finalColor = vec4(color, 1.f);
+    return finalColor;
+}
\ No newline at end of file
diff --git a/tests/rewriter/gh-160.slang b/tests/rewriter/gh-160.slang
new file mode 100644
index 000000000..c86f63c19
--- /dev/null
+++ b/tests/rewriter/gh-160.slang
@@ -0,0 +1,3 @@
+//TEST_IGNORE_FILE:
+
+// This file only exists so that `gh-160.hlsl` can import it.
-- 
cgit v1.2.3