Various IR fixes for Falcor (#280)

- Change function mangling so we use `p<parameterCount>p` instead of just `p<parameterCount>` to avoid the parameter count running into digits at the start of a mangled type name and tripping up the un-mangling logic.
  - We really need to step back at some point and define our mangling scheme a bit more carefully, especially if we are going to keep going down this road where un-mangling things is important for generating HLSL output.

- Also allow the unmangling logic to unmangle a few more cases of generic parameters, so that it can skip over them to get to the parameter count of the underlying function.

- Add a notion of an `unreachable` instruction to the IR, and emit it as the terminator (if needed) at the end of the last block for a function with a non-void return type.
  - This does *not* implement any logic to emit a diagnostic if the `unreachable` turns out to be potentially reachable

- Fix a bug in IR specialization of generics where we can't create two different specializations of the same function, because both get registered in the same hash map

With all these fixes, testing in Falcor modified to use the full Slang compiler and IR for all HLSL/Slang:

- The UI and text rendering shaders yield HLSL that compiles without error; no idea if they actually *work*

- The ModelViewer shaders yield HLSL, but there are some issues (looks like type legalization isn't applying to stuff inside constant buffers)

6 files changed, 54 insertions, 6 deletions

diff --git a/source/slang/emit.cpp b/source/slang/emit.cpp
index 84d8f113e..dccb12f53 100644
--- a/source/slang/emit.cpp
+++ b/source/slang/emit.cpp
@@ -4743,9 +4743,15 @@ emitDeclImpl(decl, nullptr);
             switch(peek())
             {
             case 'T':
+            case 'C':
                 get();
                 break;
 
+            case 'v':
+                get();
+                readType();
+                break;
+
             default:
                 SLANG_UNEXPECTED("bad name mangling");
                 break;
@@ -4862,7 +4868,9 @@ emitDeclImpl(decl, nullptr);
         UInt readParamCount()
         {
             expect("p");
-            return readCount();
+            UInt count = readCount();
+            expect("p");
+            return count;
         }
     };
 
@@ -5438,6 +5446,9 @@ emitDeclImpl(decl, nullptr);
                 SLANG_UNEXPECTED("terminator inst");
                 return;
 
+            case kIROp_unreachable:
+                return;
+
             case kIROp_ReturnVal:
             case kIROp_ReturnVoid:
             case kIROp_discard:
diff --git a/source/slang/ir-inst-defs.h b/source/slang/ir-inst-defs.h
index dbdb697a4..ffad04467 100644
--- a/source/slang/ir-inst-defs.h
+++ b/source/slang/ir-inst-defs.h
@@ -184,6 +184,7 @@ INST(loopTest, loopTest, 3, 0)
 INST(switch, switch, 3, 0)
 
 INST(discard, discard, 0, 0)
+INST(unreachable, unreachable, 0, 0)
 
 INST(Add, add, 2, 0)
 INST(Sub, sub, 2, 0)
diff --git a/source/slang/ir-insts.h b/source/slang/ir-insts.h
index b4335656e..52acf6576 100644
--- a/source/slang/ir-insts.h
+++ b/source/slang/ir-insts.h
@@ -138,6 +138,13 @@ struct IRReturnVoid : IRReturn
 struct IRDiscard : IRTerminatorInst
 {};
 
+// Signals that this point in the code should be unreachable.
+// We can/should emit a dataflow error if we can ever determine
+// that a block ending in one of these can actually be
+// executed.
+struct IRUnreachable : IRTerminatorInst
+{};
+
 struct IRBlock;
 
 struct IRUnconditionalBranch : IRTerminatorInst
@@ -487,6 +494,8 @@ struct IRBuilder
 
     IRInst* emitDiscard();
 
+    IRInst* emitUnreachable();
+
     IRInst* emitBranch(
         IRBlock*    block);
 
diff --git a/source/slang/ir.cpp b/source/slang/ir.cpp
index 313fd258b..92bcb6707 100644
--- a/source/slang/ir.cpp
+++ b/source/slang/ir.cpp
@@ -168,6 +168,7 @@ namespace Slang
         case kIROp_loopTest:
         case kIROp_discard:
         case kIROp_switch:
+        case kIROp_unreachable:
             return true;
         }
     }
@@ -1152,6 +1153,16 @@ namespace Slang
         return inst;
     }
 
+    IRInst* IRBuilder::emitUnreachable()
+    {
+        auto inst = createInst<IRUnreachable>(
+            this,
+            kIROp_unreachable,
+            nullptr);
+        addInst(inst);
+        return inst;
+    }
+
     IRInst* IRBuilder::emitDiscard()
     {
         auto inst = createInst<IRDiscard>(
@@ -3457,6 +3468,14 @@ namespace Slang
         return clonedFunc;
     }
 
+    IRFunc* cloneSimpleFuncWithoutRegistering(IRSpecContextBase* context, IRFunc* originalFunc)
+    {
+        auto clonedFunc = context->builder->createFunc();
+        cloneFunctionCommon(context, clonedFunc, originalFunc);
+        return clonedFunc;
+    }
+
+
     IRFunc* cloneSimpleFunc(IRSpecContextBase* context, IRFunc* originalFunc)
     {
         auto clonedFunc = context->builder->createFunc();
@@ -4017,7 +4036,7 @@ namespace Slang
 
         // TODO: other initialization is needed here...
 
-        auto specFunc = cloneSimpleFunc(&context, genericFunc);
+        auto specFunc = cloneSimpleFuncWithoutRegistering(&context, genericFunc);
 
         // Set up the clone to recognize that it is no longer generic
         specFunc->mangledName = specMangledName;
diff --git a/source/slang/lower-to-ir.cpp b/source/slang/lower-to-ir.cpp
index f642b316e..827504122 100644
--- a/source/slang/lower-to-ir.cpp
+++ b/source/slang/lower-to-ir.cpp
@@ -3242,8 +3242,7 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
                     // this by putting an `unreachable` terminator here,
                     // and then emit a dataflow error if this block
                     // can't be eliminated.
-                    SLANG_UNEXPECTED("Needed a return here");
-                    UNREACHABLE(subContext->irBuilder->emitReturn());
+                    subContext->irBuilder->emitUnreachable();
                 }
             }
         }
diff --git a/source/slang/mangle.cpp b/source/slang/mangle.cpp
index 38c2b29a8..68fa7f31b 100644
--- a/source/slang/mangle.cpp
+++ b/source/slang/mangle.cpp
@@ -164,6 +164,13 @@ namespace Slang
             // parameter are they at the specified depth).
             emitName(context, genericParamIntVal->declRef.GetName());
         }
+        else if( auto constantIntVal = dynamic_cast<ConstantIntVal*>(val) )
+        {
+            // TODO: need to figure out what prefix/suffix is needed
+            // to allow demangling later.
+            emitRaw(context, "k");
+            emit(context, (UInt) constantIntVal->value);
+        }
         else
         {
             SLANG_UNEXPECTED("unimplemented case in mangling");
@@ -277,11 +284,13 @@ namespace Slang
         //
         if( auto callableDeclRef = declRef.As<CallableDecl>())
         {
-            emitRaw(context, "p");
-
             auto parameters = GetParameters(callableDeclRef);
             UInt parameterCount = parameters.Count();
+
+            emitRaw(context, "p");
             emit(context, parameterCount);
+            emitRaw(context, "p");
+
             for(auto paramDeclRef : parameters)
             {
                 emitType(context, GetType(paramDeclRef));