Added NativeStringType (#2252)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Use TerminatedUnownedStringSlice for literals in output C++.

* Remove Escape/Unescape functions used in slang-token-reader.cpp
Add target type of 'host-cpp' etc to map to the target types.

* Fix some corner cases around string encoding.

* Added unit test for string escaping.
Fixed some assorted escaping bugs.

* Updated test output.

* Added decode test.

* Stop using hex output, to get around 'greedy' aspect. Use octal instead.

14 files changed, 101 insertions, 106 deletions

diff --git a/source/slang/core.meta.slang b/source/slang/core.meta.slang
index 476e88e3f..41cfea6af 100644
--- a/source/slang/core.meta.slang
+++ b/source/slang/core.meta.slang
@@ -378,6 +378,11 @@ __intrinsic_type($(kIROp_StringType))
 struct String
 {};
 
+__magic_type(NativeStringType)
+__intrinsic_type($(kIROp_NativeStringType))
+struct NativeString
+{};
+
 __magic_type(DynamicType)
 __intrinsic_type($(kIROp_DynamicType))
 struct __Dynamic
diff --git a/source/slang/slang-ast-builder.cpp b/source/slang/slang-ast-builder.cpp
index caf4c020d..7ac039187 100644
--- a/source/slang/slang-ast-builder.cpp
+++ b/source/slang/slang-ast-builder.cpp
@@ -90,6 +90,16 @@ Type* SharedASTBuilder::getStringType()
     return m_stringType;
 }
 
+Type* SharedASTBuilder::getNativeStringType()
+{
+    if (!m_nativeStringType)
+    {
+        auto nativeStringTypeDecl = findMagicDecl("NativeStringType");
+        m_nativeStringType = DeclRefType::create(m_astBuilder, makeDeclRef<Decl>(nativeStringTypeDecl));
+    }
+    return m_nativeStringType;
+}
+
 Type* SharedASTBuilder::getEnumTypeType()
 {
     if (!m_enumTypeType)
diff --git a/source/slang/slang-ast-builder.h b/source/slang/slang-ast-builder.h
index 0642455c3..97aefd118 100644
--- a/source/slang/slang-ast-builder.h
+++ b/source/slang/slang-ast-builder.h
@@ -23,6 +23,10 @@ public:
 
         /// Get the string type
     Type* getStringType();
+
+        /// Get the native string type
+    Type* getNativeStringType();
+
         /// Get the enum type type
     Type* getEnumTypeType();
         /// Get the __Dynamic type
@@ -65,6 +69,7 @@ protected:
     // TODO(tfoley): These should really belong to the compilation context!
     //
     Type* m_stringType = nullptr;
+    Type* m_nativeStringType = nullptr;
     Type* m_enumTypeType = nullptr;
     Type* m_dynamicType = nullptr;
     Type* m_nullPtrType = nullptr;
diff --git a/source/slang/slang-ast-type.h b/source/slang/slang-ast-type.h
index fee7f7cac..7aa1a36ab 100644
--- a/source/slang/slang-ast-type.h
+++ b/source/slang/slang-ast-type.h
@@ -460,12 +460,24 @@ private:
     Type* rowType = nullptr;
 };
 
-// The built-in `String` type
-class StringType : public BuiltinType 
+// Base class for built in string types
+class StringTypeBase : public BuiltinType
+{
+    SLANG_AST_CLASS(StringTypeBase)
+};
+
+// The regular built-in `String` type
+class StringType : public StringTypeBase
 {
     SLANG_AST_CLASS(StringType)
 };
 
+// The string type native to the target
+class NativeStringType : public StringTypeBase
+{
+    SLANG_AST_CLASS(NativeStringType)
+};
+
 // The built-in `__Dynamic` type
 class DynamicType : public BuiltinType
 {
diff --git a/source/slang/slang-check-conversion.cpp b/source/slang/slang-check-conversion.cpp
index a1935d65c..44bb8a610 100644
--- a/source/slang/slang-check-conversion.cpp
+++ b/source/slang/slang-check-conversion.cpp
@@ -639,6 +639,16 @@ namespace Slang
             return true;
         }
 
+        // If both are string types we assume they are convertable in both directions
+        if (as<StringTypeBase>(fromType) && as<StringTypeBase>(toType))
+        {
+            if (outToExpr)
+                *outToExpr = fromExpr;
+            if (outCost)
+                *outCost = kConversionCost_None;
+            return true;
+        }
+
         // Another important case is when either the "to" or "from" type
         // represents an error. In such a case we must have already
         // reporeted the error, so it is better to allow the conversion
diff --git a/source/slang/slang-emit-cpp.cpp b/source/slang/slang-emit-cpp.cpp
index 9887f1ba6..482ada394 100644
--- a/source/slang/slang-emit-cpp.cpp
+++ b/source/slang/slang-emit-cpp.cpp
@@ -532,6 +532,11 @@ SlangResult CPPSourceEmitter::calcTypeName(IRType* type, CodeGenTarget target, S
             out << "TypeInfo*";
             return SLANG_OK;
         }
+        case kIROp_NativeStringType:
+        {
+            out << "const char*";
+            return SLANG_OK;
+        }
         case kIROp_StringType:
         {
             out << "String";
@@ -2411,8 +2416,15 @@ bool CPPSourceEmitter::tryEmitInstExprImpl(IRInst* inst, const EmitOpInfo& inOut
         }
         case kIROp_StringLit:
         {
-            m_writer->emit("String(");
-            m_writer->emit(Slang::Misc::EscapeStringLiteral(as<IRStringLit>(inst)->getStringSlice()));
+            m_writer->emit("toTerminatedSlice(");
+
+            auto handler = StringEscapeUtil::getHandler(StringEscapeUtil::Style::Cpp);
+            
+            StringBuilder buf;
+            const auto slice = as<IRStringLit>(inst)->getStringSlice();
+            StringEscapeUtil::appendQuoted(handler, slice, buf);
+            m_writer->emit(buf);
+
             m_writer->emit(")");
             return true;
         }
diff --git a/source/slang/slang-emit-glsl.cpp b/source/slang/slang-emit-glsl.cpp
index c1bbf813b..b0c10fdc2 100644
--- a/source/slang/slang-emit-glsl.cpp
+++ b/source/slang/slang-emit-glsl.cpp
@@ -1967,7 +1967,12 @@ void GLSLSourceEmitter::emitSimpleTypeImpl(IRType* type)
             }
             return;
         }
-        case kIROp_StringType: m_writer->emit("int"); return;
+        case kIROp_NativeStringType:
+        case kIROp_StringType: 
+        {
+            m_writer->emit("int"); 
+            return;
+        }
         default: break;
     }
 
diff --git a/source/slang/slang-emit-hlsl.cpp b/source/slang/slang-emit-hlsl.cpp
index 2d42aef83..48fe86fff 100644
--- a/source/slang/slang-emit-hlsl.cpp
+++ b/source/slang/slang-emit-hlsl.cpp
@@ -853,7 +853,12 @@ void HLSLSourceEmitter::emitSimpleTypeImpl(IRType* type)
             }
             return;
         }
-        case kIROp_StringType: m_writer->emit("int"); return;
+        case kIROp_NativeStringType:
+        case kIROp_StringType: 
+        {
+            m_writer->emit("int"); 
+            return;
+        }
         default: break;
     }
 
diff --git a/source/slang/slang-ir-collect-global-uniforms.cpp b/source/slang/slang-ir-collect-global-uniforms.cpp
index 87b21c819..ca5e56b53 100644
--- a/source/slang/slang-ir-collect-global-uniforms.cpp
+++ b/source/slang/slang-ir-collect-global-uniforms.cpp
@@ -69,6 +69,11 @@ struct CollectGlobalUniformParametersContext
     //
     void processModule()
     {
+        if (!globalScopeVarLayout)
+        {
+            return;
+        }
+
         // We start by looking at the layout that was computed for the global-scope
         // parameters to determine how the parameters are supposed to be pacakged.
         //
diff --git a/source/slang/slang-ir-inst-defs.h b/source/slang/slang-ir-inst-defs.h
index f9e0a5f34..c617a0218 100644
--- a/source/slang/slang-ir-inst-defs.h
+++ b/source/slang/slang-ir-inst-defs.h
@@ -24,7 +24,10 @@ INST(Nop, nop, 0, 0)
 
     INST_RANGE(BasicType, VoidType, AfterBaseType)
 
-    INST(StringType, String, 0, 0)
+    /* StringTypeBase */
+        INST(StringType, String, 0, 0)
+        INST(NativeStringType, NativeString, 0, 0)
+    INST_RANGE(StringTypeBase, StringType, NativeStringType)
 
     INST(CapabilitySetType, CapabilitySet, 0, 0)
 
diff --git a/source/slang/slang-ir-insts.h b/source/slang/slang-ir-insts.h
index 0e54802e5..77b3eabc0 100644
--- a/source/slang/slang-ir-insts.h
+++ b/source/slang/slang-ir-insts.h
@@ -2080,6 +2080,7 @@ public:
     IRBasicType* getUInt64Type();
     IRBasicType* getCharType();
     IRStringType* getStringType();
+    IRNativeStringType* getNativeStringType();
 
     IRType* getCapabilitySetType();
 
diff --git a/source/slang/slang-ir-link.cpp b/source/slang/slang-ir-link.cpp
index b67d95abf..7984c5037 100644
--- a/source/slang/slang-ir-link.cpp
+++ b/source/slang/slang-ir-link.cpp
@@ -1477,10 +1477,13 @@ LinkedIR linkIR(
     // need to operate on all the global parameters can do so.
     //
     IRVarLayout* irGlobalScopeVarLayout = nullptr;
-    if( auto irGlobalScopeLayoutDecoration = irModuleForLayout->getModuleInst()->findDecoration<IRLayoutDecoration>() )
+    if (irModuleForLayout)
     {
-        auto irOriginalGlobalScopeVarLayout = irGlobalScopeLayoutDecoration->getLayout();
-        irGlobalScopeVarLayout = cast<IRVarLayout>(cloneValue(context, irOriginalGlobalScopeVarLayout));
+        if( auto irGlobalScopeLayoutDecoration = irModuleForLayout->getModuleInst()->findDecoration<IRLayoutDecoration>() )
+        {
+            auto irOriginalGlobalScopeVarLayout = irGlobalScopeLayoutDecoration->getLayout();
+            irGlobalScopeVarLayout = cast<IRVarLayout>(cloneValue(context, irOriginalGlobalScopeVarLayout));
+        }
     }
 
     // Bindings for global generic parameters are currently represented
diff --git a/source/slang/slang-ir.cpp b/source/slang/slang-ir.cpp
index 9de2f5b4f..d454333e6 100644
--- a/source/slang/slang-ir.cpp
+++ b/source/slang/slang-ir.cpp
@@ -2508,6 +2508,12 @@ namespace Slang
         return (IRStringType*)getType(kIROp_StringType);
     }
 
+    IRNativeStringType* IRBuilder::getNativeStringType()
+    {
+        return (IRNativeStringType*)getType(kIROp_NativeStringType);
+    }
+
+
     IRType* IRBuilder::getCapabilitySetType()
     {
         return getType(kIROp_CapabilitySetType);
@@ -4676,106 +4682,13 @@ namespace Slang
         dumpDebugID(context, inst);
     }
     
-    struct StringEncoder
-    {
-        static char getHexChar(int v)
-        {
-            return (v <= 9) ? char(v + '0') : char(v - 10 + 'A');
-        }
-
-        void flush(const char* pos)
-        {
-            if (pos > m_runStart)
-            {
-                m_builder->append(m_runStart, pos);
-            }
-            m_runStart = pos + 1;
-        }
-
-        void appendEscapedChar(const char* pos, char encodeChar)
-        {
-            flush(pos);
-            const char chars[] = { '\\', encodeChar };
-            m_builder->Append(chars, 2);
-        }
-        
-        void appendAsHex(const char* pos)
-        {
-            flush(pos);
-
-            const int v = *(const uint8_t*)pos;
-
-            char buf[5];
-            buf[0] = '\\';
-            buf[1] = 'x';
-            buf[2] = '0';
-
-            buf[3] = getHexChar(v >> 4);
-            buf[4] = getHexChar(v & 0xf);
-
-            m_builder->Append(buf, 5);
-        }
-
-        StringEncoder(StringBuilder* builder, const char* start):
-            m_runStart(start),
-            m_builder(builder)
-        {}
-
-        StringBuilder* m_builder;
-        const char* m_runStart;
-    };
-
     static void dumpEncodeString(
         IRDumpContext*  context, 
         const UnownedStringSlice& slice)
     {
-        // https://msdn.microsoft.com/en-us/library/69ze775t.aspx
-
+        auto handler = StringEscapeUtil::getHandler(StringEscapeUtil::Style::Slang);
         StringBuilder& builder = *context->builder;
-        builder.Append('"');
-        
-        {
-            const char* cur = slice.begin();
-            StringEncoder encoder(&builder, cur);
-            const char* end = slice.end();
-
-            for (; cur < end; cur++)
-            {
-                const int8_t c = uint8_t(*cur);
-                switch (c)
-                {
-                    case '\\':
-                        encoder.appendEscapedChar(cur, '\\');
-                        break;
-                    case '"':
-                        encoder.appendEscapedChar(cur, '"');
-                        break;
-                    case '\n': 
-                        encoder.appendEscapedChar(cur, 'n');
-                        break;
-                    case '\t':
-                        encoder.appendEscapedChar(cur, 't');
-                        break;
-                    case '\r':
-                        encoder.appendEscapedChar(cur, 'r');
-                        break;
-                    case '\0':
-                        encoder.appendEscapedChar(cur, '0');
-                        break;
-                    default:
-                    {
-                        if (c < 32)
-                        {
-                            encoder.appendAsHex(cur);
-                        }
-                        break;
-                    }
-                }
-            }
-            encoder.flush(end);
-        }
-        
-        builder.Append('"');
+        StringEscapeUtil::appendQuoted(handler, slice, builder);
     }
 
     static void dumpType(
diff --git a/source/slang/slang-ir.h b/source/slang/slang-ir.h
index 7a1a0b8aa..6c766542f 100644
--- a/source/slang/slang-ir.h
+++ b/source/slang/slang-ir.h
@@ -786,7 +786,13 @@ struct IRBoolType : IRBasicType
     IR_LEAF_ISA(BoolType)
 };
 
-SIMPLE_IR_TYPE(StringType, Type)
+struct IRStringTypeBase : IRType
+{
+    IR_PARENT_ISA(StringTypeBase)
+};
+
+SIMPLE_IR_TYPE(StringType, StringTypeBase)
+SIMPLE_IR_TYPE(NativeStringType, StringTypeBase)
 
 SIMPLE_IR_TYPE(DynamicType, Type)