Support raw string literals. (#2405)

* Support raw string literals.

* Use raw string literal in tests.

Co-authored-by: Yong He <yhe@nvidia.com>

5 files changed, 132 insertions, 12 deletions

diff --git a/source/compiler-core/slang-lexer-diagnostic-defs.h b/source/compiler-core/slang-lexer-diagnostic-defs.h
index 666ab057f..ceeb62455 100644
--- a/source/compiler-core/slang-lexer-diagnostic-defs.h
+++ b/source/compiler-core/slang-lexer-diagnostic-defs.h
@@ -29,5 +29,7 @@ DIAGNOSTIC(10003, Error, invalidDigitForBase, "invalid digit for base-$1 literal
 
 DIAGNOSTIC(10004, Error, endOfFileInLiteral, "end of file in literal")
 DIAGNOSTIC(10005, Error, newlineInLiteral, "newline in literal")
+DIAGNOSTIC(10010, Error, quoteCannotBeDelimiter, "'\"' encountered before '(' in raw string literal. '\"' cannot be a part of a delimiter.")
+
 
 #undef DIAGNOSTIC
diff --git a/source/compiler-core/slang-lexer.cpp b/source/compiler-core/slang-lexer.cpp
index 4a6d7d392..a9d20471a 100644
--- a/source/compiler-core/slang-lexer.cpp
+++ b/source/compiler-core/slang-lexer.cpp
@@ -784,11 +784,80 @@ namespace Slang
         }
     }
 
+    static void _lexRawStringLiteralBody(Lexer* lexer)
+    {
+        const char* start = lexer->m_cursor;
+        const char* endOfDelimiter = nullptr;
+        for (;;)
+        {
+            int c = _peek(lexer);
+            if (c == '(' && endOfDelimiter == nullptr)
+                endOfDelimiter = lexer->m_cursor;
+            if (c == '\"')
+            {
+                if (!endOfDelimiter)
+                {
+                    if (auto sink = lexer->getDiagnosticSink())
+                    {
+                        sink->diagnose(_getSourceLoc(lexer), LexerDiagnostics::quoteCannotBeDelimiter);
+                    }
+                }
+                else
+                {
+                    auto testStart = lexer->m_cursor - (endOfDelimiter - start);
+                    if (testStart > endOfDelimiter)
+                    {
+                        auto testDelimiter = UnownedStringSlice(testStart, lexer->m_cursor);
+                        auto delimiter = UnownedStringSlice(start, endOfDelimiter);
+                        if (*(testStart - 1) == ')' && testDelimiter == delimiter)
+                        {
+                            _advance(lexer);
+                            return;
+                        }
+                    }
+                }
+            }
+
+            switch (c)
+            {
+            case kEOF:
+                if (auto sink = lexer->getDiagnosticSink())
+                {
+                    sink->diagnose(_getSourceLoc(lexer), LexerDiagnostics::endOfFileInLiteral);
+                }
+                return;
+            default:
+                _advance(lexer);
+                continue;
+            }
+        }
+    }
+
+    UnownedStringSlice getRawStringLiteralTokenValue(Token const& token)
+    {
+        auto content = token.getContent();
+        if (content.getLength() <= 5)
+            return UnownedStringSlice();
+        auto start = content.begin() + 2;
+        auto delimEnd = start;
+        while (delimEnd < content.end() && *delimEnd != '(')
+            delimEnd++;
+        auto delimLength = delimEnd - start;
+        auto contentEnd = content.end() - delimLength - 2;
+        auto contentBegin = start + delimLength + 1;
+        if (contentEnd <= contentBegin)
+            return UnownedStringSlice();
+        return UnownedStringSlice(contentBegin, contentEnd);
+    }
+
     String getStringLiteralTokenValue(Token const& token)
     {
         SLANG_ASSERT(token.type == TokenType::StringLiteral
             || token.type == TokenType::CharLiteral);
 
+        if (token.getContent().startsWith("R"))
+            return getRawStringLiteralTokenValue(token);
+
         const UnownedStringSlice content = token.getContent();
 
         char const* cursor = content.begin();
@@ -1016,12 +1085,24 @@ namespace Slang
         case 'A': case 'B': case 'C': case 'D': case 'E':
         case 'F': case 'G': case 'H': case 'I': case 'J':
         case 'K': case 'L': case 'M': case 'N': case 'O':
-        case 'P': case 'Q': case 'R': case 'S': case 'T':
+        case 'P': case 'Q': case 'S': case 'T':
         case 'U': case 'V': case 'W': case 'X': case 'Y':
         case 'Z':
         case '_':
             _lexIdentifier(lexer);
             return TokenType::Identifier;
+        case 'R':
+            _advance(lexer);
+            switch (_peek(lexer))
+            {
+            default:
+                _lexIdentifier(lexer);
+                return TokenType::Identifier;
+            case '\"':
+                _advance(lexer);
+                _lexRawStringLiteralBody(lexer);
+                return TokenType::StringLiteral;
+            }
 
         case '\"':
             _advance(lexer);
@@ -1033,6 +1114,7 @@ namespace Slang
             _lexStringLiteralBody(lexer, '\'');
             return TokenType::CharLiteral;
 
+
         case '+':
             _advance(lexer);
             switch(_peek(lexer))
diff --git a/tests/cpu-program/gfx-smoke.slang b/tests/cpu-program/gfx-smoke.slang
index 2ad33052f..37beede7e 100644
--- a/tests/cpu-program/gfx-smoke.slang
+++ b/tests/cpu-program/gfx-smoke.slang
@@ -23,17 +23,17 @@ public __extern_cpp int main()
     device.value.createCommandQueue(&queueDesc, queue);
 
     gfx.ShaderProgramDesc2 programDesc = {};
-    NativeString s =
-        "[shader(\"compute\")]"
-        "[numthreads(4, 1, 1)]"
-        "void computeMain("
-        "    uint3 sv_dispatchThreadID: SV_DispatchThreadID,"
-        "    uniform RWStructuredBuffer<float> buffer"
-        "    )"
-        "{"
-        "    var input = buffer[sv_dispatchThreadID.x];"
-        "    buffer[sv_dispatchThreadID.x] = sv_dispatchThreadID.x;"
-        "}";
+    NativeString s = R"(
+        [shader("compute")]
+        [numthreads(4, 1, 1)]
+        void computeMain(
+            uint3 sv_dispatchThreadID: SV_DispatchThreadID,
+            uniform RWStructuredBuffer<float> buffer
+            )
+        {
+            var input = buffer[sv_dispatchThreadID.x];
+            buffer[sv_dispatchThreadID.x] = sv_dispatchThreadID.x;
+        })";
     programDesc.sourceData = s;
     programDesc.sourceType = gfx.ShaderModuleSourceType.SlangSource;
     programDesc.sourceDataSize = s.length;
diff --git a/tests/front-end/raw-string-literal.slang b/tests/front-end/raw-string-literal.slang
new file mode 100644
index 000000000..321f183c7
--- /dev/null
+++ b/tests/front-end/raw-string-literal.slang
@@ -0,0 +1,28 @@
+// Test raw string literals.
+
+//TEST:EXECUTABLE:
+__target_intrinsic(cpp, R"delim(printf("%c", $0))delim")
+void writeChar(int8_t ch);
+
+void printLines(NativeString text)
+{
+    NativeString str = text;
+    int8_t *ptr = bit_cast<Ptr<int8_t>>(str);
+    for (int i = 0; i < text.length; i++)
+    {
+        if (ptr[i] != 13)
+        {
+            writeChar(ptr[i]);
+        }
+    }
+}
+
+public __extern_cpp int main()
+{
+    printLines(
+        R"(This is line 1.
+        This is line 2.
+        "Hello World"
+        )");
+    return 0;
+}
\ No newline at end of file
diff --git a/tests/front-end/raw-string-literal.slang.expected b/tests/front-end/raw-string-literal.slang.expected
new file mode 100644
index 000000000..684398fb1
--- /dev/null
+++ b/tests/front-end/raw-string-literal.slang.expected
@@ -0,0 +1,8 @@
+result code = 0
+standard error = {
+}
+standard output = {
+This is line 1.
+        This is line 2.
+        "Hello World"
+        }