Merge pull request #31 from tfoleyNV/escaped-newlines

Bug fix for newline escaping.

1 files changed, 32 insertions, 1 deletions

diff --git a/source/slang/lexer.cpp b/source/slang/lexer.cpp
index cb718b538..f81ead87c 100644
--- a/source/slang/lexer.cpp
+++ b/source/slang/lexer.cpp
@@ -1052,8 +1052,39 @@ namespace Slang
             // Note(tfoley): `StringBuilder::Append()` seems to crash when appending zero bytes
             if(textEnd != textBegin)
             {
+                // HACK(tfoley): "scrubbing" token value here to remove escaped newlines...
+                //
+                // TODO: Only perform this work if we encountered an escaped newline
+                // while lexing this token (e.g., keep a flag on the lexer), or
+                // do it on-demand when the actual value of the token is needed.
+
                 StringBuilder valueBuilder;
-                valueBuilder.Append(textBegin, int(textEnd - textBegin));
+                auto tt = textBegin;
+                while(tt != textEnd)
+                {
+                    char c = *tt++;
+                    if(c == '\\')
+                    {
+                        char d = *tt;
+                        switch(d)
+                        {
+                        case '\r': case '\n':
+                            {
+                                tt++;
+                                char e = *tt;
+                                if((d ^ e) == ('\r' ^ '\n'))
+                                {
+                                    tt++;
+                                }
+                            }
+                            continue;
+
+                        default:
+                            break;
+                        }
+                    }
+                    valueBuilder.Append(c);
+                }
                 token.Content = valueBuilder.ProduceString();
             }