StringUtil::extractLine (#989)

* Added extractLine line parsing to StringUtil. Use for matching lines instead of calcLines. calcLines uses extractLine to extract lines.
Fixed problems found in output of some tests- due to how a how final line is handled. Now a final line has a \r or \n\r combination, but nothing else after it, it is considered the last line (not the line after it).

* Use StringUtil::extractLine in slang-generate.

* Improved comment on extractLine

* Remove test code from StringUtil::extractLine

* Made StringUtil::extractLine act as if line terminators are 'separators'.
Added unit-test-string.cpp - to check behavior.

* Adding LineParser - not entirely necessary, but slightly easier to use.

* Improved LineParser::Iterator end testing.
Added improved tests for LineParser.

* Move line comparson after termination case - to fix problem with gcc release build.

* Make UnownedStringSlice handle comparison when begin is nullptr - as it uses memcmp and passing nullptr to memcmp is undefined, leading to optimizer being able to do some unfortunate optimizations on gcc.

4 files changed, 110 insertions, 18 deletions

diff --git a/tools/slang-generate/main.cpp b/tools/slang-generate/main.cpp
index 046b2063c..7c0e3672a 100644
--- a/tools/slang-generate/main.cpp
+++ b/tools/slang-generate/main.cpp
@@ -554,30 +554,26 @@ void emitSimpleText(
     FILE*               stream,
     StringSpan const&   span)
 {
-    char const* cursor = span.begin();
-    char const* end = span.end();
-
-    while (cursor != end)
+    UnownedStringSlice content = span;
+    while (true)
     {
-        int c = *cursor++;
-        switch (c)
+        const auto line = StringUtil::extractLine(content);
+        if (line.begin() == nullptr)
         {
-        default:
-            fprintf(stream, "%c", c);
             break;
+        }
 
-        case '\r': case '\n':
-            if (cursor != end)
-            {
-                int d = *cursor;
-                if ((c ^ d) == ('\r' ^ '\n'))
-                {
-                    cursor++;
-                }
-                fprintf(stream, "\n");
-            }
+        // Write the line
+        fwrite(line.begin(), 1, line.size(), stream);
+
+        // Specially handle the 'final line', excluding an empty line after \n.
+        // We can detect, as if input ends with 'cr/lf' combination, content.begin == span.end(), else if content.begin() == nullptr.
+        if (content.begin() == nullptr || content.begin() == span.end())
+        {
             break;
         }
+
+        fprintf(stream, "\n");
     }
 }
 
diff --git a/tools/slang-test/slang-test.vcxproj b/tools/slang-test/slang-test.vcxproj
index 62b79d627..8f6096f86 100644
--- a/tools/slang-test/slang-test.vcxproj
+++ b/tools/slang-test/slang-test.vcxproj
@@ -179,6 +179,7 @@
     <ClCompile Include="unit-test-free-list.cpp" />
     <ClCompile Include="unit-test-memory-arena.cpp" />
     <ClCompile Include="unit-test-path.cpp" />
+    <ClCompile Include="unit-test-string.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\source\core\core.vcxproj">
diff --git a/tools/slang-test/slang-test.vcxproj.filters b/tools/slang-test/slang-test.vcxproj.filters
index e50c56819..e5cd23212 100644
--- a/tools/slang-test/slang-test.vcxproj.filters
+++ b/tools/slang-test/slang-test.vcxproj.filters
@@ -56,5 +56,8 @@
     <ClCompile Include="unit-test-path.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="unit-test-string.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/tools/slang-test/unit-test-string.cpp b/tools/slang-test/unit-test-string.cpp
new file mode 100644
index 000000000..6110ca55e
--- /dev/null
+++ b/tools/slang-test/unit-test-string.cpp
@@ -0,0 +1,92 @@
+// unit-test-path.cpp
+
+#include "../../source/core/slang-string-util.h"
+
+#include "test-context.h"
+
+using namespace Slang;
+
+static bool _areEqual(const List<UnownedStringSlice>& lines, const UnownedStringSlice* checkLines, Int checkLinesCount)
+{
+    if (checkLinesCount != lines.getCount())
+    {
+        return false;
+    }
+
+    for (Int i = 0; i < checkLinesCount; ++i)
+    {
+        if (lines[i] != checkLines[i])
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
+static bool _checkLines(const UnownedStringSlice& input, const UnownedStringSlice* checkLines, Int checkLinesCount)
+{
+    List<UnownedStringSlice> lines;
+    UnownedStringSlice text(input);
+    while (true)
+    {
+        UnownedStringSlice line = StringUtil::extractLine(text);
+        if (line.begin() == nullptr)
+        {
+            return _areEqual(lines, checkLines, checkLinesCount);
+        }
+        lines.add(line);
+    }
+}
+
+static bool _checkLineParser(const UnownedStringSlice& input)
+{
+    UnownedStringSlice remaining(input);
+    for (const auto line : LineParser(input))
+    {
+        UnownedStringSlice extractLine = StringUtil::extractLine(remaining);
+        if (line != extractLine)
+        {
+            return false;
+        }
+        // Handle hitting the end
+        if (line.begin() == nullptr || extractLine.begin() == nullptr)
+        {
+            return line.begin() == extractLine.begin();
+        }
+    }
+
+    return remaining.begin() == nullptr;
+}
+
+static void stringUnitTest()
+{
+    {
+        UnownedStringSlice checkLines[] = { UnownedStringSlice::fromLiteral("") };
+        SLANG_CHECK(_checkLines(UnownedStringSlice::fromLiteral(""), checkLines, SLANG_COUNT_OF(checkLines)));
+    }
+    {
+        // Will emit no lines
+        SLANG_CHECK(_checkLines(UnownedStringSlice(nullptr, nullptr), nullptr, 0));
+    }
+    {
+        // Two lines - both empty
+        UnownedStringSlice checkLines[] = { UnownedStringSlice(), UnownedStringSlice()};
+        SLANG_CHECK(_checkLines(UnownedStringSlice::fromLiteral("\n"), checkLines, SLANG_COUNT_OF(checkLines)));
+    }
+    {
+        UnownedStringSlice checkLines[] = { UnownedStringSlice::fromLiteral("Hello"), UnownedStringSlice::fromLiteral("World!") };
+        SLANG_CHECK(_checkLines(UnownedStringSlice::fromLiteral("Hello\nWorld!"), checkLines, SLANG_COUNT_OF(checkLines)));
+    }
+    {
+        UnownedStringSlice checkLines[] = { UnownedStringSlice::fromLiteral("Hello"), UnownedStringSlice::fromLiteral("World!"), UnownedStringSlice() };
+        SLANG_CHECK(_checkLines(UnownedStringSlice::fromLiteral("Hello\n\rWorld!\n"), checkLines, SLANG_COUNT_OF(checkLines)));
+    }
+
+    {
+        SLANG_CHECK(_checkLineParser(UnownedStringSlice::fromLiteral("Hello\n\rWorld!\n")));
+        SLANG_CHECK(_checkLineParser(UnownedStringSlice::fromLiteral("\n")));
+        SLANG_CHECK(_checkLineParser(UnownedStringSlice::fromLiteral("")));
+    }
+}
+
+SLANG_UNIT_TEST("String", stringUnitTest);