StringUtil::extractLine (#989)

* Added extractLine line parsing to StringUtil. Use for matching lines instead of calcLines. calcLines uses extractLine to extract lines.
Fixed problems found in output of some tests- due to how a how final line is handled. Now a final line has a \r or \n\r combination, but nothing else after it, it is considered the last line (not the line after it).

* Use StringUtil::extractLine in slang-generate.

* Improved comment on extractLine

* Remove test code from StringUtil::extractLine

* Made StringUtil::extractLine act as if line terminators are 'separators'.
Added unit-test-string.cpp - to check behavior.

* Adding LineParser - not entirely necessary, but slightly easier to use.

* Improved LineParser::Iterator end testing.
Added improved tests for LineParser.

* Move line comparson after termination case - to fix problem with gcc release build.

* Make UnownedStringSlice handle comparison when begin is nullptr - as it uses memcmp and passing nullptr to memcmp is undefined, leading to optimizer being able to do some unfortunate optimizations on gcc.

1 files changed, 49 insertions, 26 deletions

diff --git a/source/core/slang-string-util.cpp b/source/core/slang-string-util.cpp
index 60ebc45ba..fa96e4435 100644
--- a/source/core/slang-string-util.cpp
+++ b/source/core/slang-string-util.cpp
@@ -199,15 +199,18 @@ ComPtr<ISlangBlob> StringUtil::createStringBlob(const String& string)
     return (fromChar == toChar || string.indexOf(fromChar) == Index(-1)) ? string : calcCharReplaced(string.getUnownedSlice(), fromChar, toChar);
 }
 
-/* static */void StringUtil::calcLines(const UnownedStringSlice& textIn, List<UnownedStringSlice>& outLines)
+/* static */UnownedStringSlice StringUtil::extractLine(UnownedStringSlice& ioText)
 {
-    char const* begin = textIn.begin();
-    char const* end = textIn.end();
-
-    char const* cursor = begin;
+    char const*const begin = ioText.begin();
+    char const*const end = ioText.end();
 
-    const char* lineStart = cursor;
+    // If we have hit the end then return the 'special' terminator
+    if (begin == nullptr)
+    {
+        return UnownedStringSlice(nullptr, nullptr);
+    }
 
+    char const* cursor = begin;
     while (cursor < end)
     {
         int c = *cursor++;
@@ -215,56 +218,76 @@ ComPtr<ISlangBlob> StringUtil::createStringBlob(const String& string)
         {
             case '\r': case '\n':
             {
-                outLines.add(UnownedStringSlice(lineStart, cursor - 1));
+                // Remember the end of the line
+                const char*const lineEnd = cursor - 1;
 
                 // When we see a line-break character we need
                 // to record the line break, but we also need
                 // to deal with the annoying issue of encodings,
                 // where a multi-byte sequence might encode
                 // the line break.
-
                 if (cursor < end)
                 {
                     int d = *cursor;
                     if ((c ^ d) == ('\r' ^ '\n'))
                         cursor++;
                 }
-                lineStart = cursor;
-                break;
+
+                ioText = UnownedStringSlice(cursor, end);
+                return UnownedStringSlice(begin, lineEnd);
             }
             default:
                 break;
         }
     }
 
-    if (cursor > lineStart)
-    {
-        outLines.add(UnownedStringSlice(lineStart, cursor));
-    }
+    // There is nothing remaining
+    ioText = UnownedStringSlice(nullptr, nullptr);
+
+    // Could be empty, or the remaining line (without line end terminators of)
+    SLANG_ASSERT(begin <= cursor);
+
+    return UnownedStringSlice(begin, cursor);
 }
 
-/* static */bool StringUtil::areLinesEqual(const UnownedStringSlice& a, const UnownedStringSlice& b)
+/* static */void StringUtil::calcLines(const UnownedStringSlice& textIn, List<UnownedStringSlice>& outLines)
 {
-    List<UnownedStringSlice> slicesA;
-    List<UnownedStringSlice> slicesB;
-
-    calcLines(a, slicesA);
-    calcLines(b, slicesB);
+    outLines.clear();
 
-    const auto linesCount = slicesA.getCount();
-    if (linesCount != slicesB.getCount())
+    UnownedStringSlice text(textIn);
+    while (true)
     {
-        return false;
+        UnownedStringSlice line = extractLine(text);
+        if (line.begin() == nullptr)
+        {
+            return;
+        }
+        outLines.add(line);
     }
+}
 
-    for (Index i = 0; i < linesCount; ++i)
+/* static */bool StringUtil::areLinesEqual(const UnownedStringSlice& inA, const UnownedStringSlice& inB)
+{
+    UnownedStringSlice a(inA);
+    UnownedStringSlice b(inB);
+
+    while (true)
     {
-        if (slicesA[i] != slicesB[i])
+        const UnownedStringSlice lineA = extractLine(a);
+        const UnownedStringSlice lineB = extractLine(b);
+
+        // If either has ended, they both must have ended
+        if (lineA.begin() == nullptr || lineB.begin() == nullptr)
+        {
+            return lineA.begin() == lineB.begin();
+        }
+
+        // The lines must be equal
+        if (lineA != lineB)
         {
             return false;
         }
     }
-    return true;
 }
 
 } // namespace Slang