Parsing CPP Compiler output (#994)

* Added extractLine line parsing to StringUtil. Use for matching lines instead of calcLines. calcLines uses extractLine to extract lines.
Fixed problems found in output of some tests- due to how a how final line is handled. Now a final line has a \r or \n\r combination, but nothing else after it, it is considered the last line (not the line after it).

* Use StringUtil::extractLine in slang-generate.

* Improved comment on extractLine

* Remove test code from StringUtil::extractLine

* Made StringUtil::extractLine act as if line terminators are 'separators'.
Added unit-test-string.cpp - to check behavior.

* Adding LineParser - not entirely necessary, but slightly easier to use.

* Hack to output start of tests.

* WIP parsing CPPCompiler output.

* Make extractLine return a bool.

* First attempt at Visual Studio output parsing.

* Add handling for checking error returning from CPPCompiler.

* First pass parsing output of Gcc/Clang.

* Split out VisualStudioCompilerUtil and GCCCompilerUtil.
Simplified parsing of versions.

* Simplify CPPCompiler::Output interface.

* Fix problem with cpp-compiler on linux targets.

* Add shared library link error.

* Improving GCC/Clang parsing output.

* Make cpp compiler parsing function able to return a SlangResult.

* Handling for 'info' on clang

* Add expected result for c-compile-shared-library-error.c

* * Add flags such that link errors on shared libraries are supported.
* Added StringUtil::join

* Turn off the link shared library unfound symbol option on MacOS because it causes an error (and it's not needed on that target).

* Add natvis inclusion back to visual studio projects.

* Display message to try and determine crash problem on travisbuild.

* Fix bug in handling continuations for clang.
Disabled output of exception text.

* WIP: See what clang is outputting that is parsing incorrectly on travis.

* More handling for travis clang parsing issue.

* Restore natvis to core.vcxproj

* Fix visual studio project such that it still as natvis.

1 files changed, 460 insertions, 0 deletions

diff --git a/source/core/slang-gcc-compiler-util.cpp b/source/core/slang-gcc-compiler-util.cpp
new file mode 100644
index 000000000..9f22526a1
--- /dev/null
+++ b/source/core/slang-gcc-compiler-util.cpp
@@ -0,0 +1,460 @@
+// slang-gcc-compiler-util.cpp
+#include "slang-gcc-compiler-util.h"
+
+#include "slang-common.h"
+#include "../../slang-com-helper.h"
+#include "slang-string-util.h"
+
+#include "slang-io.h"
+#include "slang-shared-library.h"
+
+namespace Slang
+{
+
+/* static */SlangResult GCCCompilerUtil::parseVersion(const UnownedStringSlice& text, const UnownedStringSlice& prefix, CPPCompiler::Desc& outDesc)
+{
+    List<UnownedStringSlice> lines;
+    StringUtil::calcLines(text, lines);
+
+    for (auto line : lines)
+    {
+        if (line.startsWith(prefix))
+        {
+            const UnownedStringSlice remainingSlice = UnownedStringSlice(line.begin() + prefix.size(), line.end()).trim();
+            const Index versionEndIndex = remainingSlice.indexOf(' ');
+            if (versionEndIndex < 0)
+            {
+                return SLANG_FAIL;
+            }
+
+            const UnownedStringSlice versionSlice(remainingSlice.begin(), remainingSlice.begin() + versionEndIndex);
+
+            // Version is in format 0.0.0 
+            List<UnownedStringSlice> split;
+            StringUtil::split(versionSlice, '.', split);
+            List<Int> digits;
+
+            for (auto v : split)
+            {
+                Int version;
+                SLANG_RETURN_ON_FAIL(StringUtil::parseInt(v, version));
+                digits.add(version);
+            }
+
+            if (digits.getCount() < 2)
+            {
+                return SLANG_FAIL;
+            }
+
+            outDesc.majorVersion = digits[0];
+            outDesc.minorVersion = digits[1];
+            return SLANG_OK;
+        }
+    }
+
+    return SLANG_FAIL;
+}
+
+SlangResult GCCCompilerUtil::calcVersion(const String& exeName, CPPCompiler::Desc& outDesc)
+{
+    CommandLine cmdLine;
+    cmdLine.setExecutableFilename(exeName);
+    cmdLine.addArg("-v");
+
+    ExecuteResult exeRes;
+    SLANG_RETURN_ON_FAIL(ProcessUtil::execute(cmdLine, exeRes));
+
+    const UnownedStringSlice prefixes[] =
+    {
+        UnownedStringSlice::fromLiteral("clang version"),
+        UnownedStringSlice::fromLiteral("gcc version"),
+        UnownedStringSlice::fromLiteral("Apple LLVM version"),
+    };
+    const CPPCompiler::CompilerType types[] =
+    {
+        CPPCompiler::CompilerType::Clang,
+        CPPCompiler::CompilerType::GCC,
+        CPPCompiler::CompilerType::Clang,
+    };
+
+    SLANG_COMPILE_TIME_ASSERT(SLANG_COUNT_OF(prefixes) == SLANG_COUNT_OF(types));
+
+    for (Index i = 0; i < SLANG_COUNT_OF(prefixes); ++i)
+    {
+        // Set the type
+        outDesc.type = types[i];
+        // Extract the version
+        if (SLANG_SUCCEEDED(parseVersion(exeRes.standardError.getUnownedSlice(), prefixes[i], outDesc)))
+        {
+            return SLANG_OK;
+        }
+    }
+    return SLANG_FAIL;
+}
+
+static SlangResult _parseErrorType(const UnownedStringSlice& in, CPPCompiler::OutputMessage::Type& outType)
+{
+    typedef CPPCompiler::OutputMessage::Type Type;
+
+    if (in == "error" || in == "fatal error")
+    {
+        outType = Type::Error;
+    }
+    else if (in == "warning")
+    {
+        outType = Type::Warning;
+    }
+    else if (in == "info" || in == "note")
+    {
+        outType = Type::Info;
+    }
+    else
+    {
+        return SLANG_FAIL;
+    }
+    return SLANG_OK;
+}
+
+namespace { // anonymous
+
+enum class LineParseResult
+{
+    Single,             ///< It's a single line
+    Start,              ///< Line was the start of a message
+    Continuation,       ///< Not totally clear, add to previous line if nothing else hit
+    Ignore,             ///< Ignore the line
+};
+    
+} // anonymous
+    
+static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParseResult& outLineParseResult, CPPCompiler::OutputMessage& outMsg)
+{
+    typedef CPPCompiler::OutputMessage OutputMessage;
+    typedef OutputMessage::Type Type;
+    
+    // Set to default case
+    outLineParseResult = LineParseResult::Ignore;
+    
+    /* example error output from different scenarios */
+    
+    /*
+        tests/cpp-compiler/c-compile-error.c: In function ‘int main(int, char**)’:
+        tests/cpp-compiler/c-compile-error.c:8:13: error: ‘b’ was not declared in this scope
+        int a = b + c;
+        ^
+        tests/cpp-compiler/c-compile-error.c:8:17: error: ‘c’ was not declared in this scope
+        int a = b + c;
+        ^
+    */
+
+    /* /tmp/ccS0JCWe.o:c-compile-link-error.c:(.rdata$.refptr.thing[.refptr.thing]+0x0): undefined reference to `thing'
+       collect2: error: ld returned 1 exit status*/
+
+    /*
+     clang: warning: treating 'c' input as 'c++' when in C++ mode, this behavior is deprecated [-Wdeprecated]
+     Undefined symbols for architecture x86_64:
+     "_thing", referenced from:
+     _main in c-compile-link-error-a83ace.o
+     ld: symbol(s) not found for architecture x86_64
+     clang: error: linker command failed with exit code 1 (use -v to see invocation) */
+
+     /* /tmp/c-compile-link-error-ccf151.o: In function `main':
+      c-compile-link-error.c:(.text+0x19): undefined reference to `thing'
+     clang: error: linker command failed with exit code 1 (use -v to see invocation)
+     */
+
+     /* /tmp/c-compile-link-error-301c8c.o: In function `main':
+        /home/travis/build/shader-slang/slang/tests/cpp-compiler/c-compile-link-error.c:10: undefined reference to `thing'
+        clang-7: error: linker command failed with exit code 1 (use -v to see invocation)*/
+
+    outMsg.stage = OutputMessage::Stage::Compile;
+
+    List<UnownedStringSlice> split;
+    StringUtil::split(line, ':', split);
+
+    if (split.getCount() == 2)
+    {
+        const auto split0 = split[0].trim();
+        if (split0 == UnownedStringSlice::fromLiteral("ld"))
+        {
+            // We'll ignore for now
+            outMsg.stage = OutputMessage::Stage::Link;
+            outMsg.type = Type::Info;
+            outMsg.text = split[1].trim();
+            outLineParseResult = LineParseResult::Start;
+            return SLANG_OK;
+        }
+        outLineParseResult = LineParseResult::Ignore;
+        return SLANG_OK;
+    }
+    else if (split.getCount() == 3)
+    {
+        const auto split0 = split[0].trim();
+        const auto split1 = split[1].trim();
+        const auto text = split[2].trim();
+
+        // Check for special handling for clang
+        if (split0.startsWith(UnownedStringSlice::fromLiteral("clang")))
+        {
+            // Extract the type
+            SLANG_RETURN_ON_FAIL(_parseErrorType(split[1].trim(), outMsg.type));
+
+            if (text.startsWith("linker command failed"))
+            {
+                outMsg.stage = OutputMessage::Stage::Link;
+            }
+
+            outMsg.text = text;
+            outLineParseResult = LineParseResult::Start;
+            return SLANG_OK;
+        }
+        else if (split1.startsWith("(.text"))
+        {
+            // This is a little weak... but looks like it's a link error
+            outMsg.filePath = split[0];
+            outMsg.type = Type::Error;
+            outMsg.stage = OutputMessage::Stage::Link;
+            outMsg.text = text;
+            outLineParseResult = LineParseResult::Single;
+            return SLANG_OK;
+        }
+        else if (text.startsWith("ld returned"))
+        {
+            outMsg.stage = CPPCompiler::OutputMessage::Stage::Link;
+            SLANG_RETURN_ON_FAIL(_parseErrorType(split[1].trim(), outMsg.type));
+            outMsg.text = line;
+            outLineParseResult = LineParseResult::Single;
+            return SLANG_OK;
+        }
+        else if (text == "")
+        {
+            // This is probably a prelude line, we'll just ignore it
+            outLineParseResult = LineParseResult::Ignore;
+            return SLANG_OK;
+        }
+    }
+    else if (split.getCount() == 4)
+    {
+        // Probably a link error, give the source line
+        String ext = Path::getFileExt(split[0]);
+
+        // Maybe a bit fragile -> but probably okay for now
+        if (ext != "o" && ext != "obj")
+        {
+            outLineParseResult = LineParseResult::Ignore;
+            return SLANG_OK;
+        }
+        else
+        {
+            outMsg.filePath = split[1];
+            outMsg.fileLine = 0;
+            outMsg.type = OutputMessage::Type::Error;
+            outMsg.stage = OutputMessage::Stage::Link;
+            outMsg.text = split[3];
+            
+            outLineParseResult = LineParseResult::Start;
+            return SLANG_OK;
+        }
+    }
+    else if (split.getCount() == 5)
+    {
+        // Probably a regular error line
+        SLANG_RETURN_ON_FAIL(_parseErrorType(split[3].trim(), outMsg.type));
+
+        outMsg.filePath = split[0];
+        SLANG_RETURN_ON_FAIL(StringUtil::parseInt(split[1], outMsg.fileLine));
+        outMsg.text = split[4];
+
+        outLineParseResult = LineParseResult::Start;
+        return SLANG_OK;
+    }
+    
+    // Assume it's a continuation
+    outLineParseResult = LineParseResult::Continuation;
+    return SLANG_OK;
+}
+
+/* static */SlangResult GCCCompilerUtil::parseOutput(const ExecuteResult& exeRes, CPPCompiler::Output& outOutput)
+{
+    LineParseResult prevLineResult = LineParseResult::Ignore;
+    
+    outOutput.reset();
+
+    for (auto line : LineParser(exeRes.standardError.getUnownedSlice()))
+    {
+        CPPCompiler::OutputMessage msg;
+        LineParseResult lineRes;
+        
+        SLANG_RETURN_ON_FAIL(_parseGCCFamilyLine(line, lineRes, msg));
+        
+        switch (lineRes)
+        {
+            case LineParseResult::Start:
+            {
+                // It's start of a new message
+                outOutput.messages.add(msg);
+                prevLineResult = LineParseResult::Start;
+                break;
+            }
+            case LineParseResult::Single:
+            {
+                // It's a single message, without anything following
+                outOutput.messages.add(msg);
+                prevLineResult = LineParseResult::Ignore;
+                break;
+            }
+            case LineParseResult::Continuation:
+            {
+                if (prevLineResult == LineParseResult::Start || prevLineResult == LineParseResult::Continuation)
+                {
+                    if (outOutput.messages.getCount() > 0)
+                    {
+                        // We are now in a continuation, add to the last
+                        auto& text = outOutput.messages.getLast().text;
+                        text.append("\n");
+                        text.append(line);
+                    }
+                    prevLineResult = LineParseResult::Continuation;
+                }
+                break;
+            }
+            case LineParseResult::Ignore:
+            {
+                prevLineResult = lineRes;
+                break;
+            }
+            default: return SLANG_FAIL;
+        }
+    }
+
+    if (outOutput.has(CPPCompiler::OutputMessage::Type::Error) || exeRes.resultCode != 0)
+    {
+        outOutput.result = SLANG_FAIL;
+    }
+
+    return SLANG_OK;
+}
+
+/* static */void GCCCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine)
+{
+    cmdLine.addArg("-fvisibility=hidden");
+    // Use shared libraries
+    //cmdLine.addArg("-shared");
+
+    switch (options.optimizationLevel)
+    {
+        case OptimizationLevel::Debug:
+        {
+            // No optimization
+            cmdLine.addArg("-O0");
+            break;
+        }
+        case OptimizationLevel::Normal:
+        {
+            cmdLine.addArg("-Os");
+            break;
+        }
+        default: break;
+    }
+
+    if (options.debugInfoType != DebugInfoType::None)
+    {
+        cmdLine.addArg("-g");
+    }
+
+    switch (options.targetType)
+    {
+        case TargetType::SharedLibrary:
+        {
+            // Shared library
+            cmdLine.addArg("-shared");
+            // Position independent
+            cmdLine.addArg("-fPIC");
+
+            String sharedLibraryPath = SharedLibrary::calcPlatformPath(options.modulePath.getUnownedSlice());
+
+            cmdLine.addArg("-o");
+            cmdLine.addArg(sharedLibraryPath);
+            break;
+        }
+        case TargetType::Executable:
+        {
+            cmdLine.addArg("-o");
+
+            StringBuilder builder;
+            builder << options.modulePath;
+            builder << ProcessUtil::getExecutableSuffix();
+
+            cmdLine.addArg(options.modulePath);
+            break;
+        }
+        case TargetType::Object:
+        {
+            // Don't link, just produce object file
+            cmdLine.addArg("-c");
+            break;
+        }
+        default: break;
+    }
+
+    // Add defines
+    for (const auto& define : options.defines)
+    {
+        StringBuilder builder;
+        builder << define.nameWithSig;
+        if (define.value.getLength())
+        {
+            builder << "=" << define.value;
+        }
+
+        cmdLine.addArg(builder);
+    }
+
+    // Add includes
+    for (const auto& include : options.includePaths)
+    {
+        cmdLine.addArg("-I");
+        cmdLine.addArg(include);
+    }
+
+    // Link options
+    if (0) // && options.targetType != TargetType::Object)
+    {
+        //linkOptions << "-Wl,";
+        //cmdLine.addArg(linkOptions);
+    }
+
+    if (options.targetType == TargetType::SharedLibrary)
+    {
+#if !SLANG_APPLE_FAMILY
+        // On MacOS, this linker option is not supported. That's ok though in
+        // so far as on MacOS it does report any unfound symbols without the option.
+
+        // Linker flag to report any undefined symbols as a link error
+        cmdLine.addArg("-Wl,--no-undefined");
+#endif
+    }
+
+    // Files to compile
+    for (const auto& sourceFile : options.sourceFiles)
+    {
+        cmdLine.addArg(sourceFile);
+    }
+
+    for (const auto& libPath : options.libraryPaths)
+    {
+        // Note that any escaping of the path is handled in the ProcessUtil::
+        cmdLine.addArg("-L");
+        cmdLine.addArg(libPath);
+        cmdLine.addArg("-F");
+        cmdLine.addArg(libPath);
+    }
+
+    if (options.sourceType == SourceType::CPP)
+    {
+        // Make STD libs available
+        cmdLine.addArg("-lstdc++");
+    }
+}
+
+}