From 714b0881974965e8fcfbc57b452ef648290d14a1 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Fri, 21 Jun 2019 17:39:32 -0400 Subject: Parsing CPP Compiler output (#994) * Added extractLine line parsing to StringUtil. Use for matching lines instead of calcLines. calcLines uses extractLine to extract lines. Fixed problems found in output of some tests- due to how a how final line is handled. Now a final line has a \r or \n\r combination, but nothing else after it, it is considered the last line (not the line after it). * Use StringUtil::extractLine in slang-generate. * Improved comment on extractLine * Remove test code from StringUtil::extractLine * Made StringUtil::extractLine act as if line terminators are 'separators'. Added unit-test-string.cpp - to check behavior. * Adding LineParser - not entirely necessary, but slightly easier to use. * Hack to output start of tests. * WIP parsing CPPCompiler output. * Make extractLine return a bool. * First attempt at Visual Studio output parsing. * Add handling for checking error returning from CPPCompiler. * First pass parsing output of Gcc/Clang. * Split out VisualStudioCompilerUtil and GCCCompilerUtil. Simplified parsing of versions. * Simplify CPPCompiler::Output interface. * Fix problem with cpp-compiler on linux targets. * Add shared library link error. * Improving GCC/Clang parsing output. * Make cpp compiler parsing function able to return a SlangResult. * Handling for 'info' on clang * Add expected result for c-compile-shared-library-error.c * * Add flags such that link errors on shared libraries are supported. * Added StringUtil::join * Turn off the link shared library unfound symbol option on MacOS because it causes an error (and it's not needed on that target). * Add natvis inclusion back to visual studio projects. * Display message to try and determine crash problem on travisbuild. * Fix bug in handling continuations for clang. Disabled output of exception text. * WIP: See what clang is outputting that is parsing incorrectly on travis. * More handling for travis clang parsing issue. * Restore natvis to core.vcxproj * Fix visual studio project such that it still as natvis. --- source/core/slang-cpp-compiler.cpp | 409 +++++++++---------------------------- 1 file changed, 101 insertions(+), 308 deletions(-) (limited to 'source/core/slang-cpp-compiler.cpp') diff --git a/source/core/slang-cpp-compiler.cpp b/source/core/slang-cpp-compiler.cpp index 18cb9cdae..0ed0a1ba8 100644 --- a/source/core/slang-cpp-compiler.cpp +++ b/source/core/slang-cpp-compiler.cpp @@ -8,374 +8,167 @@ #include "slang-io.h" #include "slang-shared-library.h" +// if Visual Studio import the visual studio platform specific header #if SLANG_VC # include "windows/slang-win-visual-studio-util.h" #endif +#include "slang-visual-studio-compiler-util.h" +#include "slang-gcc-compiler-util.h" + namespace Slang { -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! GenericCPPCompiler !!!!!!!!!!!!!!!!!!!!!!*/ +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPCompiler::OutputMessage !!!!!!!!!!!!!!!!!!!!!!*/ -SlangResult GenericCPPCompiler::compile(const CompileOptions& options, ExecuteResult& outResult) +/* static */UnownedStringSlice CPPCompiler::OutputMessage::getTypeText(OutputMessage::Type type) { - // Copy the command line options - CommandLine cmdLine(m_cmdLine); - - // Append command line args to the end of cmdLine using the target specific function for the specified options - m_func(options, cmdLine); - -#if 0 - // Test - { - String line = ProcessUtil::getCommandLineString(cmdLine); - printf("%s", line.getBuffer()); - } -#endif - - SlangResult res = ProcessUtil::execute(cmdLine, outResult); - -#if 0 + typedef OutputMessage::Type Type; + switch (type) { - printf("stdout=\"%s\"\nstderr=\"%s\"\nret=%d\n", outResult.standardOutput.getBuffer(), outResult.standardError.getBuffer(), int(outResult.resultCode)); + default: return UnownedStringSlice::fromLiteral("Unknown"); + case Type::Info: return UnownedStringSlice::fromLiteral("Info"); + case Type::Warning: return UnownedStringSlice::fromLiteral("Warning"); + case Type::Error: return UnownedStringSlice::fromLiteral("Error"); } -#endif - - return res; } -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPCompilerUtil !!!!!!!!!!!!!!!!!!!!!!*/ -static bool _isDigit(char c) -{ - return c >= '0' && c <= '9'; -} +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPCompiler::Output !!!!!!!!!!!!!!!!!!!!!!*/ -static bool _isWhiteSpace(char c) +Index CPPCompiler::Output::getCountByType(OutputMessage::Type type) const { - return c == ' ' || c == '\t' || c == '\n' || c == '\r'; -} - -/* static */SlangResult CPPCompilerUtil::parseGCCFamilyVersion(const UnownedStringSlice& text, const UnownedStringSlice& prefix, CPPCompiler::Desc& outDesc) -{ - List lines; - StringUtil::calcLines(text, lines); - - for (auto line : lines) + Index count = 0; + for (const auto& msg : messages) { - // TODO(JS): Ugh - having to turn into a string to do this test isn't great. - if (String(line).startsWith(prefix)) - { - UnownedStringSlice versionSlice(line.begin() + prefix.size(), line.end()); - - List digits; - - const char* cur = versionSlice.begin(); - const char* end = versionSlice.end(); - - // Consume white space - while (cur < end && _isWhiteSpace(*cur)) cur++; - - // Version is in format 0.0.0 - while (true) - { - Int value = 0; - const char* start = cur; - while (cur < end && _isDigit(*cur)) - { - value = value * 10 + (*cur - '0'); - cur++; - } - - if (cur <= start) - { - break; - } - - digits.add(value); - - if (cur < end && *cur == '.') - { - cur++; - } - } - - if (digits.getCount() < 2) - { - return SLANG_FAIL; - } - - outDesc.majorVersion = digits[0]; - outDesc.minorVersion = digits[1]; - return SLANG_OK; - } + count += Index(msg.type == type); } - - return SLANG_FAIL; + return count; } -SlangResult CPPCompilerUtil::calcGCCFamilyVersion(const String& exeName, CPPCompiler::Desc& outDesc) +Int CPPCompiler::Output::countByStage(OutputMessage::Stage stage, Index counts[Int(OutputMessage::Type::CountOf)]) const { - CommandLine cmdLine; - cmdLine.setExecutableFilename(exeName); - cmdLine.addArg("-v"); - - ExecuteResult exeRes; - SLANG_RETURN_ON_FAIL(ProcessUtil::execute(cmdLine, exeRes)); - - const UnownedStringSlice prefixes[] = - { - UnownedStringSlice::fromLiteral("clang version"), - UnownedStringSlice::fromLiteral("gcc version"), - UnownedStringSlice::fromLiteral("Apple LLVM version"), - }; - const CPPCompiler::Type types[] = + Int count = 0; + ::memset(counts, 0, sizeof(Index) * Int(OutputMessage::Type::CountOf)); + for (const auto& msg : messages) { - CPPCompiler::Type::Clang, - CPPCompiler::Type::GCC, - CPPCompiler::Type::Clang, - }; - - SLANG_COMPILE_TIME_ASSERT(SLANG_COUNT_OF(prefixes) == SLANG_COUNT_OF(types)); - - for (Index i = 0; i < SLANG_COUNT_OF(prefixes); ++i) - { - // Set the type - outDesc.type = types[i]; - // Extract the version - if (SLANG_SUCCEEDED(parseGCCFamilyVersion(exeRes.standardError.getUnownedSlice(), prefixes[i], outDesc))) + if (msg.stage == stage) { - return SLANG_OK; + count++; + counts[Index(msg.type)]++; } } - return SLANG_FAIL; + return count++; } -/* static */void CPPCompilerUtil::calcVisualStudioArgs(const CompileOptions& options, CommandLine& cmdLine) +static void _appendCounts(const Index counts[Int(CPPCompiler::OutputMessage::Type::CountOf)], StringBuilder& out) { - // https://docs.microsoft.com/en-us/cpp/build/reference/compiler-options-listed-alphabetically?view=vs-2019 - - cmdLine.addArg("/nologo"); - // Generate complete debugging information - cmdLine.addArg("/Zi"); - // Display full path of source files in diagnostics - cmdLine.addArg("/FC"); + typedef CPPCompiler::OutputMessage::Type Type; - if (options.flags & CompileOptions::Flag::EnableExceptionHandling) + for (Index i = 0; i < Int(Type::CountOf); i++) { - if (options.sourceType == SourceType::CPP) + if (counts[i] > 0) { - // https://docs.microsoft.com/en-us/cpp/build/reference/eh-exception-handling-model?view=vs-2019 - // Assumes c functions cannot throw - cmdLine.addArg("/EHsc"); + out << CPPCompiler::OutputMessage::getTypeText(Type(i)) << "(" << counts[i] << ") "; } } +} - switch (options.optimizationLevel) - { - case OptimizationLevel::Debug: - { - // No optimization - cmdLine.addArg("/Od"); - - cmdLine.addArg("/MDd"); - break; - } - case OptimizationLevel::Normal: - { - cmdLine.addArg("/O2"); - // Multithreaded DLL - cmdLine.addArg("/MD"); - break; - } - default: break; - } - - // /Fd - followed by name of the pdb file - if (options.debugInfoType != DebugInfoType::None) - { - cmdLine.addPrefixPathArg("/Fd", options.modulePath, ".pdb"); - } - - switch (options.targetType) +static void _appendSimplified(const Index counts[Int(CPPCompiler::OutputMessage::Type::CountOf)], StringBuilder& out) +{ + typedef CPPCompiler::OutputMessage::Type Type; + for (Index i = 0; i < Int(Type::CountOf); i++) { - case TargetType::SharedLibrary: + if (counts[i] > 0) { - // Create dynamic link library - if (options.optimizationLevel == OptimizationLevel::Debug) - { - cmdLine.addArg("/LDd"); - } - else - { - cmdLine.addArg("/LD"); - } - - cmdLine.addPrefixPathArg("/Fe", options.modulePath, ".dll"); - break; + out << CPPCompiler::OutputMessage::getTypeText(Type(i)) << " "; } - case TargetType::Executable: - { - cmdLine.addPrefixPathArg("/Fe", options.modulePath, ".exe"); - break; - } - default: break; } +} - // Object file specify it's location - needed if we are out - cmdLine.addPrefixPathArg("/Fo", options.modulePath, ".obj"); - - // Add defines - for (const auto& define : options.defines) +void CPPCompiler::Output::appendSummary(StringBuilder& out) const +{ + Index counts[Int(OutputMessage::Type::CountOf)]; + if (countByStage(OutputMessage::Stage::Compile, counts) > 0) { - StringBuilder builder; - builder << define.nameWithSig; - if (define.value.getLength()) - { - builder << "=" << define.value; - } - - cmdLine.addArg(builder); + out << "Compile: "; + _appendCounts(counts, out); + out << "\n"; } - - // Add includes - for (const auto& include : options.includePaths) + if (countByStage(OutputMessage::Stage::Link, counts) > 0) { - cmdLine.addArg("/I"); - cmdLine.addArg(include); + out << "Link: "; + _appendCounts(counts, out); + out << "\n"; } +} - // https://docs.microsoft.com/en-us/cpp/build/reference/eh-exception-handling-model?view=vs-2019 - // /Eha - Specifies the model of exception handling. (a, s, c, r are options) - - // Files to compile - for (const auto& sourceFile : options.sourceFiles) +void CPPCompiler::Output::appendSimplifiedSummary(StringBuilder& out) const +{ + Index counts[Int(OutputMessage::Type::CountOf)]; + if (countByStage(OutputMessage::Stage::Compile, counts) > 0) { - cmdLine.addArg(sourceFile); + out << "Compile: "; + _appendSimplified(counts, out); + out << "\n"; } - - // Link options (parameters past /link go to linker) - cmdLine.addArg("/link"); - - for (const auto& libPath : options.libraryPaths) + if (countByStage(OutputMessage::Stage::Link, counts) > 0) { - // Note that any escaping of the path is handled in the ProcessUtil:: - cmdLine.addPrefixPathArg("/LIBPATH:", libPath); + out << "Link: "; + _appendSimplified(counts, out); + out << "\n"; } } -/* static */void CPPCompilerUtil::calcGCCFamilyArgs(const CompileOptions& options, CommandLine& cmdLine) +void CPPCompiler::Output::removeByType(OutputMessage::Type type) { - cmdLine.addArg("-fvisibility=hidden"); - // Use shared libraries - //cmdLine.addArg("-shared"); - - switch (options.optimizationLevel) + Index count = messages.getCount(); + for (Index i = 0; i < count; ++i) { - case OptimizationLevel::Debug: - { - // No optimization - cmdLine.addArg("-O0"); - break; - } - case OptimizationLevel::Normal: + if (messages[i].type == type) { - cmdLine.addArg("-Os"); - break; + messages.removeAt(i); + i--; + count--; } - default: break; } +} - if (options.debugInfoType != DebugInfoType::None) - { - cmdLine.addArg("-g"); - } - - switch (options.targetType) - { - case TargetType::SharedLibrary: - { - // Shared library - cmdLine.addArg("-shared"); - // Position independent - cmdLine.addArg("-fPIC"); - - String sharedLibraryPath = SharedLibrary::calcPlatformPath(options.modulePath.getUnownedSlice()); - - cmdLine.addArg("-o"); - cmdLine.addArg(sharedLibraryPath); - break; - } - case TargetType::Executable: - { - cmdLine.addArg("-o"); - - StringBuilder builder; - builder << options.modulePath; - builder << ProcessUtil::getExecutableSuffix(); +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! GenericCPPCompiler !!!!!!!!!!!!!!!!!!!!!!*/ - cmdLine.addArg(options.modulePath); - break; - } - case TargetType::Object: - { - // Don't link, just produce object file - cmdLine.addArg("-c"); - break; - } - default: break; - } +SlangResult GenericCPPCompiler::compile(const CompileOptions& options, Output& outOutput) +{ + outOutput.reset(); - // Add defines - for (const auto& define : options.defines) - { - StringBuilder builder; - builder << define.nameWithSig; - if (define.value.getLength()) - { - builder << "=" << define.value; - } + // Copy the command line options + CommandLine cmdLine(m_cmdLine); - cmdLine.addArg(builder); - } + // Append command line args to the end of cmdLine using the target specific function for the specified options + m_calcArgsFunc(options, cmdLine); - // Add includes - for (const auto& include : options.includePaths) - { - cmdLine.addArg("-I"); - cmdLine.addArg(include); - } + ExecuteResult exeRes; - // Link options - if (0) +#if 0 + // Test { - StringBuilder linkOptions; - linkOptions << "Wl,"; - cmdLine.addArg(linkOptions); + String line = ProcessUtil::getCommandLineString(cmdLine); + printf("%s", line.getBuffer()); } +#endif - // Files to compile - for (const auto& sourceFile : options.sourceFiles) - { - cmdLine.addArg(sourceFile); - } + SLANG_RETURN_ON_FAIL(ProcessUtil::execute(cmdLine, exeRes)); - for (const auto& libPath : options.libraryPaths) +#if 0 { - // Note that any escaping of the path is handled in the ProcessUtil:: - cmdLine.addArg("-L"); - cmdLine.addArg(libPath); - cmdLine.addArg("-F"); - cmdLine.addArg(libPath); + printf("stdout=\"%s\"\nstderr=\"%s\"\nret=%d\n", exeRes.standardOutput.getBuffer(), exeRes.standardError.getBuffer(), int(exeRes.resultCode)); } +#endif - if (options.sourceType == SourceType::CPP) - { - // Make STD libs available - cmdLine.addArg("-lstdc++"); - } + return m_parseOutputFunc(exeRes, outOutput); } +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPCompilerUtil !!!!!!!!!!!!!!!!!!!!!!*/ + static CPPCompiler::Desc _calcCompiledWithDesc() { CPPCompiler::Desc desc = {}; @@ -383,19 +176,19 @@ static CPPCompiler::Desc _calcCompiledWithDesc() #if SLANG_VC desc = WinVisualStudioUtil::getDesc(WinVisualStudioUtil::getCompiledVersion()); #elif SLANG_CLANG - desc.type = CPPCompiler::Type::Clang; + desc.type = CPPCompiler::CompilerType::Clang; desc.majorVersion = Int(__clang_major__); desc.minorVersion = Int(__clang_minor__); #elif SLANG_SNC - desc.type = CPPCompiler::Type::SNC; + desc.type = CPPCompiler::CompilerType::SNC; #elif SLANG_GHS - desc.type = CPPCompiler::Type::GHS; + desc.type = CPPCompiler::CompilerType::GHS; #elif SLANG_GCC - desc.type = CPPCompiler::Type::GCC; + desc.type = CPPCompiler::CompilerType::GCC; desc.majorVersion = Int(__GNUC__); desc.minorVersion = Int(__GNUC_MINOR__); #else - desc.type = CPPCompiler::Type::Unknown; + desc.type = CPPCompiler::CompilerType::Unknown; #endif return desc; @@ -418,7 +211,7 @@ const CPPCompiler::Desc& CPPCompilerUtil::getCompiledWithDesc() { Int bestIndex = -1; - const CPPCompiler::Type type = desc.type; + const CPPCompiler::CompilerType type = desc.type; Int maxVersionValue = 0; Int minVersionDiff = 0x7fffffff; @@ -488,10 +281,10 @@ const CPPCompiler::Desc& CPPCompilerUtil::getCompiledWithDesc() } // If we are gcc, we can try clang and vice versa - if (desc.type == CPPCompiler::Type::GCC || desc.type == CPPCompiler::Type::Clang) + if (desc.type == CPPCompiler::CompilerType::GCC || desc.type == CPPCompiler::CompilerType::Clang) { CPPCompiler::Desc compatible = desc; - compatible.type = (compatible.type == CPPCompiler::Type::Clang) ? CPPCompiler::Type::GCC : CPPCompiler::Type::Clang; + compatible.type = (compatible.type == CPPCompiler::CompilerType::Clang) ? CPPCompiler::CompilerType::GCC : CPPCompiler::CompilerType::Clang; compiler = findCompiler(compilers, MatchType::MinGreaterEqual, compatible); if (compiler) @@ -513,9 +306,9 @@ const CPPCompiler::Desc& CPPCompilerUtil::getCompiledWithDesc() static void _addGCCFamilyCompiler(const String& exeName, CPPCompilerSet* compilerSet) { CPPCompiler::Desc desc; - if (SLANG_SUCCEEDED(CPPCompilerUtil::calcGCCFamilyVersion(exeName, desc))) + if (SLANG_SUCCEEDED(GCCCompilerUtil::calcVersion(exeName, desc))) { - RefPtr compiler(new GenericCPPCompiler(desc, exeName, &CPPCompilerUtil::calcGCCFamilyArgs)); + RefPtr compiler(new GenericCPPCompiler(desc, exeName, &GCCCompilerUtil::calcArgs, &GCCCompilerUtil::parseOutput)); compilerSet->addCompiler(compiler); } } -- cgit v1.2.3