From 9d514e65f00dde0e309f33591f31fbf7f132a005 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Wed, 12 Jun 2019 09:05:40 -0400 Subject: Runtime execution of Visual Studio Compiler (#978) * Work in progress to be able to invoke VS from within code. * First pass at windows version of refactor of OSProcessSpawner * Closer to getting VS path lookup working. * Make OSString assignable/ctor able * Work out program files directory directly, so don't have to expand %%. * WIP: Improve handling of process spawning. * Add support for splitting input by line. * * Correctly locates visual studio install * Added functionality to invoke vs via cmd * Add option to execute the command line. * Handle in ProcessUtil for windows -> WinHandle. * Rename files slang-win-visual-studio-util.cpp/.h and slang-process-util.h * First pass at unix/linux version of ProcessUtil. * Fix reading Visual Studio path from the registry. * Get compiling on linux with. * Fix vcvarsall.bat name * Use ProcessUtil to execute external code. * Remove OSProcessSpawner. * Remove includes for "os.h" where no longer needed. * Fix tabbing issue in premake5.lua Remove test code from slang-test-main.cpp * Fix premake4.lua tabbing issue. * Small fixes to slang-process-util.h Init ExecuteResult on Win execute. * Improve comments. * Fix bug in StringUtil::calcLines - with oddly terminated source input being able to read past end. Make slang-generate use StringUtil over it's own impl. * Fix off by one bug in working out Visual Studio version. * Fix bug in calculating Visual Studio Version * Fix compilation on linux with string parameter being passed to messageFormat. * Remove erroneous use of kOSError codes - use Result. --- source/core/windows/slang-win-process-util.cpp | 338 ++++++++++++++++++++ .../core/windows/slang-win-visual-studio-util.cpp | 339 +++++++++++++++++++++ source/core/windows/slang-win-visual-studio-util.h | 50 +++ 3 files changed, 727 insertions(+) create mode 100644 source/core/windows/slang-win-process-util.cpp create mode 100644 source/core/windows/slang-win-visual-studio-util.cpp create mode 100644 source/core/windows/slang-win-visual-studio-util.h (limited to 'source/core/windows') diff --git a/source/core/windows/slang-win-process-util.cpp b/source/core/windows/slang-win-process-util.cpp new file mode 100644 index 000000000..424b87128 --- /dev/null +++ b/source/core/windows/slang-win-process-util.cpp @@ -0,0 +1,338 @@ +// slang-win-process-util.cpp +#include "../slang-process-util.h" + +#include "../slang-string.h" + +#ifdef _WIN32 +// Include Windows header in a way that minimized namespace pollution. +// TODO: We could try to avoid including this at all, but it would +// mean trying to hide certain struct layouts, which would add +// more dynamic allocation. +# define WIN32_LEAN_AND_MEAN +# define NOMINMAX +# include +# undef WIN32_LEAN_AND_MEAN +# undef NOMINMAX +#endif + +#include +#include + +namespace Slang { + +namespace { // anonymous + +struct ThreadInfo +{ + HANDLE file; + String output; +}; + +// Has behavior very similar to unique_ptr - assignment is a move. +class WinHandle +{ +public: + /// Detach the encapsulated handle. Returns the handle (which now must be externally handled) + HANDLE detach() { HANDLE handle = m_handle; m_handle = nullptr; return handle; } + + /// Return as a handle + operator HANDLE() const { return m_handle; } + + /// Assign + void operator=(HANDLE handle) { setNull(); m_handle = handle; } + void operator=(WinHandle&& rhs) { HANDLE handle = m_handle; m_handle = rhs.m_handle; rhs.m_handle = handle; } + + /// Get ready for writing + SLANG_FORCE_INLINE HANDLE* writeRef() { setNull(); return &m_handle; } + /// Get for read access + SLANG_FORCE_INLINE const HANDLE* readRef() const { return &m_handle; } + + void setNull() + { + if (m_handle) + { + CloseHandle(m_handle); + m_handle = nullptr; + } + } + + /// Ctor + WinHandle(HANDLE handle = nullptr):m_handle(handle) {} + WinHandle(WinHandle&& rhs):m_handle(rhs.m_handle) { rhs.m_handle = nullptr; } + + /// Dtor + ~WinHandle() { setNull(); } + +private: + + WinHandle(const WinHandle&) = delete; + void operator=(const WinHandle& rhs) = delete; + + HANDLE m_handle; +}; + +} // anonymous + +static DWORD WINAPI _readerThreadProc(LPVOID threadParam) +{ + ThreadInfo* info = (ThreadInfo*)threadParam; + HANDLE file = info->file; + + static const int kChunkSize = 1024; + char buffer[kChunkSize]; + + StringBuilder outputBuilder; + + // We need to re-write the output to deal with line + // endings, so we check for paired '\r' and '\n' + // characters, which may span chunks. + int prevChar = -1; + + for (;;) + { + DWORD bytesRead = 0; + BOOL readResult = ReadFile(file, buffer, kChunkSize, &bytesRead, nullptr); + + const DWORD lastError = GetLastError(); + if (lastError == ERROR_BROKEN_PIPE) + { + break; + } + + if (!readResult) + { + break; + } + + + // walk the buffer and rewrite to eliminate '\r' '\n' pairs + char* readCursor = buffer; + char const* end = buffer + bytesRead; + char* writeCursor = buffer; + + while (readCursor != end) + { + int p = prevChar; + int c = *readCursor++; + prevChar = c; + switch (c) + { + case '\r': case '\n': + // swallow input if '\r' and '\n' appear in sequence + if ((p ^ c) == ('\r' ^ '\n')) + { + // but don't swallow the next byte + prevChar = -1; + continue; + } + // always replace '\r' with '\n' + c = '\n'; + break; + + default: + break; + } + + *writeCursor++ = (char)c; + } + bytesRead = (DWORD)(writeCursor - buffer); + + // Note: Current "core" implementation gives no way to know + // the length of the buffer, so we ultimately have + // to just assume null termination... + outputBuilder.Append(buffer, bytesRead); + } + + info->output = outputBuilder.ProduceString(); + + return 0; +} + + +/* static */UnownedStringSlice ProcessUtil::getExecutableSuffix() +{ + return UnownedStringSlice::fromLiteral(".exe"); +} + +static void _appendEscaped(const UnownedStringSlice& slice, StringBuilder& out) +{ + // TODO(JS): This escaping is not complete... ! + if (slice.indexOf(' ') >= 0 || slice.indexOf('"') >= 0) + { + out << "\""; + + const char* cur = slice.begin(); + const char* end = slice.end(); + + while (cur < end) + { + char c= *cur++; + switch (c) + { + case '\"': + { + // Escape quotes. + out << "\\\""; + break; + } + default: + out.append(c); + } + } + + out << "\""; + + return; + } + + out << slice; +} + +/* static */String ProcessUtil::getCommandLineString(const CommandLine& commandLine) +{ + StringBuilder cmd; + _appendEscaped(commandLine.m_executable.getUnownedSlice(), cmd); + for (const auto& arg : commandLine.m_args) + { + cmd << " "; + _appendEscaped(arg.getUnownedSlice(), cmd); + } + return cmd.ToString(); +} + +#define SLANG_RETURN_FAIL_ON_FALSE(x) if (!(x)) return SLANG_FAIL; + +/* static */SlangResult ProcessUtil::execute(const CommandLine& commandLine, ExecuteResult& outExecuteResult) +{ + outExecuteResult.init(); + + SECURITY_ATTRIBUTES securityAttributes; + securityAttributes.nLength = sizeof(securityAttributes); + securityAttributes.lpSecurityDescriptor = nullptr; + securityAttributes.bInheritHandle = true; + + WinHandle childStdOutRead; + WinHandle childStdErrRead; + WinHandle childStdInWrite; + + // Now we can actually get around to starting a process + PROCESS_INFORMATION processInfo; + ZeroMemory(&processInfo, sizeof(processInfo)); + { + WinHandle childStdOutWrite; + WinHandle childStdErrWrite; + WinHandle childStdInRead; + + { + WinHandle childStdOutReadTmp; + WinHandle childStdErrReadTmp; + WinHandle childStdInWriteTmp; + // create stdout pipe for child process + SLANG_RETURN_FAIL_ON_FALSE(CreatePipe(childStdOutReadTmp.writeRef(), childStdOutWrite.writeRef(), &securityAttributes, 0)); + // create stderr pipe for child process + SLANG_RETURN_FAIL_ON_FALSE(CreatePipe(childStdErrReadTmp.writeRef(), childStdErrWrite.writeRef(), &securityAttributes, 0)); + // create stdin pipe for child process + SLANG_RETURN_FAIL_ON_FALSE(CreatePipe(childStdInRead.writeRef(), childStdInWriteTmp.writeRef(), &securityAttributes, 0)); + + HANDLE currentProcess = GetCurrentProcess(); + + // create a non-inheritable duplicate of the stdout reader + SLANG_RETURN_FAIL_ON_FALSE(DuplicateHandle(currentProcess, childStdOutReadTmp, currentProcess, childStdOutRead.writeRef(), 0, FALSE, DUPLICATE_SAME_ACCESS)); + // create a non-inheritable duplicate of the stderr reader + SLANG_RETURN_FAIL_ON_FALSE(DuplicateHandle(currentProcess, childStdErrReadTmp, currentProcess, childStdErrRead.writeRef(), 0, FALSE, DUPLICATE_SAME_ACCESS)); + // create a non-inheritable duplicate of the stdin writer + SLANG_RETURN_FAIL_ON_FALSE(DuplicateHandle(currentProcess, childStdInWriteTmp, currentProcess, childStdInWrite.writeRef(), 0, FALSE, DUPLICATE_SAME_ACCESS)); + } + + + // TODO: switch to proper wide-character versions of these... + STARTUPINFOW startupInfo; + ZeroMemory(&startupInfo, sizeof(startupInfo)); + startupInfo.cb = sizeof(startupInfo); + startupInfo.hStdError = childStdErrWrite; + startupInfo.hStdOutput = childStdOutWrite; + startupInfo.hStdInput = childStdInRead; + startupInfo.dwFlags = STARTF_USESTDHANDLES; + + OSString pathBuffer; + LPCWSTR path = nullptr; + + if (commandLine.m_executableType == CommandLine::ExecutableType::Path) + { + StringBuilder cmd; + _appendEscaped(commandLine.m_executable.getUnownedSlice(), cmd); + + pathBuffer = cmd.toWString(); + path = pathBuffer.begin(); + } + + // Produce the command line string + String cmdString = getCommandLineString(commandLine); + OSString cmdStringBuffer = cmdString.toWString(); + + // https://docs.microsoft.com/en-us/windows/desktop/api/processthreadsapi/nf-processthreadsapi-createprocessa + // `CreateProcess` requires write access to this, for some reason... + BOOL success = CreateProcessW( + path, + (LPWSTR)cmdStringBuffer.begin(), + nullptr, + nullptr, + true, + CREATE_NO_WINDOW, + nullptr, // TODO: allow specifying environment variables? + nullptr, + &startupInfo, + &processInfo); + + if (!success) + { + DWORD err = GetLastError(); + SLANG_UNUSED(err); + + return SLANG_FAIL; + } + + // close handles we are now done with + CloseHandle(processInfo.hThread); + } + + // Create a thread to read from the child's stdout. + ThreadInfo stdOutThreadInfo; + stdOutThreadInfo.file = childStdOutRead; + WinHandle stdOutThread = CreateThread(nullptr, 0, &_readerThreadProc, (LPVOID)&stdOutThreadInfo, 0, nullptr); + + // Create a thread to read from the child's stderr. + ThreadInfo stdErrThreadInfo; + stdErrThreadInfo.file = childStdErrRead; + WinHandle stdErrThread = CreateThread(nullptr, 0, &_readerThreadProc, (LPVOID)&stdErrThreadInfo, 0, nullptr); + + // wait for the process to exit + // TODO: set a timeout as a safety measure... + WaitForSingleObject(processInfo.hProcess, INFINITE); + + // get exit code for process + // https://docs.microsoft.com/en-us/windows/desktop/api/processthreadsapi/nf-processthreadsapi-getexitcodeprocess + + DWORD childExitCode = 0; + if (!GetExitCodeProcess(processInfo.hProcess, &childExitCode)) + { + // TODO(JS): Do we want to close here? It seems plausible because just because reading the exit code failed, doesn't mean the handle is closed + CloseHandle(processInfo.hProcess); + + return SLANG_FAIL; + } + + // wait for the reader threads + WaitForSingleObject(stdOutThread, INFINITE); + WaitForSingleObject(stdErrThread, INFINITE); + + CloseHandle(processInfo.hProcess); + + outExecuteResult.standardOutput = stdOutThreadInfo.output; + outExecuteResult.standardError = stdErrThreadInfo.output; + outExecuteResult.resultCode = childExitCode; + + return SLANG_OK; +} + +} diff --git a/source/core/windows/slang-win-visual-studio-util.cpp b/source/core/windows/slang-win-visual-studio-util.cpp new file mode 100644 index 000000000..edf33110c --- /dev/null +++ b/source/core/windows/slang-win-visual-studio-util.cpp @@ -0,0 +1,339 @@ +#include "slang-win-visual-studio-util.h" + +#include "../slang-common.h" +#include "../slang-process-util.h" +#include "../slang-string-util.h" + +#ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# define NOMINMAX +# include +# undef WIN32_LEAN_AND_MEAN +# undef NOMINMAX + +# include + +#endif + +// The method used to invoke VS was originally inspired by some ideas in +// https://github.com/RuntimeCompiledCPlusPlus/RuntimeCompiledCPlusPlus/ + +namespace Slang { + +// Information on VS versioning can be found here +// https://en.wikipedia.org/wiki/Microsoft_Visual_C%2B%2B#Internal_version_numbering + + +namespace { // anonymous + +typedef WinVisualStudioUtil::Version Version; + +struct RegistryInfo +{ + const char* regName; ///< The name of the entry in the registry + const char* pathFix; ///< With the value from the registry how to fix the path +}; + +struct VersionInfo +{ + Version version; ///< The version + const char* name; ///< The name of the registry key +}; + +} // anonymous + + +static SlangResult _readRegistryKey(const char* path, const char* keyName, String& outString) +{ + // https://docs.microsoft.com/en-us/windows/desktop/api/winreg/nf-winreg-regopenkeyexa + HKEY key; + LONG ret = RegOpenKeyExA(HKEY_LOCAL_MACHINE, path, 0, KEY_READ | KEY_WOW64_32KEY, &key); + if (ret != ERROR_SUCCESS) + { + return SLANG_FAIL; + } + + char value[MAX_PATH]; + DWORD size = MAX_PATH; + + // https://docs.microsoft.com/en-us/windows/desktop/api/winreg/nf-winreg-regqueryvalueexa + ret = RegQueryValueExA(key, keyName, nullptr, nullptr, (LPBYTE)value, &size); + RegCloseKey(key); + + if (ret != ERROR_SUCCESS) + { + return SLANG_FAIL; + } + + outString = value; + return SLANG_OK; +} + +// Make easier to set up the array +static Version _makeVersion(int main, int dot = 0) { return WinVisualStudioUtil::makeVersion(main, dot); } + +VersionInfo _makeVersionInfo(const char* name, int high, int dot = 0) +{ + VersionInfo info; + info.name = name; + info.version = WinVisualStudioUtil::makeVersion(high, dot); + return info; +} + +static const VersionInfo s_versionInfos[] = +{ + _makeVersionInfo("VS 2005", 8), + _makeVersionInfo("VS 2008", 9), + _makeVersionInfo("VS 2010", 10), + _makeVersionInfo("VS 2012", 11), + _makeVersionInfo("VS 2013", 12), + _makeVersionInfo("VS 2015", 14), + _makeVersionInfo("VS 2017", 15), + _makeVersionInfo("VS 2019", 16), +}; + +// When trying to figure out how this stuff works by running regedit - care is needed, +// because what regedit displays varies on which version of regedit is used. +// In order to use the registry paths used here it's necessary to use Start/Run with +// %systemroot%\syswow64\regedit to view 32 bit keys + +static const RegistryInfo s_regInfos[] = +{ + {"SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VC7", "" }, + {"SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VS7", "VC\\Auxiliary\\Build\\" }, +}; + +static bool _canUseVSWhere(Version version) +{ + // If greater than 15.0 we can use vswhere tool + return (int(version) >= int(_makeVersion(15))); +} + +static int _getRegistryKeyIndex(Version version) +{ + if (int(version) >= int(_makeVersion(15))) + { + return 1; + } + return 0; +} + +/* static */void WinVisualStudioUtil::getVersions(List& outVersions) +{ + const int count = SLANG_COUNT_OF(s_versionInfos); + outVersions.setCount(count); + + Version* dst = outVersions.begin(); + for (int i = 0; i < count; ++i) + { + dst[i] = s_versionInfos[i].version; + } +} + +/* static */WinVisualStudioUtil::Version WinVisualStudioUtil::getCompiledVersion() +{ + // Get the version of visual studio used to compile this source + const uint32_t version = _MSC_VER; + + switch (version) + { + case 1400: return _makeVersion(8); + case 1500: return _makeVersion(9); + case 1600: return _makeVersion(10); + case 1700: return _makeVersion(11); + case 1800: return _makeVersion(12); + case 1900: + { + return _makeVersion(14); + } + case 1911: + case 1912: + case 1913: + case 1914: + case 1915: + case 1916: + { + return _makeVersion(15); + } + case 1920: + { + return _makeVersion(16); + } + default: + { + int lastKnownVersion = 1920; + if (version > lastKnownVersion) + { + // Its an unknown newer version + return Version::Future; + } + break; + } + } + + // Unknown version + return Version::Unknown; +} + +static SlangResult _find(int versionIndex, WinVisualStudioUtil::VersionPath& outPath) +{ + const auto& versionInfo = s_versionInfos[versionIndex]; + + auto version = versionInfo.version; + + outPath.version = version; + outPath.vcvarsPath = String(); + + if (_canUseVSWhere(version)) + { + CommandLine cmd; + + // Lookup directly %ProgramFiles(x86)% path + // https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetfolderpatha + HWND hwnd = GetConsoleWindow(); + + char programFilesPath[_MAX_PATH]; + SHGetFolderPathA(hwnd, CSIDL_PROGRAM_FILESX86, NULL, 0, programFilesPath); + + String vswherePath = programFilesPath; + vswherePath.append("\\Microsoft Visual Studio\\Installer\\vswhere"); + + cmd.setExecutableFilename(vswherePath); + + StringBuilder versionName; + WinVisualStudioUtil::append(version, versionName); + + String args[] = { "-version", versionName, "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "-property", "installationPath" }; + cmd.addArgs(args, SLANG_COUNT_OF(args)); + + ExecuteResult exeRes; + if (SLANG_SUCCEEDED(ProcessUtil::execute(cmd, exeRes))) + { + // We need to chopoff CR/LF if there is one + List lines; + StringUtil::calcLines(exeRes.standardOutput.getUnownedSlice(), lines); + + if (lines.getCount()) + { + outPath.vcvarsPath = lines[0]; + outPath.vcvarsPath.append("\\VC\\Auxiliary\\Build\\"); + return SLANG_OK; + } + } + } + + const Int keyIndex = _getRegistryKeyIndex(version); + if (keyIndex >= 0) + { + SLANG_ASSERT(keyIndex < SLANG_COUNT_OF(s_regInfos)); + + // Try reading the key + const auto& keyInfo = s_regInfos[keyIndex]; + + StringBuilder keyName; + WinVisualStudioUtil::append(versionInfo.version, keyName); + + String value; + if (SLANG_SUCCEEDED(_readRegistryKey(keyInfo.regName, keyName.getBuffer(), value))) + { + outPath.vcvarsPath = value; + return SLANG_OK; + } + } + + return SLANG_FAIL; +} + +/* static */SlangResult WinVisualStudioUtil::find(List& outVersionPaths) +{ + outVersionPaths.clear(); + + const int versionCount = SLANG_COUNT_OF(s_versionInfos); + + for (int i = versionCount - 1; i >= 0; --i) + { + VersionPath versionPath; + if (SLANG_SUCCEEDED(_find(i, versionPath))) + { + outVersionPaths.add(versionPath); + } + } + + return SLANG_OK; +} + +/* static */SlangResult WinVisualStudioUtil::find(Version version, VersionPath& outPath) +{ + const int versionCount = SLANG_COUNT_OF(s_versionInfos); + + for (int i = 0; i < versionCount; ++i) + { + const auto& versionInfo = s_versionInfos[i]; + if (versionInfo.version == version) + { + return _find(i, outPath); + } + } + return SLANG_FAIL; +} + +/* static */SlangResult WinVisualStudioUtil::executeCompiler(const VersionPath& versionPath, const CommandLine& commandLine, ExecuteResult& outResult) +{ + // To invoke cl we need to run the suitable vcvars. In order to run this we have to have MS CommandLine. + // So here we build up a cl command line that is run by first running vcvars, and then executing cl with the parameters as passed to commandLine + + StringBuilder builder; + CommandLine cmdLine; + + cmdLine.setExecutableFilename("cmd.exe"); + { + String options[] = { "/q", "/c", "@prompt", "$" }; + cmdLine.addArgs(options, SLANG_COUNT_OF(options)); + } + + cmdLine.addArg("&&"); + + { + StringBuilder path; + path << versionPath.vcvarsPath; + path << "\\vcvarsall.bat"; + cmdLine.addArg(path); + } + +#if SLANG_PTR_IS_32 + cmdLine.addArg("x86"); +#else + cmdLine.addArg("x86_amd64"); +#endif + + cmdLine.addArg("&&"); + cmdLine.addArg("cl"); + + // Append the command line options + cmdLine.addArgs(commandLine.m_args.getBuffer(), commandLine.m_args.getCount()); + + return ProcessUtil::execute(cmdLine, outResult); +} + +/* static */void WinVisualStudioUtil::append(Version version, StringBuilder& outBuilder) +{ + switch (version) + { + case Version::Unknown: + { + outBuilder << "unknown"; + } + case Version::Future: + { + outBuilder << "future"; + break; + } + default: + { + outBuilder << (int(version) / 10) << "." << (int(version) % 10); + break; + } + } +} + +} // namespace Slang diff --git a/source/core/windows/slang-win-visual-studio-util.h b/source/core/windows/slang-win-visual-studio-util.h new file mode 100644 index 000000000..aec4493df --- /dev/null +++ b/source/core/windows/slang-win-visual-studio-util.h @@ -0,0 +1,50 @@ +#ifndef SLANG_WIN_VISUAL_STUDIO_UTIL_H +#define SLANG_WIN_VISUAL_STUDIO_UTIL_H + +#include "../slang-list.h" +#include "../slang-string.h" + +#include "../slang-process-util.h" + +namespace Slang { + +struct WinVisualStudioUtil +{ + enum class Version: uint32_t + { + Unknown = 0, ///< This is an unknown (and not later) version + Future = 0xff * 10, ///< This is a version 'from the future' - that isn't specifically known. Will be treated as latest + }; + + struct VersionPath + { + Version version; ///< The visual studio version + String vcvarsPath; ///< The path to vcvars bat files, that need to be executed before executing the compiler + }; + + /// Find all the installations + static SlangResult find(List& outVersionPaths); + + /// Given a version find it's path + static SlangResult find(Version version, VersionPath& outPath); + + /// Run visual studio on specified path with the parameters specified on the command line. Output placed in outResult. + static SlangResult executeCompiler(const VersionPath& versionPath, const CommandLine& commandLine, ExecuteResult& outResult); + + /// Get all the known version numbers + static void getVersions(List& outVersions); + + /// Gets the msc compiler used to compile this version. Returning Version(0) means unknown + static Version getCompiledVersion(); + + /// Create a version from a high and low indices + static Version makeVersion(int high, int low = 0) { SLANG_ASSERT(low >= 0 && low <= 9); return Version(high * 10 + low); } + + /// Convert a version number into a string + static void append(Version version, StringBuilder& outBuilder); + +}; + +} // namespace Slang + +#endif -- cgit v1.2.3