Support around JSON-RPC (#2014)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Use 'Process' to communicate with an command line tool.

* Remove slang-win-stream

* Tidy up windows ProcessUtil.

* First version of BufferedReadStream.

* Windows working IPC for steams.

* Test proxy count option.

* Split Process/ProcessUtil. Process is platform dependant. ProcessUtil are functions that are platform independent.

* First implementation of Unix Process interface.

* Unix process compiles on cygwin.

* Fix typo in unix process.

* Separate unix pipe stream error of invalid access, from pipe availability.

* Fix in standard line extraction.

* Make fd non blocking.

* Fix issues with Windows Process streams.

* Added UnixPipe.

* Some fixes around UnixPipeStream.

* Make a unix stream closed explicit.

* Hack to debug linux process/stream.

* Revert to old linux pipe handling.

* Pass executable path for unit tests.
Split out CommandLine into own source.

* Small improvements in process/command line.

* Check process behavior with crash.

* Make stderr and stdout unbuffered for crash testing.

* Only turn disable buffering in crash test.

* Disable crash test on CI.

* Fix crash on clang/linux.

* Enable crash test.
Remove _appendBuffer as can use StreamUtil functionality.

* Added inital processing for http headers.

* Small improvements to HttpHeader.

* First pass HTTPPacketConnection working on windows.

* Enable other Process communication tests.

* Update comments.

* WIP JSON RPC.

* Add terminate to Process.
Made JSONRPC a Util.

* Small tidy up around HTTPPacketConnection.

* Improve process termination options.

Co-authored-by: Yong He <yonghe@outlook.com>

15 files changed, 294 insertions, 28 deletions

diff --git a/source/compiler-core/slang-diagnostic-sink.cpp b/source/compiler-core/slang-diagnostic-sink.cpp
index c92610e85..db2885f05 100644
--- a/source/compiler-core/slang-diagnostic-sink.cpp
+++ b/source/compiler-core/slang-diagnostic-sink.cpp
@@ -446,6 +446,15 @@ void DiagnosticSink::init(SourceManager* sourceManager, SourceLocationLexer sour
     }
 }
 
+void DiagnosticSink::reset()
+{
+    m_errorCount = 0;
+    m_internalErrorLocsNoted = 0;
+
+    outputBuffer.Clear();
+}
+
+
 void DiagnosticSink::noteInternalErrorLoc(SourceLoc const& loc)
 {
     // Don't consider invalid source locations.
diff --git a/source/compiler-core/slang-diagnostic-sink.h b/source/compiler-core/slang-diagnostic-sink.h
index 7f7ea7c4e..f83672679 100644
--- a/source/compiler-core/slang-diagnostic-sink.h
+++ b/source/compiler-core/slang-diagnostic-sink.h
@@ -228,6 +228,10 @@ public:
     void setParentSink(DiagnosticSink* parentSink) { m_parentSink = parentSink; }
     DiagnosticSink* getParentSink() const { return m_parentSink; }
 
+        /// Reset state.
+        /// Resets error counts. Resets the output buffer.
+    void reset();
+
         /// Initialize state. 
     void init(SourceManager* sourceManager, SourceLocationLexer sourceLocationLexer);
 
diff --git a/source/compiler-core/slang-json-lexer.cpp b/source/compiler-core/slang-json-lexer.cpp
index 9f96ef4f5..428afca63 100644
--- a/source/compiler-core/slang-json-lexer.cpp
+++ b/source/compiler-core/slang-json-lexer.cpp
@@ -10,6 +10,34 @@ https://www.json.org/json-en.html
 
 namespace Slang {
 
+/* static */UnownedStringSlice JSONLexer::calcLexemeLocation(const UnownedStringSlice& text)
+{
+    SourceManager sourceManager;
+    sourceManager.initialize(nullptr, nullptr);
+    DiagnosticSink sink;
+    sink.init(&sourceManager, nullptr);
+
+    String contents(text);
+    SourceFile* sourceFile = sourceManager.createSourceFileWithString(PathInfo::makeUnknown(), contents);
+    SourceView* sourceView = sourceManager.createSourceView(sourceFile, nullptr, SourceLoc());
+
+    JSONLexer lexer;
+
+    lexer.init(sourceView, &sink);
+
+    if (lexer.peekType() != JSONTokenType::Invalid)
+    {
+        // Get the start offset
+        auto offset = sourceView->getRange().getOffset(lexer.peekLoc());
+
+        return text.subString(offset, lexer.peekLexeme().getLength());
+    }
+    else
+    {
+        return text.head(0);
+    }
+}
+
 SlangResult JSONLexer::init(SourceView* sourceView, DiagnosticSink* sink) 
 {
     m_sourceView = sourceView;
diff --git a/source/compiler-core/slang-json-lexer.h b/source/compiler-core/slang-json-lexer.h
index 6f81ae5fd..6e4a42a32 100644
--- a/source/compiler-core/slang-json-lexer.h
+++ b/source/compiler-core/slang-json-lexer.h
@@ -66,6 +66,9 @@ public:
         /// Must be called before use
     SlangResult init(SourceView* sourceView, DiagnosticSink* sink);
 
+        /// Determines the first token from text. Useful for diagnostics on DiagnosticSink
+    static UnownedStringSlice calcLexemeLocation(const UnownedStringSlice& text);
+
 protected:
     struct LexResult
     {
diff --git a/source/compiler-core/slang-json-rpc.cpp b/source/compiler-core/slang-json-rpc.cpp
new file mode 100644
index 000000000..15da92ff0
--- /dev/null
+++ b/source/compiler-core/slang-json-rpc.cpp
@@ -0,0 +1,48 @@
+#include "slang-json-rpc.h"
+
+namespace Slang {
+
+// https://www.jsonrpc.org/specification
+
+//    m_sourceManager.initialize(nullptr, nullptr);
+//    m_diagnosticSink.init(&m_sourceManager, &JSONLexer::calcLexemeLocation);
+
+/* static */SlangResult JSONRPCUtil::parseJSON(const UnownedStringSlice& slice, JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue)
+{
+    SourceManager* sourceManager = sink->getSourceManager();
+
+    // Now need to parse as JSON
+    String contents(slice);
+    SourceFile* sourceFile = sourceManager->createSourceFileWithString(PathInfo::makeUnknown(), contents);
+    SourceView* sourceView = sourceManager->createSourceView(sourceFile, nullptr, SourceLoc());
+
+    JSONLexer lexer;
+    lexer.init(sourceView, sink);
+
+    JSONBuilder builder(container);
+
+    JSONParser parser;
+    SLANG_RETURN_ON_FAIL(parser.parse(&lexer, sourceView, &builder, sink));
+
+    outValue = builder.getRootValue();
+    return SLANG_OK;
+}
+
+SlangResult JSONRPCUtil::parseJSONAndConsume(HTTPPacketConnection* connection, JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue)
+{
+    if (!connection->hasContent())
+    {
+        return SLANG_FAIL;
+    }
+
+    auto content = connection->getContent();
+
+    UnownedStringSlice text((const char*)content.begin(), content.getCount());
+    SlangResult res = parseJSON(text, container, sink, outValue);
+
+    // Consume the content
+    connection->consumeContent();
+    return res;
+}
+
+} // namespace Slang
diff --git a/source/compiler-core/slang-json-rpc.h b/source/compiler-core/slang-json-rpc.h
new file mode 100644
index 000000000..3ed3e5fee
--- /dev/null
+++ b/source/compiler-core/slang-json-rpc.h
@@ -0,0 +1,29 @@
+#ifndef SLANG_COMPILER_CORE_JSON_RPC_H
+#define SLANG_COMPILER_CORE_JSON_RPC_H
+
+#include "../../slang.h"
+#include "../../slang-com-helper.h"
+#include "../../slang-com-ptr.h"
+
+#include "slang-json-value.h"
+
+#include "slang-json-parser.h"
+#include "../core/slang-http.h"
+
+namespace Slang {
+
+/// Send and receive messages as JSON
+class JSONRPCUtil
+{
+public:
+    
+        /// Parse slice into JSONContainer. outValue is the root of the hierarchy.
+    static SlangResult parseJSON(const UnownedStringSlice& slice, JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue);
+
+        /// Parse content from stream, and consume the packet
+    static SlangResult parseJSONAndConsume(HTTPPacketConnection* connection, JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue);
+};
+
+} // namespace Slang
+
+#endif // SLANG_COMPILER_CORE_JSON_RPC_H
diff --git a/source/compiler-core/slang-json-value.cpp b/source/compiler-core/slang-json-value.cpp
index 5b1ee0047..9a2bb37f4 100644
--- a/source/compiler-core/slang-json-value.cpp
+++ b/source/compiler-core/slang-json-value.cpp
@@ -137,6 +137,17 @@ JSONContainer::JSONContainer(SourceManager* sourceManager):
     _addRange(Range::Type::None, 0, 0);
 }
 
+void JSONContainer::reset()
+{
+    m_slicePool.clear();
+
+    m_freeRangeIndices.clear();
+    m_arrayValues.clear();
+    m_objectValues.clear();
+
+    _addRange(Range::Type::None, 0, 0);
+}
+
 /* static */bool JSONContainer::areKeysUnique(const JSONKeyValue* keyValues, Index keyValueCount)
 {
     for (Index i = 1; i < keyValueCount; ++i)
diff --git a/source/compiler-core/slang-json-value.h b/source/compiler-core/slang-json-value.h
index acff3ef6e..d008d5f18 100644
--- a/source/compiler-core/slang-json-value.h
+++ b/source/compiler-core/slang-json-value.h
@@ -66,6 +66,7 @@ struct JSONValue
     static JSONValue makeEmptyArray(SourceLoc loc = SourceLoc()) { JSONValue value; value.type = Type::Array; value.loc = loc; value.rangeIndex = 0; return value; }
     static JSONValue makeEmptyObject(SourceLoc loc = SourceLoc()) { JSONValue value; value.type = Type::Object; value.loc = loc; value.rangeIndex = 0; return value; }
 
+    static JSONValue makeInvalid(SourceLoc loc = SourceLoc()) { JSONValue value; value.type = Type::Invalid; value.loc = loc; return value; }
     // The following functions only work if the value is stored directly NOT as a lexeme. Use the methods on the container
     // to access values if it is potentially stored as a lexeme
 
@@ -203,6 +204,11 @@ public:
 
         /// Returns the source manager used. 
     SourceManager* getSourceManager() const { return m_sourceManager; } 
+        /// Set the source manager
+    void setSourceManager(SourceManager* sourceManger) { m_sourceManager = sourceManger;  }
+
+        /// Reset the state
+    void reset();
 
         // Ctor
     JSONContainer(SourceManager* sourceManger);
diff --git a/source/compiler-core/slang-source-loc.cpp b/source/compiler-core/slang-source-loc.cpp
index 29fc0465a..3033d9626 100644
--- a/source/compiler-core/slang-source-loc.cpp
+++ b/source/compiler-core/slang-source-loc.cpp
@@ -425,14 +425,24 @@ void SourceManager::initialize(
 
     m_parent = p;
 
-    if( p )
+    _resetLoc();
+}
+
+SourceManager::~SourceManager()
+{
+    _resetSource();
+}
+
+void SourceManager::_resetLoc()
+{
+    if (m_parent)
     {
         // If we have a parent source manager, then we assume that all code at that level
         // has already been loaded, and it is safe to start our own source locations
         // right after those from the parent.
         //
         // TODO: more clever allocation in cases where that might not be reasonable
-        m_startLoc = p->m_nextLoc;
+        m_startLoc = m_parent->m_nextLoc;
     }
     else
     {
@@ -444,7 +454,7 @@ void SourceManager::initialize(
     m_nextLoc = m_startLoc;
 }
 
-SourceManager::~SourceManager()
+void SourceManager::_resetSource()
 {
     for (auto item : m_sourceViews)
     {
@@ -455,6 +465,18 @@ SourceManager::~SourceManager()
     {
         delete item;
     }
+
+    m_sourceViews.clear();
+    m_sourceFiles.clear();
+
+    m_sourceFileMap.Clear();
+}
+
+
+void SourceManager::reset()
+{
+    _resetSource();
+    _resetLoc();
 }
 
 UnownedStringSlice SourceManager::allocateStringSlice(const UnownedStringSlice& slice)
diff --git a/source/compiler-core/slang-source-loc.h b/source/compiler-core/slang-source-loc.h
index 5093e8de0..24471f3b0 100644
--- a/source/compiler-core/slang-source-loc.h
+++ b/source/compiler-core/slang-source-loc.h
@@ -449,6 +449,9 @@ struct SourceManager
         /// Get the source views
     const List<SourceView*>& getSourceViews() const { return m_sourceViews; }
 
+        /// Resets state. Will release all views/source 
+    void reset();
+
     SourceManager() :
         m_memoryArena(2048),
         m_slicePool(StringSlicePool::Style::Default)
@@ -457,6 +460,9 @@ struct SourceManager
 
     protected:
 
+    void _resetLoc();
+    void _resetSource();
+
     // The first location available to this source manager
     // (may not be the first location of all, because we might
     // have a parent source manager)
diff --git a/source/core/slang-http.cpp b/source/core/slang-http.cpp
index ea76d82a9..43be26c1c 100644
--- a/source/core/slang-http.cpp
+++ b/source/core/slang-http.cpp
@@ -287,7 +287,7 @@ SlangResult HTTPPacketConnection::update()
     return m_readResult;
 }
 
-SlangResult HTTPPacketConnection::waitForContent()
+SlangResult HTTPPacketConnection::waitForResult()
 {
     while (m_readState == ReadState::Header ||
         m_readState == ReadState::Content)
diff --git a/source/core/slang-http.h b/source/core/slang-http.h
index a7c6f6628..a298e9e27 100644
--- a/source/core/slang-http.h
+++ b/source/core/slang-http.h
@@ -123,8 +123,8 @@ public:
         /// Write. Will potentially block if write stream is blocking.
     SlangResult write(const void* content, size_t sizeInBytes);
 
-        /// Blocks until full read packet is available or the stream is appropriately closed
-    SlangResult waitForContent();
+        /// Blocks until some result - a packet, closure, or some kind of error
+    SlangResult waitForResult();
         /// Consume the content - so can read next content
     void consumeContent();
 
diff --git a/source/core/slang-process.h b/source/core/slang-process.h
index ddbe7cbf0..7cd7596ae 100644
--- a/source/core/slang-process.h
+++ b/source/core/slang-process.h
@@ -35,13 +35,27 @@ public:
 
         /// Get the stream for the type
     Stream* getStream(StreamType type) const { return m_streams[Index(type)]; }
+
+        /// Get the value returned from the process when it exited/returned. 
     int32_t getReturnValue() const { return m_returnValue;  }
 
         /// True if the process has terminated
     virtual bool isTerminated() = 0;
-        /// Blocks until the process has completed
-    virtual void waitForTermination() = 0;
 
+        /// Blocks until the process has terminated or the timeout completes
+        /// Can optionally supply a timeout time. -1 means 'infinite' and is the default.
+        /// Note that the timeOut is only used approximately. 
+        /// Returns true if has terminated.
+    virtual bool waitForTermination(Int timeOutInMs = -1) = 0;
+
+        /// Terminate the process gracefully.
+        /// After calling it may take time before the process actually terminates
+        /// Ie calling isTerminated directly after `terminate` may return false.
+        /// The return code depending on implementation/termination style, may not be set.
+    virtual void terminate(int32_t returnCode) = 0;
+
+        /// Kill the process - attempt to terminate immediately. 
+    virtual void kill(int32_t returnCode) = 0;
 
         /// The quoting style used for the command line on this target. Currently just uses Space,
         /// but in future may take into account platform sec
@@ -54,13 +68,15 @@ public:
     static SlangResult create(const CommandLine& commandLine, Process::Flags flags, RefPtr<Process>& outProcess);
 
         /// Sleep the current thread for time specified in milliseconds. 0 indicates to OS ok to yield this thread.
-    static void sleepCurrentThread(Index timeInMs);
+    static void sleepCurrentThread(Int timeInMs);
 
         /// Get a standard stream 
     static SlangResult getStdStream(StreamType type, RefPtr<Stream>& out);
 
+        /// Get the clock frequency
     static uint64_t getClockFrequency();
 
+        /// Get the clock tick.
     static uint64_t getClockTick();
 
 protected:
diff --git a/source/core/unix/slang-unix-process.cpp b/source/core/unix/slang-unix-process.cpp
index 083160f02..5131e37ec 100644
--- a/source/core/unix/slang-unix-process.cpp
+++ b/source/core/unix/slang-unix-process.cpp
@@ -28,7 +28,9 @@ class UnixProcess : public Process
 public:
     // Process 
     virtual bool isTerminated() SLANG_OVERRIDE;
-    virtual void waitForTermination() SLANG_OVERRIDE;
+    virtual bool waitForTermination(Int timeInMs) SLANG_OVERRIDE;
+    virtual void terminate(int32_t returnValue) SLANG_OVERRIDE;
+    virtual void kill(int32_t returnValue) SLANG_OVERRIDE;
 
     UnixProcess(pid_t pid, Stream*const* streams);
 
@@ -133,9 +135,64 @@ bool UnixProcess::isTerminated()
     return _updateTerminationState(WNOHANG);
 }
 
-void UnixProcess::waitForTermination()
+bool UnixProcess::waitForTermination(Int timeInMs)
 {
-    while (!_updateTerminationState(0));
+    // If < 0 we will wait blocking until terminated
+    if (timeInMs < 0)
+    {
+        while (!_updateTerminationState(0));
+        return true;
+    }
+
+    // Note that the amount of time waiting is very approximate (we are relying on sleeps time and don't take into
+    // account time outside of sleeping)
+
+    // How often to test
+    const Int checkRateMs = 100;            /// Check every 0.1 seconds
+
+    while (timeInMs > 0)
+    {
+        if (_updateTerminationState(WNOHANG))
+        {
+            return true;
+        }
+
+        // Work out how long to sleep for
+        const Int sleepMs = (timeInMs >= checkRateMs) ? checkRateMs : timeInMs;
+
+        // Sleep
+        sleepCurrentThread(sleepMs);
+
+        timeInMs -= sleepMs;
+    }
+
+    return _updateTerminationState(WNOHANG);
+}
+
+void UnixProcess::terminate(int32_t returnValue)
+{
+    // Using this mechanism, we can't set a returnValue so just ignore
+    SLANG_UNUSED(returnValue);
+
+    if (!isTerminated())
+    {
+        // Request the process terminates
+        ::kill(m_pid, SIGTERM);
+    }
+}
+
+void UnixProcess::kill(int32_t returnValue)
+{
+    if (!isTerminated())
+    {
+        // We waited, lets just terminate with kill
+        ::kill(m_pid, SIGKILL);
+
+        // Set the return value
+        m_returnValue = returnValue;
+        // Mark as terminated
+        m_isTerminated = true;
+    }
 }
 
 /* !!!!!!!!!!!!!!!!!!!!!! UnixPipeStream !!!!!!!!!!!!!!!!!!!!!!!!!!!! */
@@ -386,15 +443,20 @@ SlangResult UnixPipeStream::write(const void* buffer, size_t length)
     return uint64_t(now.tv_sec) * 1000000000 + now.tv_nsec;
 }
 
-/* static */void Process::sleepCurrentThread(Index timeInMs)
+/* static */void Process::sleepCurrentThread(Int timeInMs)
 {
     struct timespec timeSpec;
 
-    if (timeInMs > 0)
+    if (timeInMs >= 1000)
     {
         timeSpec.tv_sec = timeInMs / 1000;
         timeSpec.tv_nsec = (timeInMs % 1000) * 1000 * 1000;
     }
+    else if (timeInMs > 0)
+    {
+        timeSpec.tv_sec = 0;
+        timeSpec.tv_nsec = timeInMs * 1000 * 1000;
+    }
     else
     {
         timeSpec.tv_sec = 0;
diff --git a/source/core/windows/slang-win-process.cpp b/source/core/windows/slang-win-process.cpp
index c37936a30..ed06f5692 100644
--- a/source/core/windows/slang-win-process.cpp
+++ b/source/core/windows/slang-win-process.cpp
@@ -111,8 +111,11 @@ class WinProcess : public Process
 {
 public:
 
+    // Process
     virtual bool isTerminated() SLANG_OVERRIDE;
-    virtual void waitForTermination() SLANG_OVERRIDE;
+    virtual bool waitForTermination(Int timeInMs) SLANG_OVERRIDE;
+    virtual void terminate(int32_t returnCode) SLANG_OVERRIDE;
+    virtual void kill(int32_t returnCode) SLANG_OVERRIDE;
 
     WinProcess(HANDLE handle, Stream* const* streams) :
         m_processHandle(handle)
@@ -300,7 +303,7 @@ void WinProcess::_hasTerminated()
         // https://docs.microsoft.com/en-us/windows/desktop/api/processthreadsapi/nf-processthreadsapi-getexitcodeprocess
 
         DWORD childExitCode = 0;
-        if (GetExitCodeProcess(m_processHandle, &childExitCode))
+        if (::GetExitCodeProcess(m_processHandle, &childExitCode))
         {
             m_returnValue = int32_t(childExitCode);
         }
@@ -308,35 +311,54 @@ void WinProcess::_hasTerminated()
     }
 }
 
-void WinProcess::waitForTermination()
+bool WinProcess::waitForTermination(Int timeInMs)
 {
     if (m_processHandle.isNull())
     {
-        return;
+        return true;
     }
 
+    const DWORD timeOutTime = (timeInMs < 0) ? INFINITE : DWORD(timeInMs);
+
     // wait for the process to exit
     // TODO: set a timeout as a safety measure...
-    WaitForSingleObject(m_processHandle, INFINITE);
+    auto res = ::WaitForSingleObject(m_processHandle, timeOutTime);
+
+    if (res == WAIT_TIMEOUT)
+    {
+        return false;
+    }
 
     _hasTerminated();
+    return true;
 }
 
 bool WinProcess::isTerminated()
 {
-    if (m_processHandle.isNull())
+    return waitForTermination(0);
+}
+
+void WinProcess::terminate(int32_t returnCode)
+{
+    if (!isTerminated())
     {
-        return true;
+        // If it's not terminated, try terminating.
+        // Might take time, so use isTerminated to check
+        ::TerminateProcess(m_processHandle, UINT32(returnCode));
     }
+}
 
-    auto res = WaitForSingleObject(m_processHandle, 0);
-
-    if (res == WAIT_TIMEOUT)
+void WinProcess::kill(int32_t returnCode)
+{
+    if (!isTerminated())
     {
-        return false;
+        // https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-terminateprocess
+        ::TerminateProcess(m_processHandle, UINT32(returnCode));
+
+        // Just assume it's done and set the return code
+        m_returnValue = returnCode;
+        m_processHandle.setNull();
     }
-    _hasTerminated();
-    return true;
 }
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
@@ -505,7 +527,7 @@ bool WinProcess::isTerminated()
     return SLANG_OK;
 }
 
-/* static */void Process::sleepCurrentThread(Index timeInMs)
+/* static */void Process::sleepCurrentThread(Int timeInMs)
 {
     ::Sleep(DWORD(timeInMs));
 }