Support around JSON-RPC (#2014)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Use 'Process' to communicate with an command line tool.

* Remove slang-win-stream

* Tidy up windows ProcessUtil.

* First version of BufferedReadStream.

* Windows working IPC for steams.

* Test proxy count option.

* Split Process/ProcessUtil. Process is platform dependant. ProcessUtil are functions that are platform independent.

* First implementation of Unix Process interface.

* Unix process compiles on cygwin.

* Fix typo in unix process.

* Separate unix pipe stream error of invalid access, from pipe availability.

* Fix in standard line extraction.

* Make fd non blocking.

* Fix issues with Windows Process streams.

* Added UnixPipe.

* Some fixes around UnixPipeStream.

* Make a unix stream closed explicit.

* Hack to debug linux process/stream.

* Revert to old linux pipe handling.

* Pass executable path for unit tests.
Split out CommandLine into own source.

* Small improvements in process/command line.

* Check process behavior with crash.

* Make stderr and stdout unbuffered for crash testing.

* Only turn disable buffering in crash test.

* Disable crash test on CI.

* Fix crash on clang/linux.

* Enable crash test.
Remove _appendBuffer as can use StreamUtil functionality.

* Added inital processing for http headers.

* Small improvements to HttpHeader.

* First pass HTTPPacketConnection working on windows.

* Enable other Process communication tests.

* Update comments.

* WIP JSON RPC.

* Add terminate to Process.
Made JSONRPC a Util.

* Small tidy up around HTTPPacketConnection.

* Improve process termination options.

Co-authored-by: Yong He <yonghe@outlook.com>

10 files changed, 169 insertions, 3 deletions

diff --git a/source/compiler-core/slang-diagnostic-sink.cpp b/source/compiler-core/slang-diagnostic-sink.cpp
index c92610e85..db2885f05 100644
--- a/source/compiler-core/slang-diagnostic-sink.cpp
+++ b/source/compiler-core/slang-diagnostic-sink.cpp
@@ -446,6 +446,15 @@ void DiagnosticSink::init(SourceManager* sourceManager, SourceLocationLexer sour
     }
 }
 
+void DiagnosticSink::reset()
+{
+    m_errorCount = 0;
+    m_internalErrorLocsNoted = 0;
+
+    outputBuffer.Clear();
+}
+
+
 void DiagnosticSink::noteInternalErrorLoc(SourceLoc const& loc)
 {
     // Don't consider invalid source locations.
diff --git a/source/compiler-core/slang-diagnostic-sink.h b/source/compiler-core/slang-diagnostic-sink.h
index 7f7ea7c4e..f83672679 100644
--- a/source/compiler-core/slang-diagnostic-sink.h
+++ b/source/compiler-core/slang-diagnostic-sink.h
@@ -228,6 +228,10 @@ public:
     void setParentSink(DiagnosticSink* parentSink) { m_parentSink = parentSink; }
     DiagnosticSink* getParentSink() const { return m_parentSink; }
 
+        /// Reset state.
+        /// Resets error counts. Resets the output buffer.
+    void reset();
+
         /// Initialize state. 
     void init(SourceManager* sourceManager, SourceLocationLexer sourceLocationLexer);
 
diff --git a/source/compiler-core/slang-json-lexer.cpp b/source/compiler-core/slang-json-lexer.cpp
index 9f96ef4f5..428afca63 100644
--- a/source/compiler-core/slang-json-lexer.cpp
+++ b/source/compiler-core/slang-json-lexer.cpp
@@ -10,6 +10,34 @@ https://www.json.org/json-en.html
 
 namespace Slang {
 
+/* static */UnownedStringSlice JSONLexer::calcLexemeLocation(const UnownedStringSlice& text)
+{
+    SourceManager sourceManager;
+    sourceManager.initialize(nullptr, nullptr);
+    DiagnosticSink sink;
+    sink.init(&sourceManager, nullptr);
+
+    String contents(text);
+    SourceFile* sourceFile = sourceManager.createSourceFileWithString(PathInfo::makeUnknown(), contents);
+    SourceView* sourceView = sourceManager.createSourceView(sourceFile, nullptr, SourceLoc());
+
+    JSONLexer lexer;
+
+    lexer.init(sourceView, &sink);
+
+    if (lexer.peekType() != JSONTokenType::Invalid)
+    {
+        // Get the start offset
+        auto offset = sourceView->getRange().getOffset(lexer.peekLoc());
+
+        return text.subString(offset, lexer.peekLexeme().getLength());
+    }
+    else
+    {
+        return text.head(0);
+    }
+}
+
 SlangResult JSONLexer::init(SourceView* sourceView, DiagnosticSink* sink) 
 {
     m_sourceView = sourceView;
diff --git a/source/compiler-core/slang-json-lexer.h b/source/compiler-core/slang-json-lexer.h
index 6f81ae5fd..6e4a42a32 100644
--- a/source/compiler-core/slang-json-lexer.h
+++ b/source/compiler-core/slang-json-lexer.h
@@ -66,6 +66,9 @@ public:
         /// Must be called before use
     SlangResult init(SourceView* sourceView, DiagnosticSink* sink);
 
+        /// Determines the first token from text. Useful for diagnostics on DiagnosticSink
+    static UnownedStringSlice calcLexemeLocation(const UnownedStringSlice& text);
+
 protected:
     struct LexResult
     {
diff --git a/source/compiler-core/slang-json-rpc.cpp b/source/compiler-core/slang-json-rpc.cpp
new file mode 100644
index 000000000..15da92ff0
--- /dev/null
+++ b/source/compiler-core/slang-json-rpc.cpp
@@ -0,0 +1,48 @@
+#include "slang-json-rpc.h"
+
+namespace Slang {
+
+// https://www.jsonrpc.org/specification
+
+//    m_sourceManager.initialize(nullptr, nullptr);
+//    m_diagnosticSink.init(&m_sourceManager, &JSONLexer::calcLexemeLocation);
+
+/* static */SlangResult JSONRPCUtil::parseJSON(const UnownedStringSlice& slice, JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue)
+{
+    SourceManager* sourceManager = sink->getSourceManager();
+
+    // Now need to parse as JSON
+    String contents(slice);
+    SourceFile* sourceFile = sourceManager->createSourceFileWithString(PathInfo::makeUnknown(), contents);
+    SourceView* sourceView = sourceManager->createSourceView(sourceFile, nullptr, SourceLoc());
+
+    JSONLexer lexer;
+    lexer.init(sourceView, sink);
+
+    JSONBuilder builder(container);
+
+    JSONParser parser;
+    SLANG_RETURN_ON_FAIL(parser.parse(&lexer, sourceView, &builder, sink));
+
+    outValue = builder.getRootValue();
+    return SLANG_OK;
+}
+
+SlangResult JSONRPCUtil::parseJSONAndConsume(HTTPPacketConnection* connection, JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue)
+{
+    if (!connection->hasContent())
+    {
+        return SLANG_FAIL;
+    }
+
+    auto content = connection->getContent();
+
+    UnownedStringSlice text((const char*)content.begin(), content.getCount());
+    SlangResult res = parseJSON(text, container, sink, outValue);
+
+    // Consume the content
+    connection->consumeContent();
+    return res;
+}
+
+} // namespace Slang
diff --git a/source/compiler-core/slang-json-rpc.h b/source/compiler-core/slang-json-rpc.h
new file mode 100644
index 000000000..3ed3e5fee
--- /dev/null
+++ b/source/compiler-core/slang-json-rpc.h
@@ -0,0 +1,29 @@
+#ifndef SLANG_COMPILER_CORE_JSON_RPC_H
+#define SLANG_COMPILER_CORE_JSON_RPC_H
+
+#include "../../slang.h"
+#include "../../slang-com-helper.h"
+#include "../../slang-com-ptr.h"
+
+#include "slang-json-value.h"
+
+#include "slang-json-parser.h"
+#include "../core/slang-http.h"
+
+namespace Slang {
+
+/// Send and receive messages as JSON
+class JSONRPCUtil
+{
+public:
+    
+        /// Parse slice into JSONContainer. outValue is the root of the hierarchy.
+    static SlangResult parseJSON(const UnownedStringSlice& slice, JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue);
+
+        /// Parse content from stream, and consume the packet
+    static SlangResult parseJSONAndConsume(HTTPPacketConnection* connection, JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue);
+};
+
+} // namespace Slang
+
+#endif // SLANG_COMPILER_CORE_JSON_RPC_H
diff --git a/source/compiler-core/slang-json-value.cpp b/source/compiler-core/slang-json-value.cpp
index 5b1ee0047..9a2bb37f4 100644
--- a/source/compiler-core/slang-json-value.cpp
+++ b/source/compiler-core/slang-json-value.cpp
@@ -137,6 +137,17 @@ JSONContainer::JSONContainer(SourceManager* sourceManager):
     _addRange(Range::Type::None, 0, 0);
 }
 
+void JSONContainer::reset()
+{
+    m_slicePool.clear();
+
+    m_freeRangeIndices.clear();
+    m_arrayValues.clear();
+    m_objectValues.clear();
+
+    _addRange(Range::Type::None, 0, 0);
+}
+
 /* static */bool JSONContainer::areKeysUnique(const JSONKeyValue* keyValues, Index keyValueCount)
 {
     for (Index i = 1; i < keyValueCount; ++i)
diff --git a/source/compiler-core/slang-json-value.h b/source/compiler-core/slang-json-value.h
index acff3ef6e..d008d5f18 100644
--- a/source/compiler-core/slang-json-value.h
+++ b/source/compiler-core/slang-json-value.h
@@ -66,6 +66,7 @@ struct JSONValue
     static JSONValue makeEmptyArray(SourceLoc loc = SourceLoc()) { JSONValue value; value.type = Type::Array; value.loc = loc; value.rangeIndex = 0; return value; }
     static JSONValue makeEmptyObject(SourceLoc loc = SourceLoc()) { JSONValue value; value.type = Type::Object; value.loc = loc; value.rangeIndex = 0; return value; }
 
+    static JSONValue makeInvalid(SourceLoc loc = SourceLoc()) { JSONValue value; value.type = Type::Invalid; value.loc = loc; return value; }
     // The following functions only work if the value is stored directly NOT as a lexeme. Use the methods on the container
     // to access values if it is potentially stored as a lexeme
 
@@ -203,6 +204,11 @@ public:
 
         /// Returns the source manager used. 
     SourceManager* getSourceManager() const { return m_sourceManager; } 
+        /// Set the source manager
+    void setSourceManager(SourceManager* sourceManger) { m_sourceManager = sourceManger;  }
+
+        /// Reset the state
+    void reset();
 
         // Ctor
     JSONContainer(SourceManager* sourceManger);
diff --git a/source/compiler-core/slang-source-loc.cpp b/source/compiler-core/slang-source-loc.cpp
index 29fc0465a..3033d9626 100644
--- a/source/compiler-core/slang-source-loc.cpp
+++ b/source/compiler-core/slang-source-loc.cpp
@@ -425,14 +425,24 @@ void SourceManager::initialize(
 
     m_parent = p;
 
-    if( p )
+    _resetLoc();
+}
+
+SourceManager::~SourceManager()
+{
+    _resetSource();
+}
+
+void SourceManager::_resetLoc()
+{
+    if (m_parent)
     {
         // If we have a parent source manager, then we assume that all code at that level
         // has already been loaded, and it is safe to start our own source locations
         // right after those from the parent.
         //
         // TODO: more clever allocation in cases where that might not be reasonable
-        m_startLoc = p->m_nextLoc;
+        m_startLoc = m_parent->m_nextLoc;
     }
     else
     {
@@ -444,7 +454,7 @@ void SourceManager::initialize(
     m_nextLoc = m_startLoc;
 }
 
-SourceManager::~SourceManager()
+void SourceManager::_resetSource()
 {
     for (auto item : m_sourceViews)
     {
@@ -455,6 +465,18 @@ SourceManager::~SourceManager()
     {
         delete item;
     }
+
+    m_sourceViews.clear();
+    m_sourceFiles.clear();
+
+    m_sourceFileMap.Clear();
+}
+
+
+void SourceManager::reset()
+{
+    _resetSource();
+    _resetLoc();
 }
 
 UnownedStringSlice SourceManager::allocateStringSlice(const UnownedStringSlice& slice)
diff --git a/source/compiler-core/slang-source-loc.h b/source/compiler-core/slang-source-loc.h
index 5093e8de0..24471f3b0 100644
--- a/source/compiler-core/slang-source-loc.h
+++ b/source/compiler-core/slang-source-loc.h
@@ -449,6 +449,9 @@ struct SourceManager
         /// Get the source views
     const List<SourceView*>& getSourceViews() const { return m_sourceViews; }
 
+        /// Resets state. Will release all views/source 
+    void reset();
+
     SourceManager() :
         m_memoryArena(2048),
         m_slicePool(StringSlicePool::Style::Default)
@@ -457,6 +460,9 @@ struct SourceManager
 
     protected:
 
+    void _resetLoc();
+    void _resetSource();
+
     // The first location available to this source manager
     // (may not be the first location of all, because we might
     // have a parent source manager)