JSONBuilder (#1865)

* #include an absolute path didn't work - because paths were taken to always be relative.

* WIP JSONWriter/JSONParser.

* Checking different Layout styles for JSON.

* Add slang-json-parser.h/.cpp

* WIP JSONValue.

* Added JSONValue::destroy/Recursive.

* Improvement to JSONValue.

* Improve text double conversion precision. Testing.

* Simplify double parsing (just use atof).
JSON comparison
More testing of conversions and start of JSONValue.

* Add <math.h> for isnan, isinf etc.

* Small improvement with object comparison.

* Fix typo in getArgsByName.

* Removed use of isnan and isinf as includes don't work on linux.

* Improve JSON unit test.

* Added asInteger/asFloat/asBool to JSONValue.

* Add SourceLoc to JSONListener.

* Added ability to walk the JSONValue

* JSONBuilder.

* Add converting from lexemes via JSONBuilder.

* Fix VS warning.

* Fix warning for res not being used.

6 files changed, 555 insertions, 45 deletions

diff --git a/source/compiler-core/slang-json-lexer.cpp b/source/compiler-core/slang-json-lexer.cpp
index 261d5f413..9f96ef4f5 100644
--- a/source/compiler-core/slang-json-lexer.cpp
+++ b/source/compiler-core/slang-json-lexer.cpp
@@ -80,6 +80,17 @@ bool JSONLexer::advanceIf(JSONTokenType type)
     return false;
 }
 
+bool JSONLexer::advanceIf(JSONTokenType type, JSONToken& out)
+{
+    if (type == peekType())
+    {
+        out = m_token;
+        advance();
+        return true;
+    }
+    return false;
+}
+
 UnownedStringSlice JSONLexer::getLexeme(const JSONToken& tok) const
 {
     auto offset = m_sourceView->getRange().getOffset(tok.loc);
diff --git a/source/compiler-core/slang-json-lexer.h b/source/compiler-core/slang-json-lexer.h
index ee4b60f75..6f81ae5fd 100644
--- a/source/compiler-core/slang-json-lexer.h
+++ b/source/compiler-core/slang-json-lexer.h
@@ -61,6 +61,7 @@ public:
 
         /// Returns true and advances if current token is type
     bool advanceIf(JSONTokenType type);
+    bool advanceIf(JSONTokenType type, JSONToken& out);
 
         /// Must be called before use
     SlangResult init(SourceView* sourceView, DiagnosticSink* sink);
diff --git a/source/compiler-core/slang-json-parser.cpp b/source/compiler-core/slang-json-parser.cpp
index 478b02fb8..a38afc418 100644
--- a/source/compiler-core/slang-json-parser.cpp
+++ b/source/compiler-core/slang-json-parser.cpp
@@ -3,6 +3,8 @@
 
 #include "slang-json-diagnostics.h"
 
+#include "../core/slang-string-escape-util.h"
+
 /*
 https://www.json.org/json-en.html
 */
@@ -11,21 +13,26 @@ namespace Slang {
 
 SlangResult JSONParser::_parseObject()
 {
-    SLANG_RETURN_ON_FAIL(m_lexer->expect(JSONTokenType::LBrace));
-
-    m_listener->startObject();
+    {
+        const SourceLoc loc = m_lexer->peekLoc();
+        SLANG_RETURN_ON_FAIL(m_lexer->expect(JSONTokenType::LBrace));
+        m_listener->startObject(loc);
+    }
 
-    if (m_lexer->advanceIf(JSONTokenType::RBrace))
     {
-        m_listener->endObject();
-        return SLANG_OK;
+        const SourceLoc loc = m_lexer->peekLoc();
+        if (m_lexer->advanceIf(JSONTokenType::RBrace))
+        {
+            m_listener->endObject(loc);
+            return SLANG_OK;
+        }
     }
 
     while (true)
     {
         JSONToken keyToken;
         SLANG_RETURN_ON_FAIL(m_lexer->expect(JSONTokenType::StringLiteral, keyToken));
-        m_listener->addLexemeKey(m_lexer->getLexeme(keyToken));
+        m_listener->addKey(m_lexer->getLexeme(keyToken), keyToken.loc);
 
         SLANG_RETURN_ON_FAIL(m_lexer->expect(JSONTokenType::Colon));
 
@@ -38,21 +45,29 @@ SlangResult JSONParser::_parseObject()
         break;
     }
 
-    SLANG_RETURN_ON_FAIL(m_lexer->expect(JSONTokenType::RBrace));
-    m_listener->endObject();
+    {
+        const SourceLoc loc = m_lexer->peekLoc();
+        SLANG_RETURN_ON_FAIL(m_lexer->expect(JSONTokenType::RBrace));
+        m_listener->endObject(loc);
+    }
     return SLANG_OK;
 }
 
 SlangResult JSONParser::_parseArray()
 {
-    SLANG_RETURN_ON_FAIL(m_lexer->expect(JSONTokenType::LBracket));
-
-    m_listener->startArray();
+    {
+        const SourceLoc loc = m_lexer->peekLoc();
+        SLANG_RETURN_ON_FAIL(m_lexer->expect(JSONTokenType::LBracket));
+        m_listener->startArray(loc);
+    }
 
-    if (m_lexer->advanceIf(JSONTokenType::RBracket))
     {
-        m_listener->endArray();
-        return SLANG_OK;
+        const SourceLoc loc = m_lexer->peekLoc();
+        if (m_lexer->advanceIf(JSONTokenType::RBracket))
+        {
+            m_listener->endArray(loc);
+            return SLANG_OK;
+        }
     }
 
     while (true)
@@ -65,8 +80,11 @@ SlangResult JSONParser::_parseArray()
         break;
     }
 
-    SLANG_RETURN_ON_FAIL(m_lexer->expect(JSONTokenType::RBracket));
-    m_listener->endArray();
+    {
+        const SourceLoc loc = m_lexer->peekLoc();
+        SLANG_RETURN_ON_FAIL(m_lexer->expect(JSONTokenType::RBracket));
+        m_listener->endArray(loc);
+    }
     return SLANG_OK;
 }
 
@@ -81,7 +99,8 @@ SlangResult JSONParser::_parseValue()
         case JSONTokenType::FloatLiteral:
         case JSONTokenType::StringLiteral:
         {
-            m_listener->addLexemeValue(m_lexer->peekType(), m_lexer->peekLexeme());
+            const JSONToken& tok = m_lexer->peekToken();
+            m_listener->addLexemeValue(tok.type, m_lexer->peekLexeme(), tok.loc);
             m_lexer->advance();
             return SLANG_OK;
         }
@@ -256,8 +275,9 @@ void JSONWriter::_maybeEmitFieldComma()
     }
 }
 
-void JSONWriter::startObject()
+void JSONWriter::startObject(SourceLoc loc)
 {
+    SLANG_UNUSED(loc);
     SLANG_ASSERT(m_state.canEmitValue());
 
     _maybeEmitComma();
@@ -276,8 +296,9 @@ void JSONWriter::startObject()
     m_state.m_flags = 0;
 }
 
-void JSONWriter::endObject()
+void JSONWriter::endObject(SourceLoc loc)
 {
+    SLANG_UNUSED(loc);
     SLANG_ASSERT(m_state.m_kind == State::Kind::Object);
 
     _handleFormat(Location::BeforeCloseObject);
@@ -289,8 +310,9 @@ void JSONWriter::endObject()
     m_stack.removeLast();
 }
 
-void JSONWriter::startArray()
+void JSONWriter::startArray(SourceLoc loc)
 {
+    SLANG_UNUSED(loc);
     SLANG_ASSERT(m_state.canEmitValue());
 
     _maybeEmitComma();
@@ -309,8 +331,9 @@ void JSONWriter::startArray()
     m_state.m_flags = 0;
 }
 
-void JSONWriter::endArray()
+void JSONWriter::endArray(SourceLoc loc)
 {
+    SLANG_UNUSED(loc);
     SLANG_ASSERT(m_state.m_kind == State::Kind::Array);
 
     _handleFormat(Location::BeforeCloseArray);
@@ -322,8 +345,9 @@ void JSONWriter::endArray()
     m_stack.removeLast();
 }
 
-void JSONWriter::addLexemeKey(const UnownedStringSlice& key)
+void JSONWriter::addKey(const UnownedStringSlice& key, SourceLoc loc)
 {
+    SLANG_UNUSED(loc);
     SLANG_ASSERT(m_state.m_kind == State::Kind::Object && (m_state.m_flags & State::Flag::HasKey) == 0);
 
     _maybeEmitFieldComma();
@@ -339,12 +363,27 @@ void JSONWriter::addLexemeKey(const UnownedStringSlice& key)
     m_state.m_flags &= ~State::Flag::HasPrevious;
 }
 
-void JSONWriter::addLexemeValue(JSONTokenType type, const UnownedStringSlice& value)
+void JSONWriter::_preValue(SourceLoc loc)
 {
+    SLANG_UNUSED(loc);
     SLANG_ASSERT(m_state.canEmitValue());
 
     _maybeEmitComma();
     _maybeEmitIndent();
+}
+
+void JSONWriter::_postValue()
+{
+    // We have a previous
+    m_state.m_flags |= State::Flag::HasPrevious;
+    // We don't have a key
+    m_state.m_flags &= ~State::Flag::HasKey;
+}
+
+
+void JSONWriter::addLexemeValue(JSONTokenType type, const UnownedStringSlice& value, SourceLoc loc)
+{
+    _preValue(loc);
 
     switch (type)
     {
@@ -375,10 +414,45 @@ void JSONWriter::addLexemeValue(JSONTokenType type, const UnownedStringSlice& va
             SLANG_ASSERT(!"Can only emit values");
         }
     }
-    // We have a previous
-    m_state.m_flags |= State::Flag::HasPrevious;
-    // We don't have a key
-    m_state.m_flags &= ~State::Flag::HasKey;
+
+    _postValue();
+}
+
+void JSONWriter::addIntegerValue(int64_t value, SourceLoc loc)
+{
+    _preValue(loc);
+    m_builder << value;
+    _postValue();
+}
+
+void JSONWriter::addFloatValue(double value, SourceLoc loc)
+{
+    _preValue(loc);
+    m_builder << value;
+    _postValue();
+}
+
+void JSONWriter::addBoolValue(bool inValue, SourceLoc loc)
+{
+    _preValue(loc);
+    const UnownedStringSlice slice = inValue ? UnownedStringSlice::fromLiteral("true") : UnownedStringSlice::fromLiteral("false");
+    m_builder << slice;
+    _postValue();
+}
+
+void JSONWriter::addNullValue(SourceLoc loc)
+{
+    _preValue(loc);
+    m_builder << UnownedStringSlice::fromLiteral("null");
+    _postValue();
+}
+
+void JSONWriter::addStringValue(const UnownedStringSlice& slice, SourceLoc loc)
+{
+    _preValue(loc);
+    StringEscapeHandler* handler = StringEscapeUtil::getHandler(StringEscapeUtil::Style::JSON);
+    StringEscapeUtil::appendQuoted(handler, slice, m_builder);
+    _postValue();
 }
 
 } // namespace Slang
diff --git a/source/compiler-core/slang-json-parser.h b/source/compiler-core/slang-json-parser.h
index 2e907abb2..96531aee9 100644
--- a/source/compiler-core/slang-json-parser.h
+++ b/source/compiler-core/slang-json-parser.h
@@ -11,18 +11,32 @@ class JSONListener
 {
 public:
         /// Start an object
-    virtual void startObject() = 0;
+    virtual void startObject(SourceLoc loc) = 0;
         /// End an object
-    virtual void endObject() = 0;
+    virtual void endObject(SourceLoc loc) = 0;
         /// Start an array
-    virtual void startArray() = 0;
+    virtual void startArray(SourceLoc loc) = 0;
         /// End and array
-    virtual void endArray() = 0;
+    virtual void endArray(SourceLoc loc) = 0;
 
-        /// Add the key lexeme. Must be followed by addLexemeValue.
-    virtual void addLexemeKey(const UnownedStringSlice& key) = 0;
+
+        /// Add the key. Must be followed by addXXXValue.
+    virtual void addKey(const UnownedStringSlice& key, SourceLoc loc) = 0;
         /// Can be performed in an array or after an addLexemeKey in an object
-    virtual void addLexemeValue(JSONTokenType type, const UnownedStringSlice& value) = 0; 
+    virtual void addLexemeValue(JSONTokenType type, const UnownedStringSlice& value, SourceLoc loc) = 0;
+
+        /// An integer value
+    virtual void addIntegerValue(int64_t value, SourceLoc loc) = 0;
+        /// Add a floating point value
+    virtual void addFloatValue(double value, SourceLoc loc) = 0;
+        /// Add a boolean value
+    virtual void addBoolValue(bool value, SourceLoc loc) = 0;
+
+        /// Add a string value. NOTE! string is unescaped/quoted
+    virtual void addStringValue(const UnownedStringSlice& string, SourceLoc loc) = 0;
+
+        /// Add a null value
+    virtual void addNullValue(SourceLoc loc) = 0;
 };
 
 class JSONWriter : public JSONListener
@@ -75,12 +89,17 @@ public:
     static bool isAfter(Location loc) { return isObjectLike(loc) && (Index(loc) & 2) != 0; }
 
     // Implement JSONListener
-    virtual void startObject() SLANG_OVERRIDE;
-    virtual void endObject() SLANG_OVERRIDE;
-    virtual void startArray() SLANG_OVERRIDE;
-    virtual void endArray() SLANG_OVERRIDE;
-    virtual void addLexemeKey(const UnownedStringSlice& key) SLANG_OVERRIDE;
-    virtual void addLexemeValue(JSONTokenType type, const UnownedStringSlice& value) SLANG_OVERRIDE;
+    virtual void startObject(SourceLoc loc) SLANG_OVERRIDE;
+    virtual void endObject(SourceLoc loc) SLANG_OVERRIDE;
+    virtual void startArray(SourceLoc loc) SLANG_OVERRIDE;
+    virtual void endArray(SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addKey(const UnownedStringSlice& key, SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addLexemeValue(JSONTokenType type, const UnownedStringSlice& value, SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addIntegerValue(int64_t value, SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addFloatValue(double value, SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addBoolValue(bool value, SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addStringValue(const UnownedStringSlice& string, SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addNullValue(SourceLoc loc) SLANG_OVERRIDE;
 
         /// Get the builder
     StringBuilder& getBuilder() { return m_builder; }
@@ -142,6 +161,9 @@ protected:
     void _maybeEmitComma();
     void _maybeEmitFieldComma();
 
+    void _preValue(SourceLoc loc);
+    void _postValue();
+
     void _indent() { m_currentIndent++; }
     void _dedent() { --m_currentIndent; SLANG_ASSERT(m_currentIndent >= 0); }
 
diff --git a/source/compiler-core/slang-json-value.cpp b/source/compiler-core/slang-json-value.cpp
index 3b74c00dd..5b1ee0047 100644
--- a/source/compiler-core/slang-json-value.cpp
+++ b/source/compiler-core/slang-json-value.cpp
@@ -350,7 +350,7 @@ int64_t JSONContainer::asInteger(const JSONValue& value)
         {
             UnownedStringSlice slice = getLexeme(value);
             int64_t intValue;
-            if (SLANG_SUCCEEDED(StringUtil::parseInt64(slice, intValue)) && slice.getLength() == 0)
+            if (SLANG_SUCCEEDED(StringUtil::parseInt64(slice, intValue)))
             {
                 return intValue;
             }
@@ -371,7 +371,7 @@ double JSONContainer::asFloat(const JSONValue& value)
         {
             UnownedStringSlice slice = getLexeme(value);
             double floatValue;
-            if (SLANG_SUCCEEDED(StringUtil::parseDouble(slice, floatValue)) && slice.getLength() == 0)
+            if (SLANG_SUCCEEDED(StringUtil::parseDouble(slice, floatValue)))
             {
                 return floatValue;
             }
@@ -808,4 +808,318 @@ bool JSONContainer::areEqual(const JSONValue& a, const JSONValue& b)
     return false;
 }
 
+void JSONContainer::traverseRecursively(const JSONValue& value, JSONListener* listener)
+{
+    typedef JSONValue::Type Type;
+
+    switch (value.type)
+    {    
+        case Type::True:            return listener->addBoolValue(true, value.loc);
+        case Type::False:           return listener->addBoolValue(false, value.loc);
+        case Type::Null:            return listener->addNullValue(value.loc);
+
+        case Type::StringLexeme:    return listener->addLexemeValue(JSONTokenType::StringLiteral, getLexeme(value), value.loc);
+        case Type::IntegerLexeme:   return listener->addLexemeValue(JSONTokenType::IntegerLiteral, getLexeme(value), value.loc);
+        case Type::FloatLexeme:     return listener->addLexemeValue(JSONTokenType::FloatLiteral, getLexeme(value), value.loc);
+
+        case Type::IntegerValue:    return listener->addIntegerValue(value.intValue, value.loc); 
+        case Type::FloatValue:      return listener->addFloatValue(value.floatValue, value.loc);
+        case Type::StringValue:
+        {
+            const auto slice = getStringFromKey(value.stringKey);
+            return listener->addStringValue(slice, value.loc);
+        }
+        case Type::Array:
+        {
+            listener->startArray(value.loc);
+
+            const auto arr = getArray(value);
+
+            for (const auto& arrayValue : arr)
+            {
+                traverseRecursively(arrayValue, listener);
+            }
+
+            listener->endArray(SourceLoc());
+            break;
+        }
+        case Type::Object:
+        {
+            listener->startObject(value.loc);
+
+            const auto obj = getObject(value);
+
+            for (const auto& objKeyValue : obj)
+            {
+                // Emit the key
+                const auto keyString = getStringFromKey(objKeyValue.key);
+                listener->addKey(keyString, objKeyValue.keyLoc);
+
+                // Emit the value associated with the key
+                traverseRecursively(objKeyValue.value, listener);
+            }
+
+            listener->endObject(SourceLoc());
+            break;
+        }
+        default:
+        {
+            SLANG_ASSERT(!"Invalid type");
+            return;
+        }
+    }
+}
+
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                          JSONBuilder
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
+
+JSONBuilder::JSONBuilder(JSONContainer* container, Flags flags):
+    m_container(container),
+    m_flags(flags)
+{
+    m_state.m_kind = State::Kind::Root;
+    m_state.m_startIndex = 0;
+
+    m_keyValue.reset();
+
+    m_rootValue.reset();
+}
+
+void JSONBuilder::reset()
+{
+    // Reset the state
+    m_state.m_kind = State::Kind::Root;
+    m_state.m_startIndex = 0;
+
+    // Clear the work values
+    m_keyValue.reset();
+    m_rootValue.reset();
+
+    // Clear the lists
+    m_stateStack.clear();
+    m_values.clear();
+    m_keyValues.clear();
+}
+
+void JSONBuilder::_popState()
+{
+    SLANG_ASSERT(m_stateStack.getCount() > 0);
+
+    // Reset the end depending on typpe
+    switch (m_state.m_kind)
+    {
+        case State::Kind::Array:
+        {
+            m_values.setCount(m_state.m_startIndex);
+            break;
+        }
+        case State::Kind::Object:
+        {
+            m_keyValues.setCount(m_state.m_startIndex);
+            break;
+        }
+    }
+
+    // Pop from the stack
+    m_state = m_stateStack.getLast();
+    m_stateStack.removeLast();
+}
+
+Index JSONBuilder::_findKeyIndex(JSONKey key) const
+{
+    SLANG_ASSERT(m_state.m_kind == State::Kind::Object);
+    const Index count = m_keyValues.getCount();
+    for (Index i = m_state.m_startIndex; i < count; ++i)
+    {
+        auto& keyValue = m_keyValues[i];
+        // If we find the key return it's index
+        if (keyValue.key == key)
+        {
+            return i;
+        }
+    }
+    return -1;
+}
+
+void JSONBuilder::_add(const JSONValue& value)
+{
+    SLANG_ASSERT(value.isValid());
+    switch (m_state.m_kind)
+    {
+        case State::Kind::Root:
+        {
+            SLANG_ASSERT(!m_rootValue.isValid());
+            m_rootValue = value;
+            break;
+        }
+        case State::Kind::Array:
+        {
+            m_values.add(value);
+            break;
+        }
+        case State::Kind::Object:
+        {
+            m_keyValue.value = value;
+            const Index index = _findKeyIndex(m_keyValue.key);
+            if (index >= 0)
+            {
+                m_keyValues[index] = m_keyValue;
+            }
+            else
+            {
+                m_keyValues.add(m_keyValue);
+            }
+            m_keyValue.reset();
+            break;
+        }
+    }
+}
+
+void JSONBuilder::startObject(SourceLoc loc)
+{
+    m_stateStack.add(m_state);
+    m_state.m_kind = State::Kind::Object;
+    m_state.m_startIndex = m_keyValues.getCount();
+    m_state.m_loc = loc;
+}
+
+void JSONBuilder::endObject(SourceLoc loc)
+{
+    SLANG_UNUSED(loc);
+
+    SLANG_ASSERT(m_state.m_kind == State::Kind::Object);
+
+    const Index count = m_keyValues.getCount() - m_state.m_startIndex;
+    const JSONValue value = m_container->createObject(m_keyValues.getBuffer() + m_state.m_startIndex, count, m_state.m_loc);
+
+    // Pop current state
+    _popState();
+    // Add the value to the current state
+    _add(value);
+}
+
+void JSONBuilder::startArray(SourceLoc loc)
+{
+    m_stateStack.add(m_state);
+    m_state.m_kind = State::Kind::Array;
+    m_state.m_startIndex = m_values.getCount();
+    m_state.m_loc = loc;
+}
+
+void JSONBuilder::endArray(SourceLoc loc)
+{
+    SLANG_UNUSED(loc);
+
+    SLANG_ASSERT(m_state.m_kind == State::Kind::Array);
+
+    const Index count = m_values.getCount() - m_state.m_startIndex;
+    const JSONValue value = m_container->createArray(m_values.getBuffer() + m_state.m_startIndex, count, m_state.m_loc);
+
+    // Pop current state
+    _popState();
+    // Add the value to the current state
+    _add(value);
+}
+
+void JSONBuilder::addKey(const UnownedStringSlice& key, SourceLoc loc)
+{
+    SLANG_ASSERT(m_keyValue.key == JSONKey(0));
+    m_keyValue.key = m_container->getKey(key);
+    m_keyValue.keyLoc = loc;
+}
+
+void JSONBuilder::addLexemeValue(JSONTokenType type, const UnownedStringSlice& value, SourceLoc loc)
+{
+    switch (type)
+    {
+        case JSONTokenType::True:       return _add(JSONValue::makeBool(true, loc)); 
+        case JSONTokenType::False:      return _add(JSONValue::makeBool(false, loc));
+        case JSONTokenType::Null:       return _add(JSONValue::makeNull(loc));
+        
+        case JSONTokenType::IntegerLiteral:
+        {
+            if (m_flags & Flag::ConvertLexemes)
+            {
+                int64_t intValue = -1;
+                auto res = StringUtil::parseInt64(value, intValue);
+                SLANG_UNUSED(res);
+                SLANG_ASSERT(SLANG_SUCCEEDED(res));
+                _add(JSONValue::makeInt(intValue, loc));
+            }
+            else
+            {
+                SLANG_ASSERT(loc.isValid());
+                _add(JSONValue::makeLexeme(JSONValue::Type::IntegerLexeme, loc, value.getLength()));
+            }
+            break;
+        }
+        case JSONTokenType::FloatLiteral:
+        {
+            if (m_flags & Flag::ConvertLexemes)
+            {
+                double floatValue = 0;
+                auto res = StringUtil::parseDouble(value, floatValue);
+                SLANG_UNUSED(res);
+                SLANG_ASSERT(SLANG_SUCCEEDED(res));
+                _add(JSONValue::makeFloat(floatValue, loc));
+            }
+            else
+            {
+                SLANG_ASSERT(loc.isValid());
+                _add(JSONValue::makeLexeme(JSONValue::Type::FloatLexeme, loc, value.getLength()));
+            }
+            break;
+        }
+        case JSONTokenType::StringLiteral:
+        {
+            if (m_flags & Flag::ConvertLexemes)
+            {
+                auto handler = StringEscapeUtil::getHandler(StringEscapeUtil::Style::JSON);
+                StringBuilder buf;
+                StringEscapeUtil::appendUnquoted(handler, value, buf);
+
+                _add(m_container->createString(buf.getUnownedSlice(), loc));
+            }
+            else
+            {
+                SLANG_ASSERT(loc.isValid());
+                _add(JSONValue::makeLexeme(JSONValue::Type::StringLexeme, loc, value.getLength()));
+            }
+            break;
+        }
+        default:
+        {
+            SLANG_ASSERT(!"Unhandled type");
+        }
+    }
+}
+
+void JSONBuilder::addIntegerValue(int64_t value, SourceLoc loc)
+{
+    _add(JSONValue::makeInt(value, loc));
+}
+
+void JSONBuilder::addFloatValue(double value, SourceLoc loc)
+{
+    _add(JSONValue::makeFloat(value, loc));
+}
+
+void JSONBuilder::addBoolValue(bool value, SourceLoc loc)
+{
+    _add(JSONValue::makeBool(value, loc));
+}
+
+void JSONBuilder::addStringValue(const UnownedStringSlice& slice, SourceLoc loc)
+{
+    _add(m_container->createString(slice, loc));
+}
+
+void JSONBuilder::addNullValue(SourceLoc loc)
+{
+    _add(JSONValue::makeNull(loc));
+}
+
 } // namespace Slang
diff --git a/source/compiler-core/slang-json-value.h b/source/compiler-core/slang-json-value.h
index 48cde5750..acff3ef6e 100644
--- a/source/compiler-core/slang-json-value.h
+++ b/source/compiler-core/slang-json-value.h
@@ -7,6 +7,8 @@
 #include "slang-source-loc.h"
 #include "slang-diagnostic-sink.h"
 
+#include "slang-json-parser.h"
+
 namespace Slang {
 
 typedef uint32_t JSONKey;
@@ -74,7 +76,7 @@ struct JSONValue
         /// As a float value
     double asFloat() const;
 
-        /// True if this is a object like (array or object)
+        /// True if this is a object like
     bool isObjectLike() const { return Index(type) >= Index(Type::Array); }
 
         /// True if this appears to be a valid value
@@ -86,6 +88,12 @@ struct JSONValue
         /// Get the kind
     SLANG_FORCE_INLINE Kind getKind() const { return getKindForType(type); }
 
+    void reset()
+    {
+        type = Type::Invalid;
+        loc = SourceLoc();
+    }
+
         /// Given a type return the associated kind
     static Kind getKindForType(Type type) { return g_typeToKind[Index(type)]; }
 
@@ -109,6 +117,13 @@ struct JSONKeyValue
         /// True if it's valid
     bool isValid() const { return value.type != JSONValue::Type::Invalid; }
 
+    void reset()
+    {
+        key = JSONKey(0);
+        keyLoc = SourceLoc();
+        value.reset();
+    }
+
     JSONKey key;
     SourceLoc keyLoc;
     JSONValue value;
@@ -183,7 +198,13 @@ public:
         /// Destroy recursively from value
     void destroyRecursively(JSONValue& value);
 
-        //
+        /// Traverse a JSON hierarchy from value, outputting to the listener
+    void traverseRecursively(const JSONValue& value, JSONListener* listener);
+
+        /// Returns the source manager used. 
+    SourceManager* getSourceManager() const { return m_sourceManager; } 
+
+        // Ctor
     JSONContainer(SourceManager* sourceManger);
 
         /// Returns true if all the keys are unique
@@ -235,7 +256,74 @@ protected:
     List<Index> m_freeRangeIndices;
     List<JSONValue> m_arrayValues;
     List<JSONKeyValue> m_objectValues;
+};
+
+class JSONBuilder : public JSONListener
+{
+public:
+
+    typedef uint32_t Flags;
+    struct Flag
+    {
+        enum Enum : Flags
+        {
+            ConvertLexemes = 0x01,
+        };
+    };
+
+
+    virtual void startObject(SourceLoc loc) SLANG_OVERRIDE;
+    virtual void endObject(SourceLoc loc) SLANG_OVERRIDE;
+    virtual void startArray(SourceLoc loc) SLANG_OVERRIDE;
+    virtual void endArray(SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addKey(const UnownedStringSlice& key, SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addLexemeValue(JSONTokenType type, const UnownedStringSlice& value, SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addIntegerValue(int64_t value, SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addFloatValue(double value, SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addBoolValue(bool value, SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addStringValue(const UnownedStringSlice& string, SourceLoc loc) SLANG_OVERRIDE;
+    virtual void addNullValue(SourceLoc loc) SLANG_OVERRIDE;
+
+        /// Reset the state
+    void reset();
+
+        /// Get the root value. Will be set after valid construction
+    const JSONValue& getRootValue() const { return m_rootValue; }
+
+    JSONBuilder(JSONContainer* container, Flags flags = 0);
+
+protected:
+
+    struct State
+    {
+        enum class Kind : uint8_t
+        {
+            Root,
+            Object,
+            Array,
+        };
+        Kind m_kind;
+        Index m_startIndex;
+        SourceLoc m_loc;
+    };
+
+    void _popState();
+    void _add(const JSONValue& value);
+
+    Index _findKeyIndex(JSONKey key) const;
+
+    Flags m_flags;
+
+    List<JSONKeyValue> m_keyValues;
+    List<JSONValue> m_values;
+    List<State> m_stateStack;
+
+    State m_state;
+
+    JSONContainer* m_container;
 
+    JSONKeyValue m_keyValue;
+    JSONValue m_rootValue;
 };
 
 } // namespace Slang