Feature/serial string pool refactor (#702)

* Ongoing serialization for full debug work.

* Use StringRepresentationCache and StringSlicePool for serialization.

* Removed some older path handling for serialization which had some wrong underlying assumptions.

* Builds with refactored use of SubStringPool in ir-serialize.

* Removed prohibitedCategories because not used anywhere.

* Add category 'compatibility-issue'

* Remove work in progress on debug serialization.

6 files changed, 156 insertions, 9 deletions

diff --git a/source/core/core.vcxproj b/source/core/core.vcxproj
index 2d0c67342..f3e89b518 100644
--- a/source/core/core.vcxproj
+++ b/source/core/core.vcxproj
@@ -188,6 +188,7 @@
     <ClInclude Include="slang-io.h" />
     <ClInclude Include="slang-math.h" />
     <ClInclude Include="slang-memory-arena.h" />
+    <ClInclude Include="slang-object-scope-manager.h" />
     <ClInclude Include="slang-random-generator.h" />
     <ClInclude Include="slang-string-slice-pool.h" />
     <ClInclude Include="slang-string-util.h" />
@@ -204,6 +205,7 @@
     <ClCompile Include="slang-free-list.cpp" />
     <ClCompile Include="slang-io.cpp" />
     <ClCompile Include="slang-memory-arena.cpp" />
+    <ClCompile Include="slang-object-scope-manager.cpp" />
     <ClCompile Include="slang-random-generator.cpp" />
     <ClCompile Include="slang-string-slice-pool.cpp" />
     <ClCompile Include="slang-string-util.cpp" />
diff --git a/source/core/core.vcxproj.filters b/source/core/core.vcxproj.filters
index 1548217b4..5031e67e7 100644
--- a/source/core/core.vcxproj.filters
+++ b/source/core/core.vcxproj.filters
@@ -63,6 +63,9 @@
     <ClInclude Include="slang-memory-arena.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="slang-object-scope-manager.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="slang-random-generator.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -107,6 +110,9 @@
     <ClCompile Include="slang-memory-arena.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="slang-object-scope-manager.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="slang-random-generator.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
diff --git a/source/core/slang-object-scope-manager.cpp b/source/core/slang-object-scope-manager.cpp
new file mode 100644
index 000000000..313bd4cd5
--- /dev/null
+++ b/source/core/slang-object-scope-manager.cpp
@@ -0,0 +1,23 @@
+#include "slang-object-scope-manager.h"
+
+namespace Slang {
+
+void ObjectScopeManager::_releaseAll()
+{
+    RefObject*const* objs = m_objs.begin();
+    const int numObjs = int(m_objs.Count());
+    for (int i = 0; i < numObjs; ++i)
+    {
+        objs[i]->decreaseReference();
+    }
+}
+
+void ObjectScopeManager::clear()
+{
+    _releaseAll();
+    // Free the memory as well as resizing
+    m_objs = List<RefObject*>();
+}
+
+} // namespace Slang
+
diff --git a/source/core/slang-object-scope-manager.h b/source/core/slang-object-scope-manager.h
new file mode 100644
index 000000000..14008a490
--- /dev/null
+++ b/source/core/slang-object-scope-manager.h
@@ -0,0 +1,67 @@
+#ifndef SLANG_OBJECT_SCOPE_MANAGER_H
+#define SLANG_OBJECT_SCOPE_MANAGER_H
+
+#include "smart-pointer.h"
+#include "list.h"
+
+namespace Slang {
+
+/** Keep objects added in scope.
+
+This is currently about the most simple implementation possible. Objects are added to a list
+which members are released when ObjectScopeManager loses scope, or clear is called. 
+
+The same object can be added multiple times. This implementation will just add the same object
+multiple times. A more complex implementation might notice that the object is already in scope
+and not add a reference. 
+
+Another potential improvement would be to hold the pointers in a MemoryArena. Doing so would remove
+the requirement of a List of contiguous memory. 
+
+In implementations that can hold multiple references to the same thing, we may want to add some 
+garbage collection to remove repeat references.
+*/
+struct ObjectScopeManager
+{
+public:
+
+        /// Add an object which will be kept in scope until manager is destroyed
+    SLANG_INLINE RefObject* add(RefObject* obj);
+        /// Add an object, where it may be nullptr. If it null its a no-op
+    SLANG_INLINE RefObject* addMaybeNull(RefObject* obj);
+
+        /// Clear the contents
+    void clear();
+
+        /// Dtor
+    ~ObjectScopeManager() { _releaseAll(); }
+
+protected:
+    void _releaseAll();
+
+    List<RefObject*> m_objs;
+};
+
+// ---------------------------------------------------------------------------
+RefObject* ObjectScopeManager::addMaybeNull(RefObject* obj)
+{
+    if (obj)
+    {
+        obj->addReference();
+        m_objs.Add(obj);
+    }
+    return obj;
+}
+
+// ---------------------------------------------------------------------------
+RefObject* ObjectScopeManager::add(RefObject* obj)
+{
+    SLANG_ASSERT(obj);
+    obj->addReference();
+    m_objs.Add(obj);
+    return obj;
+}
+
+} // namespace Slang
+
+#endif // SLANG_OBJECT_SCOPE_MANAGER_H
\ No newline at end of file
diff --git a/source/core/slang-string-slice-pool.cpp b/source/core/slang-string-slice-pool.cpp
index 88963d68f..797da5a0f 100644
--- a/source/core/slang-string-slice-pool.cpp
+++ b/source/core/slang-string-slice-pool.cpp
@@ -2,6 +2,12 @@
 
 namespace Slang {
 
+/* static */ const StringSlicePool::Handle StringSlicePool::kNullHandle;
+/* static */ const StringSlicePool::Handle StringSlicePool::kEmptyHandle;
+
+/* static */const int StringSlicePool::kNumDefaultHandles;
+
+
 StringSlicePool::StringSlicePool() :
     m_arena(1024)
 {
@@ -10,18 +16,23 @@ StringSlicePool::StringSlicePool() :
 
 void StringSlicePool::clear()
 {
-    m_slices.SetSize(1);
-    m_slices[0] = UnownedStringSlice::fromLiteral("");
+    m_slices.SetSize(2);
+
+    m_slices[0] = UnownedStringSlice((const char*)nullptr, (const char*)nullptr);
+    m_slices[1] = UnownedStringSlice::fromLiteral("");
+    
+    // Add the empty entry
+    m_map.Add(m_slices[1], kEmptyHandle);
 
     m_map.Clear();
 }
 
 StringSlicePool::Handle StringSlicePool::add(const Slice& slice)
 {
-    const int* indexPtr = m_map.TryGetValue(slice);
-    if (indexPtr)
+    const Handle* handlePtr = m_map.TryGetValue(slice);
+    if (handlePtr)
     {
-        return Handle(*indexPtr);
+        return *handlePtr;
     }
 
     // Create a scoped copy
@@ -30,14 +41,37 @@ StringSlicePool::Handle StringSlicePool::add(const Slice& slice)
     const int index = int(m_slices.Count());
 
     m_slices.Add(scopePath);
-    m_map.Add(scopePath, index);
+    m_map.Add(scopePath, Handle(index));
     return Handle(index);
 }
 
+StringSlicePool::Handle StringSlicePool::add(StringRepresentation* stringRep)
+{
+    if (stringRep == nullptr)
+    {
+        return kNullHandle;
+    }
+    return add(StringRepresentation::asSlice(stringRep));
+}
+ 
+
+StringSlicePool::Handle StringSlicePool::add(const char* chars)
+{
+    if (!chars)
+    {
+        return kNullHandle;
+    }
+    if (chars[0] == 0)
+    {
+        return kEmptyHandle;
+    }
+    return add(UnownedStringSlice(chars));
+}
+
 int StringSlicePool::findIndex(const Slice& slice) const
 {
-    const int* index = m_map.TryGetValue(slice);
-    return index ? *index : -1;
+    const Handle* handlePtr = m_map.TryGetValue(slice);
+    return handlePtr ? int(*handlePtr) : -1;
 
 }
 } // namespace Slang
diff --git a/source/core/slang-string-slice-pool.h b/source/core/slang-string-slice-pool.h
index e6846b3dd..c9c8b8db9 100644
--- a/source/core/slang-string-slice-pool.h
+++ b/source/core/slang-string-slice-pool.h
@@ -12,10 +12,16 @@ namespace Slang {
 class StringSlicePool
 {
 public:
+
     /// Handle of 0 is null. If accessed will be returned as the empty string
     enum class Handle : uint32_t;
     typedef UnownedStringSlice Slice;
 
+    static const Handle kNullHandle = Handle(0);
+    static const Handle kEmptyHandle = Handle(1);
+
+    static const int kNumDefaultHandles = 2;
+
         /// Returns the index of a slice, if contained, or -1 if not found
     int findIndex(const Slice& slice) const;
 
@@ -23,6 +29,12 @@ public:
     bool has(const Slice& slice) { return findIndex(slice) >= 0; }
         /// Add a slice 
     Handle add(const Slice& slice);
+        /// Add from a string
+    Handle add(const char* chars);
+        /// Add a StringRepresentation
+    Handle add(StringRepresentation* string);
+        /// Add a string
+    Handle add(const String& string) { return add(string.getUnownedSlice()); }
 
         /// Empty contents
     void clear();
@@ -33,6 +45,9 @@ public:
         /// Get all the slices
     const List<UnownedStringSlice>& getSlices() const { return m_slices; }
 
+        /// Get the number of slices
+    int getNumSlices() const { return int(m_slices.Count()); }
+
         /// Convert a handle to and index. (A handle is just an index!) 
     static int asIndex(Handle handle) { return int(handle); }
 
@@ -41,7 +56,7 @@ public:
 
 protected:
     List<UnownedStringSlice> m_slices;
-    Dictionary<UnownedStringSlice, int> m_map;
+    Dictionary<UnownedStringSlice, Handle> m_map;
     MemoryArena m_arena;
 };