OffsetContainer serialization (#1093)

* OffsetContainer with unit tests.

* State serialization working with OffsetContainer.

* Fixes to make work with OffsetContainer.

* Added OffsetContainer documentation.

* Remove RelativeContainer.

* Fix problem with + on Offset32Ptr on windows x86 target.

* * Made OffsetBase a base class of OffsetContainer.
* Added MemoryOffsetBase to just handle being a chunk of memory.

* * Use operator[] to access contents of OffsetContainer

* Fix the type hash to work across different size_t sizes.

* Fixed some Offset type related comments.

* Fix bug around using asBase, because it returns a reference just using 'auto' will means it becomes a value type.
Remove assignment and copy ctor from OffsetBase.

* Evaluation order of assignment can lead to wrong behavior with Offset32Ptr/raw pointers. Document the fact, and fix in StateSerializeUtil.

5 files changed, 433 insertions, 482 deletions

diff --git a/source/core/core.vcxproj b/source/core/core.vcxproj
index 0a1b070fe..5b42dc81b 100644
--- a/source/core/core.vcxproj
+++ b/source/core/core.vcxproj
@@ -188,10 +188,10 @@
     <ClInclude Include="slang-math.h" />
     <ClInclude Include="slang-memory-arena.h" />
     <ClInclude Include="slang-object-scope-manager.h" />
+    <ClInclude Include="slang-offset-container.h" />
     <ClInclude Include="slang-platform.h" />
     <ClInclude Include="slang-process-util.h" />
     <ClInclude Include="slang-random-generator.h" />
-    <ClInclude Include="slang-relative-container.h" />
     <ClInclude Include="slang-render-api-util.h" />
     <ClInclude Include="slang-riff.h" />
     <ClInclude Include="slang-secure-crt.h" />
@@ -220,9 +220,9 @@
     <ClCompile Include="slang-io.cpp" />
     <ClCompile Include="slang-memory-arena.cpp" />
     <ClCompile Include="slang-object-scope-manager.cpp" />
+    <ClCompile Include="slang-offset-container.cpp" />
     <ClCompile Include="slang-platform.cpp" />
     <ClCompile Include="slang-random-generator.cpp" />
-    <ClCompile Include="slang-relative-container.cpp" />
     <ClCompile Include="slang-render-api-util.cpp" />
     <ClCompile Include="slang-riff.cpp" />
     <ClCompile Include="slang-shared-library.cpp" />
diff --git a/source/core/core.vcxproj.filters b/source/core/core.vcxproj.filters
index 56bb6b8b7..8184637f2 100644
--- a/source/core/core.vcxproj.filters
+++ b/source/core/core.vcxproj.filters
@@ -63,6 +63,9 @@
     <ClInclude Include="slang-object-scope-manager.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="slang-offset-container.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="slang-platform.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -72,9 +75,6 @@
     <ClInclude Include="slang-random-generator.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="slang-relative-container.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
     <ClInclude Include="slang-render-api-util.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -155,13 +155,13 @@
     <ClCompile Include="slang-object-scope-manager.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="slang-platform.cpp">
+    <ClCompile Include="slang-offset-container.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="slang-random-generator.cpp">
+    <ClCompile Include="slang-platform.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="slang-relative-container.cpp">
+    <ClCompile Include="slang-random-generator.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
     <ClCompile Include="slang-render-api-util.cpp">
diff --git a/source/core/slang-relative-container.cpp b/source/core/slang-offset-container.cpp
index 0b52f4268..5fed2a452 100644
--- a/source/core/slang-relative-container.cpp
+++ b/source/core/slang-offset-container.cpp
@@ -1,14 +1,11 @@
-// slang-relative-containere.cpp
-#include "slang-relative-container.h"
+// slang-offset-container.cpp
+#include "slang-offset-container.h"
 
 namespace Slang {
 
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! OffsetString !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
 
-/* static */RelativeBase RelativeBase::g_null = { nullptr };
-
-/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RelativeString !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
-
-size_t RelativeString::calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncodeSize])
+size_t OffsetString::calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncodeSize])
 {
     SLANG_ASSERT(size <= 0xffffffff);
     if (size <= kSizeBase)
@@ -32,7 +29,7 @@ size_t RelativeString::calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncod
     return num + 1;
 }
 
-/* static */const char* RelativeString::decodeSize(const char* in, size_t& outSize)
+/* static */const char* OffsetString::decodeSize(const char* in, size_t& outSize)
 {
     const uint8_t* cur = (const uint8_t*)in;
     if (*cur <= kSizeBase)
@@ -72,7 +69,7 @@ size_t RelativeString::calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncod
     }
 }
 
-/* static */size_t RelativeString::calcAllocationSize(size_t stringSize)
+/* static */size_t OffsetString::calcAllocationSize(size_t stringSize)
 {
     uint8_t encode[kMaxSizeEncodeSize];
     size_t encodeSize = calcEncodedSize(stringSize, encode);
@@ -80,12 +77,12 @@ size_t RelativeString::calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncod
     return encodeSize + stringSize + 1;
 }
 
-/* static */size_t RelativeString::calcAllocationSize(const UnownedStringSlice& slice)
+/* static */size_t OffsetString::calcAllocationSize(const UnownedStringSlice& slice)
 {
     return calcAllocationSize(slice.size());
 }
 
-UnownedStringSlice RelativeString::getSlice() const
+UnownedStringSlice OffsetString::getSlice() const
 {
     size_t size;
     const char* chars = decodeSize(m_sizeThenContents, size);
@@ -93,41 +90,43 @@ UnownedStringSlice RelativeString::getSlice() const
     return UnownedStringSlice(chars, size);
 }
 
-const char* RelativeString::getCstr() const
+const char* OffsetString::getCstr() const
 {
     return getSlice().begin();
 }
 
-/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RelativeContainer !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! OffsetContainer !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
 
-RelativeContainer::RelativeContainer()
+OffsetContainer::OffsetContainer()
 {
-    m_current = 0;
     m_capacity = 0;
-    m_base.m_data = nullptr;
+    m_data = nullptr;
+
+    // We need to allocate some of the first bytes 0 can be used for nullptr. 
+    allocateAndZero(kStartOffset, 1);
 }
 
-RelativeContainer::~RelativeContainer()
+OffsetContainer::~OffsetContainer()
 {
-    if (m_base.m_data)
+    if (m_data)
     {
-        ::free(m_base.m_data);
+        ::free(m_data);
     }
 }
 
-void* RelativeContainer::allocate(size_t size)
+void* OffsetContainer::allocate(size_t size)
 {
     return allocate(size, 1);
 }
 
-void RelativeContainer::fixAlignment(size_t alignment)
+void OffsetContainer::fixAlignment(size_t alignment)
 {
     allocate(0, alignment);
 }
 
-void* RelativeContainer::allocate(size_t size, size_t alignment)
+void* OffsetContainer::allocate(size_t size, size_t alignment)
 {
-    size_t offset = (m_current + alignment - 1) & ~(alignment - 1);
+    size_t offset = (m_dataSize + alignment - 1) & ~(alignment - 1);
 
     if (offset + size > m_capacity)
     {
@@ -148,29 +147,29 @@ void* RelativeContainer::allocate(size_t size, size_t alignment)
         size_t newSize = (calcSize < minSize) ? minSize : calcSize;
 
         // Reallocate space
-        m_base.m_data = (uint8_t*)::realloc(m_base.m_data, newSize);
+        m_data = (uint8_t*)::realloc(m_data, newSize);
         m_capacity = newSize;
     }
 
     SLANG_ASSERT(offset + size <= m_capacity);
 
-    m_current = offset + size;
-    return m_base.m_data + offset;
+    m_dataSize = offset + size;
+    return m_data + offset;
 }
 
-void* RelativeContainer::allocateAndZero(size_t size, size_t alignment)
+void* OffsetContainer::allocateAndZero(size_t size, size_t alignment)
 {
     void* data = allocate(size, alignment);
     memset(data, 0, size);
     return data;
 }
 
-Safe32Ptr<RelativeString> RelativeContainer::newString(const UnownedStringSlice& slice)
+Offset32Ptr<OffsetString> OffsetContainer::newString(const UnownedStringSlice& slice)
 {
     size_t stringSize = slice.size();
 
-    uint8_t head[RelativeString::kMaxSizeEncodeSize];
-    size_t headSize = RelativeString::calcEncodedSize(stringSize, head);
+    uint8_t head[OffsetString::kMaxSizeEncodeSize];
+    size_t headSize = OffsetString::calcEncodedSize(stringSize, head);
 
     size_t allocSize = headSize + stringSize + 1;
     uint8_t* bytes = (uint8_t*)allocate(allocSize);
@@ -181,12 +180,12 @@ Safe32Ptr<RelativeString> RelativeContainer::newString(const UnownedStringSlice&
     // 0 terminate
     bytes[headSize + stringSize] = 0;
 
-    return Safe32Ptr<RelativeString>(getOffset(bytes), &m_base);
+    return Offset32Ptr<OffsetString>(getOffset(bytes));
 }
 
-Safe32Ptr<RelativeString> RelativeContainer::newString(const char* contents)
+Offset32Ptr<OffsetString> OffsetContainer::newString(const char* contents)
 {
-    Safe32Ptr<RelativeString> relString;
+    Offset32Ptr<OffsetString> relString;
     if (contents)
     {
         relString = newString(UnownedStringSlice(contents));
@@ -194,22 +193,4 @@ Safe32Ptr<RelativeString> RelativeContainer::newString(const char* contents)
     return relString;
 }
 
-void RelativeContainer::set(void* data, size_t size)
-{
-    if (m_base.m_data)
-    {
-        ::free(m_base.m_data);
-        m_base.m_data = nullptr;
-    }
-
-    if (size > 0)
-    {
-        m_base.m_data = (uint8_t*)::malloc(size);
-        ::memcpy(m_base.m_data, data, size);
-    }
-
-    m_current = size;
-    m_capacity = size;
-}
-
 } // namespace Slang
diff --git a/source/core/slang-offset-container.h b/source/core/slang-offset-container.h
new file mode 100644
index 000000000..95c3a6589
--- /dev/null
+++ b/source/core/slang-offset-container.h
@@ -0,0 +1,393 @@
+// slang-offset-container.h
+#ifndef SLANG_OFFSET_CONTAINER_H_INCLUDED
+#define SLANG_OFFSET_CONTAINER_H_INCLUDED
+
+#include "slang-basic.h"
+
+namespace Slang {
+
+/*
+The purpose of OffsetContainer and related types is to provide a mechanism to easily serialize offset structures.
+
+The root idea here is the "offset pointer". A typical pointer in a language like C/C++ holds the absolute address
+in the current address space of the thing that is being pointed to. This introduces a problem, as when data is
+serialized in the contents will very likely be be placed at different addresses - meaning any absolute pointer
+will point to the wrong place. There is also a related issue around pointer sizes - on some targets they are 32 bits
+and on others 64 bits. 
+
+An offset pointer means a pointer that points to something 'offset' to some base address. The OffsetPtr uses a 32 bit
+offset from the pointers location in memory. This means such a pointer can address a 4Gb address space.
+
+Special care is needed when using offset pointers - both when constructing structures that contain them, reading
+them and in general usage.
+
+For simplicity here we store all offset pointers within a single contiguous allocation. This allocation is
+typically managed by the OffsetContainer for writing. When reading a MemoryOffsetBase can be used.
+
+An issue around using offset pointers, is that we cannot directly access it's contents, because it's just an
+offset to some base address. Thus to access the thing being pointed to we need to turn the offset pointer back into
+a 'raw' pointer. This is achieved via using the asRaw methods on the OffsetBase. For a convenience operator[] can also
+be used, and this is typically the preferred mechanism.
+
+NOTE! That the evaluation order of a function calls parameters is undefined in C++. That whilst it might appear doing
+
+```
+base[thing] = container.newObject<Thing>();
+```
+
+will evaluate the construction of newObject *before* the assignment, if you look at the assignment as being a function call
+(as it is when it is overloaded), then base[thing] might be evaluated *before* newObject, and if it is then the result
+could be wrong if the newObject needed to reallocate. Therefore when allocation is involved, a new (or any allocation backed
+function call from the OffsetContainer) should always place a result in a local variable. Then assign as in
+
+```
+auto anotherThing = container.newObject<Thing>();
+base[thing] = anotherThing; 
+```
+
+When creating structures - unless you know the allocated space (in the OffsetContainer or some other piece of memory)
+is larger than required, then special care is needed, because when a new larger piece of memory is allocated to hold
+everything, raw pointers pointers will likely be invalidated. When reading there is typically no need to move
+the base address, so raw pointers remain valid through out. When doing writing if a call is made to something that
+allocates memory on the OffsetContainer - any raw pointer should be assumed invalid. 
+
+For example 
+
+```
+
+struct Thing
+{
+    Offset32Ptr<OffsetString> text;
+    int value;
+};
+
+void func()
+{
+    OffsetContainer container;
+    OffsetBase& base = container.asBase();
+
+    {
+        // We can allocate on the heap. BUT we can't set up a offset pointer to it
+        Thing thing;
+        // BAD!! Will assert, because thing is not in the address range recorded in base.
+        Offset32Ptr<Thing> thingOffsetPtr= base->asPtr(&thing);
+    }
+
+    // Ok - this is now correct
+    Offset32Ptr<Thing> thing = container.newObject<Thing>();
+
+    // To write values, we need a raw pointer
+    {
+        // To get the raw pointer we can use 'asRaw'
+        auto rawThing = base->asRaw(thing);
+
+        // Or more perhaps slightly more conveniently []
+        auto rawThing = base[thing];
+        
+        // We can write and read things via the Safe32Ptr
+        rawThing->value = 10;
+        const int value = rawThing->value;
+
+        SLANG_ASSERT(value == 10);
+    }
+
+    // Now lets write to it
+    {
+        // We can have raw pointer (or reference) to a thing but we need to be *careful* if we allocate 
+        Thing* rawThing = base[thing];
+        // We are okay here, nothing between getting the raw pointer and the write allocated/newed anything on the OffsetContainer
+        rawThing->value = 20;
+
+        // Lets set up name 
+        Offset32Ptr<OffsetString> text = offsetContainer.newString("Hello World!");
+
+        // BAD! The rawThing point could now be invalid because the call to newString may have had to allocate more memory
+        rawThing->text = text;
+
+        // This is okay 
+        base[thing]->text = text;
+
+        // Or we can update rawThing such that is up to date
+        rawThing = base[thing];
+        // So now this is okay again
+        rawThing->text = text;
+
+        // BAD! we don't know the evaluation order here, if the lhs is evaluate before the rhs, then it could write to the wrong area of memory. 
+        base[thing]->text = offsetContainer.newString("Hello World again!");
+
+        // So where there is allocation, and assignment to something that in held in offset ptr use a local for the allocation as in
+        {
+            auto text = offsetContainer.newString("Hello World again!");
+            base[thing]->text = text;
+        }
+    }
+}
+
+```
+*/
+
+enum
+{
+    kNull32Offset = 0,                                  
+    kStartOffset = uint32_t(sizeof(uint64_t)),          ///< The offset to the first contained thing
+};
+
+template <typename T>
+class Offset32Ref;
+
+/* A pointer to items held in OffsetContainer (or OffsetBase relative) that remains correct even if
+the memory inside OffsetContainer moves.
+*/
+template <typename T>
+class Offset32Ptr
+{
+public:
+    typedef Offset32Ptr ThisType;
+
+    const ThisType& operator=(const ThisType& rhs) { m_offset = rhs.m_offset; return *this; }
+    bool operator==(const ThisType& rhs) const { return m_offset == rhs.m_offset; }
+    bool operator!=(const ThisType& rhs) const { return m_offset != rhs.m_offset; }
+
+    bool operator<(const ThisType& rhs) const { return m_offset < rhs.m_offset; }
+    bool operator<=(const ThisType& rhs) const { return m_offset <= rhs.m_offset; }
+    bool operator>(const ThisType& rhs) const { return m_offset > rhs.m_offset; }
+    bool operator>=(const ThisType& rhs) const { return m_offset >= rhs.m_offset; }
+
+    operator bool() const { return m_offset != kNull32Offset; }
+
+    Offset32Ref<T> operator*();
+
+    ThisType& operator++() { m_offset += uint32_t(sizeof(T)); return *this; }
+    ThisType operator++(int) { const auto offset = m_offset; m_offset += uint32_t(sizeof(T)); return ThisType(offset); }
+
+    ThisType& operator--() { m_offset -= sizeof(T); return *this; }
+    ThisType operator--(int) { const auto offset = m_offset; m_offset -= uint32_t(sizeof(T)); return ThisType(offset); }
+
+    friend ThisType operator+(const ThisType& a, Index b) { return ThisType(a.m_offset + uint32_t(sizeof(T) * b)); }
+    friend ThisType operator+(Index a, const ThisType& b) { return ThisType(b.m_offset + uint32_t(sizeof(T) * a)); }
+
+    bool isNull() const { return m_offset == kNull32Offset; }
+
+    void setNull() { m_offset = kNull32Offset; }
+    Offset32Ptr():m_offset(kNull32Offset) {}
+    Offset32Ptr(const ThisType& rhs): m_offset(rhs.m_offset) {}
+    explicit Offset32Ptr(uint32_t offset): m_offset(offset) {}
+
+    uint32_t m_offset;
+};
+
+/* A reference to items held in OffsetContainer (or OffsetBase relative) that remains correct even if
+the memory inside OffsetContainer moves.
+*/
+template <typename T>
+class Offset32Ref
+{
+public:
+    typedef Offset32Ref ThisType;
+
+    const ThisType& operator=(const ThisType& rhs) { m_offset = rhs.m_offset; return *this; }
+
+    Offset32Ptr<T> operator&() { return Offset32Ptr<T>(m_offset); }
+
+    Offset32Ref(const ThisType& rhs) : m_offset(rhs.m_offset) {}
+    explicit Offset32Ref(uint32_t offset) : m_offset(offset) { SLANG_ASSERT(offset != kNull32Offset); }
+
+    uint32_t m_offset;
+};
+
+// ---------------------------------------------------------------------------
+template <typename T>
+SLANG_FORCE_INLINE Offset32Ref<T> Offset32Ptr<T>::operator*()
+{
+    return Offset32Ref<T>(m_offset);
+}
+
+
+/* Much like Offset32Ptr this is an array but whose memory is stored inside the OffsetContainer. This means elements types
+must be 'offset types'. */
+template <typename T>
+class Offset32Array
+{
+public:
+    Offset32Ptr<const T> begin() const { return Offset32Ptr<const T>(m_data.m_offset); }
+    Offset32Ptr<const T> end() const { return begin() + Index(m_count); }
+
+    Offset32Ptr<T> begin() { return m_data; }
+    Offset32Ptr<T> end() { return begin() + Index(m_count); }
+
+    Index getCount() const { return Index(m_count); }
+
+    Offset32Ref<const T> operator[](Index i) const { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return Offset32Ref<const T>((m_data + i).m_offset); }
+    Offset32Ref<T> operator[](Index i) { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return Offset32Ref<T>((m_data + i).m_offset); }
+
+    Offset32Array(Offset32Ptr<T> data, uint32_t count) :m_data(data), m_count(count) {}
+
+    Offset32Array() :m_count(0) {}
+
+    Offset32Ptr<T> m_data;
+    uint32_t m_count;
+};
+
+/** OffsetString is used for storing strings within a OffsetContainer. Strings are stored with the initial byte indicating the size
+of the string. Note that all offset strings are stored with a terminating zero, and that the terminating zero is *NOT* included in
+the encoded size. */
+struct OffsetString
+{
+    enum
+    {
+        kSizeBase = 251,
+        kMaxSizeEncodeSize = 5,
+    };
+
+        /// Get contents as a slice
+    UnownedStringSlice getSlice() const;
+        /// Get null terminated string
+    const char* getCstr() const;
+
+        /// Decode the size. Returns the start of the string text, and outSize holds the size (NOT including terminating 0)
+    static const char* decodeSize(const char* in, size_t& outSize);
+
+        /// Returns the amount of bytes used, end encoding in 'encode'
+    static size_t calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncodeSize]);
+        /// Calculate the total size needed to store the string *including* terminating 0
+    static size_t calcAllocationSize(const UnownedStringSlice& slice);
+
+        /// Calculate the total size needed to store string. Size should be passed *without* terminating 0
+    static size_t calcAllocationSize(size_t size);
+
+    char m_sizeThenContents[1];
+};
+
+/* A type that is used to hold the base address of the contiguous memory that holds either Offset32Ptr and related types>
+*/
+class OffsetBase
+{
+public:
+    typedef OffsetBase ThisType;
+
+        /// Turn an offset into a raw regular pointer or reference
+    template <typename T>
+    T* asRaw(const Offset32Ptr<T>& ptr) { return (T*)_getRaw(ptr.m_offset); }
+    template <typename T>
+    T& asRaw(const Offset32Ref<T>& ref) { return *(T*)_getRaw(ref.m_offset); }
+
+        /// A more terse way to get a raw pointer/reference. Using the [] operator can be seen as 'indexing' to access the
+        /// object the offset relates to. Unlike 'indices' that are typically used with [] offsets are generally not contiguous. 
+    template <typename T>
+    T* operator[](const Offset32Ptr<T>& ptr) { return (T*)_getRaw(ptr.m_offset); }
+    template <typename T>
+    T& operator[](const Offset32Ref<T>& ref) { return *(T*)_getRaw(ref.m_offset); }
+
+    template <typename T>
+    Offset32Ptr<T> asPtr(T* ptr) { return Offset32Ptr<T>(getOffset(ptr)); }
+        /// Note the use of ptr when setting up a reference here - it's needed because a ref does not have to be backed by a pointer.
+        /// And commonly is not when the const& and the thing referenced can be held in a word.
+    template <typename T>
+    Offset32Ref<T> asRef(T* ptr) { SLANG_ASSERT(ptr); return Offset32Ref<T>(getOffset(ptr)); }
+
+    uint32_t getOffset(const void* ptr)
+    {
+        if (ptr == nullptr)
+        {
+            return kNull32Offset;
+        }
+        ptrdiff_t diff = ((const uint8_t*)ptr) - m_data;
+        SLANG_ASSERT(diff > 0 && size_t(diff) < m_dataSize);
+        return uint32_t(diff);
+    }
+
+        /// Get the contained data
+    SLANG_FORCE_INLINE uint8_t* getData() { return m_data; }
+        /// Return the last used byte of the data
+    SLANG_FORCE_INLINE size_t getDataCount() const { return m_dataSize; }
+
+        /// Get the first allocated thing. Typically the root of the structure contained
+    void* getFirst() { return (m_dataSize < kStartOffset) ? nullptr :  (m_data + kStartOffset); }
+
+        /// Get a raw pointer from the offset
+    uint8_t* _getRaw(uint32_t offset) { return (offset == kNull32Offset) ? nullptr : (m_data + offset); }
+
+    OffsetBase():
+        m_data(nullptr),
+        m_dataSize(0)
+    {
+    }
+
+
+
+    uint8_t* m_data;
+    size_t m_dataSize;
+
+protected:
+        /// We want protected, because we don't want copies to be made of OffsetBase by default!
+    OffsetBase(const ThisType& rhs) = default;
+    ThisType& operator=(const ThisType& rhs) = default;
+};
+
+class MemoryOffsetBase : public OffsetBase
+{
+public:
+    void set(void* data, size_t dataSize)
+    {
+        m_data = (uint8_t*)data;
+        m_dataSize = dataSize;
+    }
+};
+
+/* OffsetContainer is a type designed to manage the construction structures around 'offset types'. In particular it allows
+for construction of offset structures where their total encoded size is not known at the outset.
+
+The main mechanism to make this work is via the use of OffsetXXX types, which when constructed from the OffsetContainer will
+maintain valid values, even if the underlying backing memories location is changed.
+*/
+class OffsetContainer : public OffsetBase
+{
+public:
+
+    template <typename T>
+    Offset32Ptr<T> newObject()
+    {
+        void* data = allocate(sizeof(T), SLANG_ALIGN_OF(T));
+        new (data) T();
+        return Offset32Ptr<T>(getOffset(data));
+    }
+
+    template <typename T>
+    Offset32Array<T> newArray(size_t size)
+    {
+        if (size == 0)
+        {
+            return Offset32Array<T>();
+        }
+        T* data = (T*)allocate(sizeof(T) * size, SLANG_ALIGN_OF(T));
+        for (size_t i = 0; i < size; ++i)
+        {
+            new (data + i) T();
+        }
+        return Offset32Array<T>(Offset32Ptr<T>(getOffset(data)), uint32_t(size));
+    }
+
+        /// Get the base - which is needed for turning offsets into things
+    OffsetBase& asBase() { return *this; }
+
+    /// Allocate without alignment (effectively 1)
+    void* allocate(size_t size);
+    void* allocate(size_t size, size_t alignment);
+    void* allocateAndZero(size_t size, size_t alignment);
+
+    void fixAlignment(size_t alignment);
+
+    Offset32Ptr<OffsetString> newString(const UnownedStringSlice& slice);
+    Offset32Ptr<OffsetString> newString(const char* contents);
+
+        /// Ctor
+    OffsetContainer();
+    ~OffsetContainer();
+
+protected:
+    size_t m_capacity;
+};
+
+} // namespace Slang
+
+#endif
diff --git a/source/core/slang-relative-container.h b/source/core/slang-relative-container.h
deleted file mode 100644
index 2f7e881b5..000000000
--- a/source/core/slang-relative-container.h
+++ /dev/null
@@ -1,423 +0,0 @@
-// slang-relative-container.h
-#ifndef SLANG_RELATIVE_CONTAINER_H_INCLUDED
-#define SLANG_RELATIVE_CONTAINER_H_INCLUDED
-
-#include "slang-basic.h"
-
-namespace Slang {
-
-/*
-The purpose of RelativeContainer and related types is to provide a mechanism to easily serialize relative structures.
-
-The root idea here is the "relative pointer". A typical pointer in a language like C/C++ holds the absolute address
-of the thing that is being pointed to. This introduces a problem if the structure is serialized in that
-it is highly likely that the structures will be placed at different addresses. This means that the absolute
-pointers will not point to the correct places, and so not be usable when read back from a serialization, or
-when moved to another location.
-
-A relative pointer means a pointer that points to something relative
-to the *location of the pointer*. The Relative32Ptr uses a 32 bit offset from the pointers location in memory. This
-means such a pointer can address a 4Gb address space, but more realistically it gives it a 2Gb address space as this
-is the size such that a pointer at any address can point to any other address. For specialized uses, it can be useful
-to have 8 bit, 16 bit, 64 bit and scaled relative pointers. For the purposes here though 32 bits works well enough
-for current use cases.
-
-Special care is needed when using relative pointers - both when constructing structures that contain them, reading
-them and in general usage.
-
-For simplicity here we store all relative pointers within a single contiguous allocation. This allocation is
-typically managed by the RelativeContainer. For simplicity it's easiest to claim that all relative pointers
-*can only exist* in this address space. That is no relative pointer should be decared as a variable on the stack, and
-similarly no struct, or other derived type holding a relative pointer should be held on the stack.
-
-Why? Most simply in a 64bit address space there is no guarentee that say a 32bit relative pointer *can* point to the
-memory in the RelativeContainer. For similar reasons relative ptrs cannot be held in the heap, or in a typical ADT
-container (like std::vector). In summary RelativePtr can *only* be stored in contiguous chunk of memory designed for
-the purpose - such as RelativeContainer, or a continuous chunk of memory that has been serialized in.
-
-This presents a problem - in how do we create and use such structures? For reading the process is simple - in that we
-can just turn the relative pointer into a regular raw pointer and use it. When creating structures - unless you know
-the allocated space (in the RelativeContainer or some other piece of memory) is larger than required, then special care
-is needed, because when a new larger piece of memory is allocated to hold everything, all of the absolute pointers
-will likely be invalidated.
-
-To work around this problem we have the Safe32Ptr. This is a pointer which holds a pointer relative to the start of the
-allocation as well as knowing what the base allocation is. So if there is a change in the base allocation address -
-for example when the RelativeContainer resizes the backing memory, the Safe32Ptr is aware of the change and everything continues to
-work as expected. Safe32Ptr are much more like normal pointers - and they can be stored on the stack or in other structures
-like say a vector without problems. On the other hand a Safe32Ptr *cannot* be stored within stuctures held within the RelativeContainer,
-as it would no longer have the correct properties to be serializable (it would contain an absolute pointer - the one in the SafePtr).
-
-We can divide types into to sets. 'Relative types' and 'Non relative types'. 'Relative types' are Relative pointers/arrays,
-and value types (such as float, int etc), and POD types constructed from those types. Everything else is not a 'relative type'.
-Any relative type that has a relative pointer (such as Relative32Ptr and Relative32Array) can only be allocated in a 'relative aware'
-piece or memory such as RelativeContainer, or suitable contiguous piece of memory.
-
-So in basic usage - Relative32Ptr can only be stored inside of RelativeContainer or part of contiguous memory arranged to
-support them. Conversely Safe32Ptr can only be used outside of the places RelativePtrs can be used.
-They can both point to the same things though. RelativePtrs can be thought of pointers for use in serialization, and SafePtrs
-as pointers used to construct things using RelativePtrs.
-
-With that out of the way there is one last caveat - and it is around use of Safe32Ptr. That the Safe32Ptr is safe to use
-even if the underlying memory is moved. But raw pointers (which are of course absolute) from it are only valid whilst there
-are no changes to the location of memory. That Relative32Ptr and Safe32Ptrs are convertible between each other.
-
-For example 
-
-```
-
-struct Thing
-{
-    Relative32Ptr<RelativeString> text;
-    int value;
-};
-
-void func()
-{
-    RelativeContainer container;
-
-    // BAD! Can't construct anything containing a Relative32Ptr on the stack.
-    Thing thing;
-
-    // BAD! We are closer - thing is constructed in the container, but we cannot have Relative32Ptrs held on the stack.
-    Relative32Ptr<Thing> thing = container.newObject<Thing>();
-
-    // Ok - this is now correct
-    Safe32Ptr<Thing> thing = container.newObject<Thing>();
-
-    // We can write and read things via the Safe32Ptr
-    thing->value = 10;
-    const int value = thing->value;
-
-    // Now lets write to it
-    {
-        // We can have raw pointer (or reference) to a thing but we need to be *careful* if we allocate 
-        Thing* rawThing = thing;
-        // We are okay here, nothing between getting the raw pointer and the write allocated/newed anything on the RelativeContainer
-        rawThing->value = 20;
-
-        // Lets set up name 
-        Safe32Ptr<RelativeString> text = relativeContainer.newString("Hello World!");
-
-        // BAD! Thr rawThing point could now be invalid because the call to newString may have had to allocate more memory
-        rawThing->text = text;
-
-        // This is okay because access is through the Safe32Ptr
-        thing->text = text;
-
-        // Or we can update rawThing such that is up to date
-        rawThing = thing;
-        // So now this is okay again
-        rawThing->text = text;
-    }
-}
-
-```
-
-Safe32Array and Relative32Array have very similar behaviors to Safe32Ptr and Relative32Ptr and can be used in the same places
-for the same purposes, but their use revolves around arrays. The arrays data is always allocated in the *RelativeContainer* so
-the arrays contents *even with* Safe32Array, can only contain Relative types. 
-
-For example
-
-```
-
-// BAD! The element types cannot contain any absolute pointers and that includes SafePtr
-Safe32Array<Safe32Ptr<RelativeString>> array = container.newArray<Safe32Ptr<RelativeString>>(10);
-
-// Ok
-Safe32Array<Relative32Ptr<RelativeString>> array = container.newArray<Relative32<Ptr<RelativeString>>(10);
-// I can now set array element in the normal way
-array[1] = container.newString("Hello");
-array[2] = container.newString("World!");
-```
-
-*/
-
-
-/* A type that is used to hold the base address of the contiguous memory that holds either RelativePtr and related types
-and/or is pointed to by Safe32Ptrs.
-
-The g_null member is a special singleton version that just holds m_data as nullptr, allows the representation of 'nullptr' on
-a Safe32Ptr to be that RelativeBase with an offset of 0.
-*/
-struct RelativeBase
-{
-    uint8_t* m_data;
-
-    static RelativeBase g_null;
-};
-
-/* A pointer to items held in RelativeContainer that remains correct even if the memory inside RelativeContainer moves.
-Safe32Ptr can be allocated on the stack, on the heap, used in containers such as List, std::vector.
-*/
-template <typename T>
-class Safe32Ptr
-{
-public:
-    typedef Safe32Ptr ThisType;
-
-    T& operator*() const { return *get(); }
-    T* operator->() const { return get(); }
-    operator T*() const { return get(); }
-
-    const Safe32Ptr& operator=(const ThisType& rhs) { m_offset = rhs.m_offset; m_base = rhs.m_base; return *this; }
-    SLANG_FORCE_INLINE T* get() const { return (T*)(m_base->m_data + m_offset); } 
-
-    void setNull()
-    {
-        m_offset = 0;
-        m_base = &RelativeBase::g_null; 
-    }
-
-    Safe32Ptr(const ThisType& rhs) : m_offset(rhs.m_offset), m_base(rhs.m_base) {}
-    Safe32Ptr() : m_base(&RelativeBase::g_null), m_offset(0) {}
-    Safe32Ptr(uint32_t offset, RelativeBase* base) : m_offset(offset), m_base(base) {}
-
-    RelativeBase* m_base;
-    uint32_t m_offset;
-};
-
-
-enum
-{
-    kRelative32PtrNull = int32_t(0x80000000)
-};
-
-/* A 32 bit relative pointer. It can only be held in contiguous memory designed for it's usage (like RelativeContainer). The thing
-that it points to is relative to the address *of the pointer*.
-
-This means that in normal usage should *not* be allocated on the stack, on the heap (unless as part of contiguous piece of memory
-designed for usage), or in a container such as std::vector or List.
-
-That because pointers are relative, we use a special value `kRelative32PtrNull` to indicate a pointer is nullptr. 0 could not be used
-because if we had
-
-```
-struct Thing
-{
-    RelativePtr<Thing> thingPtr;
-    int someThingElse;
-};
-```
-
-It might be valid for thingPtr to point to Thing and in that case it's offset would be 0, and this confused with nullptr if 0 was
-used to represent nullptr.
-*/
-template <typename T>
-class Relative32Ptr
-{
-public:
-    typedef Relative32Ptr ThisType;
-
-    T& operator*() const { return *get(); }
-    T* operator->() const { return get(); }
-    operator T*() const { return get(); }
-
-    T* get()
-    {
-        uint8_t* nonConstThis = (uint8_t*)this;
-        return (m_offset == kRelative32PtrNull) ? nullptr : (T*)(nonConstThis + m_offset);
-    }
-    T* get() const
-    {
-        uint8_t* nonConstThis = const_cast<uint8_t*>((const uint8_t*)this);
-        return (m_offset == kRelative32PtrNull) ? nullptr : (T*)(nonConstThis + m_offset);
-    }
-
-    T* detach() { T* ptr = get(); m_offset = kRelative32PtrNull; }
-
-    void setNull() { m_offset = kRelative32PtrNull; }
-
-    SLANG_FORCE_INLINE void set(T* ptr) { m_offset = ptr ? int32_t(((uint8_t*)ptr) - ((const uint8_t*)this)) : uint32_t(kRelative32PtrNull); }
-
-    Relative32Ptr(const Safe32Ptr<T>& rhs) { set(rhs.get()); }
-    Relative32Ptr(const ThisType& rhs) { set(rhs.get()); }
-
-    Relative32Ptr() :m_offset(kRelative32PtrNull) {}
-    Relative32Ptr(T* ptr) { set(ptr); }
-
-    const Relative32Ptr& operator=(const ThisType& rhs) { set(rhs.get()); return *this; }
-    const Relative32Ptr& operator=(const Safe32Ptr<T>& rhs) { set(rhs.get()); return *this; }
-
-    int32_t m_offset;
-};
-
-/* Much like SafePtr this is an array but whose memory is stored inside the RelativeContainer. This means elements types 
-must be 'relative types'. */
-template <typename T>
-class Safe32Array
-{
-public:
-    const T* begin() const { return m_data; }
-    const T* end() const { return begin() + m_count; }
-
-    T* begin() { return m_data; }
-    T* end() { return begin() + m_count; }
-
-    Index getCount() const { return Index(m_count); }
-    T* getData() { return m_data.get(); }
-    const T* getData() const { return m_data.get(); }
-
-    const T& operator[](Index i) const { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; }
-    T& operator[](Index i) { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; }
-
-    Safe32Array(Safe32Ptr<T> data, uint32_t count):m_data(data), m_count(count) {}
-
-    Safe32Array():m_count(0) {}
-
-    Safe32Ptr<T> m_data;
-    uint32_t m_count;
-};
-
-/* Much like a RelativePtr this is an array whose elements are stored inside the RelativeContainer. This means element types can only be 'relative types'. */
-template <typename T>
-class Relative32Array
-{
-public:
-    typedef Relative32Array ThisType;
-
-    const T* begin() const { return m_data; }
-    const T* end() const { return begin() + m_count; }
-
-    T* begin() { return m_data; }
-    T* end() { return begin() + m_count; }
-
-    Index getCount() const { return Index(m_count); }
-    T* getData() { return m_data.get(); }
-    const T* getData() const { return m_data.get(); }
-
-    const T& operator[](Index i) const { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; }
-    T& operator[](Index i) { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; }
-
-    Relative32Array(const Safe32Array<T>& rhs):
-        m_count(rhs.m_count),
-        m_data(rhs.m_data)
-    {
-    }
-
-    Relative32Array() : m_count(0) {}
-    Relative32Array(const ThisType& rhs) : m_count(rhs.m_count), m_data(rhs.m_data) {}
-
-    uint32_t m_count;               ///< the size of the data
-    Relative32Ptr<T> m_data;           ///< The data
-};
-
-/** RelativeString is used for storing strings within a RelativeContainer. Strings are stored with the initial byte indicating the size
-of the string. Note that all relative strings are stored with a terminating zero, and that the terminating zero is *NOT* included in
-the encoded size. */
-struct RelativeString
-{
-    enum
-    {
-        kSizeBase = 251,
-        kMaxSizeEncodeSize = 5,
-    };
-
-        /// Get contents as a slice
-    UnownedStringSlice getSlice() const;
-        /// Get null terminated string
-    const char* getCstr() const;
-
-        /// Decode the size. Returns the start of the string text, and outSize holds the size (NOT including terminating 0)
-    static const char* decodeSize(const char* in, size_t& outSize);
-
-        /// Returns the amount of bytes used, end encoding in 'encode'
-    static size_t calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncodeSize]);
-        /// Calculate the total size needed to store the string *including* terminating 0
-    static size_t calcAllocationSize(const UnownedStringSlice& slice);
-
-        /// Calculate the total size needed to store string. Size should be passed *without* terminating 0
-    static size_t calcAllocationSize(size_t size);
-
-    char m_sizeThenContents[1];
-};
-
-/* RelativeContainer is a type designed to manage the construction structures around 'relative types'. In particular it allows
-for construction of relative structures where their total relative encoded size is not known at the outset.
-
-The main mechanism to make this work is via the use of SafeXXX types, which when constructed from the RelativeContainer will
-maintain valid values, even if the underlying backing memories location is changed. 
-*/
-class RelativeContainer
-{
-public:
-
-    template <typename T>
-    Safe32Ptr<T> newObject()
-    {
-        void* data = allocate(sizeof(T), SLANG_ALIGN_OF(T));
-        new (data) T();
-        return Safe32Ptr<T>(getOffset(data), &m_base);
-    }
-
-    template <typename T>
-    Safe32Array<T> newArray(size_t size)
-    {
-        if (size == 0)
-        {
-            return Safe32Array<T>();
-        }
-        T* data = (T*)allocate(sizeof(T) * size, SLANG_ALIGN_OF(T));
-        for (size_t i = 0; i < size; ++i)
-        {
-            new (data + i) T();
-        }
-        return Safe32Array<T>(Safe32Ptr<T>(getOffset(data), &m_base), uint32_t(size));
-    }
-
-        /// Make a raw pointer into a safe ptr
-    template <typename T>
-    Safe32Ptr<T> toSafe(T* in)
-    {
-        Safe32Ptr<T> dst;
-        if (in)
-        {
-            dst.m_base = &m_base;
-            dst.m_offset = getOffset(in);
-        }
-        return dst;
-    }
-
-    /// Allocate without alignment (effectively 1)
-    void* allocate(size_t size);
-    void* allocate(size_t size, size_t alignment);
-    void* allocateAndZero(size_t size, size_t alignment);
-
-    void fixAlignment(size_t alignment);
-
-    SLANG_FORCE_INLINE uint32_t getOffset(const void* ptr) const
-    {
-        ptrdiff_t offset = ((const uint8_t*)ptr) - m_base.m_data; 
-        SLANG_ASSERT(offset >= 0 && size_t(offset) < m_current);
-        return uint32_t(offset);
-    }
-
-    Safe32Ptr<RelativeString> newString(const UnownedStringSlice& slice);
-    Safe32Ptr<RelativeString> newString(const char* contents);
-
-        /// Get the contained data
-    uint8_t* getData() { return m_base.m_data; }
-        /// Return the last used byte of the data
-    size_t getDataCount() const { return m_current; }
-
-        /// Set the contents
-    void set(void* data, size_t size);
-
-    RelativeBase* getBase() { return &m_base; }
-
-        /// Ctor
-    RelativeContainer();
-    ~RelativeContainer();
-
-
-protected:
-    size_t m_current;
-    size_t m_capacity;
-    RelativeBase m_base;
-};
-
-
-} // namespace Slang
-
-#endif