diff options
| author | jsmall-nvidia <jsmall@nvidia.com> | 2019-10-24 17:58:24 -0400 |
|---|---|---|
| committer | Tim Foley <tfoleyNV@users.noreply.github.com> | 2019-10-24 14:58:24 -0700 |
| commit | 89ddb50eaccc1b7b590dbde55032721762711fb2 (patch) | |
| tree | e61da2c1604e0d52d3a9915363769ccf950b62f3 | |
| parent | 58ad4b1a9ca43098a071c42bd752a4a48405bf0e (diff) | |
OffsetContainer serialization (#1093)
* OffsetContainer with unit tests.
* State serialization working with OffsetContainer.
* Fixes to make work with OffsetContainer.
* Added OffsetContainer documentation.
* Remove RelativeContainer.
* Fix problem with + on Offset32Ptr on windows x86 target.
* * Made OffsetBase a base class of OffsetContainer.
* Added MemoryOffsetBase to just handle being a chunk of memory.
* * Use operator[] to access contents of OffsetContainer
* Fix the type hash to work across different size_t sizes.
* Fixed some Offset type related comments.
* Fix bug around using asBase, because it returns a reference just using 'auto' will means it becomes a value type.
Remove assignment and copy ctor from OffsetBase.
* Evaluation order of assignment can lead to wrong behavior with Offset32Ptr/raw pointers. Document the fact, and fix in StateSerializeUtil.
| -rw-r--r-- | source/core/core.vcxproj | 4 | ||||
| -rw-r--r-- | source/core/core.vcxproj.filters | 12 | ||||
| -rw-r--r-- | source/core/slang-offset-container.cpp (renamed from source/core/slang-relative-container.cpp) | 83 | ||||
| -rw-r--r-- | source/core/slang-offset-container.h | 393 | ||||
| -rw-r--r-- | source/core/slang-relative-container.h | 423 | ||||
| -rw-r--r-- | source/slang/slang-options.cpp | 4 | ||||
| -rw-r--r-- | source/slang/slang-state-serialize.cpp | 435 | ||||
| -rw-r--r-- | source/slang/slang-state-serialize.h | 62 | ||||
| -rw-r--r-- | source/slang/slang.cpp | 10 | ||||
| -rw-r--r-- | tools/slang-test/slang-test.vcxproj | 2 | ||||
| -rw-r--r-- | tools/slang-test/slang-test.vcxproj.filters | 2 | ||||
| -rw-r--r-- | tools/slang-test/unit-offset-container.cpp | 119 | ||||
| -rw-r--r-- | tools/slang-test/unit-relative-container.cpp | 94 |
13 files changed, 843 insertions, 800 deletions
diff --git a/source/core/core.vcxproj b/source/core/core.vcxproj index 0a1b070fe..5b42dc81b 100644 --- a/source/core/core.vcxproj +++ b/source/core/core.vcxproj @@ -188,10 +188,10 @@ <ClInclude Include="slang-math.h" /> <ClInclude Include="slang-memory-arena.h" /> <ClInclude Include="slang-object-scope-manager.h" /> + <ClInclude Include="slang-offset-container.h" /> <ClInclude Include="slang-platform.h" /> <ClInclude Include="slang-process-util.h" /> <ClInclude Include="slang-random-generator.h" /> - <ClInclude Include="slang-relative-container.h" /> <ClInclude Include="slang-render-api-util.h" /> <ClInclude Include="slang-riff.h" /> <ClInclude Include="slang-secure-crt.h" /> @@ -220,9 +220,9 @@ <ClCompile Include="slang-io.cpp" /> <ClCompile Include="slang-memory-arena.cpp" /> <ClCompile Include="slang-object-scope-manager.cpp" /> + <ClCompile Include="slang-offset-container.cpp" /> <ClCompile Include="slang-platform.cpp" /> <ClCompile Include="slang-random-generator.cpp" /> - <ClCompile Include="slang-relative-container.cpp" /> <ClCompile Include="slang-render-api-util.cpp" /> <ClCompile Include="slang-riff.cpp" /> <ClCompile Include="slang-shared-library.cpp" /> diff --git a/source/core/core.vcxproj.filters b/source/core/core.vcxproj.filters index 56bb6b8b7..8184637f2 100644 --- a/source/core/core.vcxproj.filters +++ b/source/core/core.vcxproj.filters @@ -63,6 +63,9 @@ <ClInclude Include="slang-object-scope-manager.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="slang-offset-container.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="slang-platform.h"> <Filter>Header Files</Filter> </ClInclude> @@ -72,9 +75,6 @@ <ClInclude Include="slang-random-generator.h"> <Filter>Header Files</Filter> </ClInclude> - <ClInclude Include="slang-relative-container.h"> - <Filter>Header Files</Filter> - </ClInclude> <ClInclude Include="slang-render-api-util.h"> <Filter>Header Files</Filter> </ClInclude> @@ -155,13 +155,13 @@ <ClCompile Include="slang-object-scope-manager.cpp"> <Filter>Source Files</Filter> </ClCompile> - <ClCompile Include="slang-platform.cpp"> + <ClCompile Include="slang-offset-container.cpp"> <Filter>Source Files</Filter> </ClCompile> - <ClCompile Include="slang-random-generator.cpp"> + <ClCompile Include="slang-platform.cpp"> <Filter>Source Files</Filter> </ClCompile> - <ClCompile Include="slang-relative-container.cpp"> + <ClCompile Include="slang-random-generator.cpp"> <Filter>Source Files</Filter> </ClCompile> <ClCompile Include="slang-render-api-util.cpp"> diff --git a/source/core/slang-relative-container.cpp b/source/core/slang-offset-container.cpp index 0b52f4268..5fed2a452 100644 --- a/source/core/slang-relative-container.cpp +++ b/source/core/slang-offset-container.cpp @@ -1,14 +1,11 @@ -// slang-relative-containere.cpp -#include "slang-relative-container.h" +// slang-offset-container.cpp +#include "slang-offset-container.h" namespace Slang { +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! OffsetString !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ -/* static */RelativeBase RelativeBase::g_null = { nullptr }; - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RelativeString !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -size_t RelativeString::calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncodeSize]) +size_t OffsetString::calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncodeSize]) { SLANG_ASSERT(size <= 0xffffffff); if (size <= kSizeBase) @@ -32,7 +29,7 @@ size_t RelativeString::calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncod return num + 1; } -/* static */const char* RelativeString::decodeSize(const char* in, size_t& outSize) +/* static */const char* OffsetString::decodeSize(const char* in, size_t& outSize) { const uint8_t* cur = (const uint8_t*)in; if (*cur <= kSizeBase) @@ -72,7 +69,7 @@ size_t RelativeString::calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncod } } -/* static */size_t RelativeString::calcAllocationSize(size_t stringSize) +/* static */size_t OffsetString::calcAllocationSize(size_t stringSize) { uint8_t encode[kMaxSizeEncodeSize]; size_t encodeSize = calcEncodedSize(stringSize, encode); @@ -80,12 +77,12 @@ size_t RelativeString::calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncod return encodeSize + stringSize + 1; } -/* static */size_t RelativeString::calcAllocationSize(const UnownedStringSlice& slice) +/* static */size_t OffsetString::calcAllocationSize(const UnownedStringSlice& slice) { return calcAllocationSize(slice.size()); } -UnownedStringSlice RelativeString::getSlice() const +UnownedStringSlice OffsetString::getSlice() const { size_t size; const char* chars = decodeSize(m_sizeThenContents, size); @@ -93,41 +90,43 @@ UnownedStringSlice RelativeString::getSlice() const return UnownedStringSlice(chars, size); } -const char* RelativeString::getCstr() const +const char* OffsetString::getCstr() const { return getSlice().begin(); } -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RelativeContainer !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! OffsetContainer !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ -RelativeContainer::RelativeContainer() +OffsetContainer::OffsetContainer() { - m_current = 0; m_capacity = 0; - m_base.m_data = nullptr; + m_data = nullptr; + + // We need to allocate some of the first bytes 0 can be used for nullptr. + allocateAndZero(kStartOffset, 1); } -RelativeContainer::~RelativeContainer() +OffsetContainer::~OffsetContainer() { - if (m_base.m_data) + if (m_data) { - ::free(m_base.m_data); + ::free(m_data); } } -void* RelativeContainer::allocate(size_t size) +void* OffsetContainer::allocate(size_t size) { return allocate(size, 1); } -void RelativeContainer::fixAlignment(size_t alignment) +void OffsetContainer::fixAlignment(size_t alignment) { allocate(0, alignment); } -void* RelativeContainer::allocate(size_t size, size_t alignment) +void* OffsetContainer::allocate(size_t size, size_t alignment) { - size_t offset = (m_current + alignment - 1) & ~(alignment - 1); + size_t offset = (m_dataSize + alignment - 1) & ~(alignment - 1); if (offset + size > m_capacity) { @@ -148,29 +147,29 @@ void* RelativeContainer::allocate(size_t size, size_t alignment) size_t newSize = (calcSize < minSize) ? minSize : calcSize; // Reallocate space - m_base.m_data = (uint8_t*)::realloc(m_base.m_data, newSize); + m_data = (uint8_t*)::realloc(m_data, newSize); m_capacity = newSize; } SLANG_ASSERT(offset + size <= m_capacity); - m_current = offset + size; - return m_base.m_data + offset; + m_dataSize = offset + size; + return m_data + offset; } -void* RelativeContainer::allocateAndZero(size_t size, size_t alignment) +void* OffsetContainer::allocateAndZero(size_t size, size_t alignment) { void* data = allocate(size, alignment); memset(data, 0, size); return data; } -Safe32Ptr<RelativeString> RelativeContainer::newString(const UnownedStringSlice& slice) +Offset32Ptr<OffsetString> OffsetContainer::newString(const UnownedStringSlice& slice) { size_t stringSize = slice.size(); - uint8_t head[RelativeString::kMaxSizeEncodeSize]; - size_t headSize = RelativeString::calcEncodedSize(stringSize, head); + uint8_t head[OffsetString::kMaxSizeEncodeSize]; + size_t headSize = OffsetString::calcEncodedSize(stringSize, head); size_t allocSize = headSize + stringSize + 1; uint8_t* bytes = (uint8_t*)allocate(allocSize); @@ -181,12 +180,12 @@ Safe32Ptr<RelativeString> RelativeContainer::newString(const UnownedStringSlice& // 0 terminate bytes[headSize + stringSize] = 0; - return Safe32Ptr<RelativeString>(getOffset(bytes), &m_base); + return Offset32Ptr<OffsetString>(getOffset(bytes)); } -Safe32Ptr<RelativeString> RelativeContainer::newString(const char* contents) +Offset32Ptr<OffsetString> OffsetContainer::newString(const char* contents) { - Safe32Ptr<RelativeString> relString; + Offset32Ptr<OffsetString> relString; if (contents) { relString = newString(UnownedStringSlice(contents)); @@ -194,22 +193,4 @@ Safe32Ptr<RelativeString> RelativeContainer::newString(const char* contents) return relString; } -void RelativeContainer::set(void* data, size_t size) -{ - if (m_base.m_data) - { - ::free(m_base.m_data); - m_base.m_data = nullptr; - } - - if (size > 0) - { - m_base.m_data = (uint8_t*)::malloc(size); - ::memcpy(m_base.m_data, data, size); - } - - m_current = size; - m_capacity = size; -} - } // namespace Slang diff --git a/source/core/slang-offset-container.h b/source/core/slang-offset-container.h new file mode 100644 index 000000000..95c3a6589 --- /dev/null +++ b/source/core/slang-offset-container.h @@ -0,0 +1,393 @@ +// slang-offset-container.h +#ifndef SLANG_OFFSET_CONTAINER_H_INCLUDED +#define SLANG_OFFSET_CONTAINER_H_INCLUDED + +#include "slang-basic.h" + +namespace Slang { + +/* +The purpose of OffsetContainer and related types is to provide a mechanism to easily serialize offset structures. + +The root idea here is the "offset pointer". A typical pointer in a language like C/C++ holds the absolute address +in the current address space of the thing that is being pointed to. This introduces a problem, as when data is +serialized in the contents will very likely be be placed at different addresses - meaning any absolute pointer +will point to the wrong place. There is also a related issue around pointer sizes - on some targets they are 32 bits +and on others 64 bits. + +An offset pointer means a pointer that points to something 'offset' to some base address. The OffsetPtr uses a 32 bit +offset from the pointers location in memory. This means such a pointer can address a 4Gb address space. + +Special care is needed when using offset pointers - both when constructing structures that contain them, reading +them and in general usage. + +For simplicity here we store all offset pointers within a single contiguous allocation. This allocation is +typically managed by the OffsetContainer for writing. When reading a MemoryOffsetBase can be used. + +An issue around using offset pointers, is that we cannot directly access it's contents, because it's just an +offset to some base address. Thus to access the thing being pointed to we need to turn the offset pointer back into +a 'raw' pointer. This is achieved via using the asRaw methods on the OffsetBase. For a convenience operator[] can also +be used, and this is typically the preferred mechanism. + +NOTE! That the evaluation order of a function calls parameters is undefined in C++. That whilst it might appear doing + +``` +base[thing] = container.newObject<Thing>(); +``` + +will evaluate the construction of newObject *before* the assignment, if you look at the assignment as being a function call +(as it is when it is overloaded), then base[thing] might be evaluated *before* newObject, and if it is then the result +could be wrong if the newObject needed to reallocate. Therefore when allocation is involved, a new (or any allocation backed +function call from the OffsetContainer) should always place a result in a local variable. Then assign as in + +``` +auto anotherThing = container.newObject<Thing>(); +base[thing] = anotherThing; +``` + +When creating structures - unless you know the allocated space (in the OffsetContainer or some other piece of memory) +is larger than required, then special care is needed, because when a new larger piece of memory is allocated to hold +everything, raw pointers pointers will likely be invalidated. When reading there is typically no need to move +the base address, so raw pointers remain valid through out. When doing writing if a call is made to something that +allocates memory on the OffsetContainer - any raw pointer should be assumed invalid. + +For example + +``` + +struct Thing +{ + Offset32Ptr<OffsetString> text; + int value; +}; + +void func() +{ + OffsetContainer container; + OffsetBase& base = container.asBase(); + + { + // We can allocate on the heap. BUT we can't set up a offset pointer to it + Thing thing; + // BAD!! Will assert, because thing is not in the address range recorded in base. + Offset32Ptr<Thing> thingOffsetPtr= base->asPtr(&thing); + } + + // Ok - this is now correct + Offset32Ptr<Thing> thing = container.newObject<Thing>(); + + // To write values, we need a raw pointer + { + // To get the raw pointer we can use 'asRaw' + auto rawThing = base->asRaw(thing); + + // Or more perhaps slightly more conveniently [] + auto rawThing = base[thing]; + + // We can write and read things via the Safe32Ptr + rawThing->value = 10; + const int value = rawThing->value; + + SLANG_ASSERT(value == 10); + } + + // Now lets write to it + { + // We can have raw pointer (or reference) to a thing but we need to be *careful* if we allocate + Thing* rawThing = base[thing]; + // We are okay here, nothing between getting the raw pointer and the write allocated/newed anything on the OffsetContainer + rawThing->value = 20; + + // Lets set up name + Offset32Ptr<OffsetString> text = offsetContainer.newString("Hello World!"); + + // BAD! The rawThing point could now be invalid because the call to newString may have had to allocate more memory + rawThing->text = text; + + // This is okay + base[thing]->text = text; + + // Or we can update rawThing such that is up to date + rawThing = base[thing]; + // So now this is okay again + rawThing->text = text; + + // BAD! we don't know the evaluation order here, if the lhs is evaluate before the rhs, then it could write to the wrong area of memory. + base[thing]->text = offsetContainer.newString("Hello World again!"); + + // So where there is allocation, and assignment to something that in held in offset ptr use a local for the allocation as in + { + auto text = offsetContainer.newString("Hello World again!"); + base[thing]->text = text; + } + } +} + +``` +*/ + +enum +{ + kNull32Offset = 0, + kStartOffset = uint32_t(sizeof(uint64_t)), ///< The offset to the first contained thing +}; + +template <typename T> +class Offset32Ref; + +/* A pointer to items held in OffsetContainer (or OffsetBase relative) that remains correct even if +the memory inside OffsetContainer moves. +*/ +template <typename T> +class Offset32Ptr +{ +public: + typedef Offset32Ptr ThisType; + + const ThisType& operator=(const ThisType& rhs) { m_offset = rhs.m_offset; return *this; } + bool operator==(const ThisType& rhs) const { return m_offset == rhs.m_offset; } + bool operator!=(const ThisType& rhs) const { return m_offset != rhs.m_offset; } + + bool operator<(const ThisType& rhs) const { return m_offset < rhs.m_offset; } + bool operator<=(const ThisType& rhs) const { return m_offset <= rhs.m_offset; } + bool operator>(const ThisType& rhs) const { return m_offset > rhs.m_offset; } + bool operator>=(const ThisType& rhs) const { return m_offset >= rhs.m_offset; } + + operator bool() const { return m_offset != kNull32Offset; } + + Offset32Ref<T> operator*(); + + ThisType& operator++() { m_offset += uint32_t(sizeof(T)); return *this; } + ThisType operator++(int) { const auto offset = m_offset; m_offset += uint32_t(sizeof(T)); return ThisType(offset); } + + ThisType& operator--() { m_offset -= sizeof(T); return *this; } + ThisType operator--(int) { const auto offset = m_offset; m_offset -= uint32_t(sizeof(T)); return ThisType(offset); } + + friend ThisType operator+(const ThisType& a, Index b) { return ThisType(a.m_offset + uint32_t(sizeof(T) * b)); } + friend ThisType operator+(Index a, const ThisType& b) { return ThisType(b.m_offset + uint32_t(sizeof(T) * a)); } + + bool isNull() const { return m_offset == kNull32Offset; } + + void setNull() { m_offset = kNull32Offset; } + Offset32Ptr():m_offset(kNull32Offset) {} + Offset32Ptr(const ThisType& rhs): m_offset(rhs.m_offset) {} + explicit Offset32Ptr(uint32_t offset): m_offset(offset) {} + + uint32_t m_offset; +}; + +/* A reference to items held in OffsetContainer (or OffsetBase relative) that remains correct even if +the memory inside OffsetContainer moves. +*/ +template <typename T> +class Offset32Ref +{ +public: + typedef Offset32Ref ThisType; + + const ThisType& operator=(const ThisType& rhs) { m_offset = rhs.m_offset; return *this; } + + Offset32Ptr<T> operator&() { return Offset32Ptr<T>(m_offset); } + + Offset32Ref(const ThisType& rhs) : m_offset(rhs.m_offset) {} + explicit Offset32Ref(uint32_t offset) : m_offset(offset) { SLANG_ASSERT(offset != kNull32Offset); } + + uint32_t m_offset; +}; + +// --------------------------------------------------------------------------- +template <typename T> +SLANG_FORCE_INLINE Offset32Ref<T> Offset32Ptr<T>::operator*() +{ + return Offset32Ref<T>(m_offset); +} + + +/* Much like Offset32Ptr this is an array but whose memory is stored inside the OffsetContainer. This means elements types +must be 'offset types'. */ +template <typename T> +class Offset32Array +{ +public: + Offset32Ptr<const T> begin() const { return Offset32Ptr<const T>(m_data.m_offset); } + Offset32Ptr<const T> end() const { return begin() + Index(m_count); } + + Offset32Ptr<T> begin() { return m_data; } + Offset32Ptr<T> end() { return begin() + Index(m_count); } + + Index getCount() const { return Index(m_count); } + + Offset32Ref<const T> operator[](Index i) const { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return Offset32Ref<const T>((m_data + i).m_offset); } + Offset32Ref<T> operator[](Index i) { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return Offset32Ref<T>((m_data + i).m_offset); } + + Offset32Array(Offset32Ptr<T> data, uint32_t count) :m_data(data), m_count(count) {} + + Offset32Array() :m_count(0) {} + + Offset32Ptr<T> m_data; + uint32_t m_count; +}; + +/** OffsetString is used for storing strings within a OffsetContainer. Strings are stored with the initial byte indicating the size +of the string. Note that all offset strings are stored with a terminating zero, and that the terminating zero is *NOT* included in +the encoded size. */ +struct OffsetString +{ + enum + { + kSizeBase = 251, + kMaxSizeEncodeSize = 5, + }; + + /// Get contents as a slice + UnownedStringSlice getSlice() const; + /// Get null terminated string + const char* getCstr() const; + + /// Decode the size. Returns the start of the string text, and outSize holds the size (NOT including terminating 0) + static const char* decodeSize(const char* in, size_t& outSize); + + /// Returns the amount of bytes used, end encoding in 'encode' + static size_t calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncodeSize]); + /// Calculate the total size needed to store the string *including* terminating 0 + static size_t calcAllocationSize(const UnownedStringSlice& slice); + + /// Calculate the total size needed to store string. Size should be passed *without* terminating 0 + static size_t calcAllocationSize(size_t size); + + char m_sizeThenContents[1]; +}; + +/* A type that is used to hold the base address of the contiguous memory that holds either Offset32Ptr and related types> +*/ +class OffsetBase +{ +public: + typedef OffsetBase ThisType; + + /// Turn an offset into a raw regular pointer or reference + template <typename T> + T* asRaw(const Offset32Ptr<T>& ptr) { return (T*)_getRaw(ptr.m_offset); } + template <typename T> + T& asRaw(const Offset32Ref<T>& ref) { return *(T*)_getRaw(ref.m_offset); } + + /// A more terse way to get a raw pointer/reference. Using the [] operator can be seen as 'indexing' to access the + /// object the offset relates to. Unlike 'indices' that are typically used with [] offsets are generally not contiguous. + template <typename T> + T* operator[](const Offset32Ptr<T>& ptr) { return (T*)_getRaw(ptr.m_offset); } + template <typename T> + T& operator[](const Offset32Ref<T>& ref) { return *(T*)_getRaw(ref.m_offset); } + + template <typename T> + Offset32Ptr<T> asPtr(T* ptr) { return Offset32Ptr<T>(getOffset(ptr)); } + /// Note the use of ptr when setting up a reference here - it's needed because a ref does not have to be backed by a pointer. + /// And commonly is not when the const& and the thing referenced can be held in a word. + template <typename T> + Offset32Ref<T> asRef(T* ptr) { SLANG_ASSERT(ptr); return Offset32Ref<T>(getOffset(ptr)); } + + uint32_t getOffset(const void* ptr) + { + if (ptr == nullptr) + { + return kNull32Offset; + } + ptrdiff_t diff = ((const uint8_t*)ptr) - m_data; + SLANG_ASSERT(diff > 0 && size_t(diff) < m_dataSize); + return uint32_t(diff); + } + + /// Get the contained data + SLANG_FORCE_INLINE uint8_t* getData() { return m_data; } + /// Return the last used byte of the data + SLANG_FORCE_INLINE size_t getDataCount() const { return m_dataSize; } + + /// Get the first allocated thing. Typically the root of the structure contained + void* getFirst() { return (m_dataSize < kStartOffset) ? nullptr : (m_data + kStartOffset); } + + /// Get a raw pointer from the offset + uint8_t* _getRaw(uint32_t offset) { return (offset == kNull32Offset) ? nullptr : (m_data + offset); } + + OffsetBase(): + m_data(nullptr), + m_dataSize(0) + { + } + + + + uint8_t* m_data; + size_t m_dataSize; + +protected: + /// We want protected, because we don't want copies to be made of OffsetBase by default! + OffsetBase(const ThisType& rhs) = default; + ThisType& operator=(const ThisType& rhs) = default; +}; + +class MemoryOffsetBase : public OffsetBase +{ +public: + void set(void* data, size_t dataSize) + { + m_data = (uint8_t*)data; + m_dataSize = dataSize; + } +}; + +/* OffsetContainer is a type designed to manage the construction structures around 'offset types'. In particular it allows +for construction of offset structures where their total encoded size is not known at the outset. + +The main mechanism to make this work is via the use of OffsetXXX types, which when constructed from the OffsetContainer will +maintain valid values, even if the underlying backing memories location is changed. +*/ +class OffsetContainer : public OffsetBase +{ +public: + + template <typename T> + Offset32Ptr<T> newObject() + { + void* data = allocate(sizeof(T), SLANG_ALIGN_OF(T)); + new (data) T(); + return Offset32Ptr<T>(getOffset(data)); + } + + template <typename T> + Offset32Array<T> newArray(size_t size) + { + if (size == 0) + { + return Offset32Array<T>(); + } + T* data = (T*)allocate(sizeof(T) * size, SLANG_ALIGN_OF(T)); + for (size_t i = 0; i < size; ++i) + { + new (data + i) T(); + } + return Offset32Array<T>(Offset32Ptr<T>(getOffset(data)), uint32_t(size)); + } + + /// Get the base - which is needed for turning offsets into things + OffsetBase& asBase() { return *this; } + + /// Allocate without alignment (effectively 1) + void* allocate(size_t size); + void* allocate(size_t size, size_t alignment); + void* allocateAndZero(size_t size, size_t alignment); + + void fixAlignment(size_t alignment); + + Offset32Ptr<OffsetString> newString(const UnownedStringSlice& slice); + Offset32Ptr<OffsetString> newString(const char* contents); + + /// Ctor + OffsetContainer(); + ~OffsetContainer(); + +protected: + size_t m_capacity; +}; + +} // namespace Slang + +#endif diff --git a/source/core/slang-relative-container.h b/source/core/slang-relative-container.h deleted file mode 100644 index 2f7e881b5..000000000 --- a/source/core/slang-relative-container.h +++ /dev/null @@ -1,423 +0,0 @@ -// slang-relative-container.h -#ifndef SLANG_RELATIVE_CONTAINER_H_INCLUDED -#define SLANG_RELATIVE_CONTAINER_H_INCLUDED - -#include "slang-basic.h" - -namespace Slang { - -/* -The purpose of RelativeContainer and related types is to provide a mechanism to easily serialize relative structures. - -The root idea here is the "relative pointer". A typical pointer in a language like C/C++ holds the absolute address -of the thing that is being pointed to. This introduces a problem if the structure is serialized in that -it is highly likely that the structures will be placed at different addresses. This means that the absolute -pointers will not point to the correct places, and so not be usable when read back from a serialization, or -when moved to another location. - -A relative pointer means a pointer that points to something relative -to the *location of the pointer*. The Relative32Ptr uses a 32 bit offset from the pointers location in memory. This -means such a pointer can address a 4Gb address space, but more realistically it gives it a 2Gb address space as this -is the size such that a pointer at any address can point to any other address. For specialized uses, it can be useful -to have 8 bit, 16 bit, 64 bit and scaled relative pointers. For the purposes here though 32 bits works well enough -for current use cases. - -Special care is needed when using relative pointers - both when constructing structures that contain them, reading -them and in general usage. - -For simplicity here we store all relative pointers within a single contiguous allocation. This allocation is -typically managed by the RelativeContainer. For simplicity it's easiest to claim that all relative pointers -*can only exist* in this address space. That is no relative pointer should be decared as a variable on the stack, and -similarly no struct, or other derived type holding a relative pointer should be held on the stack. - -Why? Most simply in a 64bit address space there is no guarentee that say a 32bit relative pointer *can* point to the -memory in the RelativeContainer. For similar reasons relative ptrs cannot be held in the heap, or in a typical ADT -container (like std::vector). In summary RelativePtr can *only* be stored in contiguous chunk of memory designed for -the purpose - such as RelativeContainer, or a continuous chunk of memory that has been serialized in. - -This presents a problem - in how do we create and use such structures? For reading the process is simple - in that we -can just turn the relative pointer into a regular raw pointer and use it. When creating structures - unless you know -the allocated space (in the RelativeContainer or some other piece of memory) is larger than required, then special care -is needed, because when a new larger piece of memory is allocated to hold everything, all of the absolute pointers -will likely be invalidated. - -To work around this problem we have the Safe32Ptr. This is a pointer which holds a pointer relative to the start of the -allocation as well as knowing what the base allocation is. So if there is a change in the base allocation address - -for example when the RelativeContainer resizes the backing memory, the Safe32Ptr is aware of the change and everything continues to -work as expected. Safe32Ptr are much more like normal pointers - and they can be stored on the stack or in other structures -like say a vector without problems. On the other hand a Safe32Ptr *cannot* be stored within stuctures held within the RelativeContainer, -as it would no longer have the correct properties to be serializable (it would contain an absolute pointer - the one in the SafePtr). - -We can divide types into to sets. 'Relative types' and 'Non relative types'. 'Relative types' are Relative pointers/arrays, -and value types (such as float, int etc), and POD types constructed from those types. Everything else is not a 'relative type'. -Any relative type that has a relative pointer (such as Relative32Ptr and Relative32Array) can only be allocated in a 'relative aware' -piece or memory such as RelativeContainer, or suitable contiguous piece of memory. - -So in basic usage - Relative32Ptr can only be stored inside of RelativeContainer or part of contiguous memory arranged to -support them. Conversely Safe32Ptr can only be used outside of the places RelativePtrs can be used. -They can both point to the same things though. RelativePtrs can be thought of pointers for use in serialization, and SafePtrs -as pointers used to construct things using RelativePtrs. - -With that out of the way there is one last caveat - and it is around use of Safe32Ptr. That the Safe32Ptr is safe to use -even if the underlying memory is moved. But raw pointers (which are of course absolute) from it are only valid whilst there -are no changes to the location of memory. That Relative32Ptr and Safe32Ptrs are convertible between each other. - -For example - -``` - -struct Thing -{ - Relative32Ptr<RelativeString> text; - int value; -}; - -void func() -{ - RelativeContainer container; - - // BAD! Can't construct anything containing a Relative32Ptr on the stack. - Thing thing; - - // BAD! We are closer - thing is constructed in the container, but we cannot have Relative32Ptrs held on the stack. - Relative32Ptr<Thing> thing = container.newObject<Thing>(); - - // Ok - this is now correct - Safe32Ptr<Thing> thing = container.newObject<Thing>(); - - // We can write and read things via the Safe32Ptr - thing->value = 10; - const int value = thing->value; - - // Now lets write to it - { - // We can have raw pointer (or reference) to a thing but we need to be *careful* if we allocate - Thing* rawThing = thing; - // We are okay here, nothing between getting the raw pointer and the write allocated/newed anything on the RelativeContainer - rawThing->value = 20; - - // Lets set up name - Safe32Ptr<RelativeString> text = relativeContainer.newString("Hello World!"); - - // BAD! Thr rawThing point could now be invalid because the call to newString may have had to allocate more memory - rawThing->text = text; - - // This is okay because access is through the Safe32Ptr - thing->text = text; - - // Or we can update rawThing such that is up to date - rawThing = thing; - // So now this is okay again - rawThing->text = text; - } -} - -``` - -Safe32Array and Relative32Array have very similar behaviors to Safe32Ptr and Relative32Ptr and can be used in the same places -for the same purposes, but their use revolves around arrays. The arrays data is always allocated in the *RelativeContainer* so -the arrays contents *even with* Safe32Array, can only contain Relative types. - -For example - -``` - -// BAD! The element types cannot contain any absolute pointers and that includes SafePtr -Safe32Array<Safe32Ptr<RelativeString>> array = container.newArray<Safe32Ptr<RelativeString>>(10); - -// Ok -Safe32Array<Relative32Ptr<RelativeString>> array = container.newArray<Relative32<Ptr<RelativeString>>(10); -// I can now set array element in the normal way -array[1] = container.newString("Hello"); -array[2] = container.newString("World!"); -``` - -*/ - - -/* A type that is used to hold the base address of the contiguous memory that holds either RelativePtr and related types -and/or is pointed to by Safe32Ptrs. - -The g_null member is a special singleton version that just holds m_data as nullptr, allows the representation of 'nullptr' on -a Safe32Ptr to be that RelativeBase with an offset of 0. -*/ -struct RelativeBase -{ - uint8_t* m_data; - - static RelativeBase g_null; -}; - -/* A pointer to items held in RelativeContainer that remains correct even if the memory inside RelativeContainer moves. -Safe32Ptr can be allocated on the stack, on the heap, used in containers such as List, std::vector. -*/ -template <typename T> -class Safe32Ptr -{ -public: - typedef Safe32Ptr ThisType; - - T& operator*() const { return *get(); } - T* operator->() const { return get(); } - operator T*() const { return get(); } - - const Safe32Ptr& operator=(const ThisType& rhs) { m_offset = rhs.m_offset; m_base = rhs.m_base; return *this; } - SLANG_FORCE_INLINE T* get() const { return (T*)(m_base->m_data + m_offset); } - - void setNull() - { - m_offset = 0; - m_base = &RelativeBase::g_null; - } - - Safe32Ptr(const ThisType& rhs) : m_offset(rhs.m_offset), m_base(rhs.m_base) {} - Safe32Ptr() : m_base(&RelativeBase::g_null), m_offset(0) {} - Safe32Ptr(uint32_t offset, RelativeBase* base) : m_offset(offset), m_base(base) {} - - RelativeBase* m_base; - uint32_t m_offset; -}; - - -enum -{ - kRelative32PtrNull = int32_t(0x80000000) -}; - -/* A 32 bit relative pointer. It can only be held in contiguous memory designed for it's usage (like RelativeContainer). The thing -that it points to is relative to the address *of the pointer*. - -This means that in normal usage should *not* be allocated on the stack, on the heap (unless as part of contiguous piece of memory -designed for usage), or in a container such as std::vector or List. - -That because pointers are relative, we use a special value `kRelative32PtrNull` to indicate a pointer is nullptr. 0 could not be used -because if we had - -``` -struct Thing -{ - RelativePtr<Thing> thingPtr; - int someThingElse; -}; -``` - -It might be valid for thingPtr to point to Thing and in that case it's offset would be 0, and this confused with nullptr if 0 was -used to represent nullptr. -*/ -template <typename T> -class Relative32Ptr -{ -public: - typedef Relative32Ptr ThisType; - - T& operator*() const { return *get(); } - T* operator->() const { return get(); } - operator T*() const { return get(); } - - T* get() - { - uint8_t* nonConstThis = (uint8_t*)this; - return (m_offset == kRelative32PtrNull) ? nullptr : (T*)(nonConstThis + m_offset); - } - T* get() const - { - uint8_t* nonConstThis = const_cast<uint8_t*>((const uint8_t*)this); - return (m_offset == kRelative32PtrNull) ? nullptr : (T*)(nonConstThis + m_offset); - } - - T* detach() { T* ptr = get(); m_offset = kRelative32PtrNull; } - - void setNull() { m_offset = kRelative32PtrNull; } - - SLANG_FORCE_INLINE void set(T* ptr) { m_offset = ptr ? int32_t(((uint8_t*)ptr) - ((const uint8_t*)this)) : uint32_t(kRelative32PtrNull); } - - Relative32Ptr(const Safe32Ptr<T>& rhs) { set(rhs.get()); } - Relative32Ptr(const ThisType& rhs) { set(rhs.get()); } - - Relative32Ptr() :m_offset(kRelative32PtrNull) {} - Relative32Ptr(T* ptr) { set(ptr); } - - const Relative32Ptr& operator=(const ThisType& rhs) { set(rhs.get()); return *this; } - const Relative32Ptr& operator=(const Safe32Ptr<T>& rhs) { set(rhs.get()); return *this; } - - int32_t m_offset; -}; - -/* Much like SafePtr this is an array but whose memory is stored inside the RelativeContainer. This means elements types -must be 'relative types'. */ -template <typename T> -class Safe32Array -{ -public: - const T* begin() const { return m_data; } - const T* end() const { return begin() + m_count; } - - T* begin() { return m_data; } - T* end() { return begin() + m_count; } - - Index getCount() const { return Index(m_count); } - T* getData() { return m_data.get(); } - const T* getData() const { return m_data.get(); } - - const T& operator[](Index i) const { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; } - T& operator[](Index i) { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; } - - Safe32Array(Safe32Ptr<T> data, uint32_t count):m_data(data), m_count(count) {} - - Safe32Array():m_count(0) {} - - Safe32Ptr<T> m_data; - uint32_t m_count; -}; - -/* Much like a RelativePtr this is an array whose elements are stored inside the RelativeContainer. This means element types can only be 'relative types'. */ -template <typename T> -class Relative32Array -{ -public: - typedef Relative32Array ThisType; - - const T* begin() const { return m_data; } - const T* end() const { return begin() + m_count; } - - T* begin() { return m_data; } - T* end() { return begin() + m_count; } - - Index getCount() const { return Index(m_count); } - T* getData() { return m_data.get(); } - const T* getData() const { return m_data.get(); } - - const T& operator[](Index i) const { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; } - T& operator[](Index i) { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; } - - Relative32Array(const Safe32Array<T>& rhs): - m_count(rhs.m_count), - m_data(rhs.m_data) - { - } - - Relative32Array() : m_count(0) {} - Relative32Array(const ThisType& rhs) : m_count(rhs.m_count), m_data(rhs.m_data) {} - - uint32_t m_count; ///< the size of the data - Relative32Ptr<T> m_data; ///< The data -}; - -/** RelativeString is used for storing strings within a RelativeContainer. Strings are stored with the initial byte indicating the size -of the string. Note that all relative strings are stored with a terminating zero, and that the terminating zero is *NOT* included in -the encoded size. */ -struct RelativeString -{ - enum - { - kSizeBase = 251, - kMaxSizeEncodeSize = 5, - }; - - /// Get contents as a slice - UnownedStringSlice getSlice() const; - /// Get null terminated string - const char* getCstr() const; - - /// Decode the size. Returns the start of the string text, and outSize holds the size (NOT including terminating 0) - static const char* decodeSize(const char* in, size_t& outSize); - - /// Returns the amount of bytes used, end encoding in 'encode' - static size_t calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncodeSize]); - /// Calculate the total size needed to store the string *including* terminating 0 - static size_t calcAllocationSize(const UnownedStringSlice& slice); - - /// Calculate the total size needed to store string. Size should be passed *without* terminating 0 - static size_t calcAllocationSize(size_t size); - - char m_sizeThenContents[1]; -}; - -/* RelativeContainer is a type designed to manage the construction structures around 'relative types'. In particular it allows -for construction of relative structures where their total relative encoded size is not known at the outset. - -The main mechanism to make this work is via the use of SafeXXX types, which when constructed from the RelativeContainer will -maintain valid values, even if the underlying backing memories location is changed. -*/ -class RelativeContainer -{ -public: - - template <typename T> - Safe32Ptr<T> newObject() - { - void* data = allocate(sizeof(T), SLANG_ALIGN_OF(T)); - new (data) T(); - return Safe32Ptr<T>(getOffset(data), &m_base); - } - - template <typename T> - Safe32Array<T> newArray(size_t size) - { - if (size == 0) - { - return Safe32Array<T>(); - } - T* data = (T*)allocate(sizeof(T) * size, SLANG_ALIGN_OF(T)); - for (size_t i = 0; i < size; ++i) - { - new (data + i) T(); - } - return Safe32Array<T>(Safe32Ptr<T>(getOffset(data), &m_base), uint32_t(size)); - } - - /// Make a raw pointer into a safe ptr - template <typename T> - Safe32Ptr<T> toSafe(T* in) - { - Safe32Ptr<T> dst; - if (in) - { - dst.m_base = &m_base; - dst.m_offset = getOffset(in); - } - return dst; - } - - /// Allocate without alignment (effectively 1) - void* allocate(size_t size); - void* allocate(size_t size, size_t alignment); - void* allocateAndZero(size_t size, size_t alignment); - - void fixAlignment(size_t alignment); - - SLANG_FORCE_INLINE uint32_t getOffset(const void* ptr) const - { - ptrdiff_t offset = ((const uint8_t*)ptr) - m_base.m_data; - SLANG_ASSERT(offset >= 0 && size_t(offset) < m_current); - return uint32_t(offset); - } - - Safe32Ptr<RelativeString> newString(const UnownedStringSlice& slice); - Safe32Ptr<RelativeString> newString(const char* contents); - - /// Get the contained data - uint8_t* getData() { return m_base.m_data; } - /// Return the last used byte of the data - size_t getDataCount() const { return m_current; } - - /// Set the contents - void set(void* data, size_t size); - - RelativeBase* getBase() { return &m_base; } - - /// Ctor - RelativeContainer(); - ~RelativeContainer(); - - -protected: - size_t m_current; - size_t m_capacity; - RelativeBase m_base; -}; - - -} // namespace Slang - -#endif diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp index 3e2bec07d..c76993ca4 100644 --- a/source/slang/slang-options.cpp +++ b/source/slang/slang-options.cpp @@ -514,6 +514,8 @@ struct OptionsParser SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState(reproName, buffer)); auto requestState = StateSerializeUtil::getRequest(buffer); + MemoryOffsetBase base; + base.set(buffer.getBuffer(), buffer.getCount()); // If we can find a directory, that exists, we will set up a file system to load from that directory ComPtr<ISlangFileSystem> fileSystem; @@ -527,7 +529,7 @@ struct OptionsParser } } - SLANG_RETURN_ON_FAIL(StateSerializeUtil::load(requestState, fileSystem, requestImpl)); + SLANG_RETURN_ON_FAIL(StateSerializeUtil::load(base, requestState, fileSystem, requestImpl)); if (argCursor < argEnd) { diff --git a/source/slang/slang-state-serialize.cpp b/source/slang/slang-state-serialize.cpp index f7e472643..2d40e78ef 100644 --- a/source/slang/slang-state-serialize.cpp +++ b/source/slang/slang-state-serialize.cpp @@ -12,41 +12,49 @@ namespace Slang { /* static */const RiffSemanticVersion StateSerializeUtil::g_semanticVersion = RiffSemanticVersion::make(StateSerializeUtil::kMajorVersion, StateSerializeUtil::kMinorVersion, StateSerializeUtil::kPatchVersion); +// We can't just use sizeof for the sizes of these types, because the hash will be dependent on the ptr size, +// which isn't an issue for serialization (we turn all pointers into Offset32Ptr -> uint32_t). So we use an x macro +// to set up the thing to hash. +// +// Note that bool is in the list because size of bool can change between compilers. +#define SLANG_STATE_TYPES(x) \ + x(Util::FileState) \ + x(Util::PathInfoState) \ + x(Util::PathInfoState::CompressedResult) \ + x(SlangPathType) \ + x(Util::PathAndPathInfo) \ + x(Util::TargetRequestState) \ + x(Profile) \ + x(CodeGenTarget) \ + x(SlangTargetFlags) \ + x(FloatingPointMode) \ + x(Util::StringPair) \ + x(Util::SourceFileState) \ + x(PathInfo::Type) \ + x(Util::TranslationUnitRequestState) \ + x(SourceLanguage) \ + x(Util::EntryPointState) \ + x(Profile) \ + x(Util::RequestState) \ + x(SlangCompileFlags) \ + x(bool) \ + x(LineDirectiveMode) \ + x(DebugInfoLevel) \ + x(OptimizationLevel) \ + x(ContainerFormat) \ + x(PassThroughMode) \ + x(SlangMatrixLayoutMode) \ + +#define SLANG_STATE_TYPE_SIZE(x) uint32_t(sizeof(x)), + // A function to calculate the hash related in list in part to how the types used are sized. Can catch crude breaking binary differences. static uint32_t _calcTypeHash() { typedef StateSerializeUtil Util; - - const size_t sizes[] = + const uint32_t sizes[] = { - sizeof(Util::FileState), - sizeof(Util::PathInfoState), - sizeof(Util::PathInfoState::CompressedResult), - sizeof(SlangPathType), - sizeof(Util::PathAndPathInfo), - sizeof(Util::TargetRequestState), - sizeof(Profile), - sizeof(CodeGenTarget), - sizeof(SlangTargetFlags), - sizeof(FloatingPointMode), - sizeof(Util::StringPair), - sizeof(Util::SourceFileState), - sizeof(PathInfo::Type), - sizeof(Util::TranslationUnitRequestState), - sizeof(SourceLanguage), - sizeof(Util::EntryPointState), - sizeof(Profile), - sizeof(Util::RequestState), - sizeof(SlangCompileFlags), - sizeof(bool), //< Unfortunately bools size can change across compilers/versions - sizeof(LineDirectiveMode), - sizeof(DebugInfoLevel), - sizeof(OptimizationLevel), - sizeof(ContainerFormat), - sizeof(PassThroughMode), - sizeof(SlangMatrixLayoutMode), + SLANG_STATE_TYPES(SLANG_STATE_TYPE_SIZE) }; - return uint32_t(GetHashCode((const char*)&sizes, sizeof(sizes))); } @@ -65,54 +73,71 @@ struct StoreContext typedef StateSerializeUtil::SourceFileState SourceFileState; typedef StateSerializeUtil::PathInfoState PathInfoState; - StoreContext(RelativeContainer* container) + StoreContext(OffsetContainer* container) { m_container = container; } - Safe32Ptr<FileState> findFile(const String& uniqueIdentity) + Offset32Ptr<FileState> findFile(const String& uniqueIdentity) { - Safe32Ptr<FileState> file; + Offset32Ptr<FileState> file; m_uniqueToFileMap.TryGetValue(uniqueIdentity, file); return file; } - Safe32Ptr<FileState> addFile(const String& uniqueIdentity, const UnownedStringSlice* content) + Offset32Ptr<FileState> addFile(const String& uniqueIdentity, const UnownedStringSlice* content) { - Safe32Ptr<FileState> file; + OffsetBase& base = m_container->asBase(); + Offset32Ptr<FileState> file; + // Get the file, if it has an identity - if (uniqueIdentity.getLength() && m_uniqueToFileMap.TryGetValue(uniqueIdentity, file)) + if (uniqueIdentity.getLength()) { - return file; + if (!m_uniqueToFileMap.TryGetValue(uniqueIdentity, file)) + { + // If file was not found create it + // Create the file + file = m_container->newObject<FileState>(); + // Add it + m_uniqueToFileMap.Add(uniqueIdentity, file); + + // Set the identity + auto offsetUniqueIdentity = m_container->newString(uniqueIdentity.getUnownedSlice()); + base[file]->uniqueIdentity = offsetUniqueIdentity; + + // Add the file + m_files.add(file); + } } - - // If file was not found create it - // Create the file - file = m_container->newObject<FileState>(); - - if (content) + else { - file->contents = m_container->newString(*content); + // Create a file, but we know it can't have unique identity + file = m_container->newObject<FileState>(); + // Add the file + m_files.add(file); } - if (uniqueIdentity.getLength()) + + // If the contents is not set add it + if (!base[file]->contents && content) { - file->uniqueIdentity = m_container->newString(uniqueIdentity.getUnownedSlice()); - m_uniqueToFileMap.Add(uniqueIdentity, file); + auto offsetContent = m_container->newString(*content); + base[file]->contents = offsetContent; } - m_files.add(file); return file; } - Safe32Ptr<SourceFileState> addSourceFile(SourceFile* sourceFile) + Offset32Ptr<SourceFileState> addSourceFile(SourceFile* sourceFile) { if (!sourceFile) { - return Safe32Ptr<SourceFileState>(); + return Offset32Ptr<SourceFileState>(); } - Safe32Ptr<StateSerializeUtil::SourceFileState> sourceFileState; + auto& base = m_container->asBase(); + + Offset32Ptr<StateSerializeUtil::SourceFileState> sourceFileState; if (m_sourceFileMap.TryGetValue(sourceFile, sourceFileState)) { return sourceFileState; @@ -121,32 +146,35 @@ struct StoreContext const PathInfo& pathInfo = sourceFile->getPathInfo(); UnownedStringSlice content = sourceFile->getContent(); - Safe32Ptr<FileState> file = addFile(pathInfo.uniqueIdentity, &content); + Offset32Ptr<FileState> file = addFile(pathInfo.uniqueIdentity, &content); - Safe32Ptr<RelativeString> foundPath; + Offset32Ptr<OffsetString> foundPath; - if (pathInfo.foundPath.getLength() && file->foundPath == nullptr) + if (pathInfo.foundPath.getLength() && base[file]->foundPath.isNull()) { foundPath = fromString(pathInfo.foundPath.getUnownedSlice()); } // Set on the file - file->foundPath = foundPath; + base[file]->foundPath = foundPath; // Create the source file sourceFileState = m_container->newObject<SourceFileState>(); - sourceFileState->file = file; - sourceFileState->foundPath = foundPath; - sourceFileState->type = pathInfo.type; + { + auto dst = base[sourceFileState]; + dst->file = file; + dst->foundPath = foundPath; + dst->type = pathInfo.type; + } m_sourceFileMap.Add(sourceFile, sourceFileState); return sourceFileState; } - Safe32Ptr<RelativeString> fromString(const String& in) + Offset32Ptr<OffsetString> fromString(const String& in) { - Safe32Ptr<RelativeString> value; + Offset32Ptr<OffsetString> value; if (m_stringMap.TryGetValue(in, value)) { @@ -156,39 +184,43 @@ struct StoreContext m_stringMap.Add(in, value); return value; } - Safe32Ptr<RelativeString> fromName(Name* name) + Offset32Ptr<OffsetString> fromName(Name* name) { if (name) { return fromString(name->text); } - return Safe32Ptr<RelativeString>(); + return Offset32Ptr<OffsetString>(); } - Safe32Ptr<PathInfoState> addPathInfo(const CacheFileSystem::PathInfo* srcPathInfo) + Offset32Ptr<PathInfoState> addPathInfo(const CacheFileSystem::PathInfo* srcPathInfo) { if (!srcPathInfo) { - return Safe32Ptr<PathInfoState>(); + return Offset32Ptr<PathInfoState>(); } - Safe32Ptr<PathInfoState> pathInfo; + OffsetBase& base = m_container->asBase(); + + Offset32Ptr<PathInfoState> pathInfo; if (!m_pathInfoMap.TryGetValue(srcPathInfo, pathInfo)) { // Get the associated file - Safe32Ptr<FileState> fileState; + Offset32Ptr<FileState> fileState; // Only store as file if we have the contents - if(srcPathInfo->m_fileBlob) + if(ISlangBlob* fileBlob = srcPathInfo->m_fileBlob) { - fileState = addFile(srcPathInfo->getUniqueIdentity(), nullptr); + UnownedStringSlice content((const char*)fileBlob->getBufferPointer(), fileBlob->getBufferSize()); + + fileState = addFile(srcPathInfo->getUniqueIdentity(), &content); } // Save the rest of the state pathInfo = m_container->newObject<PathInfoState>(); - PathInfoState& dst = *pathInfo; + PathInfoState& dst = base[*pathInfo]; - pathInfo->file = fileState; + dst.file = fileState; // Save any other info dst.getCanonicalPathResult = srcPathInfo->m_getCanonicalPathResult; @@ -200,36 +232,41 @@ struct StoreContext } // Fill in info on the file - Safe32Ptr<FileState> fileState(m_container->toSafe(pathInfo->file.get())); + auto fileState(base[pathInfo]->file); // If have fileState add any missing element if (fileState) { - if (srcPathInfo->m_fileBlob && fileState->contents == nullptr) + if (srcPathInfo->m_fileBlob && base[fileState]->contents.isNull()) { UnownedStringSlice contents((const char*)srcPathInfo->m_fileBlob->getBufferPointer(), srcPathInfo->m_fileBlob->getBufferSize()); - fileState->contents = m_container->newString(contents); + auto offsetContents = m_container->newString(contents); + base[fileState]->contents = offsetContents; } - if (srcPathInfo->m_canonicalPath && fileState->canonicalPath == nullptr) + if (srcPathInfo->m_canonicalPath && base[fileState]->canonicalPath.isNull()) { - fileState->canonicalPath = fromString(srcPathInfo->m_canonicalPath->getString()); + auto offsetCanonicalPath = fromString(srcPathInfo->m_canonicalPath->getString()); + base[fileState]->canonicalPath = offsetCanonicalPath; } - if (srcPathInfo->m_uniqueIdentity && fileState->uniqueIdentity == nullptr) + if (srcPathInfo->m_uniqueIdentity && base[fileState]->uniqueIdentity.isNull()) { - fileState->uniqueIdentity = fromString(srcPathInfo->m_uniqueIdentity->getString()); + auto offsetUniqueIdentity = fromString(srcPathInfo->m_uniqueIdentity->getString()); + base[fileState]->uniqueIdentity = offsetUniqueIdentity; } } return pathInfo; } - const Safe32Array<StateSerializeUtil::StringPair> calcDefines(const Dictionary<String, String>& srcDefines) + const Offset32Array<StateSerializeUtil::StringPair> calcDefines(const Dictionary<String, String>& srcDefines) { typedef StateSerializeUtil::StringPair StringPair; - Safe32Array<StringPair> dstDefines = m_container->newArray<StringPair>(srcDefines.Count()); + Offset32Array<StringPair> dstDefines = m_container->newArray<StringPair>(srcDefines.Count()); + + OffsetBase& base = m_container->asBase(); Index index = 0; for (const auto& srcDefine : srcDefines) @@ -238,7 +275,7 @@ struct StoreContext auto key = fromString(srcDefine.Key); auto value = fromString(srcDefine.Value); - auto& dstDefine = dstDefines[index]; + auto& dstDefine = base[dstDefines[index]]; dstDefine.first = key; dstDefine.second = value; @@ -248,27 +285,29 @@ struct StoreContext return dstDefines; } - const Safe32Array<Relative32Ptr<RelativeString>> fromList(const List<String>& src) + const Offset32Array<Offset32Ptr<OffsetString>> fromList(const List<String>& src) { - Safe32Array<Relative32Ptr<RelativeString>> dst = m_container->newArray<Relative32Ptr<RelativeString>>(src.getCount()); + Offset32Array<Offset32Ptr<OffsetString>> dst = m_container->newArray<Offset32Ptr<OffsetString>>(src.getCount()); + OffsetBase& base = m_container->asBase(); + for (Index j = 0; j < src.getCount(); ++j) { - dst[j] = fromString(src[j]); + base[dst[j]] = fromString(src[j]); } return dst; } - Dictionary<String, Safe32Ptr<RelativeString> > m_stringMap; + Dictionary<String, Offset32Ptr<OffsetString> > m_stringMap; - Dictionary<SourceFile*, Safe32Ptr<StateSerializeUtil::SourceFileState> > m_sourceFileMap; + Dictionary<SourceFile*, Offset32Ptr<StateSerializeUtil::SourceFileState> > m_sourceFileMap; - Dictionary<String, Safe32Ptr<StateSerializeUtil::FileState> > m_uniqueToFileMap; + Dictionary<String, Offset32Ptr<StateSerializeUtil::FileState> > m_uniqueToFileMap; - Dictionary<const CacheFileSystem::PathInfo*, Safe32Ptr<PathInfoState> > m_pathInfoMap; + Dictionary<const CacheFileSystem::PathInfo*, Offset32Ptr<PathInfoState> > m_pathInfoMap; - List<Safe32Ptr<StateSerializeUtil::FileState> > m_files; + List<Offset32Ptr<StateSerializeUtil::FileState> > m_files; - RelativeContainer* m_container; + OffsetContainer* m_container; }; } // @@ -288,16 +327,18 @@ static bool _isStorable(const PathInfo::Type type) } } -/* static */SlangResult StateSerializeUtil::store(EndToEndCompileRequest* request, RelativeContainer& inOutContainer, Safe32Ptr<RequestState>& outRequest) +/* static */SlangResult StateSerializeUtil::store(EndToEndCompileRequest* request, OffsetContainer& inOutContainer, Offset32Ptr<RequestState>& outRequest) { StoreContext context(&inOutContainer); + OffsetBase& base = inOutContainer.asBase(); + auto linkage = request->getLinkage(); - Safe32Ptr<RequestState> requestState = inOutContainer.newObject<RequestState>(); + Offset32Ptr<RequestState> requestState = inOutContainer.newObject<RequestState>(); { - RequestState* dst = requestState; + RequestState* dst = base[requestState]; dst->compileFlags = request->getFrontEndReq()->compileFlags; dst->shouldDumpIntermediates = request->getBackEndReq()->shouldDumpIntermediates; @@ -322,7 +363,7 @@ static bool _isStorable(const PathInfo::Type type) SLANG_ASSERT(srcEntryPoints.getCount() == srcEndToEndEntryPoints.getCount()); - Safe32Array<EntryPointState> dstEntryPoints = inOutContainer.newArray<EntryPointState>(srcEntryPoints.getCount()); + Offset32Array<EntryPointState> dstEntryPoints = inOutContainer.newArray<EntryPointState>(srcEntryPoints.getCount()); for (Index i = 0; i < srcEntryPoints.getCount(); ++i) { @@ -330,9 +371,9 @@ static bool _isStorable(const PathInfo::Type type) const auto& srcEndToEndEntryPoint = srcEndToEndEntryPoints[i]; auto dstSpecializationArgStrings = context.fromList(srcEndToEndEntryPoint.specializationArgStrings); - Safe32Ptr<RelativeString> dstName = context.fromName(srcEntryPoint->getName()); + Offset32Ptr<OffsetString> dstName = context.fromName(srcEntryPoint->getName()); - EntryPointState& dst = dstEntryPoints[i]; + EntryPointState& dst = base[dstEntryPoints[i]]; dst.profile = srcEntryPoint->getProfile(); dst.translationUnitIndex = uint32_t(srcEntryPoint->getTranslationUnitIndex()); @@ -340,7 +381,7 @@ static bool _isStorable(const PathInfo::Type type) dst.name = dstName; } - requestState->entryPoints = dstEntryPoints; + base[requestState]->entryPoints = dstEntryPoints; } @@ -361,7 +402,7 @@ static bool _isStorable(const PathInfo::Type type) // Add all the target requests { - Safe32Array<TargetRequestState> dstTargets = inOutContainer.newArray<TargetRequestState>(linkage->targets.getCount()); + Offset32Array<TargetRequestState> dstTargets = inOutContainer.newArray<TargetRequestState>(linkage->targets.getCount()); for (Index i = 0; i < linkage->targets.getCount(); ++i) { @@ -369,7 +410,7 @@ static bool _isStorable(const PathInfo::Type type) // Copy the simple stuff { - auto& dst = dstTargets[i]; + auto& dst = base[dstTargets[i]]; dst.target = srcTargetRequest->getTarget(); dst.profile = srcTargetRequest->getTargetProfile(); dst.targetFlags = srcTargetRequest->targetFlags; @@ -386,14 +427,14 @@ static bool _isStorable(const PathInfo::Type type) const auto& entryPointOutputPaths = infos->entryPointOutputPaths; - Safe32Array<OutputState> dstOutputStates = inOutContainer.newArray<OutputState>(entryPointOutputPaths.Count()); + Offset32Array<OutputState> dstOutputStates = inOutContainer.newArray<OutputState>(entryPointOutputPaths.Count()); Index index = 0; for (const auto& pair : entryPointOutputPaths) { - Safe32Ptr<RelativeString> outputPath = inOutContainer.newString(pair.Value.getUnownedSlice()); + Offset32Ptr<OffsetString> outputPath = inOutContainer.newString(pair.Value.getUnownedSlice()); - auto& dstOutputState = dstOutputStates[index]; + auto& dstOutputState = base[dstOutputStates[index]]; dstOutputState.entryPointIndex = int32_t(pair.Key); dstOutputState.outputPath = outputPath; @@ -401,35 +442,35 @@ static bool _isStorable(const PathInfo::Type type) index++; } - dstTargets[i].outputStates = dstOutputStates; + base[dstTargets[i]].outputStates = dstOutputStates; } } } // Save the result - requestState->targetRequests = dstTargets; + base[requestState]->targetRequests = dstTargets; } // Add the search paths { const auto& srcPaths = linkage->searchDirectories.searchDirectories; - Safe32Array<Relative32Ptr<RelativeString> > dstPaths = inOutContainer.newArray<Relative32Ptr<RelativeString> >(srcPaths.getCount()); + Offset32Array<Offset32Ptr<OffsetString> > dstPaths = inOutContainer.newArray<Offset32Ptr<OffsetString> >(srcPaths.getCount()); // We don't handle parents here SLANG_ASSERT(linkage->searchDirectories.parent == nullptr); for (Index i = 0; i < srcPaths.getCount(); ++i) { - dstPaths[i] = context.fromString(srcPaths[i].path); + base[dstPaths[i]] = context.fromString(srcPaths[i].path); } - requestState->searchPaths = dstPaths; + base[requestState]->searchPaths = dstPaths; } // Add preprocessor definitions - requestState->preprocessorDefinitions = context.calcDefines(linkage->preprocessorDefinitions); + base[requestState]->preprocessorDefinitions = context.calcDefines(linkage->preprocessorDefinitions); { const auto& srcTranslationUnits = request->getFrontEndReq()->translationUnits; - Safe32Array<TranslationUnitRequestState> dstTranslationUnits = inOutContainer.newArray<TranslationUnitRequestState>(srcTranslationUnits.getCount()); + Offset32Array<TranslationUnitRequestState> dstTranslationUnits = inOutContainer.newArray<TranslationUnitRequestState>(srcTranslationUnits.getCount()); for (Index i = 0; i < srcTranslationUnits.getCount(); ++i) { @@ -439,18 +480,18 @@ static bool _isStorable(const PathInfo::Type type) auto defines = context.calcDefines(srcTranslationUnit->preprocessorDefinitions); auto moduleName = context.fromName(srcTranslationUnit->moduleName); - Safe32Array<Relative32Ptr<SourceFileState>> dstSourceFiles; + Offset32Array<Offset32Ptr<SourceFileState>> dstSourceFiles; { const auto& srcFiles = srcTranslationUnit->getSourceFiles(); - dstSourceFiles = inOutContainer.newArray<Relative32Ptr<SourceFileState> >(srcFiles.getCount()); + dstSourceFiles = inOutContainer.newArray<Offset32Ptr<SourceFileState> >(srcFiles.getCount()); for (Index j = 0; j < srcFiles.getCount(); ++j) { - dstSourceFiles[j] = context.addSourceFile(srcFiles[j]); + base[dstSourceFiles[j]] = context.addSourceFile(srcFiles[j]); } } - TranslationUnitRequestState& dstTranslationUnit = dstTranslationUnits[i]; + TranslationUnitRequestState& dstTranslationUnit = base[dstTranslationUnits[i]]; dstTranslationUnit.language = srcTranslationUnit->sourceLanguage; dstTranslationUnit.moduleName = moduleName; @@ -458,7 +499,7 @@ static bool _isStorable(const PathInfo::Type type) dstTranslationUnit.preprocessorDefinitions = defines; } - requestState->translationUnits = dstTranslationUnits; + base[requestState]->translationUnits = dstTranslationUnits; } // Find files from the file system, and mapping paths to files @@ -473,22 +514,22 @@ static bool _isStorable(const PathInfo::Type type) { const auto& srcFiles = cacheFileSystem->getPathMap(); - Safe32Array<PathAndPathInfo> pathMap = inOutContainer.newArray<PathAndPathInfo>(srcFiles.Count()); + Offset32Array<PathAndPathInfo> pathMap = inOutContainer.newArray<PathAndPathInfo>(srcFiles.Count()); Index index = 0; for (const auto& pair : srcFiles) { - Safe32Ptr<RelativeString> path = context.fromString(pair.Key); - Safe32Ptr<PathInfoState> pathInfo = context.addPathInfo(pair.Value); + Offset32Ptr<OffsetString> path = context.fromString(pair.Key); + Offset32Ptr<PathInfoState> pathInfo = context.addPathInfo(pair.Value); - PathAndPathInfo& dstInfo = pathMap[index]; + PathAndPathInfo& dstInfo = base[pathMap[index]]; dstInfo.path = path; dstInfo.pathInfo = pathInfo; index++; } - requestState->pathInfoMap = pathMap; + base[requestState]->pathInfoMap = pathMap; } } @@ -496,25 +537,25 @@ static bool _isStorable(const PathInfo::Type type) { Dictionary<String, int> uniqueNameMap; - auto files = inOutContainer.newArray<Relative32Ptr<FileState>>(context.m_files.getCount()); + auto files = inOutContainer.newArray<Offset32Ptr<FileState>>(context.m_files.getCount()); for (Index i = 0; i < context.m_files.getCount(); ++i) { - Safe32Ptr<FileState> file = context.m_files[i]; + Offset32Ptr<FileState> file = context.m_files[i]; // Need to come up with unique names String path; - if (file->canonicalPath) + if (auto canonicalPath = base[file]->canonicalPath) { - path = file->canonicalPath->getSlice(); + path = base[canonicalPath]->getSlice(); } - else if (file->foundPath) + else if (auto foundPath = base[file]->foundPath) { - path = file->foundPath->getSlice(); + path = base[foundPath]->getSlice(); } - else if (file->uniqueIdentity) + else if (auto uniqueIdentity = base[file]->uniqueIdentity) { - path = file->uniqueIdentity->getSlice(); + path = base[uniqueIdentity]->getSlice(); } if (path.getLength() == 0) @@ -552,26 +593,26 @@ static bool _isStorable(const PathInfo::Type type) } // Save the unique generated name - file->uniqueName = inOutContainer.newString(uniqueName.getUnownedSlice()); + base[file]->uniqueName = inOutContainer.newString(uniqueName.getUnownedSlice()); - files[i] = file; + base[files[i]] = file; } - requestState->files = files; + base[requestState]->files = files; } // Save all the SourceFile state { const auto& srcSourceFiles = context.m_sourceFileMap; - auto dstSourceFiles = inOutContainer.newArray<Relative32Ptr<SourceFileState>>(srcSourceFiles.Count()); + auto dstSourceFiles = inOutContainer.newArray<Offset32Ptr<SourceFileState>>(srcSourceFiles.Count()); Index index = 0; for (const auto& pair : srcSourceFiles) { - dstSourceFiles[index] = pair.Value; + base[dstSourceFiles[index]] = pair.Value; index++; } - requestState->sourceFiles = dstSourceFiles; + base[requestState]->sourceFiles = dstSourceFiles; } outRequest = requestState; @@ -599,13 +640,13 @@ struct LoadContext if (m_fileSystem && file->uniqueName) { // Try loading from the file system - m_fileSystem->loadFile(file->uniqueName->getCstr(), blob.writeRef()); + m_fileSystem->loadFile(m_base->asRaw(file->uniqueName)->getCstr(), blob.writeRef()); } // If wasn't loaded, and has contents, use that if (!blob && file->contents) { - blob = new StringBlob(file->contents->getSlice()); + blob = new StringBlob(m_base->asRaw(file->contents)->getSlice()); } // Add to map, even if the blob is nullptr (say from a failed read) @@ -625,7 +666,7 @@ struct LoadContext SourceFile* dstFile; if (!m_sourceFileMap.TryGetValue(sourceFile, dstFile)) { - FileState* file = sourceFile->file; + FileState* file = m_base->asRaw(sourceFile->file); ISlangBlob* blob = getFileBlob(file); PathInfo pathInfo; @@ -634,16 +675,16 @@ struct LoadContext if (sourceFile->foundPath) { - pathInfo.foundPath = sourceFile->foundPath->getSlice(); + pathInfo.foundPath = m_base->asRaw(sourceFile->foundPath)->getSlice(); } else if (file->foundPath) { - pathInfo.foundPath = file->foundPath->getSlice(); + pathInfo.foundPath = m_base->asRaw(file->foundPath)->getSlice(); } if (file->uniqueIdentity) { - pathInfo.uniqueIdentity = file->uniqueIdentity->getSlice(); + pathInfo.uniqueIdentity = m_base->asRaw(file->uniqueIdentity)->getSlice(); } dstFile = new SourceFile(m_sourceManager, pathInfo, blob->getBufferSize()); @@ -667,18 +708,18 @@ struct LoadContext } CacheFileSystem::PathInfo* dstInfo = new CacheFileSystem::PathInfo(String()); - FileState* file = srcInfo->file; + FileState* file = m_base->asRaw(srcInfo->file); if (file) { if (file->uniqueIdentity) { - String uniqueIdentity = file->uniqueIdentity->getSlice(); + String uniqueIdentity = m_base->asRaw(file->uniqueIdentity)->getSlice(); dstInfo->m_uniqueIdentity = new StringBlob(uniqueIdentity); } if (file->canonicalPath) { - dstInfo->m_canonicalPath = new StringBlob(file->canonicalPath->getSlice()); + dstInfo->m_canonicalPath = new StringBlob(m_base->asRaw(file->canonicalPath)->getSlice()); } dstInfo->m_fileBlob = getFileBlob(file); @@ -693,26 +734,40 @@ struct LoadContext return dstInfo; } - static List<const char*> toList(const Relative32Array<Relative32Ptr<RelativeString>>& src) + List<const char*> toList(const Offset32Array<Offset32Ptr<OffsetString>>& src) { List<const char*> dst; dst.setCount(src.getCount()); for (Index i = 0; i < src.getCount(); ++i) { - RelativeString* srcString = src[i]; + OffsetString* srcString = m_base->asRaw(m_base->asRaw(src[i])); dst[i] = srcString ? srcString->getCstr() : nullptr; } return dst; } - LoadContext(SourceManager* sourceManger, ISlangFileSystem* fileSystem): + + void loadDefines(const Offset32Array<StateSerializeUtil::StringPair>& in, Dictionary<String, String>& out) + { + out.Clear(); + + for (const auto& define : in) + { + out.Add(m_base->asRaw(m_base->asRaw(define).first)->getSlice(), m_base->asRaw(m_base->asRaw(define).second)->getSlice()); + } + } + + LoadContext(SourceManager* sourceManger, ISlangFileSystem* fileSystem, OffsetBase* base): m_sourceManager(sourceManger), - m_fileSystem(fileSystem) + m_fileSystem(fileSystem), + m_base(base) { } ISlangFileSystem* m_fileSystem; + OffsetBase* m_base; + SourceManager* m_sourceManager; Dictionary<SourceFileState*, SourceFile*> m_sourceFileMap; Dictionary<FileState*, ComPtr<ISlangBlob> > m_fileToBlobMap; @@ -721,22 +776,12 @@ struct LoadContext } // anonymous -static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& in, Dictionary<String, String>& out) -{ - out.Clear(); - - for (const auto& define : in) - { - out.Add(define.first->getSlice(), define.second->getSlice()); - } -} -/* static */SlangResult StateSerializeUtil::load(RequestState* requestState, ISlangFileSystem* fileSystem, EndToEndCompileRequest* request) +/* static */SlangResult StateSerializeUtil::load(OffsetBase& base, RequestState* requestState, ISlangFileSystem* fileSystem, EndToEndCompileRequest* request) { auto externalRequest = asExternal(request); - auto linkage = request->getLinkage(); // TODO(JS): Really should be more exhaustive here, and set up to initial state ideally @@ -747,7 +792,7 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& linkage->targets.clear(); } - LoadContext context(linkage->getSourceManager(), fileSystem); + LoadContext context(linkage->getSourceManager(), fileSystem, &base); // Try to set state through API - as doing so means if state stored in multiple places it will be ok @@ -772,7 +817,7 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& { for (Index i = 0; i < requestState->targetRequests.getCount(); ++i) { - TargetRequestState& src = requestState->targetRequests[i]; + TargetRequestState& src = base.asRaw(requestState->targetRequests[i]); int index = spAddCodeGenTarget(externalRequest, SlangCompileTarget(src.target)); SLANG_ASSERT(index == i); @@ -789,14 +834,16 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& RefPtr<EndToEndCompileRequest::TargetInfo> dstTargetInfo(new EndToEndCompileRequest::TargetInfo); request->targetInfos[dstTarget] = dstTargetInfo; - for (const auto& srcOutputState : src.outputStates) + for (const auto& srcOutputStateOffset : src.outputStates) { + const auto& srcOutputState = base.asRaw(srcOutputStateOffset); + SLANG_ASSERT(srcOutputState.entryPointIndex < requestState->entryPoints.getCount()); String entryPointPath; if (srcOutputState.outputPath) { - entryPointPath = srcOutputState.outputPath->getSlice(); + entryPointPath = base.asRaw(srcOutputState.outputPath)->getSlice(); } dstTargetInfo->entryPointOutputPaths.Add(srcOutputState.entryPointIndex, entryPointPath); @@ -811,11 +858,11 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& dstPaths.setCount(srcPaths.getCount()); for (Index i = 0; i < srcPaths.getCount(); ++i) { - dstPaths[i].path = srcPaths[i]->getSlice(); + dstPaths[i].path = base.asRaw(base.asRaw(srcPaths[i]))->getSlice(); } } - _loadDefines(requestState->preprocessorDefinitions, linkage->preprocessorDefinitions); + context.loadDefines(requestState->preprocessorDefinitions, linkage->preprocessorDefinitions); { auto frontEndReq = request->getFrontEndReq(); @@ -827,7 +874,7 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& for (Index i = 0; i < srcTranslationUnits.getCount(); ++i) { - const auto& srcTranslationUnit = srcTranslationUnits[i]; + const auto& srcTranslationUnit = base.asRaw(srcTranslationUnits[i]); int index = frontEndReq->addTranslationUnit(srcTranslationUnit.language); SLANG_UNUSED(index); @@ -835,12 +882,12 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& TranslationUnitRequest* dstTranslationUnit = dstTranslationUnits[i]; - _loadDefines(srcTranslationUnit.preprocessorDefinitions, dstTranslationUnit->preprocessorDefinitions); + context.loadDefines(srcTranslationUnit.preprocessorDefinitions, dstTranslationUnit->preprocessorDefinitions); Name* moduleName = nullptr; if (srcTranslationUnit.moduleName) { - moduleName = request->getNamePool()->getName(srcTranslationUnit.moduleName->getSlice()); + moduleName = request->getNamePool()->getName(base.asRaw(srcTranslationUnit.moduleName)->getSlice()); } dstTranslationUnit->moduleName = moduleName; @@ -852,7 +899,7 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& for (Index j = 0; j < srcSourceFiles.getCount(); ++j) { - SourceFile* sourceFile = context.getSourceFile(srcSourceFiles[i]); + SourceFile* sourceFile = context.getSourceFile(base.asRaw(base.asRaw(srcSourceFiles[i]))); // Add to translation unit dstTranslationUnit->addSourceFile(sourceFile); } @@ -864,9 +911,11 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& // Check there aren't any set entry point SLANG_ASSERT(request->getFrontEndReq()->m_entryPointReqs.getCount() == 0); - for (const auto& srcEntryPoint : requestState->entryPoints) + for (const auto& srcEntryPointOffset : requestState->entryPoints) { - const char* name = srcEntryPoint.name ? srcEntryPoint.name->getCstr() : nullptr; + const auto srcEntryPoint = base.asRaw(srcEntryPointOffset); + + const char* name = srcEntryPoint.name ? base.asRaw(srcEntryPoint.name)->getCstr() : nullptr; Stage stage = srcEntryPoint.profile.GetStage(); @@ -883,10 +932,11 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& // Put all the paths to path info { - for (const auto& pair : requestState->pathInfoMap) + for (const auto& pairOffset : requestState->pathInfoMap) { - CacheFileSystem::PathInfo* pathInfo = context.addPathInfo(pair.pathInfo); - dstPathMap.Add(pair.path->getSlice(), pathInfo); + const auto& pair = base.asRaw(pairOffset); + CacheFileSystem::PathInfo* pathInfo = context.addPathInfo(base.asRaw(pair.pathInfo)); + dstPathMap.Add(base.asRaw(pair.path)->getSlice(), pathInfo); } } // Put all the path infos in the cache system @@ -912,8 +962,8 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& /* static */SlangResult StateSerializeUtil::saveState(EndToEndCompileRequest* request, Stream* stream) { - RelativeContainer container; - Safe32Ptr<RequestState> requestState; + OffsetContainer container; + Offset32Ptr<RequestState> requestState; SLANG_RETURN_ON_FAIL(store(request, container, requestState)); Header header; @@ -976,7 +1026,7 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& /* static */ StateSerializeUtil::RequestState* StateSerializeUtil::getRequest(const List<uint8_t>& buffer) { - return (StateSerializeUtil::RequestState*)buffer.getBuffer(); + return (StateSerializeUtil::RequestState*)(buffer.getBuffer() + kStartOffset); } /* static */SlangResult StateSerializeUtil::calcDirectoryPathFromFilename(const String& filename, String& outPath) @@ -1005,6 +1055,9 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& List<uint8_t> buffer; SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState(filename, buffer)); + MemoryOffsetBase base; + base.set(buffer.getBuffer(), buffer.getCount()); + RequestState* requestState = StateSerializeUtil::getRequest(buffer); String dirPath; @@ -1014,60 +1067,66 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& // Set up a file system to write into this directory RelativeFileSystem relFileSystem(OSFileSystemExt::getSingleton(), dirPath); - return extractFiles(requestState, &relFileSystem); + return extractFiles(base, requestState, &relFileSystem); } -/* static */SlangResult StateSerializeUtil::extractFiles(RequestState* requestState, ISlangFileSystemExt* fileSystem) +/* static */SlangResult StateSerializeUtil::extractFiles(OffsetBase& base, RequestState* requestState, ISlangFileSystemExt* fileSystem) { StringBuilder builder; builder << "[files]\n"; - for (FileState* file : requestState->files) + for (auto fileOffset : requestState->files) { + auto file = base.asRaw(base.asRaw(fileOffset)); + if (file->contents) { - UnownedStringSlice contents = file->contents->getSlice(); + UnownedStringSlice contents = base.asRaw(file->contents)->getSlice(); - SLANG_RETURN_ON_FAIL(fileSystem->saveFile(file->uniqueName->getCstr(), contents.begin(), contents.size())); + SLANG_RETURN_ON_FAIL(fileSystem->saveFile(base.asRaw(file->uniqueName)->getCstr(), contents.begin(), contents.size())); - RelativeString* originalName = nullptr; + OffsetString* originalName = nullptr; if (file->canonicalPath) { - originalName = file->canonicalPath; + originalName = base.asRaw(file->canonicalPath); } else if (file->foundPath) { - originalName = file->foundPath; + originalName = base.asRaw(file->foundPath); } else if (file->uniqueIdentity) { - originalName = file->uniqueIdentity; + originalName = base.asRaw(file->uniqueIdentity); } - builder << file->uniqueName->getSlice() << " -> "; + builder << base.asRaw(file->uniqueName)->getSlice() << " -> "; if (originalName) { builder << originalName->getSlice(); } - else + + if (builder.getLength() == 0) { builder << "?"; } + builder << "\n"; } } builder << "[paths]\n"; - for (const PathAndPathInfo& path : requestState->pathInfoMap) + for (const auto pathOffset : requestState->pathInfoMap) { - builder << path.path->getSlice() << " -> "; + const auto& path = base.asRaw(pathOffset); + + builder << base.asRaw(path.path)->getSlice() << " -> "; - const auto pathInfo = path.pathInfo.get(); + const auto pathInfo = base.asRaw(path.pathInfo); if (pathInfo->file) { - builder << pathInfo->file->uniqueName->getSlice(); + builder << base.asRaw(base.asRaw(pathInfo->file)->uniqueName)->getSlice(); } else { diff --git a/source/slang/slang-state-serialize.h b/source/slang/slang-state-serialize.h index 42edd4080..6de792097 100644 --- a/source/slang/slang-state-serialize.h +++ b/source/slang/slang-state-serialize.h @@ -8,7 +8,7 @@ // For TranslationUnitRequest #include "slang-compiler.h" -#include "../core/slang-relative-container.h" +#include "../core/slang-offset-container.h" #include "slang-file-system.h" @@ -35,12 +35,12 @@ struct StateSerializeUtil struct FileState { - Relative32Ptr<RelativeString> uniqueIdentity; ///< The unique identity for the file (from ISlangFileSystem), or nullptr - Relative32Ptr<RelativeString> contents; ///< The contents of this file - Relative32Ptr<RelativeString> canonicalPath; ///< The canonical name of this file (or nullptr) - Relative32Ptr<RelativeString> foundPath; ///< The 'found' path + Offset32Ptr<OffsetString> uniqueIdentity; ///< The unique identity for the file (from ISlangFileSystem), or nullptr + Offset32Ptr<OffsetString> contents; ///< The contents of this file + Offset32Ptr<OffsetString> canonicalPath; ///< The canonical name of this file (or nullptr) + Offset32Ptr<OffsetString> foundPath; ///< The 'found' path - Relative32Ptr<RelativeString> uniqueName; ///< A generated unique name (not used by slang, but used as mechanism to replace files) + Offset32Ptr<OffsetString> uniqueName; ///< A generated unique name (not used by slang, but used as mechanism to replace files) }; struct PathInfoState @@ -52,19 +52,19 @@ struct StateSerializeUtil CompressedResult getPathTypeResult = CompressedResult::Uninitialized; CompressedResult getCanonicalPathResult = CompressedResult::Uninitialized; - Relative32Ptr<FileState> file; ///< File contents + Offset32Ptr<FileState> file; ///< File contents }; struct PathAndPathInfo { - Relative32Ptr<RelativeString> path; - Relative32Ptr<PathInfoState> pathInfo; + Offset32Ptr<OffsetString> path; + Offset32Ptr<PathInfoState> pathInfo; }; struct OutputState { int32_t entryPointIndex; - Relative32Ptr<RelativeString> outputPath; + Offset32Ptr<OffsetString> outputPath; }; // spSetCodeGenTarget/spAddCodeGenTarget @@ -79,20 +79,20 @@ struct StateSerializeUtil SlangTargetFlags targetFlags; FloatingPointMode floatingPointMode; - Relative32Array<OutputState> outputStates; + Offset32Array<OutputState> outputStates; }; struct StringPair { - Relative32Ptr<RelativeString> first; - Relative32Ptr<RelativeString> second; + Offset32Ptr<OffsetString> first; + Offset32Ptr<OffsetString> second; }; struct SourceFileState { PathInfo::Type type; ///< The type of this file - Relative32Ptr<RelativeString> foundPath; ///< The Path this was found along - Relative32Ptr<FileState> file; ///< The file contents + Offset32Ptr<OffsetString> foundPath; ///< The Path this was found along + Offset32Ptr<FileState> file; ///< The file contents }; // spAddTranslationUnit @@ -100,26 +100,26 @@ struct StateSerializeUtil { SourceLanguage language; - Relative32Ptr<RelativeString> moduleName; + Offset32Ptr<OffsetString> moduleName; // spTranslationUnit_addPreprocessorDefine - Relative32Array<StringPair> preprocessorDefinitions; + Offset32Array<StringPair> preprocessorDefinitions; - Relative32Array<Relative32Ptr<SourceFileState> > sourceFiles; + Offset32Array<Offset32Ptr<SourceFileState> > sourceFiles; }; struct EntryPointState { - Relative32Ptr<RelativeString> name; + Offset32Ptr<OffsetString> name; Profile profile; uint32_t translationUnitIndex; - Relative32Array<Relative32Ptr<RelativeString>> specializationArgStrings; + Offset32Array<Offset32Ptr<OffsetString>> specializationArgStrings; }; struct RequestState { - Relative32Array<Relative32Ptr<FileState>> files; ///< All of the files - Relative32Array<Relative32Ptr<SourceFileState>> sourceFiles; ///< All of the source files (from source manager) + Offset32Array<Offset32Ptr<FileState>> files; ///< All of the files + Offset32Array<Offset32Ptr<SourceFileState>> sourceFiles; ///< All of the source files (from source manager) // spSetCompileFlags SlangCompileFlags compileFlags; @@ -128,7 +128,7 @@ struct StateSerializeUtil // spSetLineDirectiveMode LineDirectiveMode lineDirectiveMode; - Relative32Array<TargetRequestState> targetRequests; + Offset32Array<TargetRequestState> targetRequests; // spSetDebugInfoLevel DebugInfoLevel debugInfoLevel; @@ -140,24 +140,24 @@ struct StateSerializeUtil PassThroughMode passThroughMode; // spAddSearchPath - Relative32Array<Relative32Ptr<RelativeString> > searchPaths; + Offset32Array<Offset32Ptr<OffsetString> > searchPaths; // spAddPreprocessorDefine - Relative32Array<StringPair> preprocessorDefinitions; + Offset32Array<StringPair> preprocessorDefinitions; bool useUnknownImageFormatAsDefault = false; bool obfuscateCode = false; - Relative32Array<PathAndPathInfo> pathInfoMap; ///< Stores all the accesses to the file system + Offset32Array<PathAndPathInfo> pathInfoMap; ///< Stores all the accesses to the file system - Relative32Array<TranslationUnitRequestState> translationUnits; + Offset32Array<TranslationUnitRequestState> translationUnits; - Relative32Array<EntryPointState> entryPoints; + Offset32Array<EntryPointState> entryPoints; SlangMatrixLayoutMode defaultMatrixLayoutMode; }; - static SlangResult store(EndToEndCompileRequest* request, RelativeContainer& inOutContainer, Safe32Ptr<RequestState>& outRequest); + static SlangResult store(EndToEndCompileRequest* request, OffsetContainer& inOutContainer, Offset32Ptr<RequestState>& outRequest); static SlangResult saveState(EndToEndCompileRequest* request, const String& filename); @@ -166,7 +166,7 @@ struct StateSerializeUtil /// Load the requestState into request /// The fileSystem is optional and can be passed as nullptr. If set, as each file is loaded /// it will attempt to load from fileSystem the *uniqueName* - static SlangResult load(RequestState* requestState, ISlangFileSystem* fileSystem, EndToEndCompileRequest* request); + static SlangResult load(OffsetBase& base, RequestState* requestState, ISlangFileSystem* fileSystem, EndToEndCompileRequest* request); static SlangResult loadState(const String& filename, List<uint8_t>& outBuffer); static SlangResult loadState(Stream* stream, List<uint8_t>& outBuffer); @@ -176,7 +176,7 @@ struct StateSerializeUtil static SlangResult extractFilesToDirectory(const String& file); - static SlangResult extractFiles(RequestState* requestState, ISlangFileSystemExt* fileSystem); + static SlangResult extractFiles(OffsetBase& base, RequestState* requestState, ISlangFileSystemExt* fileSystem); /// Given the repo file work out a suitable path static SlangResult calcDirectoryPathFromFilename(const String& filename, String& outPath); diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 8479d3202..3053b60c0 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -3465,9 +3465,12 @@ SLANG_API SlangResult spLoadRepro( List<uint8_t> buffer; SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState((const uint8_t*)data, size, buffer)); + MemoryOffsetBase base; + base.set(buffer.getBuffer(), buffer.getCount()); + StateSerializeUtil::RequestState* requestState = StateSerializeUtil::getRequest(buffer); - SLANG_RETURN_ON_FAIL(StateSerializeUtil::load(requestState, fileSystem, request)); + SLANG_RETURN_ON_FAIL(StateSerializeUtil::load(base, requestState, fileSystem, request)); return SLANG_OK; } @@ -3512,8 +3515,11 @@ SLANG_API SlangResult spExtractRepro(SlangSession* session, const void* reproDat SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState(&memoryStream, buffer)); } + MemoryOffsetBase base; + base.set(buffer.getBuffer(), buffer.getCount()); + StateSerializeUtil::RequestState* requestState = StateSerializeUtil::getRequest(buffer); - return StateSerializeUtil::extractFiles(requestState, fileSystem); + return StateSerializeUtil::extractFiles(base, requestState, fileSystem); } // Reflection API diff --git a/tools/slang-test/slang-test.vcxproj b/tools/slang-test/slang-test.vcxproj index d1326d655..b22a3fe60 100644 --- a/tools/slang-test/slang-test.vcxproj +++ b/tools/slang-test/slang-test.vcxproj @@ -175,7 +175,7 @@ <ClCompile Include="slangc-tool.cpp" /> <ClCompile Include="test-context.cpp" /> <ClCompile Include="test-reporter.cpp" /> - <ClCompile Include="unit-relative-container.cpp" /> + <ClCompile Include="unit-offset-container.cpp" /> <ClCompile Include="unit-test-byte-encode.cpp" /> <ClCompile Include="unit-test-free-list.cpp" /> <ClCompile Include="unit-test-memory-arena.cpp" /> diff --git a/tools/slang-test/slang-test.vcxproj.filters b/tools/slang-test/slang-test.vcxproj.filters index 9bf6f9b26..b88490559 100644 --- a/tools/slang-test/slang-test.vcxproj.filters +++ b/tools/slang-test/slang-test.vcxproj.filters @@ -44,7 +44,7 @@ <ClCompile Include="test-reporter.cpp"> <Filter>Source Files</Filter> </ClCompile> - <ClCompile Include="unit-relative-container.cpp"> + <ClCompile Include="unit-offset-container.cpp"> <Filter>Source Files</Filter> </ClCompile> <ClCompile Include="unit-test-byte-encode.cpp"> diff --git a/tools/slang-test/unit-offset-container.cpp b/tools/slang-test/unit-offset-container.cpp new file mode 100644 index 000000000..d0990d9bb --- /dev/null +++ b/tools/slang-test/unit-offset-container.cpp @@ -0,0 +1,119 @@ +// unit-test-path.cpp + +#include "../../source/core/slang-offset-container.h" + +#include "test-context.h" + +using namespace Slang; + +static void _checkEncodeDecode(uint32_t size) +{ + uint8_t encode[OffsetString::kMaxSizeEncodeSize]; + + size_t encodeSize = OffsetString::calcEncodedSize(size, encode); + + size_t decodedSize; + const char* chars = OffsetString::decodeSize((const char*)encode, decodedSize); + + SLANG_CHECK(decodedSize == size); + SLANG_CHECK(chars - (const char*)encode == encodeSize); +} + +namespace { // anonymous + +struct Root +{ + Offset32Array<Offset32Ptr<OffsetString> > dirs; + Offset32Ptr<OffsetString> name; + float value; +}; + +} // anonymous + +static void offsetContainerUnitTest() +{ + _checkEncodeDecode(253); + + for (int64_t i = 0; i < 0x100000000; i += (i / 2) + 1) + { + _checkEncodeDecode(uint32_t(i)); + } + + { + OffsetContainer container; + + const char* strings[] = + { + "Hello", + "World", + nullptr, + }; + + { + auto& base = container.asBase(); + + Offset32Ptr<Root> root = container.newObject<Root>(); + + auto array = container.newArray<Offset32Ptr<OffsetString>>(SLANG_COUNT_OF(strings)); + for (Int i = 0; i < SLANG_COUNT_OF(strings); ++i) + { + base[array[i]] = container.newString(strings[i]); + } + base[root]->dirs = array; + } + + { + List<uint8_t> copy; + copy.addRange(container.getData(), container.getDataCount()); + + MemoryOffsetBase base; + base.set(copy.getBuffer(), copy.getCount()); + + Root* root = (Root*)(copy.getBuffer() + kStartOffset); + + SLANG_CHECK(root->dirs.getCount() == SLANG_COUNT_OF(strings)); + + Int count = root->dirs.getCount(); + for (Int i = 0; i < count; ++i) + { + OffsetString* str = base.asRaw(base.asRaw(root->dirs[i])); + + const char* check = strings[i]; + + if (check) + { + SLANG_CHECK(str != nullptr); + const char* strCstr = str->getCstr(); + SLANG_CHECK(strcmp(strCstr, check) == 0); + } + else + { + SLANG_CHECK(str == nullptr); + } + } + + { + Index index = 0; + for (const auto v : root->dirs) + { + OffsetString* str = base.asRaw(base.asRaw(v)); + const char* check = strings[index]; + if (check) + { + SLANG_CHECK(str != nullptr); + const char* strCstr = str->getCstr(); + SLANG_CHECK(strcmp(strCstr, check) == 0); + } + else + { + SLANG_CHECK(str == nullptr); + } + + index ++; + } + } + } + } +} + +SLANG_UNIT_TEST("OffsetContainer", offsetContainerUnitTest); diff --git a/tools/slang-test/unit-relative-container.cpp b/tools/slang-test/unit-relative-container.cpp deleted file mode 100644 index a9ff7c0fe..000000000 --- a/tools/slang-test/unit-relative-container.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// unit-test-path.cpp - -#include "../../source/core/slang-relative-container.h" - -#include "test-context.h" - -using namespace Slang; - -static void _checkEncodeDecode(uint32_t size) -{ - uint8_t encode[RelativeString::kMaxSizeEncodeSize]; - - size_t encodeSize = RelativeString::calcEncodedSize(size, encode); - - size_t decodedSize; - const char* chars = RelativeString::decodeSize((const char*)encode, decodedSize); - - SLANG_CHECK(decodedSize == size); - SLANG_CHECK(chars - (const char*)encode == encodeSize); -} - -namespace { // anonymous - -struct Root -{ - Relative32Array<Relative32Ptr<RelativeString> > dirs; - Relative32Ptr<RelativeString> name; - float value; -}; - -} // anonymous - -static void relativeContainerUnitTest() -{ - _checkEncodeDecode(253); - - for (int64_t i = 0; i < 0x100000000; i += (i / 2) + 1) - { - _checkEncodeDecode(uint32_t(i)); - } - - { - RelativeContainer container; - - const char* strings[] = - { - "Hello", - "World", - nullptr, - }; - - { - Safe32Ptr<Root> root = container.newObject<Root>(); - - auto array = container.newArray<Relative32Ptr<RelativeString>>(SLANG_COUNT_OF(strings)); - for (Int i = 0; i < SLANG_COUNT_OF(strings); ++i) - { - array[i] = container.newString(strings[i]); - } - - root->dirs = array; - } - - { - RelativeContainer copy; - copy.set(container.getData(), container.getDataCount()); - - Root* root = (Root*)copy.getData(); - - SLANG_CHECK(root->dirs.getCount() == SLANG_COUNT_OF(strings)); - - Int count = root->dirs.getCount(); - for (Int i = 0; i < count; ++i) - { - RelativeString* str = root->dirs[i]; - - const char* check = strings[i]; - - if (check) - { - SLANG_CHECK(str != nullptr); - const char* strCstr = str->getCstr(); - SLANG_CHECK(strcmp(strCstr, check) == 0); - } - else - { - SLANG_CHECK(str == nullptr); - } - } - } - } -} - -SLANG_UNIT_TEST("RelativeContainer", relativeContainerUnitTest); |
