diff options
| author | jsmall-nvidia <jsmall@nvidia.com> | 2019-10-21 15:32:13 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-10-21 15:32:13 -0400 |
| commit | 5ca446888656da91165b7bf90b7b2195d1e1afac (patch) | |
| tree | 893a03930bc706089f28c156032ffe883ea0d2a1 /source/core | |
| parent | a854bf2fde6e466aa698f4132971faadc827913a (diff) | |
`Repro` functionality (#1085)
* WIP on serialize/save state.
* Relative string encoding.
* Added RelativeContainer unit test.
Split out RelativeContainer into core.
* Fix bug in RelativeString encoding.
* More work around relative container.
* Fix checks.
* Use RelativeBase for safe access.
Use malloc/free/realloc instead of List.
* Add natvis support for relative types.
* Setting up of state (not includes) writing of repro state.
* Capture after spCompile.
* Writing SourceFile and file system files.
Added -dump-repo
* First pass at loading state.
* First pass at reading repro.
* Small optimization around Safe32Ptr
* Refactor how repro data is stored - to make saving off the files more simple, by having all all backed by 'files'.
Make file loading always set up PathInfo so we get uniqueIdentifier info.
* Generate unique file names.
* Added RelativeFileSystem
Added saveFile to ISlangFileSystemExt and implemented for interfaces
Added mechanism to save of files (and manifest)
* Added ability to replace files in repo with directory holding their contents.
* Add support for entry points.
* Fix problem compiling on linux.
* Added SIMPLE_EX option, where everything on command line must be specified.
* Fix typo in unit test for relative container.
* Fix another typo in unit test for RelativeContainer.
* Fix small bugs.
* Fix release unused variable issue in slang-state-serialize.cpp
* Fix checking for SIMPLE_EX in testing, else broke COMMAND_LINE_SIMPLE.
* Fix warnings on 32 bit debug build.
* Added import-subdir-search-path-repro.slang test. Although disabled for now as writes to root of slang project.
* Remove wrong version of import-subdir-search-path-repro.slang
* Added import-subdir-search-path-repro.slang
Diffstat (limited to 'source/core')
| -rw-r--r-- | source/core/core.natvis | 41 | ||||
| -rw-r--r-- | source/core/core.vcxproj | 4 | ||||
| -rw-r--r-- | source/core/core.vcxproj.filters | 12 | ||||
| -rw-r--r-- | source/core/slang-relative-container.cpp | 215 | ||||
| -rw-r--r-- | source/core/slang-relative-container.h | 254 | ||||
| -rw-r--r-- | source/core/slang-riff.cpp | 99 | ||||
| -rw-r--r-- | source/core/slang-riff.h | 41 |
7 files changed, 666 insertions, 0 deletions
diff --git a/source/core/core.natvis b/source/core/core.natvis index 9d3f52839..2b1e3ff7e 100644 --- a/source/core/core.natvis +++ b/source/core/core.natvis @@ -62,4 +62,45 @@ <ExpandedItem>pointer</ExpandedItem> </Expand> </Type> + + +<Type Name="Slang::Safe32Ptr<*>"> + <Expand> + <ExpandedItem>($T1*)(m_base->m_data + m_offset)</ExpandedItem> + </Expand> +</Type> + +<Type Name="Slang::Relative32Ptr<*>"> + <Expand> + <ExpandedItem>(m_offset == 0x80000000) ? nullptr : ($T1*)(((char*)this) + m_offset)</ExpandedItem> + </Expand> +</Type> + + +<Type Name="Slang::Safe32Array<*>"> + <Expand> + <Item Name="[count]">m_count</Item> + <ArrayItems> + <Size>m_count</Size> + <ValuePointer>($T1*)(m_data.m_base->m_data + m_data.m_offset)</ValuePointer> + </ArrayItems> + </Expand> +</Type> + + +<Type Name="Slang::Relative32Array<*>"> + <Expand> + <Item Name="[count]">m_count</Item> + <ArrayItems> + <Size>m_count</Size> + <ValuePointer>(m_data.m_offset == 0x80000000) ? nullptr : ($T1*)(((char*)&m_data) + m_data.m_offset)</ValuePointer> + </ArrayItems> + </Expand> +</Type> + +<Type Name="Slang::RelativeString"> + <DisplayString>{(m_sizeThenContents + 1),s}</DisplayString> + <StringView>(m_sizeThenContents + 1),s</StringView> +</Type> + </AutoVisualizer> diff --git a/source/core/core.vcxproj b/source/core/core.vcxproj index a5a6c1b24..0a1b070fe 100644 --- a/source/core/core.vcxproj +++ b/source/core/core.vcxproj @@ -191,7 +191,9 @@ <ClInclude Include="slang-platform.h" /> <ClInclude Include="slang-process-util.h" /> <ClInclude Include="slang-random-generator.h" /> + <ClInclude Include="slang-relative-container.h" /> <ClInclude Include="slang-render-api-util.h" /> + <ClInclude Include="slang-riff.h" /> <ClInclude Include="slang-secure-crt.h" /> <ClInclude Include="slang-shared-library.h" /> <ClInclude Include="slang-smart-pointer.h" /> @@ -220,7 +222,9 @@ <ClCompile Include="slang-object-scope-manager.cpp" /> <ClCompile Include="slang-platform.cpp" /> <ClCompile Include="slang-random-generator.cpp" /> + <ClCompile Include="slang-relative-container.cpp" /> <ClCompile Include="slang-render-api-util.cpp" /> + <ClCompile Include="slang-riff.cpp" /> <ClCompile Include="slang-shared-library.cpp" /> <ClCompile Include="slang-std-writers.cpp" /> <ClCompile Include="slang-stream.cpp" /> diff --git a/source/core/core.vcxproj.filters b/source/core/core.vcxproj.filters index fa4101506..56bb6b8b7 100644 --- a/source/core/core.vcxproj.filters +++ b/source/core/core.vcxproj.filters @@ -72,9 +72,15 @@ <ClInclude Include="slang-random-generator.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="slang-relative-container.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="slang-render-api-util.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="slang-riff.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="slang-secure-crt.h"> <Filter>Header Files</Filter> </ClInclude> @@ -155,9 +161,15 @@ <ClCompile Include="slang-random-generator.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="slang-relative-container.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="slang-render-api-util.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="slang-riff.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="slang-shared-library.cpp"> <Filter>Source Files</Filter> </ClCompile> diff --git a/source/core/slang-relative-container.cpp b/source/core/slang-relative-container.cpp new file mode 100644 index 000000000..0b52f4268 --- /dev/null +++ b/source/core/slang-relative-container.cpp @@ -0,0 +1,215 @@ +// slang-relative-containere.cpp +#include "slang-relative-container.h" + +namespace Slang { + + +/* static */RelativeBase RelativeBase::g_null = { nullptr }; + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RelativeString !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +size_t RelativeString::calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncodeSize]) +{ + SLANG_ASSERT(size <= 0xffffffff); + if (size <= kSizeBase) + { + encode[0] = uint8_t(size); + return 1; + } + // Encode + int num = 0; + while (size) + { + encode[num + 1] = uint8_t(size); + size >>= 8; + num++; + } + + // It might be one byte past the front, if its < 0x100 but greater than kSizeBase + SLANG_ASSERT(num >= 1); + + encode[0] = uint8_t(kSizeBase + num); + return num + 1; +} + +/* static */const char* RelativeString::decodeSize(const char* in, size_t& outSize) +{ + const uint8_t* cur = (const uint8_t*)in; + if (*cur <= kSizeBase) + { + outSize = *cur; + return in + 1; + } + + int numBytes = *cur - kSizeBase; + switch (numBytes) + { + case 1: + { + outSize = cur[1]; + return in + 2; + } + case 2: + { + outSize = cur[1] | (uint32_t(cur[2]) << 8); + return in + 3; + } + case 3: + { + outSize = cur[1] | (uint32_t(cur[2]) << 8) | (uint32_t(cur[3]) << 16); + return in + 4; + } + case 4: + { + outSize = cur[1] | (uint32_t(cur[2]) << 8) | (uint32_t(cur[3]) << 16) | (uint32_t(cur[4]) << 24); + return in + 5; + } + default: + { + outSize = 0; + return nullptr; + } + } +} + +/* static */size_t RelativeString::calcAllocationSize(size_t stringSize) +{ + uint8_t encode[kMaxSizeEncodeSize]; + size_t encodeSize = calcEncodedSize(stringSize, encode); + // Add 1 for terminating 0 + return encodeSize + stringSize + 1; +} + +/* static */size_t RelativeString::calcAllocationSize(const UnownedStringSlice& slice) +{ + return calcAllocationSize(slice.size()); +} + +UnownedStringSlice RelativeString::getSlice() const +{ + size_t size; + const char* chars = decodeSize(m_sizeThenContents, size); + + return UnownedStringSlice(chars, size); +} + +const char* RelativeString::getCstr() const +{ + return getSlice().begin(); +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RelativeContainer !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +RelativeContainer::RelativeContainer() +{ + m_current = 0; + m_capacity = 0; + m_base.m_data = nullptr; +} + +RelativeContainer::~RelativeContainer() +{ + if (m_base.m_data) + { + ::free(m_base.m_data); + } +} + +void* RelativeContainer::allocate(size_t size) +{ + return allocate(size, 1); +} + +void RelativeContainer::fixAlignment(size_t alignment) +{ + allocate(0, alignment); +} + +void* RelativeContainer::allocate(size_t size, size_t alignment) +{ + size_t offset = (m_current + alignment - 1) & ~(alignment - 1); + + if (offset + size > m_capacity) + { + const size_t minSize = offset + size; + + size_t calcSize = m_capacity; + if (calcSize < 2048) + { + calcSize = 2048; + } + else + { + // Expand geometrically, but lets not double in size... + calcSize = calcSize + (calcSize / 2); + } + + // We must be at least minSize + size_t newSize = (calcSize < minSize) ? minSize : calcSize; + + // Reallocate space + m_base.m_data = (uint8_t*)::realloc(m_base.m_data, newSize); + m_capacity = newSize; + } + + SLANG_ASSERT(offset + size <= m_capacity); + + m_current = offset + size; + return m_base.m_data + offset; +} + +void* RelativeContainer::allocateAndZero(size_t size, size_t alignment) +{ + void* data = allocate(size, alignment); + memset(data, 0, size); + return data; +} + +Safe32Ptr<RelativeString> RelativeContainer::newString(const UnownedStringSlice& slice) +{ + size_t stringSize = slice.size(); + + uint8_t head[RelativeString::kMaxSizeEncodeSize]; + size_t headSize = RelativeString::calcEncodedSize(stringSize, head); + + size_t allocSize = headSize + stringSize + 1; + uint8_t* bytes = (uint8_t*)allocate(allocSize); + + ::memcpy(bytes, head, headSize); + ::memcpy(bytes + headSize, slice.begin(), stringSize); + + // 0 terminate + bytes[headSize + stringSize] = 0; + + return Safe32Ptr<RelativeString>(getOffset(bytes), &m_base); +} + +Safe32Ptr<RelativeString> RelativeContainer::newString(const char* contents) +{ + Safe32Ptr<RelativeString> relString; + if (contents) + { + relString = newString(UnownedStringSlice(contents)); + } + return relString; +} + +void RelativeContainer::set(void* data, size_t size) +{ + if (m_base.m_data) + { + ::free(m_base.m_data); + m_base.m_data = nullptr; + } + + if (size > 0) + { + m_base.m_data = (uint8_t*)::malloc(size); + ::memcpy(m_base.m_data, data, size); + } + + m_current = size; + m_capacity = size; +} + +} // namespace Slang diff --git a/source/core/slang-relative-container.h b/source/core/slang-relative-container.h new file mode 100644 index 000000000..8416505d1 --- /dev/null +++ b/source/core/slang-relative-container.h @@ -0,0 +1,254 @@ +// slang-relative-container.h +#ifndef SLANG_RELATIVE_CONTAINER_H_INCLUDED +#define SLANG_RELATIVE_CONTAINER_H_INCLUDED + +#include "slang-basic.h" + +namespace Slang { + +struct RelativeBase +{ + uint8_t* m_data; + + static RelativeBase g_null; +}; + +template <typename T> +class Safe32Ptr +{ +public: + typedef Safe32Ptr ThisType; + + T& operator*() const { return *get(); } + T* operator->() const { return get(); } + operator T*() const { return get(); } + + const Safe32Ptr& operator=(const ThisType& rhs) { m_offset = rhs.m_offset; m_base = rhs.m_base; return *this; } + SLANG_FORCE_INLINE T* get() const { return (T*)(m_base->m_data + m_offset); } + + void setNull() + { + m_offset = 0; + m_base = &RelativeBase::g_null; + } + + Safe32Ptr(const ThisType& rhs) : m_offset(rhs.m_offset), m_base(rhs.m_base) {} + Safe32Ptr() : m_base(&RelativeBase::g_null), m_offset(0) {} + Safe32Ptr(uint32_t offset, RelativeBase* base) : m_offset(offset), m_base(base) {} + + RelativeBase* m_base; + uint32_t m_offset; +}; + + +enum +{ + kRelative32PtrNull = int32_t(0x80000000) +}; + +template <typename T> +class Relative32Ptr +{ +public: + typedef Relative32Ptr ThisType; + + T& operator*() const { return *get(); } + T* operator->() const { return get(); } + operator T*() const { return get(); } + + T* get() + { + uint8_t* nonConstThis = (uint8_t*)this; + return (m_offset == kRelative32PtrNull) ? nullptr : (T*)(nonConstThis + m_offset); + } + T* get() const + { + uint8_t* nonConstThis = const_cast<uint8_t*>((const uint8_t*)this); + return (m_offset == kRelative32PtrNull) ? nullptr : (T*)(nonConstThis + m_offset); + } + + T* detach() { T* ptr = get(); m_offset = kRelative32PtrNull; } + + void setNull() { m_offset = kRelative32PtrNull; } + + SLANG_FORCE_INLINE void set(T* ptr) { m_offset = ptr ? int32_t(((uint8_t*)ptr) - ((const uint8_t*)this)) : uint32_t(kRelative32PtrNull); } + + Relative32Ptr(const Safe32Ptr<T>& rhs) { set(rhs.get()); } + Relative32Ptr(const ThisType& rhs) { set(rhs.get()); } + + Relative32Ptr() :m_offset(kRelative32PtrNull) {} + Relative32Ptr(T* ptr) { set(ptr); } + + const Relative32Ptr& operator=(const ThisType& rhs) { set(rhs.get()); return *this; } + const Relative32Ptr& operator=(const Safe32Ptr<T>& rhs) { set(rhs.get()); return *this; } + + int32_t m_offset; +}; + +template <typename T> +class Safe32Array +{ +public: + const T* begin() const { return m_data; } + const T* end() const { return begin() + m_count; } + + T* begin() { return m_data; } + T* end() { return begin() + m_count; } + + Index getCount() const { return Index(m_count); } + T* getData() { return m_data.get(); } + const T* getData() const { return m_data.get(); } + + const T& operator[](Index i) const { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; } + T& operator[](Index i) { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; } + + Safe32Array(Safe32Ptr<T> data, uint32_t count):m_data(data), m_count(count) {} + + Safe32Array():m_count(0) {} + + Safe32Ptr<T> m_data; + uint32_t m_count; +}; + + +template <typename T> +class Relative32Array +{ +public: + typedef Relative32Array ThisType; + + const T* begin() const { return m_data; } + const T* end() const { return begin() + m_count; } + + T* begin() { return m_data; } + T* end() { return begin() + m_count; } + + Index getCount() const { return Index(m_count); } + T* getData() { return m_data.get(); } + const T* getData() const { return m_data.get(); } + + const T& operator[](Index i) const { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; } + T& operator[](Index i) { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; } + + Relative32Array(const Safe32Array<T>& rhs): + m_count(rhs.m_count), + m_data(rhs.m_data) + { + } + + Relative32Array() : m_count(0) {} + Relative32Array(const ThisType& rhs) : m_count(rhs.m_count), m_data(rhs.m_data) {} + + uint32_t m_count; ///< the size of the data + Relative32Ptr<T> m_data; ///< The data +}; + +struct RelativeString +{ + enum + { + kSizeBase = 251, + kMaxSizeEncodeSize = 5, + }; + + /// Get contents as a slice + UnownedStringSlice getSlice() const; + /// Get null terminated string + const char* getCstr() const; + + /// Decode the size. Returns the start of the string text, and outSize holds the size (NOT including terminating 0) + static const char* decodeSize(const char* in, size_t& outSize); + + /// Returns the amount of bytes used, end encoding in 'encode' + static size_t calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncodeSize]); + /// Calculate the total size needed to store the string *including* terminating 0 + static size_t calcAllocationSize(const UnownedStringSlice& slice); + + /// Calculate the total size needed to store string. Size should be passed *without* terminating 0 + static size_t calcAllocationSize(size_t size); + + char m_sizeThenContents[1]; +}; + +class RelativeContainer +{ +public: + + template <typename T> + Safe32Ptr<T> allocate() + { + void* data = allocate(sizeof(T), SLANG_ALIGN_OF(T)); + new (data) T(); + return Safe32Ptr<T>(getOffset(data), &m_base); + } + + template <typename T> + Safe32Array<T> allocateArray(size_t size) + { + if (size == 0) + { + return Safe32Array<T>(); + } + T* data = (T*)allocate(sizeof(T) * size, SLANG_ALIGN_OF(T)); + for (size_t i = 0; i < size; ++i) + { + new (data + i) T(); + } + return Safe32Array<T>(Safe32Ptr<T>(getOffset(data), &m_base), uint32_t(size)); + } + + /// Make a pointer into a safe ptr + template <typename T> + Safe32Ptr<T> toSafe(T* in) + { + Safe32Ptr<T> dst; + if (in) + { + dst.m_base = &m_base; + dst.m_offset = getOffset(in); + } + return dst; + } + + /// Allocate without alignment (effectively 1) + void* allocate(size_t size); + void* allocate(size_t size, size_t alignment); + void* allocateAndZero(size_t size, size_t alignment); + + void fixAlignment(size_t alignment); + + SLANG_FORCE_INLINE uint32_t getOffset(const void* ptr) const + { + ptrdiff_t offset = ((const uint8_t*)ptr) - m_base.m_data; + SLANG_ASSERT(offset >= 0 && size_t(offset) < m_current); + return uint32_t(offset); + } + + Safe32Ptr<RelativeString> newString(const UnownedStringSlice& slice); + Safe32Ptr<RelativeString> newString(const char* contents); + + /// Get the contained data + uint8_t* getData() { return m_base.m_data; } + /// Return the last used byte of the data + size_t getDataCount() const { return m_current; } + + /// Set the contents + void set(void* data, size_t size); + + RelativeBase* getBase() { return &m_base; } + + /// Ctor + RelativeContainer(); + ~RelativeContainer(); + + +protected: + size_t m_current; + size_t m_capacity; + RelativeBase m_base; +}; + + +} // namespace Slang + +#endif diff --git a/source/core/slang-riff.cpp b/source/core/slang-riff.cpp new file mode 100644 index 000000000..df2076013 --- /dev/null +++ b/source/core/slang-riff.cpp @@ -0,0 +1,99 @@ +#include "slang-riff.h" + +#include "../../slang-com-helper.h" + +namespace Slang +{ + +/* static */int64_t RiffUtil::calcChunkTotalSize(const RiffChunk& chunk) +{ + int64_t size = chunk.m_size + sizeof(RiffChunk); + return (size + 3) & ~int64_t(3); +} + +/* static */SlangResult RiffUtil::skip(const RiffChunk& chunk, Stream* stream, int64_t* remainingBytesInOut) +{ + int64_t chunkSize = calcChunkTotalSize(chunk); + if (remainingBytesInOut) + { + *remainingBytesInOut -= chunkSize; + } + + // Skip the payload (we don't need to skip the Chunk because that was already read + stream->Seek(SeekOrigin::Current, chunkSize - sizeof(RiffChunk)); + return SLANG_OK; +} + + +/* static */SlangResult RiffUtil::readChunk(Stream* stream, RiffChunk& outChunk) +{ + try + { + stream->Read(&outChunk, sizeof(RiffChunk)); + } + catch (IOException&) + { + return SLANG_FAIL; + } + + // TODO(JS): Could handle endianness issues here... + + return SLANG_OK; +} + + +/* static */SlangResult RiffUtil::writeData(uint32_t riffType, const void* data, size_t size, Stream* out) +{ + SLANG_ASSERT(uint64_t(size) <= uint64_t(0xfffffffff)); + + // TODO(JS): Could handle endianness here + RiffChunk chunk; + chunk.m_type = riffType; + chunk.m_size = uint32_t(size); + + try + { + out->Write(&chunk, sizeof(chunk)); + out->Write(data, size); + size_t remaining = size & 3; + if (remaining) + { + uint8_t end[4] = { 0, 0, 0, 0}; + out->Write(end, 4 - remaining); + } + } + catch (IOException&) + { + return SLANG_FAIL; + } + + return SLANG_OK; +} + + +/* static */SlangResult RiffUtil::readData(Stream* stream, RiffChunk& outChunk, List<uint8_t>& data) +{ + SLANG_RETURN_ON_FAIL(readChunk(stream, outChunk)); + + data.setCount(outChunk.m_size); + + try + { + stream->Read(data.getBuffer(), outChunk.m_size); + + // Skip to the alignment + uint32_t remaining = outChunk.m_size & 3; + if (remaining) + { + stream->Seek(SeekOrigin::Current, 4 - remaining); + } + } + catch (IOException&) + { + return SLANG_FAIL; + } + + return SLANG_OK; +} + +} diff --git a/source/core/slang-riff.h b/source/core/slang-riff.h new file mode 100644 index 000000000..f6854310b --- /dev/null +++ b/source/core/slang-riff.h @@ -0,0 +1,41 @@ +#ifndef SLANG_RIFF_H +#define SLANG_RIFF_H + +#include "../core/slang-basic.h" +#include "../core/slang-stream.h" + +namespace Slang +{ + +// http://fileformats.archiveteam.org/wiki/RIFF +// http://www.fileformat.info/format/riff/egff.htm + +#define SLANG_FOUR_CC(c0, c1, c2, c3) ((uint32_t(c0) << 0) | (uint32_t(c1) << 8) | (uint32_t(c2) << 16) | (uint32_t(c3) << 24)) + +struct RiffFourCC +{ + static const uint32_t kRiff = SLANG_FOUR_CC('R', 'I', 'F', 'F'); +}; + +struct RiffChunk +{ + uint32_t m_type; ///< The FourCC code that identifies this chunk + uint32_t m_size; ///< Size does *NOT* include the riff chunk size. The size can be byte sized, but on storage it will always be treated as aligned up by 4. +}; + +struct RiffUtil +{ + static int64_t calcChunkTotalSize(const RiffChunk& chunk); + + static SlangResult skip(const RiffChunk& chunk, Stream* stream, int64_t* remainingBytesInOut); + + static SlangResult readChunk(Stream* stream, RiffChunk& outChunk); + + + static SlangResult writeData(uint32_t riffType, const void* data, size_t size, Stream* out); + static SlangResult readData(Stream* stream, RiffChunk& outChunk, List<uint8_t>& data); +}; + +} + +#endif |
