summaryrefslogtreecommitdiffstats
path: root/source
diff options
context:
space:
mode:
authorjsmall-nvidia <jsmall@nvidia.com>2019-10-21 15:32:13 -0400
committerGitHub <noreply@github.com>2019-10-21 15:32:13 -0400
commit5ca446888656da91165b7bf90b7b2195d1e1afac (patch)
tree893a03930bc706089f28c156032ffe883ea0d2a1 /source
parenta854bf2fde6e466aa698f4132971faadc827913a (diff)
`Repro` functionality (#1085)
* WIP on serialize/save state. * Relative string encoding. * Added RelativeContainer unit test. Split out RelativeContainer into core. * Fix bug in RelativeString encoding. * More work around relative container. * Fix checks. * Use RelativeBase for safe access. Use malloc/free/realloc instead of List. * Add natvis support for relative types. * Setting up of state (not includes) writing of repro state. * Capture after spCompile. * Writing SourceFile and file system files. Added -dump-repo * First pass at loading state. * First pass at reading repro. * Small optimization around Safe32Ptr * Refactor how repro data is stored - to make saving off the files more simple, by having all all backed by 'files'. Make file loading always set up PathInfo so we get uniqueIdentifier info. * Generate unique file names. * Added RelativeFileSystem Added saveFile to ISlangFileSystemExt and implemented for interfaces Added mechanism to save of files (and manifest) * Added ability to replace files in repo with directory holding their contents. * Add support for entry points. * Fix problem compiling on linux. * Added SIMPLE_EX option, where everything on command line must be specified. * Fix typo in unit test for relative container. * Fix another typo in unit test for RelativeContainer. * Fix small bugs. * Fix release unused variable issue in slang-state-serialize.cpp * Fix checking for SIMPLE_EX in testing, else broke COMMAND_LINE_SIMPLE. * Fix warnings on 32 bit debug build. * Added import-subdir-search-path-repro.slang test. Although disabled for now as writes to root of slang project. * Remove wrong version of import-subdir-search-path-repro.slang * Added import-subdir-search-path-repro.slang
Diffstat (limited to 'source')
-rw-r--r--source/core/core.natvis41
-rw-r--r--source/core/core.vcxproj4
-rw-r--r--source/core/core.vcxproj.filters12
-rw-r--r--source/core/slang-relative-container.cpp215
-rw-r--r--source/core/slang-relative-container.h254
-rw-r--r--source/core/slang-riff.cpp99
-rw-r--r--source/core/slang-riff.h41
-rw-r--r--source/slang/slang-compiler.h39
-rw-r--r--source/slang/slang-diagnostic-defs.h2
-rw-r--r--source/slang/slang-file-system.cpp126
-rw-r--r--source/slang/slang-file-system.h134
-rw-r--r--source/slang/slang-ir-serialize.cpp87
-rw-r--r--source/slang/slang-ir-serialize.h25
-rw-r--r--source/slang/slang-options.cpp46
-rw-r--r--source/slang/slang-source-loc.h7
-rw-r--r--source/slang/slang-state-serialize.cpp989
-rw-r--r--source/slang/slang-state-serialize.h176
-rw-r--r--source/slang/slang.cpp133
-rw-r--r--source/slang/slang.vcxproj2
-rw-r--r--source/slang/slang.vcxproj.filters8
20 files changed, 2316 insertions, 124 deletions
diff --git a/source/core/core.natvis b/source/core/core.natvis
index 9d3f52839..2b1e3ff7e 100644
--- a/source/core/core.natvis
+++ b/source/core/core.natvis
@@ -62,4 +62,45 @@
<ExpandedItem>pointer</ExpandedItem>
</Expand>
</Type>
+
+
+<Type Name="Slang::Safe32Ptr&lt;*&gt;">
+ <Expand>
+ <ExpandedItem>($T1*)(m_base->m_data + m_offset)</ExpandedItem>
+ </Expand>
+</Type>
+
+<Type Name="Slang::Relative32Ptr&lt;*&gt;">
+ <Expand>
+ <ExpandedItem>(m_offset == 0x80000000) ? nullptr : ($T1*)(((char*)this) + m_offset)</ExpandedItem>
+ </Expand>
+</Type>
+
+
+<Type Name="Slang::Safe32Array&lt;*&gt;">
+ <Expand>
+ <Item Name="[count]">m_count</Item>
+ <ArrayItems>
+ <Size>m_count</Size>
+ <ValuePointer>($T1*)(m_data.m_base->m_data + m_data.m_offset)</ValuePointer>
+ </ArrayItems>
+ </Expand>
+</Type>
+
+
+<Type Name="Slang::Relative32Array&lt;*&gt;">
+ <Expand>
+ <Item Name="[count]">m_count</Item>
+ <ArrayItems>
+ <Size>m_count</Size>
+ <ValuePointer>(m_data.m_offset == 0x80000000) ? nullptr : ($T1*)(((char*)&amp;m_data) + m_data.m_offset)</ValuePointer>
+ </ArrayItems>
+ </Expand>
+</Type>
+
+<Type Name="Slang::RelativeString">
+ <DisplayString>{(m_sizeThenContents + 1),s}</DisplayString>
+ <StringView>(m_sizeThenContents + 1),s</StringView>
+</Type>
+
</AutoVisualizer>
diff --git a/source/core/core.vcxproj b/source/core/core.vcxproj
index a5a6c1b24..0a1b070fe 100644
--- a/source/core/core.vcxproj
+++ b/source/core/core.vcxproj
@@ -191,7 +191,9 @@
<ClInclude Include="slang-platform.h" />
<ClInclude Include="slang-process-util.h" />
<ClInclude Include="slang-random-generator.h" />
+ <ClInclude Include="slang-relative-container.h" />
<ClInclude Include="slang-render-api-util.h" />
+ <ClInclude Include="slang-riff.h" />
<ClInclude Include="slang-secure-crt.h" />
<ClInclude Include="slang-shared-library.h" />
<ClInclude Include="slang-smart-pointer.h" />
@@ -220,7 +222,9 @@
<ClCompile Include="slang-object-scope-manager.cpp" />
<ClCompile Include="slang-platform.cpp" />
<ClCompile Include="slang-random-generator.cpp" />
+ <ClCompile Include="slang-relative-container.cpp" />
<ClCompile Include="slang-render-api-util.cpp" />
+ <ClCompile Include="slang-riff.cpp" />
<ClCompile Include="slang-shared-library.cpp" />
<ClCompile Include="slang-std-writers.cpp" />
<ClCompile Include="slang-stream.cpp" />
diff --git a/source/core/core.vcxproj.filters b/source/core/core.vcxproj.filters
index fa4101506..56bb6b8b7 100644
--- a/source/core/core.vcxproj.filters
+++ b/source/core/core.vcxproj.filters
@@ -72,9 +72,15 @@
<ClInclude Include="slang-random-generator.h">
<Filter>Header Files</Filter>
</ClInclude>
+ <ClInclude Include="slang-relative-container.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
<ClInclude Include="slang-render-api-util.h">
<Filter>Header Files</Filter>
</ClInclude>
+ <ClInclude Include="slang-riff.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
<ClInclude Include="slang-secure-crt.h">
<Filter>Header Files</Filter>
</ClInclude>
@@ -155,9 +161,15 @@
<ClCompile Include="slang-random-generator.cpp">
<Filter>Source Files</Filter>
</ClCompile>
+ <ClCompile Include="slang-relative-container.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
<ClCompile Include="slang-render-api-util.cpp">
<Filter>Source Files</Filter>
</ClCompile>
+ <ClCompile Include="slang-riff.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
<ClCompile Include="slang-shared-library.cpp">
<Filter>Source Files</Filter>
</ClCompile>
diff --git a/source/core/slang-relative-container.cpp b/source/core/slang-relative-container.cpp
new file mode 100644
index 000000000..0b52f4268
--- /dev/null
+++ b/source/core/slang-relative-container.cpp
@@ -0,0 +1,215 @@
+// slang-relative-containere.cpp
+#include "slang-relative-container.h"
+
+namespace Slang {
+
+
+/* static */RelativeBase RelativeBase::g_null = { nullptr };
+
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RelativeString !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
+
+size_t RelativeString::calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncodeSize])
+{
+ SLANG_ASSERT(size <= 0xffffffff);
+ if (size <= kSizeBase)
+ {
+ encode[0] = uint8_t(size);
+ return 1;
+ }
+ // Encode
+ int num = 0;
+ while (size)
+ {
+ encode[num + 1] = uint8_t(size);
+ size >>= 8;
+ num++;
+ }
+
+ // It might be one byte past the front, if its < 0x100 but greater than kSizeBase
+ SLANG_ASSERT(num >= 1);
+
+ encode[0] = uint8_t(kSizeBase + num);
+ return num + 1;
+}
+
+/* static */const char* RelativeString::decodeSize(const char* in, size_t& outSize)
+{
+ const uint8_t* cur = (const uint8_t*)in;
+ if (*cur <= kSizeBase)
+ {
+ outSize = *cur;
+ return in + 1;
+ }
+
+ int numBytes = *cur - kSizeBase;
+ switch (numBytes)
+ {
+ case 1:
+ {
+ outSize = cur[1];
+ return in + 2;
+ }
+ case 2:
+ {
+ outSize = cur[1] | (uint32_t(cur[2]) << 8);
+ return in + 3;
+ }
+ case 3:
+ {
+ outSize = cur[1] | (uint32_t(cur[2]) << 8) | (uint32_t(cur[3]) << 16);
+ return in + 4;
+ }
+ case 4:
+ {
+ outSize = cur[1] | (uint32_t(cur[2]) << 8) | (uint32_t(cur[3]) << 16) | (uint32_t(cur[4]) << 24);
+ return in + 5;
+ }
+ default:
+ {
+ outSize = 0;
+ return nullptr;
+ }
+ }
+}
+
+/* static */size_t RelativeString::calcAllocationSize(size_t stringSize)
+{
+ uint8_t encode[kMaxSizeEncodeSize];
+ size_t encodeSize = calcEncodedSize(stringSize, encode);
+ // Add 1 for terminating 0
+ return encodeSize + stringSize + 1;
+}
+
+/* static */size_t RelativeString::calcAllocationSize(const UnownedStringSlice& slice)
+{
+ return calcAllocationSize(slice.size());
+}
+
+UnownedStringSlice RelativeString::getSlice() const
+{
+ size_t size;
+ const char* chars = decodeSize(m_sizeThenContents, size);
+
+ return UnownedStringSlice(chars, size);
+}
+
+const char* RelativeString::getCstr() const
+{
+ return getSlice().begin();
+}
+
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RelativeContainer !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
+
+RelativeContainer::RelativeContainer()
+{
+ m_current = 0;
+ m_capacity = 0;
+ m_base.m_data = nullptr;
+}
+
+RelativeContainer::~RelativeContainer()
+{
+ if (m_base.m_data)
+ {
+ ::free(m_base.m_data);
+ }
+}
+
+void* RelativeContainer::allocate(size_t size)
+{
+ return allocate(size, 1);
+}
+
+void RelativeContainer::fixAlignment(size_t alignment)
+{
+ allocate(0, alignment);
+}
+
+void* RelativeContainer::allocate(size_t size, size_t alignment)
+{
+ size_t offset = (m_current + alignment - 1) & ~(alignment - 1);
+
+ if (offset + size > m_capacity)
+ {
+ const size_t minSize = offset + size;
+
+ size_t calcSize = m_capacity;
+ if (calcSize < 2048)
+ {
+ calcSize = 2048;
+ }
+ else
+ {
+ // Expand geometrically, but lets not double in size...
+ calcSize = calcSize + (calcSize / 2);
+ }
+
+ // We must be at least minSize
+ size_t newSize = (calcSize < minSize) ? minSize : calcSize;
+
+ // Reallocate space
+ m_base.m_data = (uint8_t*)::realloc(m_base.m_data, newSize);
+ m_capacity = newSize;
+ }
+
+ SLANG_ASSERT(offset + size <= m_capacity);
+
+ m_current = offset + size;
+ return m_base.m_data + offset;
+}
+
+void* RelativeContainer::allocateAndZero(size_t size, size_t alignment)
+{
+ void* data = allocate(size, alignment);
+ memset(data, 0, size);
+ return data;
+}
+
+Safe32Ptr<RelativeString> RelativeContainer::newString(const UnownedStringSlice& slice)
+{
+ size_t stringSize = slice.size();
+
+ uint8_t head[RelativeString::kMaxSizeEncodeSize];
+ size_t headSize = RelativeString::calcEncodedSize(stringSize, head);
+
+ size_t allocSize = headSize + stringSize + 1;
+ uint8_t* bytes = (uint8_t*)allocate(allocSize);
+
+ ::memcpy(bytes, head, headSize);
+ ::memcpy(bytes + headSize, slice.begin(), stringSize);
+
+ // 0 terminate
+ bytes[headSize + stringSize] = 0;
+
+ return Safe32Ptr<RelativeString>(getOffset(bytes), &m_base);
+}
+
+Safe32Ptr<RelativeString> RelativeContainer::newString(const char* contents)
+{
+ Safe32Ptr<RelativeString> relString;
+ if (contents)
+ {
+ relString = newString(UnownedStringSlice(contents));
+ }
+ return relString;
+}
+
+void RelativeContainer::set(void* data, size_t size)
+{
+ if (m_base.m_data)
+ {
+ ::free(m_base.m_data);
+ m_base.m_data = nullptr;
+ }
+
+ if (size > 0)
+ {
+ m_base.m_data = (uint8_t*)::malloc(size);
+ ::memcpy(m_base.m_data, data, size);
+ }
+
+ m_current = size;
+ m_capacity = size;
+}
+
+} // namespace Slang
diff --git a/source/core/slang-relative-container.h b/source/core/slang-relative-container.h
new file mode 100644
index 000000000..8416505d1
--- /dev/null
+++ b/source/core/slang-relative-container.h
@@ -0,0 +1,254 @@
+// slang-relative-container.h
+#ifndef SLANG_RELATIVE_CONTAINER_H_INCLUDED
+#define SLANG_RELATIVE_CONTAINER_H_INCLUDED
+
+#include "slang-basic.h"
+
+namespace Slang {
+
+struct RelativeBase
+{
+ uint8_t* m_data;
+
+ static RelativeBase g_null;
+};
+
+template <typename T>
+class Safe32Ptr
+{
+public:
+ typedef Safe32Ptr ThisType;
+
+ T& operator*() const { return *get(); }
+ T* operator->() const { return get(); }
+ operator T*() const { return get(); }
+
+ const Safe32Ptr& operator=(const ThisType& rhs) { m_offset = rhs.m_offset; m_base = rhs.m_base; return *this; }
+ SLANG_FORCE_INLINE T* get() const { return (T*)(m_base->m_data + m_offset); }
+
+ void setNull()
+ {
+ m_offset = 0;
+ m_base = &RelativeBase::g_null;
+ }
+
+ Safe32Ptr(const ThisType& rhs) : m_offset(rhs.m_offset), m_base(rhs.m_base) {}
+ Safe32Ptr() : m_base(&RelativeBase::g_null), m_offset(0) {}
+ Safe32Ptr(uint32_t offset, RelativeBase* base) : m_offset(offset), m_base(base) {}
+
+ RelativeBase* m_base;
+ uint32_t m_offset;
+};
+
+
+enum
+{
+ kRelative32PtrNull = int32_t(0x80000000)
+};
+
+template <typename T>
+class Relative32Ptr
+{
+public:
+ typedef Relative32Ptr ThisType;
+
+ T& operator*() const { return *get(); }
+ T* operator->() const { return get(); }
+ operator T*() const { return get(); }
+
+ T* get()
+ {
+ uint8_t* nonConstThis = (uint8_t*)this;
+ return (m_offset == kRelative32PtrNull) ? nullptr : (T*)(nonConstThis + m_offset);
+ }
+ T* get() const
+ {
+ uint8_t* nonConstThis = const_cast<uint8_t*>((const uint8_t*)this);
+ return (m_offset == kRelative32PtrNull) ? nullptr : (T*)(nonConstThis + m_offset);
+ }
+
+ T* detach() { T* ptr = get(); m_offset = kRelative32PtrNull; }
+
+ void setNull() { m_offset = kRelative32PtrNull; }
+
+ SLANG_FORCE_INLINE void set(T* ptr) { m_offset = ptr ? int32_t(((uint8_t*)ptr) - ((const uint8_t*)this)) : uint32_t(kRelative32PtrNull); }
+
+ Relative32Ptr(const Safe32Ptr<T>& rhs) { set(rhs.get()); }
+ Relative32Ptr(const ThisType& rhs) { set(rhs.get()); }
+
+ Relative32Ptr() :m_offset(kRelative32PtrNull) {}
+ Relative32Ptr(T* ptr) { set(ptr); }
+
+ const Relative32Ptr& operator=(const ThisType& rhs) { set(rhs.get()); return *this; }
+ const Relative32Ptr& operator=(const Safe32Ptr<T>& rhs) { set(rhs.get()); return *this; }
+
+ int32_t m_offset;
+};
+
+template <typename T>
+class Safe32Array
+{
+public:
+ const T* begin() const { return m_data; }
+ const T* end() const { return begin() + m_count; }
+
+ T* begin() { return m_data; }
+ T* end() { return begin() + m_count; }
+
+ Index getCount() const { return Index(m_count); }
+ T* getData() { return m_data.get(); }
+ const T* getData() const { return m_data.get(); }
+
+ const T& operator[](Index i) const { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; }
+ T& operator[](Index i) { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; }
+
+ Safe32Array(Safe32Ptr<T> data, uint32_t count):m_data(data), m_count(count) {}
+
+ Safe32Array():m_count(0) {}
+
+ Safe32Ptr<T> m_data;
+ uint32_t m_count;
+};
+
+
+template <typename T>
+class Relative32Array
+{
+public:
+ typedef Relative32Array ThisType;
+
+ const T* begin() const { return m_data; }
+ const T* end() const { return begin() + m_count; }
+
+ T* begin() { return m_data; }
+ T* end() { return begin() + m_count; }
+
+ Index getCount() const { return Index(m_count); }
+ T* getData() { return m_data.get(); }
+ const T* getData() const { return m_data.get(); }
+
+ const T& operator[](Index i) const { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; }
+ T& operator[](Index i) { SLANG_ASSERT(i >= 0 && uint32_t(i) < m_count); return m_data.get()[i]; }
+
+ Relative32Array(const Safe32Array<T>& rhs):
+ m_count(rhs.m_count),
+ m_data(rhs.m_data)
+ {
+ }
+
+ Relative32Array() : m_count(0) {}
+ Relative32Array(const ThisType& rhs) : m_count(rhs.m_count), m_data(rhs.m_data) {}
+
+ uint32_t m_count; ///< the size of the data
+ Relative32Ptr<T> m_data; ///< The data
+};
+
+struct RelativeString
+{
+ enum
+ {
+ kSizeBase = 251,
+ kMaxSizeEncodeSize = 5,
+ };
+
+ /// Get contents as a slice
+ UnownedStringSlice getSlice() const;
+ /// Get null terminated string
+ const char* getCstr() const;
+
+ /// Decode the size. Returns the start of the string text, and outSize holds the size (NOT including terminating 0)
+ static const char* decodeSize(const char* in, size_t& outSize);
+
+ /// Returns the amount of bytes used, end encoding in 'encode'
+ static size_t calcEncodedSize(size_t size, uint8_t encode[kMaxSizeEncodeSize]);
+ /// Calculate the total size needed to store the string *including* terminating 0
+ static size_t calcAllocationSize(const UnownedStringSlice& slice);
+
+ /// Calculate the total size needed to store string. Size should be passed *without* terminating 0
+ static size_t calcAllocationSize(size_t size);
+
+ char m_sizeThenContents[1];
+};
+
+class RelativeContainer
+{
+public:
+
+ template <typename T>
+ Safe32Ptr<T> allocate()
+ {
+ void* data = allocate(sizeof(T), SLANG_ALIGN_OF(T));
+ new (data) T();
+ return Safe32Ptr<T>(getOffset(data), &m_base);
+ }
+
+ template <typename T>
+ Safe32Array<T> allocateArray(size_t size)
+ {
+ if (size == 0)
+ {
+ return Safe32Array<T>();
+ }
+ T* data = (T*)allocate(sizeof(T) * size, SLANG_ALIGN_OF(T));
+ for (size_t i = 0; i < size; ++i)
+ {
+ new (data + i) T();
+ }
+ return Safe32Array<T>(Safe32Ptr<T>(getOffset(data), &m_base), uint32_t(size));
+ }
+
+ /// Make a pointer into a safe ptr
+ template <typename T>
+ Safe32Ptr<T> toSafe(T* in)
+ {
+ Safe32Ptr<T> dst;
+ if (in)
+ {
+ dst.m_base = &m_base;
+ dst.m_offset = getOffset(in);
+ }
+ return dst;
+ }
+
+ /// Allocate without alignment (effectively 1)
+ void* allocate(size_t size);
+ void* allocate(size_t size, size_t alignment);
+ void* allocateAndZero(size_t size, size_t alignment);
+
+ void fixAlignment(size_t alignment);
+
+ SLANG_FORCE_INLINE uint32_t getOffset(const void* ptr) const
+ {
+ ptrdiff_t offset = ((const uint8_t*)ptr) - m_base.m_data;
+ SLANG_ASSERT(offset >= 0 && size_t(offset) < m_current);
+ return uint32_t(offset);
+ }
+
+ Safe32Ptr<RelativeString> newString(const UnownedStringSlice& slice);
+ Safe32Ptr<RelativeString> newString(const char* contents);
+
+ /// Get the contained data
+ uint8_t* getData() { return m_base.m_data; }
+ /// Return the last used byte of the data
+ size_t getDataCount() const { return m_current; }
+
+ /// Set the contents
+ void set(void* data, size_t size);
+
+ RelativeBase* getBase() { return &m_base; }
+
+ /// Ctor
+ RelativeContainer();
+ ~RelativeContainer();
+
+
+protected:
+ size_t m_current;
+ size_t m_capacity;
+ RelativeBase m_base;
+};
+
+
+} // namespace Slang
+
+#endif
diff --git a/source/core/slang-riff.cpp b/source/core/slang-riff.cpp
new file mode 100644
index 000000000..df2076013
--- /dev/null
+++ b/source/core/slang-riff.cpp
@@ -0,0 +1,99 @@
+#include "slang-riff.h"
+
+#include "../../slang-com-helper.h"
+
+namespace Slang
+{
+
+/* static */int64_t RiffUtil::calcChunkTotalSize(const RiffChunk& chunk)
+{
+ int64_t size = chunk.m_size + sizeof(RiffChunk);
+ return (size + 3) & ~int64_t(3);
+}
+
+/* static */SlangResult RiffUtil::skip(const RiffChunk& chunk, Stream* stream, int64_t* remainingBytesInOut)
+{
+ int64_t chunkSize = calcChunkTotalSize(chunk);
+ if (remainingBytesInOut)
+ {
+ *remainingBytesInOut -= chunkSize;
+ }
+
+ // Skip the payload (we don't need to skip the Chunk because that was already read
+ stream->Seek(SeekOrigin::Current, chunkSize - sizeof(RiffChunk));
+ return SLANG_OK;
+}
+
+
+/* static */SlangResult RiffUtil::readChunk(Stream* stream, RiffChunk& outChunk)
+{
+ try
+ {
+ stream->Read(&outChunk, sizeof(RiffChunk));
+ }
+ catch (IOException&)
+ {
+ return SLANG_FAIL;
+ }
+
+ // TODO(JS): Could handle endianness issues here...
+
+ return SLANG_OK;
+}
+
+
+/* static */SlangResult RiffUtil::writeData(uint32_t riffType, const void* data, size_t size, Stream* out)
+{
+ SLANG_ASSERT(uint64_t(size) <= uint64_t(0xfffffffff));
+
+ // TODO(JS): Could handle endianness here
+ RiffChunk chunk;
+ chunk.m_type = riffType;
+ chunk.m_size = uint32_t(size);
+
+ try
+ {
+ out->Write(&chunk, sizeof(chunk));
+ out->Write(data, size);
+ size_t remaining = size & 3;
+ if (remaining)
+ {
+ uint8_t end[4] = { 0, 0, 0, 0};
+ out->Write(end, 4 - remaining);
+ }
+ }
+ catch (IOException&)
+ {
+ return SLANG_FAIL;
+ }
+
+ return SLANG_OK;
+}
+
+
+/* static */SlangResult RiffUtil::readData(Stream* stream, RiffChunk& outChunk, List<uint8_t>& data)
+{
+ SLANG_RETURN_ON_FAIL(readChunk(stream, outChunk));
+
+ data.setCount(outChunk.m_size);
+
+ try
+ {
+ stream->Read(data.getBuffer(), outChunk.m_size);
+
+ // Skip to the alignment
+ uint32_t remaining = outChunk.m_size & 3;
+ if (remaining)
+ {
+ stream->Seek(SeekOrigin::Current, 4 - remaining);
+ }
+ }
+ catch (IOException&)
+ {
+ return SLANG_FAIL;
+ }
+
+ return SLANG_OK;
+}
+
+}
diff --git a/source/core/slang-riff.h b/source/core/slang-riff.h
new file mode 100644
index 000000000..f6854310b
--- /dev/null
+++ b/source/core/slang-riff.h
@@ -0,0 +1,41 @@
+#ifndef SLANG_RIFF_H
+#define SLANG_RIFF_H
+
+#include "../core/slang-basic.h"
+#include "../core/slang-stream.h"
+
+namespace Slang
+{
+
+// http://fileformats.archiveteam.org/wiki/RIFF
+// http://www.fileformat.info/format/riff/egff.htm
+
+#define SLANG_FOUR_CC(c0, c1, c2, c3) ((uint32_t(c0) << 0) | (uint32_t(c1) << 8) | (uint32_t(c2) << 16) | (uint32_t(c3) << 24))
+
+struct RiffFourCC
+{
+ static const uint32_t kRiff = SLANG_FOUR_CC('R', 'I', 'F', 'F');
+};
+
+struct RiffChunk
+{
+ uint32_t m_type; ///< The FourCC code that identifies this chunk
+ uint32_t m_size; ///< Size does *NOT* include the riff chunk size. The size can be byte sized, but on storage it will always be treated as aligned up by 4.
+};
+
+struct RiffUtil
+{
+ static int64_t calcChunkTotalSize(const RiffChunk& chunk);
+
+ static SlangResult skip(const RiffChunk& chunk, Stream* stream, int64_t* remainingBytesInOut);
+
+ static SlangResult readChunk(Stream* stream, RiffChunk& outChunk);
+
+
+ static SlangResult writeData(uint32_t riffType, const void* data, size_t size, Stream* out);
+ static SlangResult readData(Stream* stream, RiffChunk& outChunk, List<uint8_t>& data);
+};
+
+}
+
+#endif
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index 143d3bdaf..9b361064d 100644
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -13,6 +13,8 @@
#include "slang-profile.h"
#include "slang-syntax.h"
+#include "slang-file-system.h"
+
#include "../../slang.h"
namespace Slang
@@ -195,6 +197,9 @@ namespace Slang
/// Get the profile that the entry point is to be compiled for
Profile getProfile() { return m_profile; }
+ /// Get the index to the translation unit
+ int getTranslationUnitIndex() const { return m_translationUnitIndex; }
+
private:
// The parent compile request
FrontEndCompileRequest* m_compileRequest;
@@ -1174,6 +1179,9 @@ namespace Slang
/// or a wrapped impl that makes fileSystem operate as fileSystemExt
ComPtr<ISlangFileSystemExt> fileSystemExt;
+ /// Set if fileSystemExt is a cache file system
+ RefPtr<CacheFileSystem> cacheFileSystem;
+
ISlangFileSystemExt* getFileSystemExt() { return fileSystemExt; }
/// Load a file into memory using the configured file system.
@@ -1182,7 +1190,7 @@ namespace Slang
/// @param outBlob A destination pointer to receive the loaded blob
/// @returns A `SlangResult` to indicate success or failure.
///
- SlangResult loadFile(String const& path, ISlangBlob** outBlob);
+ SlangResult loadFile(String const& path, PathInfo& outPathInfo, ISlangBlob** outBlob);
RefPtr<Expr> parseTypeString(String typeStr, RefPtr<Scope> scope);
@@ -1235,6 +1243,8 @@ namespace Slang
m_sourceManager = sourceManager;
}
+ void setRequireCacheFileSystem(bool requireCacheFileSystem);
+
void setFileSystem(ISlangFileSystem* fileSystem);
/// The layout to use for matrices by default (row/column major)
@@ -1245,6 +1255,8 @@ namespace Slang
OptimizationLevel optimizationLevel = OptimizationLevel::Default;
+
+ bool m_requireCacheFileSystem = false;
bool m_useFalcorCustomSharedKeywordSemantics = false;
private:
@@ -1313,7 +1325,7 @@ namespace Slang
SourceManager* getSourceManager() { return getLinkage()->getSourceManager(); }
NamePool* getNamePool() { return getLinkage()->getNamePool(); }
ISlangFileSystemExt* getFileSystemExt() { return getLinkage()->getFileSystemExt(); }
- SlangResult loadFile(String const& path, ISlangBlob** outBlob) { return getLinkage()->loadFile(path, outBlob); }
+ SlangResult loadFile(String const& path, PathInfo& outPathInfo, ISlangBlob** outBlob) { return getLinkage()->loadFile(path, outPathInfo, outBlob); }
bool shouldDumpIR = false;
bool shouldValidateIR = false;
@@ -1710,6 +1722,10 @@ namespace Slang
String mDiagnosticOutput;
+
+ // If set, will dump the compilation state
+ String dumpRepro;
+
/// A blob holding the diagnostic output
ComPtr<ISlangBlob> diagnosticOutputBlob;
@@ -1744,6 +1760,7 @@ namespace Slang
void setWriter(WriterChannel chan, ISlangWriter* writer);
ISlangWriter* getWriter(WriterChannel chan) const { return m_writers[int(chan)]; }
+ /// The end to end request can be passed as nullptr, if not driven by one
SlangResult executeActionsInner();
SlangResult executeActions();
@@ -2002,6 +2019,9 @@ namespace Slang
~Session();
private:
+
+ SlangResult _loadRequest(EndToEndCompileRequest* request, const void* data, size_t size);
+
/// Linkage used for all built-in (stdlib) code.
RefPtr<Linkage> m_builtinLinkage;
@@ -2089,6 +2109,21 @@ SLANG_FORCE_INLINE EndToEndCompileRequest* asInternal(SlangCompileRequest* reque
return reinterpret_cast<EndToEndCompileRequest*>(request);
}
+class ListBlob : public ISlangBlob, public RefObject
+{
+public:
+ SLANG_REF_OBJECT_IUNKNOWN_ALL
+
+ // ISlangBlob
+ SLANG_NO_THROW void const* SLANG_MCALL getBufferPointer() SLANG_OVERRIDE { return m_data.getBuffer(); }
+ SLANG_NO_THROW size_t SLANG_MCALL getBufferSize() SLANG_OVERRIDE { return m_data.getCount(); }
+
+ List<uint8_t> m_data;
+
+protected:
+ ISlangUnknown* getInterface(const Guid& guid);
+};
+
}
#endif
diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h
index 067a65159..008b1ef04 100644
--- a/source/slang/slang-diagnostic-defs.h
+++ b/source/slang/slang-diagnostic-defs.h
@@ -108,6 +108,8 @@ DIAGNOSTIC( 70, Error, cannotMatchOutputFileToEntryPoint, "the output path '$
DIAGNOSTIC( 80, Error, duplicateOutputPathsForEntryPointAndTarget, "multiple output paths have been specified entry point '$0' on target '$1'")
+DIAGNOSTIC( 81, Error, parametersAfterLoadReproIgnored, "Parameters after -load-repro [file] are ignored")
+
//
// 1xxxx - Lexical anaylsis
//
diff --git a/source/slang/slang-file-system.cpp b/source/slang/slang-file-system.cpp
index 975d35f2c..4ee961df5 100644
--- a/source/slang/slang-file-system.cpp
+++ b/source/slang/slang-file-system.cpp
@@ -13,6 +13,7 @@ namespace Slang
static const Guid IID_ISlangUnknown = SLANG_UUID_ISlangUnknown;
static const Guid IID_ISlangFileSystem = SLANG_UUID_ISlangFileSystem;
static const Guid IID_ISlangFileSystemExt = SLANG_UUID_ISlangFileSystemExt;
+static const Guid IID_SlangCacheFileSystem = SLANG_UUID_CacheFileSystem;
// Cacluate a combined path, just using Path:: string processing
static SlangResult _calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut)
@@ -129,6 +130,30 @@ SlangResult OSFileSystemExt::loadFile(char const* pathIn, ISlangBlob** outBlob)
return SLANG_E_CANNOT_OPEN;
}
+SLANG_NO_THROW SlangResult SLANG_MCALL OSFileSystemExt::saveFile(const char* pathIn, const void* data, size_t size)
+{
+ const String path = _fixPathDelimiters(pathIn);
+
+ try
+ {
+ FileStream stream(pathIn, FileMode::Create, FileAccess::Write, FileShare::ReadWrite);
+
+ int64_t numWritten = stream.Write(data, size);
+
+ if (numWritten != int64_t(size))
+ {
+ return SLANG_FAIL;
+ }
+
+ }
+ catch (IOException&)
+ {
+ return SLANG_E_CANNOT_OPEN;
+ }
+
+ return SLANG_OK;
+}
+
// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CacheFileSystem !!!!!!!!!!!!!!!!!!!!!!!!!!!
/* static */ const Result CacheFileSystem::s_compressedResultToResult[] =
@@ -159,13 +184,35 @@ ISlangUnknown* CacheFileSystem::getInterface(const Guid& guid)
return _getInterface(this, guid);
}
+SLANG_NO_THROW SlangResult SLANG_MCALL CacheFileSystem::queryInterface(SlangUUID const& uuid, void** outObject)
+{
+ if (uuid == IID_SlangCacheFileSystem)
+ {
+ *outObject = this;
+ return SLANG_OK;
+ }
+
+ ISlangUnknown* intf = getInterface(uuid);
+ if (intf)
+ {
+ addReference();
+ *outObject = intf;
+ return SLANG_OK;
+ }
+ return SLANG_E_NO_INTERFACE;
+}
+
+
CacheFileSystem::CacheFileSystem(ISlangFileSystem* fileSystem, UniqueIdentityMode uniqueIdentityMode, PathStyle pathStyle) :
m_fileSystem(fileSystem),
m_uniqueIdentityMode(uniqueIdentityMode),
m_pathStyle(pathStyle)
{
- // Try to get the more sophisticated interface
- fileSystem->queryInterface(IID_ISlangFileSystemExt, (void**)m_fileSystemExt.writeRef());
+ if (fileSystem)
+ {
+ // Try to get the more sophisticated interface
+ fileSystem->queryInterface(IID_ISlangFileSystemExt, (void**)m_fileSystemExt.writeRef());
+ }
switch (uniqueIdentityMode)
{
@@ -263,6 +310,12 @@ SlangResult CacheFileSystem::_calcUniqueIdentity(const String& path, String& out
case UniqueIdentityMode::SimplifyPathAndHash:
case UniqueIdentityMode::Hash:
{
+ // If we don't have a file system -> assume cannot be found
+ if (m_fileSystem == nullptr)
+ {
+ return SLANG_E_NOT_FOUND;
+ }
+
// I can only see if this is the same file as already loaded by loading the file and doing a hash
Result res = m_fileSystem->loadFile(path.getBuffer(), outFileContents.writeRef());
if (SLANG_FAILED(res) || outFileContents == nullptr)
@@ -513,4 +566,73 @@ SlangResult CacheFileSystem::getCanonicalPath(const char* path, ISlangBlob** out
return SLANG_OK;
}
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RelativeFileSystem !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
+
+ISlangUnknown* RelativeFileSystem::getInterface(const Guid& guid)
+{
+ return _getInterface(this, guid);
+}
+
+SlangResult RelativeFileSystem::_getFixedPath(const char* path, String& outPath)
+{
+ ComPtr<ISlangBlob> blob;
+ SLANG_RETURN_ON_FAIL(m_fileSystem->calcCombinedPath(SLANG_PATH_TYPE_DIRECTORY, m_relativePath.getBuffer(), path, blob.writeRef()));
+
+ outPath = StringUtil::getString(blob);
+ return SLANG_OK;
+}
+
+SLANG_NO_THROW SlangResult SLANG_MCALL RelativeFileSystem::loadFile(char const* path, ISlangBlob** outBlob)
+{
+ String fixedPath;
+ SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath));
+ return m_fileSystem->loadFile(fixedPath.getBuffer(), outBlob);
+}
+
+SLANG_NO_THROW SlangResult SLANG_MCALL RelativeFileSystem::getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity)
+{
+ String fixedPath;
+ SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath));
+ return m_fileSystem->getFileUniqueIdentity(fixedPath.getBuffer(), outUniqueIdentity);
+}
+
+SLANG_NO_THROW SlangResult SLANG_MCALL RelativeFileSystem::calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** outPath)
+{
+ String fixedFromPath;
+ SLANG_RETURN_ON_FAIL(_getFixedPath(fromPath, fixedFromPath));
+
+ return m_fileSystem->calcCombinedPath(fromPathType, fixedFromPath.getBuffer(), path, outPath);
+}
+
+SLANG_NO_THROW SlangResult SLANG_MCALL RelativeFileSystem::getPathType(const char* path, SlangPathType* outPathType)
+{
+ String fixedPath;
+ SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath));
+ return m_fileSystem->getPathType(fixedPath.getBuffer(), outPathType);
+}
+
+SLANG_NO_THROW SlangResult SLANG_MCALL RelativeFileSystem::getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath)
+{
+ return m_fileSystem->getSimplifiedPath(path, outSimplifiedPath);
+}
+
+SLANG_NO_THROW SlangResult SLANG_MCALL RelativeFileSystem::getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath)
+{
+ String fixedPath;
+ SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath));
+ return m_fileSystem->getCanonicalPath(fixedPath.getBuffer(), outCanonicalPath);
+}
+
+SLANG_NO_THROW void SLANG_MCALL RelativeFileSystem::clearCache()
+{
+ m_fileSystem->clearCache();
+}
+
+SLANG_NO_THROW SlangResult SLANG_MCALL RelativeFileSystem::saveFile(const char* path, const void* data, size_t size)
+{
+ String fixedPath;
+ SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath));
+ return m_fileSystem->saveFile(fixedPath.getBuffer(), data, size);
+}
+
}
diff --git a/source/slang/slang-file-system.h b/source/slang/slang-file-system.h
index d91d8ff65..71596be5b 100644
--- a/source/slang/slang-file-system.h
+++ b/source/slang/slang-file-system.h
@@ -51,6 +51,12 @@ public:
virtual SLANG_NO_THROW void SLANG_MCALL clearCache() SLANG_OVERRIDE {}
+ virtual SLANG_NO_THROW SlangResult SLANG_MCALL saveFile(
+ const char* path,
+ const void* data,
+ size_t size) SLANG_OVERRIDE ;
+
+
/// Get a default instance
static ISlangFileSystemExt* getSingleton() { return &s_singleton; }
@@ -93,6 +99,9 @@ private:
static OSFileSystem s_singleton;
};
+
+ #define SLANG_UUID_CacheFileSystem { 0x2f4d1d03, 0xa0d1, 0x434b, { 0x87, 0x7a, 0x65, 0x5, 0xa4, 0xa0, 0x9a, 0x3b } };
+
/* Wraps an underlying ISlangFileSystem or ISlangFileSystemExt and provides caching,
as well as emulation of methods if only has ISlangFileSystem interface. Will query capabilities
of the interface on the constructor.
@@ -133,8 +142,40 @@ class CacheFileSystem: public ISlangFileSystemExt, public RefObject
CountOf,
};
- // ISlangUnknown
- SLANG_REF_OBJECT_IUNKNOWN_ALL
+ struct PathInfo
+ {
+ PathInfo(const String& uniqueIdentity)
+ {
+ m_uniqueIdentity = new StringBlob(uniqueIdentity);
+ m_uniqueIdentity->addRef();
+
+ m_loadFileResult = CompressedResult::Uninitialized;
+ m_getPathTypeResult = CompressedResult::Uninitialized;
+ m_getCanonicalPathResult = CompressedResult::Uninitialized;
+
+ m_pathType = SLANG_PATH_TYPE_FILE;
+ }
+
+ /// Get the unique identity path as a string
+ const String& getUniqueIdentity() const { SLANG_ASSERT(m_uniqueIdentity); return m_uniqueIdentity->getString(); }
+
+ RefPtr<StringBlob> m_uniqueIdentity;
+ CompressedResult m_loadFileResult;
+ CompressedResult m_getPathTypeResult;
+ CompressedResult m_getCanonicalPathResult;
+
+ SlangPathType m_pathType;
+ ComPtr<ISlangBlob> m_fileBlob;
+ RefPtr<StringBlob> m_canonicalPath;
+ };
+
+ Dictionary<String, PathInfo*>& getPathMap() { return m_pathMap; }
+ Dictionary<String, PathInfo*>& getUniqueMap() { return m_uniqueIdentityMap; }
+
+ // ISlangUnknown
+ SLANG_NO_THROW SlangResult SLANG_MCALL queryInterface(SlangUUID const& uuid, void** outObject) SLANG_OVERRIDE;
+ SLANG_REF_OBJECT_IUNKNOWN_ADD_REF
+ SLANG_REF_OBJECT_IUNKNOWN_RELEASE
// ISlangFileSystem
virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile(
@@ -166,6 +207,15 @@ class CacheFileSystem: public ISlangFileSystemExt, public RefObject
virtual SLANG_NO_THROW void SLANG_MCALL clearCache() SLANG_OVERRIDE;
+ virtual SLANG_NO_THROW SlangResult SLANG_MCALL saveFile(const char* path, const void* data, size_t size) SLANG_OVERRIDE
+ {
+ SLANG_UNUSED(path);
+ SLANG_UNUSED(data);
+ SLANG_UNUSED(size);
+
+ return SLANG_E_NOT_IMPLEMENTED;
+ }
+
/// Ctor
CacheFileSystem(ISlangFileSystem* fileSystem, UniqueIdentityMode uniqueIdentityMode = UniqueIdentityMode::Default, PathStyle pathStyle = PathStyle::Default);
/// Dtor
@@ -178,32 +228,7 @@ class CacheFileSystem: public ISlangFileSystemExt, public RefObject
protected:
ISlangUnknown* getInterface(const Guid& guid);
- struct PathInfo
- {
- PathInfo(const String& uniqueIdentity)
- {
- m_uniqueIdentity = new StringBlob(uniqueIdentity);
- m_uniqueIdentity->addRef();
-
- m_loadFileResult = CompressedResult::Uninitialized;
- m_getPathTypeResult = CompressedResult::Uninitialized;
- m_getCanonicalPathResult = CompressedResult::Uninitialized;
-
- m_pathType = SLANG_PATH_TYPE_FILE;
- }
-
- /// Get the unique identity path as a string
- const String& getUniqueIdentity() const { SLANG_ASSERT(m_uniqueIdentity); return m_uniqueIdentity->getString(); }
-
- RefPtr<StringBlob> m_uniqueIdentity;
- CompressedResult m_loadFileResult;
- CompressedResult m_getPathTypeResult;
- CompressedResult m_getCanonicalPathResult;
-
- SlangPathType m_pathType;
- ComPtr<ISlangBlob> m_fileBlob;
- RefPtr<StringBlob> m_canonicalPath;
- };
+
/// Given a path, works out a uniqueIdentity, based on the uniqueIdentityMode. outFileContents will be set if file had to be read to produce the uniqueIdentity (ie with Hash)
SlangResult _calcUniqueIdentity(const String& path, String& outUniqueIdentity, ComPtr<ISlangBlob>& outFileContents);
@@ -229,6 +254,59 @@ protected:
ComPtr<ISlangFileSystemExt> m_fileSystemExt; ///< Optionally set -> if nullptr will fall back on the m_fileSystem and emulate all the other methods of ISlangFileSystemExt
};
+class RelativeFileSystem : public ISlangFileSystemExt, public RefObject
+{
+public:
+ SLANG_REF_OBJECT_IUNKNOWN_ALL
+
+ // ISlangFileSystem
+ virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile(
+ char const* path,
+ ISlangBlob** outBlob) SLANG_OVERRIDE;
+
+ // ISlangFileSystemExt
+ virtual SLANG_NO_THROW SlangResult SLANG_MCALL getFileUniqueIdentity(
+ const char* path,
+ ISlangBlob** outUniqueIdentity) SLANG_OVERRIDE;
+
+ virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcCombinedPath(
+ SlangPathType fromPathType,
+ const char* fromPath,
+ const char* path,
+ ISlangBlob** pathOut) SLANG_OVERRIDE;
+
+ virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType(
+ const char* path,
+ SlangPathType* outPathType) SLANG_OVERRIDE;
+
+ virtual SLANG_NO_THROW SlangResult SLANG_MCALL getSimplifiedPath(
+ const char* path,
+ ISlangBlob** outSimplifiedPath) SLANG_OVERRIDE;
+
+ virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanonicalPath(
+ const char* path,
+ ISlangBlob** outCanonicalPath) SLANG_OVERRIDE;
+
+ virtual SLANG_NO_THROW void SLANG_MCALL clearCache() SLANG_OVERRIDE;
+
+ virtual SLANG_NO_THROW SlangResult SLANG_MCALL saveFile(const char* path, const void* data, size_t size) SLANG_OVERRIDE;
+
+ RelativeFileSystem(ISlangFileSystemExt* fileSystem, const String& relativePath):
+ m_fileSystem(fileSystem),
+ m_relativePath(relativePath)
+ {
+ }
+
+protected:
+
+ SlangResult _getFixedPath(const char* path, String& outPath);
+
+ ISlangUnknown* getInterface(const Guid& guid);
+
+ ComPtr<ISlangFileSystemExt> m_fileSystem;
+ String m_relativePath;
+};
+
}
#endif // SLANG_FILE_SYSTEM_H_INCLUDED
diff --git a/source/slang/slang-ir-serialize.cpp b/source/slang/slang-ir-serialize.cpp
index cbb774794..02ed3142d 100644
--- a/source/slang/slang-ir-serialize.cpp
+++ b/source/slang/slang-ir-serialize.cpp
@@ -840,7 +840,7 @@ static Result _writeArrayChunk(IRSerialBinary::CompressionType compressionType,
{
case Bin::CompressionType::None:
{
- payloadSize = sizeof(Bin::ArrayHeader) - sizeof(Bin::Chunk) + typeSize * numEntries;
+ payloadSize = sizeof(Bin::ArrayHeader) - sizeof(RiffChunk) + typeSize * numEntries;
Bin::ArrayHeader header;
header.m_chunk.m_type = chunkId;
@@ -860,7 +860,7 @@ static Result _writeArrayChunk(IRSerialBinary::CompressionType compressionType,
ByteEncodeUtil::encodeLiteUInt32((const uint32_t*)data, numCompressedEntries, compressedPayload);
- payloadSize = sizeof(Bin::CompressedArrayHeader) - sizeof(Bin::Chunk) + compressedPayload.getCount();
+ payloadSize = sizeof(Bin::CompressedArrayHeader) - sizeof(RiffChunk) + compressedPayload.getCount();
Bin::CompressedArrayHeader header;
header.m_chunk.m_type = SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId);
@@ -1005,7 +1005,7 @@ Result _writeInstArrayChunk(IRSerialBinary::CompressionType compressionType, uin
List<uint8_t> compressedPayload;
SLANG_RETURN_ON_FAIL(_encodeInsts(compressionType, array, compressedPayload));
- size_t payloadSize = sizeof(Bin::CompressedArrayHeader) - sizeof(Bin::Chunk) + compressedPayload.getCount();
+ size_t payloadSize = sizeof(Bin::CompressedArrayHeader) - sizeof(RiffChunk) + compressedPayload.getCount();
Bin::CompressedArrayHeader header;
header.m_chunk.m_type = SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId);
@@ -1124,7 +1124,7 @@ static size_t _calcInstChunkSize(IRSerialBinary::CompressionType compressionType
}
{
- Bin::Chunk riffHeader;
+ RiffChunk riffHeader;
riffHeader.m_type = Bin::kRiffFourCc;
riffHeader.m_size = uint32_t(totalSize);
@@ -1133,7 +1133,7 @@ static size_t _calcInstChunkSize(IRSerialBinary::CompressionType compressionType
{
Bin::SlangHeader slangHeader;
slangHeader.m_chunk.m_type = Bin::kSlangFourCc;
- slangHeader.m_chunk.m_size = uint32_t(sizeof(slangHeader) - sizeof(Bin::Chunk));
+ slangHeader.m_chunk.m_size = uint32_t(sizeof(slangHeader) - sizeof(RiffChunk));
slangHeader.m_compressionType = uint32_t(Bin::CompressionType::VariableByteLite);
stream->Write(&slangHeader, sizeof(slangHeader));
@@ -1192,7 +1192,7 @@ class ListResizerForType: public ListResizer
List<T>& m_list;
};
-static Result _readArrayChunk(IRSerialBinary::CompressionType compressionType, const IRSerialBinary::Chunk& chunk, Stream* stream, size_t* numReadInOut, ListResizer& listOut)
+static Result _readArrayChunk(IRSerialBinary::CompressionType compressionType, const RiffChunk& chunk, Stream* stream, size_t* numReadInOut, ListResizer& listOut)
{
typedef IRSerialBinary Bin;
@@ -1206,13 +1206,13 @@ static Result _readArrayChunk(IRSerialBinary::CompressionType compressionType, c
Bin::CompressedArrayHeader header;
header.m_chunk = chunk;
- stream->Read(&header.m_chunk + 1, sizeof(header) - sizeof(Bin::Chunk));
- *numReadInOut += sizeof(header) - sizeof(Bin::Chunk);
+ stream->Read(&header.m_chunk + 1, sizeof(header) - sizeof(RiffChunk));
+ *numReadInOut += sizeof(header) - sizeof(RiffChunk);
void* data = listOut.setSize(header.m_numEntries);
// Need to read all the compressed data...
- size_t payloadSize = header.m_chunk.m_size - (sizeof(header) - sizeof(Bin::Chunk));
+ size_t payloadSize = header.m_chunk.m_size - (sizeof(header) - sizeof(RiffChunk));
List<uint8_t> compressedPayload;
compressedPayload.setCount(payloadSize);
@@ -1232,8 +1232,8 @@ static Result _readArrayChunk(IRSerialBinary::CompressionType compressionType, c
Bin::ArrayHeader header;
header.m_chunk = chunk;
- stream->Read(&header.m_chunk + 1, sizeof(header) - sizeof(Bin::Chunk));
- *numReadInOut += sizeof(header) - sizeof(Bin::Chunk);
+ stream->Read(&header.m_chunk + 1, sizeof(header) - sizeof(RiffChunk));
+ *numReadInOut += sizeof(header) - sizeof(RiffChunk);
const size_t payloadSize = header.m_numEntries * typeSize;
@@ -1260,7 +1260,7 @@ static Result _readArrayChunk(IRSerialBinary::CompressionType compressionType, c
}
template <typename T>
-Result _readArrayChunk(const IRSerialBinary::SlangHeader& header, const IRSerialBinary::Chunk& chunk, Stream* stream, size_t* numReadInOut, List<T>& arrayOut)
+static Result _readArrayChunk(const IRSerialBinary::SlangHeader& header, const RiffChunk& chunk, Stream* stream, size_t* numReadInOut, List<T>& arrayOut)
{
typedef IRSerialBinary Bin;
@@ -1276,7 +1276,7 @@ Result _readArrayChunk(const IRSerialBinary::SlangHeader& header, const IRSerial
}
template <typename T>
-Result _readArrayUncompressedChunk(const IRSerialBinary::SlangHeader& header, const IRSerialBinary::Chunk& chunk, Stream* stream, size_t* numReadInOut, List<T>& arrayOut)
+static Result _readArrayUncompressedChunk(const IRSerialBinary::SlangHeader& header, const RiffChunk& chunk, Stream* stream, size_t* numReadInOut, List<T>& arrayOut)
{
typedef IRSerialBinary Bin;
SLANG_UNUSED(header);
@@ -1351,7 +1351,7 @@ static Result _decodeInsts(IRSerialBinary::CompressionType compressionType, cons
return SLANG_OK;
}
-Result _readInstArrayChunk(const IRSerialBinary::SlangHeader& slangHeader, const IRSerialBinary::Chunk& chunk, Stream* stream, size_t* numReadInOut, List<IRSerialData::Inst>& arrayOut)
+static Result _readInstArrayChunk(const IRSerialBinary::SlangHeader& slangHeader, const RiffChunk& chunk, Stream* stream, size_t* numReadInOut, List<IRSerialData::Inst>& arrayOut)
{
typedef IRSerialBinary Bin;
@@ -1374,11 +1374,11 @@ Result _readInstArrayChunk(const IRSerialBinary::SlangHeader& slangHeader, const
Bin::CompressedArrayHeader header;
header.m_chunk = chunk;
- stream->Read(&header.m_chunk + 1, sizeof(header) - sizeof(Bin::Chunk));
- *numReadInOut += sizeof(header) - sizeof(Bin::Chunk);
+ stream->Read(&header.m_chunk + 1, sizeof(header) - sizeof(RiffChunk));
+ *numReadInOut += sizeof(header) - sizeof(RiffChunk);
// Need to read all the compressed data...
- size_t payloadSize = header.m_chunk.m_size - (sizeof(header) - sizeof(Bin::Chunk));
+ size_t payloadSize = header.m_chunk.m_size - (sizeof(header) - sizeof(RiffChunk));
List<uint8_t> compressedPayload;
compressedPayload.setCount(payloadSize);
@@ -1409,26 +1409,6 @@ Result _readInstArrayChunk(const IRSerialBinary::SlangHeader& slangHeader, const
return SLANG_OK;
}
-int64_t _calcChunkTotalSize(const IRSerialBinary::Chunk& chunk)
-{
- int64_t size = chunk.m_size + sizeof(IRSerialBinary::Chunk);
- return (size + 3) & ~int64_t(3);
-}
-
-/* static */Result IRSerialReader::_skip(const IRSerialBinary::Chunk& chunk, Stream* stream, int64_t* remainingBytesInOut)
-{
- typedef IRSerialBinary Bin;
- int64_t chunkSize = _calcChunkTotalSize(chunk);
- if (remainingBytesInOut)
- {
- *remainingBytesInOut -= chunkSize;
- }
-
- // Skip the payload (we don't need to skip the Chunk because that was already read
- stream->Seek(SeekOrigin::Current, chunkSize - sizeof(IRSerialBinary::Chunk));
- return SLANG_OK;
-}
-
/* static */Result IRSerialReader::readStream(Stream* stream, IRSerialData* dataOut)
{
typedef IRSerialBinary Bin;
@@ -1437,8 +1417,8 @@ int64_t _calcChunkTotalSize(const IRSerialBinary::Chunk& chunk)
int64_t remainingBytes = 0;
{
- Bin::Chunk header;
- stream->Read(&header, sizeof(header));
+ RiffChunk header;
+ SLANG_RETURN_ON_FAIL(RiffUtil::readChunk(stream, header));
if (header.m_type != Bin::kRiffFourCc)
{
return SLANG_FAIL;
@@ -1454,9 +1434,8 @@ int64_t _calcChunkTotalSize(const IRSerialBinary::Chunk& chunk)
while (remainingBytes > 0)
{
- Bin::Chunk chunk;
-
- stream->Read(&chunk, sizeof(chunk));
+ RiffChunk chunk;
+ SLANG_RETURN_ON_FAIL(RiffUtil::readChunk(stream, chunk));
size_t bytesRead = sizeof(chunk);
@@ -1472,77 +1451,77 @@ int64_t _calcChunkTotalSize(const IRSerialBinary::Chunk& chunk)
stream->Read(&slangHeader.m_chunk + 1, sizeof(slangHeader) - sizeof(chunk));
- remainingBytes -= _calcChunkTotalSize(chunk);
+ remainingBytes -= RiffUtil::calcChunkTotalSize(chunk);
break;
}
case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kInstFourCc):
case Bin::kInstFourCc:
{
SLANG_RETURN_ON_FAIL(_readInstArrayChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_insts));
- remainingBytes -= _calcChunkTotalSize(chunk);
+ remainingBytes -= RiffUtil::calcChunkTotalSize(chunk);
break;
}
case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kChildRunFourCc):
case Bin::kChildRunFourCc:
{
SLANG_RETURN_ON_FAIL(_readArrayChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_childRuns));
- remainingBytes -= _calcChunkTotalSize(chunk);
+ remainingBytes -= RiffUtil::calcChunkTotalSize(chunk);
break;
}
case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kExternalOperandsFourCc):
case Bin::kExternalOperandsFourCc:
{
SLANG_RETURN_ON_FAIL(_readArrayChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_externalOperands));
- remainingBytes -= _calcChunkTotalSize(chunk);
+ remainingBytes -= RiffUtil::calcChunkTotalSize(chunk);
break;
}
case Bin::kStringFourCc:
{
SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_stringTable));
- remainingBytes -= _calcChunkTotalSize(chunk);
+ remainingBytes -= RiffUtil::calcChunkTotalSize(chunk);
break;
}
case Bin::kUInt32SourceLocFourCc:
{
SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_rawSourceLocs));
- remainingBytes -= _calcChunkTotalSize(chunk);
+ remainingBytes -= RiffUtil::calcChunkTotalSize(chunk);
break;
}
case Bin::kDebugStringFourCc:
{
SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugStringTable));
- remainingBytes -= _calcChunkTotalSize(chunk);
+ remainingBytes -= RiffUtil::calcChunkTotalSize(chunk);
break;
}
case Bin::kDebugLineInfoFourCc:
{
SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugLineInfos));
- remainingBytes -= _calcChunkTotalSize(chunk);
+ remainingBytes -= RiffUtil::calcChunkTotalSize(chunk);
break;
}
case Bin::kDebugAdjustedLineInfoFourCc:
{
SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugAdjustedLineInfos));
- remainingBytes -= _calcChunkTotalSize(chunk);
+ remainingBytes -= RiffUtil::calcChunkTotalSize(chunk);
break;
}
case Bin::kDebugSourceInfoFourCc:
{
SLANG_RETURN_ON_FAIL(_readArrayChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugSourceInfos));
- remainingBytes -= _calcChunkTotalSize(chunk);
+ remainingBytes -= RiffUtil::calcChunkTotalSize(chunk);
break;
}
case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kDebugSourceLocRunFourCc):
case Bin::kDebugSourceLocRunFourCc:
{
SLANG_RETURN_ON_FAIL(_readArrayChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugSourceLocRuns));
- remainingBytes -= _calcChunkTotalSize(chunk);
+ remainingBytes -= RiffUtil::calcChunkTotalSize(chunk);
break;
}
default:
{
- SLANG_RETURN_ON_FAIL(_skip(chunk, stream, &remainingBytes));
+ SLANG_RETURN_ON_FAIL(RiffUtil::skip(chunk, stream, &remainingBytes));
break;
}
}
diff --git a/source/slang/slang-ir-serialize.h b/source/slang/slang-ir-serialize.h
index 4f4301666..3cd4098e0 100644
--- a/source/slang/slang-ir-serialize.h
+++ b/source/slang/slang-ir-serialize.h
@@ -2,10 +2,8 @@
#ifndef SLANG_IR_SERIALIZE_H_INCLUDED
#define SLANG_IR_SERIALIZE_H_INCLUDED
-#include "../core/slang-basic.h"
-#include "../core/slang-stream.h"
-
#include "../core/slang-object-scope-manager.h"
+#include "../core/slang-riff.h"
#include "slang-ir.h"
@@ -349,20 +347,10 @@ SLANG_FORCE_INLINE int IRSerialData::getOperands(const Inst& inst, const InstInd
}
-#define SLANG_FOUR_CC(c0, c1, c2, c3) ((uint32_t(c0) << 0) | (uint32_t(c1) << 8) | (uint32_t(c2) << 16) | (uint32_t(c3) << 24))
-
#define SLANG_MAKE_COMPRESSED_FOUR_CC(fourCc) (((fourCc) & 0xffff00ff) | (uint32_t('c') << 8))
struct IRSerialBinary
{
- // http://fileformats.archiveteam.org/wiki/RIFF
- // http://www.fileformat.info/format/riff/egff.htm
-
- struct Chunk
- {
- uint32_t m_type;
- uint32_t m_size;
- };
enum class CompressionType
{
@@ -370,8 +358,7 @@ struct IRSerialBinary
VariableByteLite,
};
-
- static const uint32_t kRiffFourCc = SLANG_FOUR_CC('R', 'I', 'F', 'F');
+ static const uint32_t kRiffFourCc = RiffFourCC::kRiff;
static const uint32_t kSlangFourCc = SLANG_FOUR_CC('S', 'L', 'N', 'G'); ///< Holds all the slang specific chunks
static const uint32_t kInstFourCc = SLANG_FOUR_CC('S', 'L', 'i', 'n');
@@ -394,17 +381,17 @@ struct IRSerialBinary
struct SlangHeader
{
- Chunk m_chunk;
+ RiffChunk m_chunk;
uint32_t m_compressionType; ///< Holds the compression type used (if used at all)
};
struct ArrayHeader
{
- Chunk m_chunk;
+ RiffChunk m_chunk;
uint32_t m_numEntries;
};
struct CompressedArrayHeader
{
- Chunk m_chunk;
+ RiffChunk m_chunk;
uint32_t m_numEntries; ///< The number of entries
uint32_t m_numCompressedEntries; ///< The amount of compressed entries
};
@@ -526,8 +513,6 @@ struct IRSerialReader
protected:
- static Result _skip(const IRSerialBinary::Chunk& chunk, Stream* stream, int64_t* remainingBytesInOut);
-
StringRepresentationCache m_stringRepresentationCache;
const IRSerialData* m_serialData;
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index 31a5c7c7f..2e1b625d9 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -10,6 +10,8 @@
#include "slang-file-system.h"
+#include "slang-state-serialize.h"
+
#include <assert.h>
namespace Slang {
@@ -491,6 +493,50 @@ struct OptionsParser
requestImpl->getFrontEndReq()->shouldDumpIR = true;
requestImpl->getBackEndReq()->shouldDumpIR = true;
}
+ else if (argStr == "-dump-repro")
+ {
+ SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, requestImpl->dumpRepro));
+ requestImpl->getLinkage()->setRequireCacheFileSystem(true);
+ }
+ else if (argStr == "-extract-repro")
+ {
+ String reproName;
+ SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, reproName));
+
+ SLANG_RETURN_ON_FAIL(StateSerializeUtil::extractFilesToDirectory(reproName));
+ }
+ else if(argStr == "-load-repro")
+ {
+ String reproName;
+ SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, reproName));
+
+ List<uint8_t> buffer;
+ SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState(reproName, buffer));
+
+ auto requestState = StateSerializeUtil::getRequest(buffer);
+
+ // If we can find a directory, that exists, we will set up a file system to load from that directory
+ ComPtr<ISlangFileSystem> fileSystem;
+ String dirPath;
+ if (SLANG_SUCCEEDED(StateSerializeUtil::calcDirectoryPathFromFilename(reproName, dirPath)))
+ {
+ SlangPathType pathType;
+ if (SLANG_SUCCEEDED(Path::getPathType(dirPath, &pathType)) && pathType == SLANG_PATH_TYPE_DIRECTORY)
+ {
+ fileSystem = new RelativeFileSystem(OSFileSystemExt::getSingleton(), dirPath);
+ }
+ }
+
+ SLANG_RETURN_ON_FAIL(StateSerializeUtil::load(requestState, fileSystem, requestImpl));
+
+ if (argCursor < argEnd)
+ {
+ sink->diagnose(SourceLoc(), Diagnostics::parametersAfterLoadReproIgnored);
+ return SLANG_FAIL;
+ }
+
+ return SLANG_OK;
+ }
else if (argStr == "-serial-ir")
{
requestImpl->getFrontEndReq()->useSerialIRBottleneck = true;
diff --git a/source/slang/slang-source-loc.h b/source/slang/slang-source-loc.h
index 30affeaab..45b94bb5b 100644
--- a/source/slang/slang-source-loc.h
+++ b/source/slang/slang-source-loc.h
@@ -41,7 +41,7 @@ interpretation for that lex/parse.
struct PathInfo
{
/// To be more rigorous about where a path comes from, the type identifies what a paths origin is
- enum class Type
+ enum class Type : uint8_t
{
Unknown, ///< The path is not known
Normal, ///< Normal has both path and uniqueIdentity
@@ -371,7 +371,10 @@ struct SourceManager
/// Allocate a string slice
UnownedStringSlice allocateStringSlice(const UnownedStringSlice& slice);
-
+
+ /// Get all of the source files
+ const List<SourceFile*>& getSourceFiles() const { return m_sourceFiles; }
+
SourceManager() :
m_memoryArena(2048)
{}
diff --git a/source/slang/slang-state-serialize.cpp b/source/slang/slang-state-serialize.cpp
new file mode 100644
index 000000000..b37529121
--- /dev/null
+++ b/source/slang/slang-state-serialize.cpp
@@ -0,0 +1,989 @@
+// slang-state-serialize.cpp
+#include "slang-state-serialize.h"
+
+#include "../core/slang-text-io.h"
+
+#include "../core/slang-math.h"
+
+#include "slang-source-loc.h"
+
+namespace Slang {
+
+
+namespace { // anonymous
+
+struct StoreContext
+{
+ typedef StateSerializeUtil::FileState FileState;
+ typedef StateSerializeUtil::SourceFileState SourceFileState;
+ typedef StateSerializeUtil::PathInfoState PathInfoState;
+
+ StoreContext(RelativeContainer* container)
+ {
+ m_container = container;
+ }
+
+ Safe32Ptr<FileState> findFile(const String& uniqueIdentity)
+ {
+ Safe32Ptr<FileState> file;
+ m_uniqueToFileMap.TryGetValue(uniqueIdentity, file);
+ return file;
+ }
+
+ Safe32Ptr<FileState> addFile(const String& uniqueIdentity, const UnownedStringSlice* content)
+ {
+ Safe32Ptr<FileState> file;
+
+ // Get the file, if it has an identity
+ if (uniqueIdentity.getLength() && m_uniqueToFileMap.TryGetValue(uniqueIdentity, file))
+ {
+ return file;
+ }
+
+ // If file was not found create it
+ // Create the file
+ file = m_container->allocate<FileState>();
+
+ if (content)
+ {
+ file->contents = m_container->newString(*content);
+ }
+ if (uniqueIdentity.getLength())
+ {
+ file->uniqueIdentity = m_container->newString(uniqueIdentity.getUnownedSlice());
+ m_uniqueToFileMap.Add(uniqueIdentity, file);
+ }
+
+ m_files.add(file);
+ return file;
+ }
+
+ Safe32Ptr<SourceFileState> addSourceFile(SourceFile* sourceFile)
+ {
+ if (!sourceFile)
+ {
+ return Safe32Ptr<SourceFileState>();
+ }
+
+ Safe32Ptr<StateSerializeUtil::SourceFileState> sourceFileState;
+ if (m_sourceFileMap.TryGetValue(sourceFile, sourceFileState))
+ {
+ return sourceFileState;
+ }
+
+ const PathInfo& pathInfo = sourceFile->getPathInfo();
+
+ UnownedStringSlice content = sourceFile->getContent();
+ Safe32Ptr<FileState> file = addFile(pathInfo.uniqueIdentity, &content);
+
+ Safe32Ptr<RelativeString> foundPath;
+
+ if (pathInfo.foundPath.getLength() && file->foundPath == nullptr)
+ {
+ foundPath = fromString(pathInfo.foundPath.getUnownedSlice());
+ }
+ // Set on the file
+ file->foundPath = foundPath;
+
+ // Create the source file
+ sourceFileState = m_container->allocate<SourceFileState>();
+
+ sourceFileState->file = file;
+ sourceFileState->foundPath = foundPath;
+ sourceFileState->type = pathInfo.type;
+
+ m_sourceFileMap.Add(sourceFile, sourceFileState);
+
+ return sourceFileState;
+ }
+
+ Safe32Ptr<RelativeString> fromString(const String& in)
+ {
+ Safe32Ptr<RelativeString> value;
+
+ if (m_stringMap.TryGetValue(in, value))
+ {
+ return value;
+ }
+ value = m_container->newString(in.getUnownedSlice());
+ m_stringMap.Add(in, value);
+ return value;
+ }
+ Safe32Ptr<RelativeString> fromName(Name* name)
+ {
+ if (name)
+ {
+ return fromString(name->text);
+ }
+ return Safe32Ptr<RelativeString>();
+ }
+
+ Safe32Ptr<PathInfoState> addPathInfo(const CacheFileSystem::PathInfo* srcPathInfo)
+ {
+ if (!srcPathInfo)
+ {
+ return Safe32Ptr<PathInfoState>();
+ }
+
+ Safe32Ptr<PathInfoState> pathInfo;
+ if (!m_pathInfoMap.TryGetValue(srcPathInfo, pathInfo))
+ {
+ // Get the associated file
+ Safe32Ptr<FileState> fileState;
+
+ // Only store as file if we have the contents
+ if(srcPathInfo->m_fileBlob)
+ {
+ fileState = addFile(srcPathInfo->getUniqueIdentity(), nullptr);
+ }
+
+ // Save the rest of the state
+ pathInfo = m_container->allocate<PathInfoState>();
+ PathInfoState& dst = *pathInfo;
+
+ pathInfo->file = fileState;
+
+ // Save any other info
+ dst.getCanonicalPathResult = srcPathInfo->m_getCanonicalPathResult;
+ dst.getPathTypeResult = srcPathInfo->m_getPathTypeResult;
+ dst.loadFileResult = srcPathInfo->m_loadFileResult;
+ dst.pathType = srcPathInfo->m_pathType;
+
+ m_pathInfoMap.Add(srcPathInfo, pathInfo);
+ }
+
+ // Fill in info on the file
+ Safe32Ptr<FileState> fileState(m_container->toSafe(pathInfo->file.get()));
+
+ // If have fileState add any missing element
+ if (fileState)
+ {
+ if (srcPathInfo->m_fileBlob && fileState->contents == nullptr)
+ {
+ UnownedStringSlice contents((const char*)srcPathInfo->m_fileBlob->getBufferPointer(), srcPathInfo->m_fileBlob->getBufferSize());
+ fileState->contents = m_container->newString(contents);
+ }
+
+ if (srcPathInfo->m_canonicalPath && fileState->canonicalPath == nullptr)
+ {
+ fileState->canonicalPath = fromString(srcPathInfo->m_canonicalPath->getString());
+ }
+
+ if (srcPathInfo->m_uniqueIdentity && fileState->uniqueIdentity == nullptr)
+ {
+ fileState->uniqueIdentity = fromString(srcPathInfo->m_uniqueIdentity->getString());
+ }
+ }
+
+ return pathInfo;
+ }
+
+ const Safe32Array<StateSerializeUtil::StringPair> calcDefines(const Dictionary<String, String>& srcDefines)
+ {
+ typedef StateSerializeUtil::StringPair StringPair;
+
+ Safe32Array<StringPair> dstDefines = m_container->allocateArray<StringPair>(srcDefines.Count());
+
+ Index index = 0;
+ for (const auto& srcDefine : srcDefines)
+ {
+ // Do allocation before setting
+ auto key = fromString(srcDefine.Key);
+ auto value = fromString(srcDefine.Value);
+
+ auto& dstDefine = dstDefines[index];
+ dstDefine.first = key;
+ dstDefine.second = value;
+
+ index++;
+ }
+
+ return dstDefines;
+ }
+
+ const Safe32Array<Relative32Ptr<RelativeString>> fromList(const List<String>& src)
+ {
+ Safe32Array<Relative32Ptr<RelativeString>> dst = m_container->allocateArray<Relative32Ptr<RelativeString>>(src.getCount());
+ for (Index j = 0; j < src.getCount(); ++j)
+ {
+ dst[j] = fromString(src[j]);
+ }
+ return dst;
+ }
+
+ Dictionary<String, Safe32Ptr<RelativeString> > m_stringMap;
+
+ Dictionary<SourceFile*, Safe32Ptr<StateSerializeUtil::SourceFileState> > m_sourceFileMap;
+
+ Dictionary<String, Safe32Ptr<StateSerializeUtil::FileState> > m_uniqueToFileMap;
+
+ Dictionary<const CacheFileSystem::PathInfo*, Safe32Ptr<PathInfoState> > m_pathInfoMap;
+
+ List<Safe32Ptr<StateSerializeUtil::FileState> > m_files;
+
+ RelativeContainer* m_container;
+};
+
+} //
+
+static bool _isStorable(const PathInfo::Type type)
+{
+ switch (type)
+ {
+ case PathInfo::Type::Unknown:
+ case PathInfo::Type::Normal:
+ case PathInfo::Type::FoundPath:
+ case PathInfo::Type::FromString:
+ {
+ return true;
+ }
+ default: return false;
+ }
+}
+
+/* static */SlangResult StateSerializeUtil::store(EndToEndCompileRequest* request, RelativeContainer& inOutContainer, Safe32Ptr<RequestState>& outRequest)
+{
+ StoreContext context(&inOutContainer);
+
+ auto linkage = request->getLinkage();
+
+ Safe32Ptr<RequestState> requestState = inOutContainer.allocate<RequestState>();
+
+ {
+ RequestState* dst = requestState;
+
+ dst->compileFlags = request->getFrontEndReq()->compileFlags;
+ dst->shouldDumpIntermediates = request->getBackEndReq()->shouldDumpIntermediates;
+ dst->lineDirectiveMode = request->getBackEndReq()->lineDirectiveMode;
+
+ dst->debugInfoLevel = linkage->debugInfoLevel;
+ dst->optimizationLevel = linkage->optimizationLevel;
+ dst->containerFormat = request->containerFormat;
+ dst->passThroughMode = request->passThrough;
+
+
+ dst->useUnknownImageFormatAsDefault = request->getBackEndReq()->useUnknownImageFormatAsDefault;
+ dst->obfuscateCode = request->getBackEndReq()->obfuscateCode;
+
+ dst->defaultMatrixLayoutMode = linkage->defaultMatrixLayoutMode;
+ }
+
+ // Entry points
+ {
+ const auto& srcEntryPoints = request->getFrontEndReq()->m_entryPointReqs;
+ const auto& srcEndToEndEntryPoints = request->entryPoints;
+
+ SLANG_ASSERT(srcEntryPoints.getCount() == srcEndToEndEntryPoints.getCount());
+
+ Safe32Array<EntryPointState> dstEntryPoints = inOutContainer.allocateArray<EntryPointState>(srcEntryPoints.getCount());
+
+ for (Index i = 0; i < srcEntryPoints.getCount(); ++i)
+ {
+ FrontEndEntryPointRequest* srcEntryPoint = srcEntryPoints[i];
+ const auto& srcEndToEndEntryPoint = srcEndToEndEntryPoints[i];
+
+ auto dstSpecializationArgStrings = context.fromList(srcEndToEndEntryPoint.specializationArgStrings);
+ Safe32Ptr<RelativeString> dstName = context.fromName(srcEntryPoint->getName());
+
+ EntryPointState& dst = dstEntryPoints[i];
+
+ dst.profile = srcEntryPoint->getProfile();
+ dst.translationUnitIndex = uint32_t(srcEntryPoint->getTranslationUnitIndex());
+ dst.specializationArgStrings = dstSpecializationArgStrings;
+ dst.name = dstName;
+ }
+
+ requestState->entryPoints = dstEntryPoints;
+ }
+
+
+ // Add all of the source files
+ {
+ SourceManager* sourceManager = request->getFrontEndReq()->getSourceManager();
+ const List<SourceFile*>& sourceFiles = sourceManager->getSourceFiles();
+
+ for (SourceFile* sourceFile : sourceFiles)
+ {
+ const PathInfo& pathInfo = sourceFile->getPathInfo();
+ if (_isStorable(pathInfo.type))
+ {
+ context.addSourceFile(sourceFile);
+ }
+ }
+ }
+
+ // Add all the target requests
+ {
+ Safe32Array<TargetRequestState> dstTargets = inOutContainer.allocateArray<TargetRequestState>(linkage->targets.getCount());
+
+ for (Index i = 0; i < linkage->targets.getCount(); ++i)
+ {
+ auto& dst = dstTargets[i];
+ TargetRequest* targetRequest = linkage->targets[i];
+
+ dst.target = targetRequest->getTarget();
+ dst.profile = targetRequest->getTargetProfile();
+ dst.targetFlags = targetRequest->targetFlags;
+ dst.floatingPointMode = targetRequest->floatingPointMode;
+ }
+
+ requestState->targetRequests = dstTargets;
+ }
+
+ // Add the search paths
+ {
+ const auto& srcPaths = linkage->searchDirectories.searchDirectories;
+ Safe32Array<Relative32Ptr<RelativeString> > dstPaths = inOutContainer.allocateArray<Relative32Ptr<RelativeString> >(srcPaths.getCount());
+
+ // We don't handle parents here
+ SLANG_ASSERT(linkage->searchDirectories.parent == nullptr);
+ for (Index i = 0; i < srcPaths.getCount(); ++i)
+ {
+ dstPaths[i] = context.fromString(srcPaths[i].path);
+ }
+ requestState->searchPaths = dstPaths;
+ }
+
+ // Add preprocessor definitions
+ requestState->preprocessorDefinitions = context.calcDefines(linkage->preprocessorDefinitions);
+
+ {
+ const auto& srcTranslationUnits = request->getFrontEndReq()->translationUnits;
+ Safe32Array<TranslationUnitRequestState> dstTranslationUnits = inOutContainer.allocateArray<TranslationUnitRequestState>(srcTranslationUnits.getCount());
+
+ for (Index i = 0; i < srcTranslationUnits.getCount(); ++i)
+ {
+ TranslationUnitRequest* srcTranslationUnit = srcTranslationUnits[i];
+
+ // Do before setting, because this can allocate, and therefore break, the following section
+ auto defines = context.calcDefines(srcTranslationUnit->preprocessorDefinitions);
+ auto moduleName = context.fromName(srcTranslationUnit->moduleName);
+
+ Safe32Array<Relative32Ptr<SourceFileState>> dstSourceFiles;
+ {
+ const auto& srcFiles = srcTranslationUnit->getSourceFiles();
+ dstSourceFiles = inOutContainer.allocateArray<Relative32Ptr<SourceFileState> >(srcFiles.getCount());
+
+ for (Index j = 0; j < srcFiles.getCount(); ++j)
+ {
+ dstSourceFiles[j] = context.addSourceFile(srcFiles[j]);
+ }
+ }
+
+ TranslationUnitRequestState& dstTranslationUnit = dstTranslationUnits[i];
+
+ dstTranslationUnit.language = srcTranslationUnit->sourceLanguage;
+ dstTranslationUnit.moduleName = moduleName;
+ dstTranslationUnit.sourceFiles = dstSourceFiles;
+ dstTranslationUnit.preprocessorDefinitions = defines;
+ }
+
+ requestState->translationUnits = dstTranslationUnits;
+ }
+
+ // Find files from the file system, and mapping paths to files
+ {
+ CacheFileSystem* cacheFileSystem = linkage->cacheFileSystem;
+ // Traverse the references (in process we will construct the map from PathInfo)
+ {
+ const auto& srcFiles = cacheFileSystem->getPathMap();
+
+ Safe32Array<PathAndPathInfo> pathMap = inOutContainer.allocateArray<PathAndPathInfo>(srcFiles.Count());
+
+ Index index = 0;
+ for (const auto& pair : srcFiles)
+ {
+ Safe32Ptr<RelativeString> path = context.fromString(pair.Key);
+ Safe32Ptr<PathInfoState> pathInfo = context.addPathInfo(pair.Value);
+
+ PathAndPathInfo& dstInfo = pathMap[index];
+ dstInfo.path = path;
+ dstInfo.pathInfo = pathInfo;
+
+ index++;
+ }
+
+ requestState->pathInfoMap = pathMap;
+ }
+ }
+
+ // Save all of the files
+ {
+ Dictionary<String, int> uniqueNameMap;
+
+ auto files = inOutContainer.allocateArray<Relative32Ptr<FileState>>(context.m_files.getCount());
+ for (Index i = 0; i < context.m_files.getCount(); ++i)
+ {
+ Safe32Ptr<FileState> file = context.m_files[i];
+
+ // Need to come up with unique names
+ String path;
+
+ if (file->canonicalPath)
+ {
+ path = file->canonicalPath->getSlice();
+ }
+ else if (file->foundPath)
+ {
+ path = file->foundPath->getSlice();
+ }
+ else if (file->uniqueIdentity)
+ {
+ path = file->uniqueIdentity->getSlice();
+ }
+
+ if (path.getLength() == 0)
+ {
+ StringBuilder builder;
+ builder << "unnamed" << i;
+ path = builder;
+ }
+
+ String filename = Path::getFileNameWithoutExt(path);
+ String ext = Path::getFileExt(path);
+
+ StringBuilder uniqueName;
+ for (Index j = 0; j < 0x10000; j++)
+ {
+ uniqueName.Clear();
+ uniqueName << filename;
+
+ if (j > 0)
+ {
+ uniqueName << "-" << j;
+ }
+
+ if (ext.getLength())
+ {
+ uniqueName << "." << ext;
+ }
+
+ int dummy = 0;
+ if (!uniqueNameMap.TryGetValueOrAdd(uniqueName, dummy))
+ {
+ // It was added so we are done
+ break;
+ }
+ }
+
+ // Save the unique generated name
+ file->uniqueName = inOutContainer.newString(uniqueName.getUnownedSlice());
+
+ files[i] = file;
+ }
+
+ requestState->files = files;
+ }
+
+ // Save all the SourceFile state
+ {
+ const auto& srcSourceFiles = context.m_sourceFileMap;
+ auto dstSourceFiles = inOutContainer.allocateArray<Relative32Ptr<SourceFileState>>(srcSourceFiles.Count());
+
+ Index index = 0;
+ for (const auto& pair : srcSourceFiles)
+ {
+ dstSourceFiles[index] = pair.Value;
+ index++;
+ }
+ requestState->sourceFiles = dstSourceFiles;
+ }
+
+ outRequest = requestState;
+ return SLANG_OK;
+}
+
+namespace { // anonymous
+
+struct LoadContext
+{
+ typedef StateSerializeUtil::SourceFileState SourceFileState;
+ typedef StateSerializeUtil::FileState FileState;
+ typedef StateSerializeUtil::PathInfoState PathInfoState;
+
+ ISlangBlob* getFileBlob(FileState* file)
+ {
+ if (!file)
+ {
+ return nullptr;
+ }
+
+ ComPtr<ISlangBlob> blob;
+ if (!m_fileToBlobMap.TryGetValue(file, blob))
+ {
+ if (m_fileSystem && file->uniqueName)
+ {
+ // Try loading from the file system
+ m_fileSystem->loadFile(file->uniqueName->getCstr(), blob.writeRef());
+ }
+
+ // If wasn't loaded, and has contents, use that
+ if (!blob && file->contents)
+ {
+ blob = new StringBlob(file->contents->getSlice());
+ }
+
+ // Add to map, even if the blob is nullptr (say from a failed read)
+ m_fileToBlobMap.Add(file, blob);
+ }
+
+ return blob;
+ }
+
+ SourceFile* getSourceFile(SourceFileState* sourceFile)
+ {
+ if (sourceFile == nullptr)
+ {
+ return nullptr;
+ }
+
+ SourceFile* dstFile;
+ if (!m_sourceFileMap.TryGetValue(sourceFile, dstFile))
+ {
+ FileState* file = sourceFile->file;
+ ISlangBlob* blob = getFileBlob(file);
+
+ PathInfo pathInfo;
+
+ pathInfo.type = sourceFile->type;
+
+ if (sourceFile->foundPath)
+ {
+ pathInfo.foundPath = sourceFile->foundPath->getSlice();
+ }
+ else if (file->foundPath)
+ {
+ pathInfo.foundPath = file->foundPath->getSlice();
+ }
+
+ if (file->uniqueIdentity)
+ {
+ pathInfo.uniqueIdentity = file->uniqueIdentity->getSlice();
+ }
+
+ dstFile = new SourceFile(m_sourceManager, pathInfo, blob->getBufferSize());
+ dstFile->setContents(blob);
+
+ // Add to map
+ m_sourceFileMap.Add(sourceFile, dstFile);
+
+ // Add to manager
+ m_sourceManager->addSourceFile(pathInfo.uniqueIdentity, dstFile);
+ }
+ return dstFile;
+ }
+
+ CacheFileSystem::PathInfo* addPathInfo(const PathInfoState* srcInfo)
+ {
+ CacheFileSystem::PathInfo* pathInfo;
+ if (m_pathInfoMap.TryGetValue(srcInfo, pathInfo))
+ {
+ return pathInfo;
+ }
+
+ CacheFileSystem::PathInfo* dstInfo = new CacheFileSystem::PathInfo(String());
+ FileState* file = srcInfo->file;
+ if (file)
+ {
+ if (file->uniqueIdentity)
+ {
+ String uniqueIdentity = file->uniqueIdentity->getSlice();
+ dstInfo->m_uniqueIdentity = new StringBlob(uniqueIdentity);
+ }
+
+ if (file->canonicalPath)
+ {
+ dstInfo->m_canonicalPath = new StringBlob(file->canonicalPath->getSlice());
+ }
+
+ dstInfo->m_fileBlob = getFileBlob(file);
+ }
+
+ dstInfo->m_getCanonicalPathResult = srcInfo->getCanonicalPathResult;
+ dstInfo->m_getPathTypeResult = srcInfo->getPathTypeResult;
+ dstInfo->m_loadFileResult = srcInfo->loadFileResult;
+ dstInfo->m_pathType = srcInfo->pathType;
+
+ m_pathInfoMap.Add(srcInfo, dstInfo);
+ return dstInfo;
+ }
+
+ static List<const char*> toList(const Relative32Array<Relative32Ptr<RelativeString>>& src)
+ {
+ List<const char*> dst;
+ dst.setCount(src.getCount());
+ for (Index i = 0; i < src.getCount(); ++i)
+ {
+ RelativeString* srcString = src[i];
+ dst[i] = srcString ? srcString->getCstr() : nullptr;
+ }
+ return dst;
+ }
+
+ LoadContext(SourceManager* sourceManger, ISlangFileSystem* fileSystem):
+ m_sourceManager(sourceManger),
+ m_fileSystem(fileSystem)
+ {
+ }
+
+ ISlangFileSystem* m_fileSystem;
+
+ SourceManager* m_sourceManager;
+ Dictionary<SourceFileState*, SourceFile*> m_sourceFileMap;
+ Dictionary<FileState*, ComPtr<ISlangBlob> > m_fileToBlobMap;
+ Dictionary<const PathInfoState*, CacheFileSystem::PathInfo*> m_pathInfoMap;
+};
+
+} // anonymous
+
+static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>& in, Dictionary<String, String>& out)
+{
+ out.Clear();
+
+ for (const auto& define : in)
+ {
+ out.Add(define.first->getSlice(), define.second->getSlice());
+ }
+}
+
+
+/* static */SlangResult StateSerializeUtil::load(RequestState* requestState, ISlangFileSystem* fileSystem, EndToEndCompileRequest* request)
+{
+ auto externalRequest = asExternal(request);
+
+ auto linkage = request->getLinkage();
+
+ LoadContext context(linkage->getSourceManager(), fileSystem);
+
+ // Try to set state through API - as doing so means if state stored in multiple places it will be ok
+
+ {
+ spSetCompileFlags(externalRequest, (SlangCompileFlags)requestState->compileFlags);
+ spSetDumpIntermediates(externalRequest, int(requestState->shouldDumpIntermediates));
+ spSetLineDirectiveMode(externalRequest, SlangLineDirectiveMode(requestState->lineDirectiveMode));
+ spSetDebugInfoLevel(externalRequest, SlangDebugInfoLevel(requestState->debugInfoLevel));
+ spSetOptimizationLevel(externalRequest, SlangOptimizationLevel(requestState->optimizationLevel));
+ spSetOutputContainerFormat(externalRequest, SlangContainerFormat(requestState->containerFormat));
+ spSetPassThrough(externalRequest, SlangPassThrough(request->passThrough));
+
+ request->getBackEndReq()->useUnknownImageFormatAsDefault = requestState->useUnknownImageFormatAsDefault;
+ request->getBackEndReq()->obfuscateCode = requestState->obfuscateCode;
+
+ linkage->setMatrixLayoutMode(requestState->defaultMatrixLayoutMode);
+ }
+ {
+ for (Index i = 0; i < requestState->targetRequests.getCount(); ++i)
+ {
+ TargetRequestState& src = requestState->targetRequests[i];
+ int index = spAddCodeGenTarget(externalRequest, SlangCompileTarget(src.target));
+ SLANG_UNUSED(index);
+ SLANG_ASSERT(index == i);
+
+ auto dstTarget = linkage->targets[i];
+
+ SLANG_ASSERT(dstTarget->getTarget() == src.target);
+ dstTarget->targetProfile = src.profile;
+ dstTarget->targetFlags = src.targetFlags;
+ dstTarget->floatingPointMode = src.floatingPointMode;
+ }
+ }
+
+ {
+ const auto& srcPaths = requestState->searchPaths;
+ auto& dstPaths = linkage->searchDirectories.searchDirectories;
+ dstPaths.setCount(srcPaths.getCount());
+ for (Index i = 0; i < srcPaths.getCount(); ++i)
+ {
+ dstPaths[i].path = srcPaths[i]->getSlice();
+ }
+ }
+
+ _loadDefines(requestState->preprocessorDefinitions, linkage->preprocessorDefinitions);
+
+ {
+ auto frontEndReq = request->getFrontEndReq();
+
+ const auto& srcTranslationUnits = requestState->translationUnits;
+ auto& dstTranslationUnits = frontEndReq->translationUnits;
+
+ dstTranslationUnits.clear();
+
+ for (Index i = 0; i < srcTranslationUnits.getCount(); ++i)
+ {
+ const auto& srcTranslationUnit = srcTranslationUnits[i];
+
+ int index = frontEndReq->addTranslationUnit(srcTranslationUnit.language);
+ SLANG_UNUSED(index);
+ SLANG_ASSERT(index == i);
+
+ TranslationUnitRequest* dstTranslationUnit = dstTranslationUnits[i];
+
+ _loadDefines(srcTranslationUnit.preprocessorDefinitions, dstTranslationUnit->preprocessorDefinitions);
+
+ Name* moduleName = nullptr;
+ if (srcTranslationUnit.moduleName)
+ {
+ moduleName = request->getNamePool()->getName(srcTranslationUnit.moduleName->getSlice());
+ }
+
+ dstTranslationUnit->moduleName = moduleName;
+
+ const auto& srcSourceFiles = srcTranslationUnit.sourceFiles;
+ auto& dstSourceFiles = dstTranslationUnit->m_sourceFiles;
+
+ dstSourceFiles.clear();
+
+ for (Index j = 0; j < srcSourceFiles.getCount(); ++j)
+ {
+ SourceFile* sourceFile = context.getSourceFile(srcSourceFiles[i]);
+ // Add to translation unit
+ dstTranslationUnit->addSourceFile(sourceFile);
+ }
+ }
+ }
+
+ // Entry points
+ {
+ for (const auto& srcEntryPoint : requestState->entryPoints)
+ {
+ const char* name = srcEntryPoint.name ? srcEntryPoint.name->getCstr() : nullptr;
+
+ Stage stage = srcEntryPoint.profile.GetStage();
+
+ List<const char*> args = context.toList(srcEntryPoint.specializationArgStrings);
+
+ spAddEntryPointEx(externalRequest, int(srcEntryPoint.translationUnitIndex), name, SlangStage(stage), int(args.getCount()), args.getBuffer());
+ }
+ }
+
+ {
+ RefPtr<CacheFileSystem> cacheFileSystem = new CacheFileSystem(nullptr);
+ auto& dstUniqueMap = cacheFileSystem->getUniqueMap();
+ auto& dstPathMap = cacheFileSystem->getPathMap();
+
+ // Put all the paths to path info
+ {
+ for (const auto& pair : requestState->pathInfoMap)
+ {
+ CacheFileSystem::PathInfo* pathInfo = context.addPathInfo(pair.pathInfo);
+ dstPathMap.Add(pair.path->getSlice(), pathInfo);
+ }
+ }
+ // Put all the path infos in the cache system
+ {
+ for (const auto& pair : context.m_pathInfoMap)
+ {
+ CacheFileSystem::PathInfo* pathInfo = pair.Value;
+ SLANG_ASSERT(pathInfo->m_uniqueIdentity);
+ dstUniqueMap.Add(pathInfo->m_uniqueIdentity->getString(), pathInfo);
+ }
+ }
+
+ // This is a bit of a hack, we are going to replace the file system, with our one which is filled in
+ // with what was read from the file.
+
+ linkage->fileSystemExt = cacheFileSystem;
+ linkage->cacheFileSystem = cacheFileSystem;
+ }
+
+ return SLANG_OK;
+}
+
+
+/* static */SlangResult StateSerializeUtil::saveState(EndToEndCompileRequest* request, Stream* stream)
+{
+ RelativeContainer container;
+ Safe32Ptr<RequestState> requestState;
+ SLANG_RETURN_ON_FAIL(store(request, container, requestState));
+ return RiffUtil::writeData(kSlangStateFourCC, container.getData(), container.getDataCount(), stream);
+}
+
+/* static */SlangResult StateSerializeUtil::saveState(EndToEndCompileRequest* request, const String& filename)
+{
+ RefPtr<Stream> stream(new FileStream(filename, FileMode::Create, FileAccess::Write, FileShare::ReadWrite));
+ return saveState(request, stream);
+}
+
+/* static */ SlangResult StateSerializeUtil::loadState(const String& filename, List<uint8_t>& outBuffer)
+{
+ RefPtr<Stream> stream;
+ try
+ {
+ stream = new FileStream(filename, FileMode::Open, FileAccess::Read, FileShare::ReadWrite);
+ }
+ catch (IOException&)
+ {
+ return SLANG_FAIL;
+ }
+
+ return loadState(stream, outBuffer);
+}
+
+/* static */ SlangResult StateSerializeUtil::loadState(Stream* stream, List<uint8_t>& buffer)
+{
+ RiffChunk chunk;
+ SLANG_RETURN_ON_FAIL(RiffUtil::readData(stream, chunk, buffer));
+
+ if (chunk.m_type != kSlangStateFourCC)
+ {
+ return SLANG_FAIL;
+ }
+
+ return SLANG_OK;
+}
+
+/* static */SlangResult StateSerializeUtil::loadState(const uint8_t* data, size_t size, List<uint8_t>& outBuffer)
+{
+ MemoryStream stream(FileAccess::Read);
+
+ stream.m_contents.setCount(size);
+ ::memcpy(stream.m_contents.getBuffer(), data, size);
+
+ return loadState(&stream, outBuffer);
+}
+
+/* static */ StateSerializeUtil::RequestState* StateSerializeUtil::getRequest(const List<uint8_t>& buffer)
+{
+ return (StateSerializeUtil::RequestState*)buffer.getBuffer();
+}
+
+/* static */SlangResult StateSerializeUtil::calcDirectoryPathFromFilename(const String& filename, String& outPath)
+{
+ String absPath;
+ SLANG_RETURN_ON_FAIL(Path::getCanonical(filename, absPath));
+
+ String parentDir = Path::getParentDirectory(absPath);
+
+ String baseName = Path::getFileNameWithoutExt(filename);
+ String ext = Path::getFileExt(filename);
+
+ if (ext.getLength() == 0)
+ {
+ StringBuilder builder;
+ builder << baseName << "-files";
+ baseName = builder;
+ }
+
+ outPath = Path::combine(parentDir, baseName);
+ return SLANG_OK;
+}
+
+/* static */SlangResult StateSerializeUtil::extractFilesToDirectory(const String& filename)
+{
+ List<uint8_t> buffer;
+ SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState(filename, buffer));
+
+ RequestState* requestState = StateSerializeUtil::getRequest(buffer);
+
+ String dirPath;
+ SLANG_RETURN_ON_FAIL(StateSerializeUtil::calcDirectoryPathFromFilename(filename, dirPath));
+
+ Path::createDirectory(dirPath);
+ // Set up a file system to write into this directory
+ RelativeFileSystem relFileSystem(OSFileSystemExt::getSingleton(), dirPath);
+
+ return extractFiles(requestState, &relFileSystem);
+}
+
+/* static */SlangResult StateSerializeUtil::extractFiles(RequestState* requestState, ISlangFileSystemExt* fileSystem)
+{
+ StringBuilder builder;
+
+ builder << "[files]\n";
+
+ for (FileState* file : requestState->files)
+ {
+ if (file->contents)
+ {
+ UnownedStringSlice contents = file->contents->getSlice();
+
+ SLANG_RETURN_ON_FAIL(fileSystem->saveFile(file->uniqueName->getCstr(), contents.begin(), contents.size()));
+
+ RelativeString* originalName = nullptr;
+ if (file->canonicalPath)
+ {
+ originalName = file->canonicalPath;
+ }
+ else if (file->foundPath)
+ {
+ originalName = file->foundPath;
+ }
+ else if (file->uniqueIdentity)
+ {
+ originalName = file->uniqueIdentity;
+ }
+
+ builder << file->uniqueName->getSlice() << " -> ";
+ if (originalName)
+ {
+ builder << originalName->getSlice();
+ }
+ else
+ {
+ builder << "?";
+ }
+ builder << "\n";
+ }
+ }
+
+ builder << "[paths]\n";
+ for (const PathAndPathInfo& path : requestState->pathInfoMap)
+ {
+ builder << path.path->getSlice() << " -> ";
+
+ const auto pathInfo = path.pathInfo.get();
+
+ if (pathInfo->file)
+ {
+ builder << pathInfo->file->uniqueName->getSlice();
+ }
+ else
+ {
+ typedef CacheFileSystem::CompressedResult CompressedResult;
+ if (pathInfo->getPathTypeResult == CompressedResult::Ok)
+ {
+ switch (pathInfo->pathType)
+ {
+ case SLANG_PATH_TYPE_FILE: builder << "file "; break;
+ case SLANG_PATH_TYPE_DIRECTORY: builder << "directory "; break;
+ default: builder << "?"; break;
+ }
+ }
+
+ CompressedResult curRes = pathInfo->getCanonicalPathResult;
+ CompressedResult results[] =
+ {
+ pathInfo->getPathTypeResult,
+ pathInfo->loadFileResult,
+ };
+
+ for (auto compRes : results)
+ {
+ if (int(compRes) > int(curRes))
+ {
+ curRes = compRes;
+ }
+ }
+
+ switch (curRes)
+ {
+ default:
+ case CompressedResult::Uninitialized: break;
+ case CompressedResult::Ok: break;
+
+ case CompressedResult::NotFound: builder << " [not found]"; break;
+ case CompressedResult::CannotOpen: builder << "[cannot open]"; break;
+ case CompressedResult::Fail: builder << "[fail]"; break;
+ }
+
+
+ }
+
+ builder << "\n";
+ }
+
+ SLANG_RETURN_ON_FAIL(fileSystem->saveFile("manifest.txt", builder.getBuffer(), builder.getLength()));
+ return SLANG_OK;
+}
+
+} // namespace Slang
diff --git a/source/slang/slang-state-serialize.h b/source/slang/slang-state-serialize.h
new file mode 100644
index 000000000..f43cc8d45
--- /dev/null
+++ b/source/slang/slang-state-serialize.h
@@ -0,0 +1,176 @@
+// slang-state-serialize.h
+#ifndef SLANG_STATE_SERIALIZE_H_INCLUDED
+#define SLANG_STATE_SERIALIZE_H_INCLUDED
+
+#include "../core/slang-riff.h"
+#include "../core/slang-string.h"
+
+// For TranslationUnitRequest
+#include "slang-compiler.h"
+
+#include "../core/slang-relative-container.h"
+
+#include "slang-file-system.h"
+
+namespace Slang {
+
+struct StateSerializeUtil
+{
+ static const uint32_t kSlangStateFourCC = SLANG_FOUR_CC('S', 'L', 'S', 'T'); ///< Holds all the slang specific chunks
+
+ struct Header
+ {
+ RiffChunk m_chunk;
+ uint32_t m_compressionType; ///< Holds the compression type used (if used at all)
+ };
+
+ struct FileState
+ {
+ Relative32Ptr<RelativeString> uniqueIdentity; ///< The unique identity for the file (from ISlangFileSystem), or nullptr
+ Relative32Ptr<RelativeString> contents; ///< The contents of this file
+ Relative32Ptr<RelativeString> canonicalPath; ///< The canonical name of this file (or nullptr)
+ Relative32Ptr<RelativeString> foundPath; ///< The 'found' path
+
+ Relative32Ptr<RelativeString> uniqueName; ///< A generated unique name (not used by slang, but used as mechanism to replace files)
+ };
+
+ struct PathInfoState
+ {
+ typedef CacheFileSystem::CompressedResult CompressedResult;
+
+ SlangPathType pathType = SLANG_PATH_TYPE_FILE;
+ CompressedResult loadFileResult = CompressedResult::Uninitialized;
+ CompressedResult getPathTypeResult = CompressedResult::Uninitialized;
+ CompressedResult getCanonicalPathResult = CompressedResult::Uninitialized;
+
+ Relative32Ptr<FileState> file; ///< File contents
+ };
+
+ struct PathAndPathInfo
+ {
+ Relative32Ptr<RelativeString> path;
+ Relative32Ptr<PathInfoState> pathInfo;
+ };
+
+ // spSetCodeGenTarget/spAddCodeGenTarget
+ // spSetTargetProfile
+ // spSetTargetFlags
+ // spSetTargetFloatingPointMode
+ // spSetTargetMatrixLayoutMode
+ struct TargetRequestState
+ {
+ Profile profile;
+ CodeGenTarget target;
+ SlangTargetFlags targetFlags;
+ FloatingPointMode floatingPointMode;
+ };
+
+ struct FileReference
+ {
+ Relative32Ptr<RelativeString> name;
+ Relative32Ptr<FileState> file;
+ };
+
+ struct StringPair
+ {
+ Relative32Ptr<RelativeString> first;
+ Relative32Ptr<RelativeString> second;
+ };
+
+ struct SourceFileState
+ {
+ PathInfo::Type type; ///< The type of this file
+ Relative32Ptr<RelativeString> foundPath; ///< The Path this was found along
+ Relative32Ptr<FileState> file; ///< The file contents
+ };
+
+ // spAddTranslationUnit
+ struct TranslationUnitRequestState
+ {
+ SourceLanguage language;
+
+ Relative32Ptr<RelativeString> moduleName;
+
+ // spTranslationUnit_addPreprocessorDefine
+ Relative32Array<StringPair> preprocessorDefinitions;
+
+ Relative32Array<Relative32Ptr<SourceFileState> > sourceFiles;
+ };
+
+ struct EntryPointState
+ {
+ Relative32Ptr<RelativeString> name;
+ Profile profile;
+ uint32_t translationUnitIndex;
+ Relative32Array<Relative32Ptr<RelativeString>> specializationArgStrings;
+ };
+
+ struct RequestState
+ {
+ Relative32Array<Relative32Ptr<FileState>> files; ///< All of the files
+ Relative32Array<Relative32Ptr<SourceFileState>> sourceFiles; ///< All of the source files (from source manager)
+
+ // spSetCompileFlags
+ SlangCompileFlags compileFlags;
+ // spSetDumpIntermediates
+ bool shouldDumpIntermediates;
+ // spSetLineDirectiveMode
+ LineDirectiveMode lineDirectiveMode;
+
+ Relative32Array<TargetRequestState> targetRequests;
+
+ // spSetDebugInfoLevel
+ DebugInfoLevel debugInfoLevel;
+ // spSetOptimizationLevel
+ OptimizationLevel optimizationLevel;
+ // spSetOutputContainerFormat
+ ContainerFormat containerFormat;
+ // spSetPassThrough
+ PassThroughMode passThroughMode;
+
+ // spAddSearchPath
+ Relative32Array<Relative32Ptr<RelativeString> > searchPaths;
+
+ // spAddPreprocessorDefine
+ Relative32Array<StringPair> preprocessorDefinitions;
+
+ bool useUnknownImageFormatAsDefault = false;
+ bool obfuscateCode = false;
+
+ Relative32Array<PathAndPathInfo> pathInfoMap; ///< Stores all the accesses to the file system
+
+ Relative32Array<TranslationUnitRequestState> translationUnits;
+
+ Relative32Array<EntryPointState> entryPoints;
+
+ SlangMatrixLayoutMode defaultMatrixLayoutMode;
+ };
+
+ static SlangResult store(EndToEndCompileRequest* request, RelativeContainer& inOutContainer, Safe32Ptr<RequestState>& outRequest);
+
+ static SlangResult saveState(EndToEndCompileRequest* request, const String& filename);
+
+ static SlangResult saveState(EndToEndCompileRequest* request, Stream* stream);
+
+ /// Load the requestState into request
+ /// The fileSystem is optional and can be passed as nullptr. If set, as each file is loaded
+ /// it will attempt to load from fileSystem the *uniqueName*
+ static SlangResult load(RequestState* requestState, ISlangFileSystem* fileSystem, EndToEndCompileRequest* request);
+
+ static SlangResult loadState(const String& filename, List<uint8_t>& outBuffer);
+ static SlangResult loadState(Stream* stream, List<uint8_t>& outBuffer);
+ static SlangResult loadState(const uint8_t* data, size_t size, List<uint8_t>& outBuffer);
+
+ static RequestState* getRequest(const List<uint8_t>& inBuffer);
+
+ static SlangResult extractFilesToDirectory(const String& file);
+
+ static SlangResult extractFiles(RequestState* requestState, ISlangFileSystemExt* fileSystem);
+
+ /// Given the repo file work out a suitable path
+ static SlangResult calcDirectoryPathFromFilename(const String& filename, String& outPath);
+};
+
+} // namespace Slang
+
+#endif
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index 4ae3f4654..19ac6ea1f 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -14,6 +14,8 @@
#include "slang-syntax-visitors.h"
#include "slang-type-layout.h"
+#include "slang-state-serialize.h"
+
#include "slang-file-system.h"
#include "../core/slang-writer.h"
@@ -676,6 +678,10 @@ ComPtr<ISlangBlob> createRawBlob(void const* inData, size_t size)
return ComPtr<ISlangBlob>(new RawBlob(inData, size));
}
+ISlangUnknown* ListBlob::getInterface(const Guid& guid)
+{
+ return (guid == IID_ISlangUnknown || guid == IID_ISlangBlob) ? static_cast<ISlangBlob*>(this) : nullptr;
+}
//
// TargetRequest
//
@@ -790,9 +796,21 @@ void EndToEndCompileRequest::setWriter(WriterChannel chan, ISlangWriter* writer)
}
}
-SlangResult Linkage::loadFile(String const& path, ISlangBlob** outBlob)
+SlangResult Linkage::loadFile(String const& path, PathInfo& outPathInfo, ISlangBlob** outBlob)
{
- return fileSystemExt->loadFile(path.getBuffer(), outBlob);
+ outPathInfo.type = PathInfo::Type::Unknown;
+
+ SLANG_RETURN_ON_FAIL(fileSystemExt->loadFile(path.getBuffer(), outBlob));
+
+ ComPtr<ISlangBlob> uniqueIdentity;
+ // Get the unique identity
+ SLANG_RETURN_ON_FAIL(fileSystemExt->getFileUniqueIdentity(path.getBuffer(), uniqueIdentity.writeRef()));
+
+ outPathInfo.foundPath = path;
+ outPathInfo.type = PathInfo::Type::FoundPath;
+ outPathInfo.uniqueIdentity = StringUtil::getString(uniqueIdentity);
+
+ return SLANG_OK;
}
RefPtr<Expr> Linkage::parseTypeString(String typeStr, RefPtr<Scope> scope)
@@ -1097,6 +1115,7 @@ SlangResult FrontEndCompileRequest::executeActionsInner()
{
parseTranslationUnit(translationUnit.Ptr());
}
+
if (getSink()->GetErrorCount() != 0)
return SLANG_FAIL;
@@ -1375,8 +1394,10 @@ void FrontEndCompileRequest::addTranslationUnitSourceFile(
// paths were not taken into account by this function.
//
+ PathInfo pathInfo;
+
ComPtr<ISlangBlob> sourceBlob;
- SlangResult result = loadFile(path, sourceBlob.writeRef());
+ SlangResult result = loadFile(path, pathInfo, sourceBlob.writeRef());
if(SLANG_FAILED(result))
{
// Emit a diagnostic!
@@ -1388,7 +1409,6 @@ void FrontEndCompileRequest::addTranslationUnitSourceFile(
}
// Was loaded from the specified path
- const auto pathInfo = PathInfo::makePath(path);
SourceFile* sourceFile = getSourceManager()->createSourceFileWithBlob(pathInfo, sourceBlob);
addTranslationUnitSourceFile(translationUnitIndex, sourceFile);
}
@@ -2464,27 +2484,52 @@ Session* CompileRequestBase::getSession()
}
static const Slang::Guid IID_ISlangFileSystemExt = SLANG_UUID_ISlangFileSystemExt;
+static const Slang::Guid IID_SlangCacheFileSystem = SLANG_UUID_CacheFileSystem;
void Linkage::setFileSystem(ISlangFileSystem* inFileSystem)
{
// Set the fileSystem
fileSystem = inFileSystem;
+ // Release what's there
+ fileSystemExt.setNull();
+ cacheFileSystem.setNull();
+
// If nullptr passed in set up default
if (inFileSystem == nullptr)
{
- fileSystemExt = new Slang::CacheFileSystem(Slang::OSFileSystemExt::getSingleton());
+ cacheFileSystem = new Slang::CacheFileSystem(Slang::OSFileSystemExt::getSingleton());
+ fileSystemExt = cacheFileSystem;
}
else
{
- // See if we have the full ISlangFileSystemExt interface, if we do just use it
- inFileSystem->queryInterface(IID_ISlangFileSystemExt, (void**)fileSystemExt.writeRef());
-
- // If not wrap with CacheFileSystem that emulates ISlangFileSystemExt from the ISlangFileSystem interface
- if (!fileSystemExt)
+ CacheFileSystem* cacheFileSystemPtr = nullptr;
+ inFileSystem->queryInterface(IID_SlangCacheFileSystem, (void**)&cacheFileSystemPtr);
+ if (cacheFileSystemPtr)
+ {
+ cacheFileSystem = cacheFileSystemPtr;
+ fileSystemExt = cacheFileSystemPtr;
+ }
+ else
{
- // Construct a wrapper to emulate the extended interface behavior
- fileSystemExt = new Slang::CacheFileSystem(fileSystem);
+ if (m_requireCacheFileSystem)
+ {
+ cacheFileSystem = new Slang::CacheFileSystem(inFileSystem);
+ fileSystemExt = cacheFileSystem;
+ }
+ else
+ {
+ // See if we have the full ISlangFileSystemExt interface, if we do just use it
+ inFileSystem->queryInterface(IID_ISlangFileSystemExt, (void**)fileSystemExt.writeRef());
+
+ // If not wrap with CacheFileSystem that emulates ISlangFileSystemExt from the ISlangFileSystem interface
+ if (!fileSystemExt)
+ {
+ // Construct a wrapper to emulate the extended interface behavior
+ cacheFileSystem = new Slang::CacheFileSystem(fileSystem);
+ fileSystemExt = cacheFileSystem;
+ }
+ }
}
}
@@ -2492,6 +2537,19 @@ void Linkage::setFileSystem(ISlangFileSystem* inFileSystem)
getSourceManager()->setFileSystemExt(fileSystemExt);
}
+void Linkage::setRequireCacheFileSystem(bool requireCacheFileSystem)
+{
+ if (requireCacheFileSystem == m_requireCacheFileSystem)
+ {
+ return;
+ }
+
+ ComPtr<ISlangFileSystem> scopeFileSystem(fileSystem);
+ m_requireCacheFileSystem = requireCacheFileSystem;
+
+ setFileSystem(scopeFileSystem);
+}
+
RefPtr<Module> findOrImportModule(
Linkage* linkage,
Name* name,
@@ -3158,6 +3216,8 @@ SLANG_API SlangResult spCompile(
{
auto req = Slang::asInternal(request);
+ SlangResult res = SLANG_FAIL;
+
#if !defined(SLANG_DEBUG_INTERNAL_ERROR)
// By default we'd like to catch as many internal errors as possible,
// and report them to the user nicely (rather than just crash their
@@ -3168,7 +3228,7 @@ SLANG_API SlangResult spCompile(
//
// TODO: Consider supporting Windows "Structured Exception Handling"
// so that we can also recover from a wider class of crashes.
- SlangResult res = SLANG_FAIL;
+
try
{
res = req->executeActions();
@@ -3196,14 +3256,21 @@ SLANG_API SlangResult spCompile(
req->getSink()->diagnose(Slang::SourceLoc(), Slang::Diagnostics::compilationAborted);
}
req->mDiagnosticOutput = req->getSink()->outputBuffer.ProduceString();
- return res;
+
#else
// When debugging, we probably don't want to filter out any errors, since
// we are probably trying to root-cause and *fix* those errors.
{
- return req->executeActions();
+ res = req->executeActions();
}
#endif
+
+ if (req->dumpRepro.getLength())
+ {
+ SLANG_RETURN_ON_FAIL(Slang::StateSerializeUtil::saveState(req, req->dumpRepro));
+ }
+
+ return res;
}
SLANG_API int
@@ -3386,6 +3453,42 @@ SLANG_API void const* spGetCompileRequestCode(
return nullptr;
}
+SLANG_API SlangResult spLoadRepro(
+ SlangCompileRequest* inRequest,
+ ISlangFileSystem* fileSystem,
+ const void* data,
+ size_t size)
+{
+ using namespace Slang;
+ auto request = asInternal(inRequest);
+
+ List<uint8_t> buffer;
+ SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState((const uint8_t*)data, size, buffer));
+
+ StateSerializeUtil::RequestState* requestState = StateSerializeUtil::getRequest(buffer);
+
+ SLANG_RETURN_ON_FAIL(StateSerializeUtil::load(requestState, fileSystem, request));
+ return SLANG_OK;
+}
+
+SLANG_API SlangResult spSaveRepro(
+ SlangCompileRequest* inRequest,
+ ISlangBlob** outBlob)
+{
+ using namespace Slang;
+ auto request = asInternal(inRequest);
+
+ MemoryStream stream(FileAccess::Write);
+
+ SLANG_RETURN_ON_FAIL(StateSerializeUtil::saveState(request, &stream));
+
+ RefPtr<ListBlob> listBlob(new ListBlob);
+ listBlob->m_data.swapWith(stream.m_contents);
+
+ *outBlob = listBlob.detach();
+ return SLANG_OK;
+}
+
// Reflection API
SLANG_API SlangResult spCompileRequest_getProgram(
diff --git a/source/slang/slang.vcxproj b/source/slang/slang.vcxproj
index 251b46b3e..36f28705f 100644
--- a/source/slang/slang.vcxproj
+++ b/source/slang/slang.vcxproj
@@ -228,6 +228,7 @@
<ClInclude Include="slang-profile.h" />
<ClInclude Include="slang-reflection.h" />
<ClInclude Include="slang-source-loc.h" />
+ <ClInclude Include="slang-state-serialize.h" />
<ClInclude Include="slang-stmt-defs.h" />
<ClInclude Include="slang-syntax-base-defs.h" />
<ClInclude Include="slang-syntax-defs.h" />
@@ -289,6 +290,7 @@
<ClCompile Include="slang-profile.cpp" />
<ClCompile Include="slang-reflection.cpp" />
<ClCompile Include="slang-source-loc.cpp" />
+ <ClCompile Include="slang-state-serialize.cpp" />
<ClCompile Include="slang-stdlib.cpp" />
<ClCompile Include="slang-syntax.cpp" />
<ClCompile Include="slang-token.cpp" />
diff --git a/source/slang/slang.vcxproj.filters b/source/slang/slang.vcxproj.filters
index 3dd2c4a49..474f4c2bb 100644
--- a/source/slang/slang.vcxproj.filters
+++ b/source/slang/slang.vcxproj.filters
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Header Files">
@@ -183,6 +183,9 @@
<ClInclude Include="slang-source-loc.h">
<Filter>Header Files</Filter>
</ClInclude>
+ <ClInclude Include="slang-state-serialize.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
<ClInclude Include="slang-stmt-defs.h">
<Filter>Header Files</Filter>
</ClInclude>
@@ -362,6 +365,9 @@
<ClCompile Include="slang-source-loc.cpp">
<Filter>Source Files</Filter>
</ClCompile>
+ <ClCompile Include="slang-state-serialize.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
<ClCompile Include="slang-stdlib.cpp">
<Filter>Source Files</Filter>
</ClCompile>