Feature/riff improvements (#1099)

* Added RiffReadHelper

* Move type to fourCC in Chunk simplifies some code.

* Make MemoryArena able to track external blocks.
Allow ownership of Data to vary.
Changed IR serialization to use moved allocations to avoid copies.

As it turns out all of the array writes could use unowned data, but doing so requires the IRData to stay in scope longer than IRSerialData, which it does at the moment - but perhaps needs better naming or a control for the feature.

1 files changed, 25 insertions, 53 deletions

diff --git a/source/slang/slang-ir-serialize.cpp b/source/slang/slang-ir-serialize.cpp
index ebbb792f4..97b070f9c 100644
--- a/source/slang/slang-ir-serialize.cpp
+++ b/source/slang/slang-ir-serialize.cpp
@@ -827,7 +827,9 @@ static Result _writeArrayChunk(IRSerialBinary::CompressionType compressionType,
             header.m_numCompressedEntries = uint32_t(numCompressedEntries);
 
             container->write(&header, sizeof(header));
-            container->write(compressedPayload.begin(), compressedPayload.getCount());
+
+            const size_t compressedSize = compressedPayload.getCount();
+            container->moveOwned(container->addData(), compressedPayload.detachBuffer(), compressedSize);
             break;
         }
         default:
@@ -963,7 +965,10 @@ Result _writeInstArrayChunk(IRSerialBinary::CompressionType compressionType, uin
             header.m_numCompressedEntries = 0;          
 
             container->write(&header, sizeof(header));
-            container->write(compressedPayload.begin(), compressedPayload.getCount());
+
+            const size_t compressedPayloadSize = compressedPayload.getCount();
+            container->moveOwned(container->addData(), compressedPayload.detachBuffer(), compressedPayloadSize);
+
             return SLANG_OK;
         }
         default: break;
@@ -1050,56 +1055,32 @@ static Result _readArrayChunk(IRSerialBinary::CompressionType compressionType, R
 {
     typedef IRSerialBinary Bin;
 
-    RiffContainer::Data* data = dataChunk->getSingleData();
-    const uint8_t* cur = (const uint8_t*)data->getPayload();
-    const uint8_t* end = cur + data->m_size;
-
+    RiffReadHelper read = dataChunk->asReadHelper();
     const size_t typeSize = listOut.getTypeSize();
 
     switch (compressionType)
     {
         case Bin::CompressionType::VariableByteLite:
         {
-            if (cur + sizeof(Bin::CompressedArrayHeader) > end)
-            {
-                return SLANG_FAIL;
-            }
+            Bin::CompressedArrayHeader header;
+            SLANG_RETURN_ON_FAIL(read.read(header));
 
-            // We have a compressed header
-            Bin::CompressedArrayHeader header = *(const Bin::CompressedArrayHeader*)cur;
-            cur += sizeof(Bin::CompressedArrayHeader);
-            
             void* dst = listOut.setSize(header.m_numEntries);
-            
-            // Need to read all the compressed data... 
-            size_t payloadSize = size_t(end - cur);
-
-            List<uint8_t> compressedPayload;
-            compressedPayload.setCount(payloadSize);
-
             SLANG_ASSERT(header.m_numCompressedEntries == uint32_t((header.m_numEntries * typeSize) / sizeof(uint32_t)));
 
             // Decode..
-            ByteEncodeUtil::decodeLiteUInt32(cur, header.m_numCompressedEntries, (uint32_t*)dst);
+            ByteEncodeUtil::decodeLiteUInt32(read.getData(), header.m_numCompressedEntries, (uint32_t*)dst);
             break;
         }
         case Bin::CompressionType::None:
         {
             // Read uncompressed
-            if (cur + sizeof(Bin::ArrayHeader) > end)
-            {
-                return SLANG_FAIL;
-            }
-
-            Bin::ArrayHeader header = *(const Bin::ArrayHeader*)cur;
-            cur += sizeof(Bin::ArrayHeader);
-
+            Bin::ArrayHeader header;
+            SLANG_RETURN_ON_FAIL(read.read(header));
             const size_t payloadSize = header.m_numEntries * typeSize;
-
-            SLANG_ASSERT(payloadSize == size_t(end - cur));
+            SLANG_ASSERT(payloadSize == read.getRemainingSize());
             void* dst = listOut.setSize(header.m_numEntries);
-
-            ::memcpy(dst, cur, payloadSize);
+            ::memcpy(dst, read.getData(), payloadSize);
             break;
         }
     }
@@ -1113,7 +1094,7 @@ static Result _readArrayChunk(const IRSerialBinary::ModuleHeader* header, RiffCo
 
     Bin::CompressionType compressionType = Bin::CompressionType::None;
 
-    if (dataChunk->m_type == SLANG_MAKE_COMPRESSED_FOUR_CC(dataChunk->m_type))
+    if (dataChunk->m_fourCC == SLANG_MAKE_COMPRESSED_FOUR_CC(dataChunk->m_fourCC))
     {
         // If it has compression, use the compression type set in the header
         compressionType = Bin::CompressionType(header->m_compressionType);
@@ -1125,10 +1106,9 @@ static Result _readArrayChunk(const IRSerialBinary::ModuleHeader* header, RiffCo
 template <typename T>
 static Result _readArrayUncompressedChunk(const IRSerialBinary::ModuleHeader* header, RiffContainer::DataChunk* chunk, List<T>& arrayOut)
 {
-    typedef IRSerialBinary Bin;
     SLANG_UNUSED(header);
     ListResizerForType<T> resizer(arrayOut);
-    return _readArrayChunk(Bin::CompressionType::None, chunk, resizer);
+    return _readArrayChunk(IRSerialBinary::CompressionType::None, chunk, resizer);
 }
 
 static Result _decodeInsts(IRSerialBinary::CompressionType compressionType, const uint8_t* encodeCur, size_t encodeInSize, List<IRSerialData::Inst>& instsOut)
@@ -1209,7 +1189,7 @@ static Result _readInstArrayChunk(const IRSerialBinary::ModuleHeader* moduleHead
     typedef IRSerialBinary Bin;
 
     Bin::CompressionType compressionType = Bin::CompressionType::None;
-    if (chunk->m_type == SLANG_MAKE_COMPRESSED_FOUR_CC(chunk->m_type))
+    if (chunk->m_fourCC == SLANG_MAKE_COMPRESSED_FOUR_CC(chunk->m_fourCC))
     {
         compressionType = Bin::CompressionType(moduleHeader->m_compressionType);
     }
@@ -1223,22 +1203,14 @@ static Result _readInstArrayChunk(const IRSerialBinary::ModuleHeader* moduleHead
         }
         case Bin::CompressionType::VariableByteLite:
         {
-            RiffContainer::Data* data = chunk->getSingleData();
-            const uint8_t* cur = (const uint8_t*)data->getPayload();
-            const uint8_t* end = cur + data->m_size;
-            if (cur + sizeof(Bin::CompressedArrayHeader) <= end)
-            {
-                return SLANG_FAIL;
-            }
-            Bin::CompressedArrayHeader header = *(const Bin::CompressedArrayHeader*)cur;
-            cur += sizeof(Bin::CompressedArrayHeader);
-            
-            arrayOut.setCount(header.m_numEntries);
+            RiffReadHelper read = chunk->asReadHelper();
 
-            // Need to read all the compressed data... 
-            size_t payloadSize = size_t(end - cur);
+            Bin::CompressedArrayHeader header;
+            SLANG_RETURN_ON_FAIL(read.read(header));
+
+            arrayOut.setCount(header.m_numEntries);
 
-            SLANG_RETURN_ON_FAIL(_decodeInsts(compressionType, cur, payloadSize, arrayOut));
+            SLANG_RETURN_ON_FAIL(_decodeInsts(compressionType, read.getData(), read.getRemainingSize(), arrayOut));
             break;
         }
         default:
@@ -1270,7 +1242,7 @@ static Result _readInstArrayChunk(const IRSerialBinary::ModuleHeader* moduleHead
             continue;
         }
         
-        switch (dataChunk->m_type)
+        switch (dataChunk->m_fourCC)
         {
             case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kInstFourCc):
             case Bin::kInstFourCc: