Feature/ir serial debug (#657)

* * Change the layout of IROp such that 'main' IROps are 0-x.
* Removed MANUAL_RANGE instuction types, as no longer needed.

* Work in prog on optimizing.

* * Constant time lookup for IROpInfo
* Refactor and document a little more the IROp layout
* Mark ops that use 'other' bits

* Fix typo in definition of kIROpFlag_UseOther

* First pass at working out serialization structure.

* Work in progress on ir-serialize

* Storing strings in IRSerialInfo
Split out IRSerialInfo from the IRSerializer - to make more explicit what is actually saved.

* First pass at serializing out data.

* First pass at serialize reading.

* Fix riff fourcc mark order.

* First pass at reconstructing IRInst / IRDecoration from serialized data.

* Handling of TextureBaseType

* Deserializing of constants.

* Small changes around ir serialization.

* Changed StringIndex indexing to not be an offset into the m_strings array, but an index into strings in order. Doing so makes cache lookup much faster, and makes the 'indicies' themselves smaller and therefore more compressible.

* Removed the need for m_arena in IRSerialWriter. Previously it's purpose was to store the string contents that were being used to lookup UnownedStringSlice.
Now we keep the StringRepresentation in scope and reference that, and so don't need the copy.

* Don't need to construct the IRModuleInst as is created and set on createModule call.

* Remove test code for testing serialization.

* Fix problem with release build in ir-serialize causing warning.

* Use SLANG_OFFSET_OF for offsets in non pod classes to avoid gcc/clang warning.
Give storage to integral static variables to avoid linkage problems with gcc/clang.

* Fix warnings under x86 win32 debug.

* Small improvements around IR serialization.

* * Support for serializing SourceLoc.
* Small improvements around serialization.

* RawSourceLoc allows for regular SourceLoc information to be held (and serialized) as is.
This is only really useful for the 'passthru' mode as there needs to be a more compact mechanism to encode source locations.

* Small fixes around comments for SourceLoc serializing.

1 files changed, 60 insertions, 46 deletions

diff --git a/source/slang/ir-serialize.h b/source/slang/ir-serialize.h
index 31e8aa6ee..356e14055 100644
--- a/source/slang/ir-serialize.h
+++ b/source/slang/ir-serialize.h
@@ -20,7 +20,8 @@ struct IRSerialData
     enum class InstIndex : uint32_t;
     enum class StringIndex : uint32_t;
     enum class ArrayIndex : uint32_t;
-    enum class SourceLoc : uint32_t;
+
+    enum class RawSourceLoc : SourceLoc::RawValue;             ///< This is just to copy over source loc data (ie not strictly serialize)
     enum class StringOffset : uint32_t;             ///< Offset into the m_stringsBuffer
     
     typedef uint32_t SizeType;
@@ -28,11 +29,6 @@ struct IRSerialData
     static const StringIndex kNullStringIndex = StringIndex(0);
     static const StringIndex kEmptyStringIndex = StringIndex(1);
 
-    enum
-    {
-        kNumOperands = 2,
-    };
-
     /// A run of instructions
     struct InstRun
     {
@@ -53,12 +49,19 @@ struct IRSerialData
     // Decoration information is stored in m_decorationRuns
     struct Inst
     {
-        // NOTE! Can't change order or list without changing approprite s_payloadInfos
+        enum
+        {
+            kMaxOperands = 2,           ///< Maximum number of operands that can be held in an instruction (otherwise held 'externally')
+        };
+
+        // NOTE! Can't change order or list without changing appropriate s_payloadInfos
         enum class PayloadType : uint8_t
         {
+            // First 3 must be in this order so a cast from 0-2 is directly represented as number of operands 
             Empty,                          ///< Has no payload (or operands)
             Operand_1,                      ///< 1 Operand
             Operand_2,                      ///< 2 Operands
+
             OperandAndUInt32,               ///< 1 Operand and a single UInt32
             OperandExternal,                ///< Operands are held externally
             String_1,                       ///< 1 String
@@ -71,10 +74,7 @@ struct IRSerialData
         };
 
             /// Get the number of operands
-        SLANG_FORCE_INLINE int getNumOperands() const 
-        {
-            return (m_payloadType == PayloadType::OperandExternal) ? m_payload.m_externalOperand.m_size : s_payloadInfos[int(m_payloadType)].m_numOperands;
-        }
+        SLANG_FORCE_INLINE int getNumOperands() const;
 
         uint8_t m_op;                       ///< For now one of IROp 
         PayloadType m_payloadType;	 		///< The type of payload 
@@ -101,8 +101,8 @@ struct IRSerialData
             uint32_t m_uint32;                              ///< Unsigned integral value
             IRFloatingPointValue m_float;              ///< Floating point value
             IRIntegerValue m_int;                      ///< Integral value
-            InstIndex m_operands[kNumOperands];	            ///< For items that 2 or less operands it can use this.  
-            StringIndex m_stringIndices[kNumOperands];
+            InstIndex m_operands[kMaxOperands];	            ///< For items that 2 or less operands it can use this.  
+            StringIndex m_stringIndices[kMaxOperands];
             ExternalOperandPayload m_externalOperand;              ///< Operands are stored in an an index of an operand array 
             OperandAndUInt32 m_operandAndUInt32;
         };
@@ -110,44 +110,19 @@ struct IRSerialData
         Payload m_payload;
     };
 
-    /// Clear to initial state
-    void clear()
-    {
-        // First Instruction is null
-        m_insts.SetSize(1);
-        memset(&m_insts[0], 0, sizeof(Inst));
-
-        m_childRuns.Clear();
-        m_decorationRuns.Clear();
-        m_externalOperands.Clear();
-
-        m_strings.SetSize(2);
-        m_strings[int(kNullStringIndex)] = 0;
-        m_strings[int(kEmptyStringIndex)] = 0;
-
-        m_decorationBaseIndex = 0;
-    }
+        /// Clear to initial state
+    void clear();
+        /// Get the operands of an instruction
+    SLANG_FORCE_INLINE int getOperands(const Inst& inst, const InstIndex** operandsOut) const;
 
-    SLANG_FORCE_INLINE int getOperands(const Inst& inst, const InstIndex** operandsOut) const
-    {
-        if (inst.m_payloadType == Inst::PayloadType::OperandExternal)
-        {
-            *operandsOut = m_externalOperands.begin() + int(inst.m_payload.m_externalOperand.m_arrayIndex);
-            return int(inst.m_payload.m_externalOperand.m_size);
-        }
-        else
-        {
-            *operandsOut = inst.m_payload.m_operands;
-            return s_payloadInfos[int(inst.m_payloadType)].m_numOperands;
-        }
-    }
+        /// Calculate the amount of memory used by this IRSerialData
+    size_t calcSizeInBytes() const;
 
     /// Ctor
     IRSerialData() :
         m_decorationBaseIndex(0)
     {}
 
-
     List<Inst> m_insts;                         ///< The instructions
 
     List<InstRun> m_childRuns;                  ///< Holds the information about children that belong to an instruction
@@ -157,11 +132,35 @@ struct IRSerialData
 
     List<char> m_strings;                       ///< All strings. Indexed into by StringIndex
 
+    List<RawSourceLoc> m_rawSourceLocs;         ///< A source location per instruction (saved without modification from IRInst)s
+
     static const PayloadInfo s_payloadInfos[int(Inst::PayloadType::CountOf)];
     
     int m_decorationBaseIndex;                  ///< All decorations insts are at indices >= to this value
 };
 
+// --------------------------------------------------------------------------
+SLANG_FORCE_INLINE int IRSerialData::Inst::getNumOperands() const
+{
+    return (m_payloadType == PayloadType::OperandExternal) ? m_payload.m_externalOperand.m_size : s_payloadInfos[int(m_payloadType)].m_numOperands;
+}
+
+// --------------------------------------------------------------------------
+SLANG_FORCE_INLINE int IRSerialData::getOperands(const Inst& inst, const InstIndex** operandsOut) const
+{
+    if (inst.m_payloadType == Inst::PayloadType::OperandExternal)
+    {
+        *operandsOut = m_externalOperands.begin() + int(inst.m_payload.m_externalOperand.m_arrayIndex);
+        return int(inst.m_payload.m_externalOperand.m_size);
+    }
+    else
+    {
+        *operandsOut = inst.m_payload.m_operands;
+        return s_payloadInfos[int(inst.m_payloadType)].m_numOperands;
+    }
+}
+
+
 #define SLANG_FOUR_CC(c0, c1, c2, c3) ((uint32_t(c0) << 0) | (uint32_t(c1) << 8) | (uint32_t(c2) << 16) | (uint32_t(c3) << 24)) 
 
 struct IRSerialBinary
@@ -182,6 +181,10 @@ struct IRSerialBinary
     static const uint32_t kChildRunFourCc = SLANG_FOUR_CC('S', 'L', 'c', 'r');
     static const uint32_t kExternalOperandsFourCc = SLANG_FOUR_CC('S', 'L', 'e', 'o');
     static const uint32_t kStringFourCc = SLANG_FOUR_CC('S', 'L', 's', 't');
+        /// 4 bytes per entry
+    static const uint32_t kUInt32SourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '4');
+        /// 8 bytes per entry
+    static const uint32_t kUInt64SourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '8');
 
     struct SlangHeader
     {
@@ -200,7 +203,17 @@ struct IRSerialWriter
 {
     typedef IRSerialData Ser;
 
-    Result write(IRModule* module, IRSerialData* serialData);
+    struct OptionFlag
+    {
+        typedef uint32_t Type;
+        enum Enum: Type
+        {
+            RawSourceLocation = 1,
+        };
+    };
+    typedef OptionFlag::Type OptionFlags;
+
+    Result write(IRModule* module, SourceManager* sourceManager, OptionFlags options, IRSerialData* serialData);
 
     static Result writeStream(const IRSerialData& data, Stream* stream);
 
@@ -268,6 +281,7 @@ struct IRSerialReader
 
     void _calcStringStarts();
     IRDecoration* _createDecoration(const Ser::Inst& srcIns);
+    static Result _skip(const IRSerialBinary::Chunk& chunk, Stream* stream, int64_t* remainingBytesInOut);
 
     List<Ser::StringOffset> m_stringStarts;
     List<StringRepresentation*> m_stringRepresentationCache;
@@ -277,7 +291,7 @@ struct IRSerialReader
 };
 
 
-Result serializeModule(IRModule* module, Stream* stream);
+Result serializeModule(IRModule* module, SourceManager* sourceManager, Stream* stream);
 Result readModule(Session* session, Stream* stream, RefPtr<IRModule>& moduleOut);
 
 } // namespace Slang