From b9cddcb9c718f986ee5e4f7c6189ee2ebea4ace1 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Thu, 17 Sep 2020 16:47:57 -0400 Subject: Share debug information between AST and IR (#1547) * Test if blob is returned. * Rename serialize files so can be grouped. * StringRepresentationCache -> SerialStringTable * Split out SerialStringTable from slang-serialize-ir * First pass at reorganizing serialization/containers. Remain some issues about debug info. * Fix bug in calculating sourceloc. * Improve calcFixSourceLoc * Make allocations for payload RiffContainer align to at least 8 bytes. This is important for read, if the payload can contain 8 byte aligned data. Note this has no effect on Riff file format alignment rules. * Improve comments around RiffContainer and alignment. * Remove SerialStringTable, can just use StringSlicePool instead. * Typo fix for Clang/Linux. Co-authored-by: Tim Foley --- source/slang/slang-ast-serialize.cpp | 2059 ---------------------------- source/slang/slang-ast-serialize.h | 598 -------- source/slang/slang-compiler.cpp | 115 +- source/slang/slang-compiler.h | 6 +- source/slang/slang-ir-serialize-types.cpp | 333 ----- source/slang/slang-ir-serialize-types.h | 418 ------ source/slang/slang-ir-serialize.cpp | 1589 --------------------- source/slang/slang-ir-serialize.h | 163 --- source/slang/slang-options.cpp | 4 +- source/slang/slang-serialize-ast.cpp | 2008 +++++++++++++++++++++++++++ source/slang/slang-serialize-ast.h | 601 ++++++++ source/slang/slang-serialize-container.cpp | 553 ++++++++ source/slang/slang-serialize-container.h | 107 ++ source/slang/slang-serialize-debug.cpp | 404 ++++++ source/slang/slang-serialize-debug.h | 225 +++ source/slang/slang-serialize-ir-types.cpp | 102 ++ source/slang/slang-serialize-ir-types.h | 258 ++++ source/slang/slang-serialize-ir.cpp | 910 ++++++++++++ source/slang/slang-serialize-ir.h | 97 ++ source/slang/slang-serialize-types.cpp | 249 ++++ source/slang/slang-serialize-types.h | 230 ++++ source/slang/slang.cpp | 74 +- source/slang/slang.vcxproj | 18 +- source/slang/slang.vcxproj.filters | 54 +- 24 files changed, 5866 insertions(+), 5309 deletions(-) delete mode 100644 source/slang/slang-ast-serialize.cpp delete mode 100644 source/slang/slang-ast-serialize.h delete mode 100644 source/slang/slang-ir-serialize-types.cpp delete mode 100644 source/slang/slang-ir-serialize-types.h delete mode 100644 source/slang/slang-ir-serialize.cpp delete mode 100644 source/slang/slang-ir-serialize.h create mode 100644 source/slang/slang-serialize-ast.cpp create mode 100644 source/slang/slang-serialize-ast.h create mode 100644 source/slang/slang-serialize-container.cpp create mode 100644 source/slang/slang-serialize-container.h create mode 100644 source/slang/slang-serialize-debug.cpp create mode 100644 source/slang/slang-serialize-debug.h create mode 100644 source/slang/slang-serialize-ir-types.cpp create mode 100644 source/slang/slang-serialize-ir-types.h create mode 100644 source/slang/slang-serialize-ir.cpp create mode 100644 source/slang/slang-serialize-ir.h create mode 100644 source/slang/slang-serialize-types.cpp create mode 100644 source/slang/slang-serialize-types.h (limited to 'source/slang') diff --git a/source/slang/slang-ast-serialize.cpp b/source/slang/slang-ast-serialize.cpp deleted file mode 100644 index 22d5c3de2..000000000 --- a/source/slang/slang-ast-serialize.cpp +++ /dev/null @@ -1,2059 +0,0 @@ -// slang-ast-serialize.cpp -#include "slang-ast-serialize.h" - -#include "slang-ast-generated.h" -#include "slang-ast-generated-macro.h" - -#include "slang-compiler.h" -#include "slang-type-layout.h" - -#include "slang-ast-dump.h" -#include "slang-mangle.h" - -#include "slang-ast-support-types.h" - -#include "slang-legalize-types.h" - -#include "../core/slang-byte-encode-util.h" - -namespace Slang { - - -// Things stored as references: -// -// NodeBase derived types -// Array -// -// RefObject derived types: -// -// Breadcrumb -// StringRepresentation -// Scope - - - -// Helpers to convert fields treated as values - -class ASTSerialReader; -class ASTSerialWriter; - -template -static void _toSerialValue(ASTSerialWriter* writer, const NATIVE_TYPE& src, SERIAL_TYPE& dst) -{ - ASTSerialTypeInfo::toSerial(writer, &src, &dst); -} - -template -static void _toNativeValue(ASTSerialReader* reader, const SERIAL_TYPE& src, NATIVE_TYPE& dst) -{ - ASTSerialTypeInfo::toNative(reader, &src, &dst); -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ModuleASTSerialFilter !!!!!!!!!!!!!!!!!!!!!!!! - - -ASTSerialIndex ModuleASTSerialFilter::writePointer(ASTSerialWriter* writer, const NodeBase* inPtr) -{ - NodeBase* ptr = const_cast(inPtr); - SLANG_ASSERT(ptr); - - if (Decl* decl = as(ptr)) - { - ModuleDecl* moduleDecl = findModuleForDecl(decl); - SLANG_ASSERT(moduleDecl); - if (moduleDecl && moduleDecl != m_moduleDecl) - { - ASTBuilder* astBuilder = m_moduleDecl->module->getASTBuilder(); - - // It's a reference to a declaration in another module, so create an ImportExternalDecl. - - String mangledName = getMangledName(astBuilder, decl); - - ImportExternalDecl* importDecl = astBuilder->create(); - importDecl->mangledName = mangledName; - const ASTSerialIndex index = writer->writePointer(importDecl); - - // Set as the index of this - writer->setPointerIndex(ptr, index); - - return index; - } - else - { - // Okay... we can just write it out then - return writer->writePointer(ptr); - } - } - - // TODO(JS): What we really want to do here is to ignore bodies functions. - // It's not 100% clear if this is even right though - for example does type inference - // imply the body is needed to say infer a return type? - // Also not clear if statements in other scenarios (if there are others) might need to be kept. - // - // For now we just ignore all stmts - - if (Stmt* stmt = as(ptr)) - { - // - writer->setPointerIndex(stmt, ASTSerialIndex(0)); - return ASTSerialIndex(0); - } - - // For now for everything else just write it - return writer->writePointer(ptr); -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Serial <-> Native conversion !!!!!!!!!!!!!!!!!!!!!!!! - - -// We need to have a way to map between the two. -// If no mapping is needed, (just a copy), then we don't bother with the functions -template -struct ASTSerialBasicTypeInfo -{ - typedef T NativeType; - typedef T SerialType; - - // We want the alignment to be the same as the size of the type for basic types - // NOTE! Might be different from SLANG_ALIGN_OF(SerialType) - enum { SerialAlignment = sizeof(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(T*)serial = *(const T*)native; } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(T*)native = *(const T*)serial; } - - static const ASTSerialType* getType() - { - static const ASTSerialType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; - return &type; - } -}; - -template -struct ASTSerialConvertTypeInfo -{ - typedef NATIVE_T NativeType; - typedef SERIAL_T SerialType; - - enum { SerialAlignment = ASTSerialBasicTypeInfo::SerialAlignment }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SERIAL_T*)serial = SERIAL_T(*(const NATIVE_T*)native); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(NATIVE_T*)native = NATIVE_T(*(const SERIAL_T*)serial); } -}; - -template -struct ASTSerialIdentityTypeInfo -{ - typedef T NativeType; - typedef T SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(T*)serial = *(const T*)native; } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(T*)native = *(const T*)serial; } -}; - -// Don't need to convert the index type - -template <> -struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; - - -// Because is sized, we don't need to convert -template <> -struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; - -// Implement for Basic Types - -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; - -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; - -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; - -// SamplerStateFlavor - -template <> -struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; - -// TextureFlavor - -template <> -struct ASTSerialTypeInfo -{ - typedef TextureFlavor NativeType; - typedef uint16_t SerialType; - enum { SerialAlignment = sizeof(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SerialType*)serial = ((const NativeType*)native)->flavor; } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); ((NativeType*)native)->flavor = *(const SerialType*)serial; } -}; - -// Fixed arrays - -template -struct ASTSerialTypeInfo -{ - typedef ASTSerialTypeInfo ElementASTSerialType; - typedef typename ElementASTSerialType::SerialType SerialElementType; - - typedef T NativeType[N]; - typedef SerialElementType SerialType[N]; - - enum { SerialAlignment = ASTSerialTypeInfo::SerialAlignment }; - - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) - { - SerialElementType* serial = (SerialElementType*)outSerial; - const T* native = (const T*)inNative; - for (Index i = 0; i < Index(N); ++i) - { - ElementASTSerialType::toSerial(writer, native + i, serial + i); - } - } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - const SerialElementType* serial = (const SerialElementType*)inSerial; - T* native = (T*)outNative; - for (Index i = 0; i < Index(N); ++i) - { - ElementASTSerialType::toNative(reader, serial + i, native + i); - } - } -}; - -// Special case bool - as we can't rely on size alignment -template <> -struct ASTSerialTypeInfo -{ - typedef bool NativeType; - typedef uint8_t SerialType; - - enum { SerialAlignment = sizeof(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) - { - SLANG_UNUSED(writer); - *(SerialType*)outSerial = *(const NativeType*)inNative ? 1 : 0; - } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - SLANG_UNUSED(reader); - *(NativeType*)outNative = (*(const SerialType*)inSerial) != 0; - } -}; - - -// Pointer -// Could handle different pointer base types with some more template magic here, but instead went with Pointer type to keep -// things simpler. -template -struct ASTSerialTypeInfo -{ - typedef T* NativeType; - typedef ASTSerialIndex SerialType; - - enum - { - SerialAlignment = SLANG_ALIGN_OF(SerialType) - }; - - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) - { - *(SerialType*)outSerial = writer->addPointer(*(T**)inNative); - } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - *(T**)outNative = reader->getPointer(*(const SerialType*)inSerial).dynamicCast(); - } -}; - -// Special case Name -template <> -struct ASTSerialTypeInfo : public ASTSerialTypeInfo -{ - // Special case - typedef Name* NativeType; - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - *(Name**)outNative = reader->getName(*(const SerialType*)inSerial); - } -}; - -template <> -struct ASTSerialTypeInfo : public ASTSerialTypeInfo -{ -}; - - -struct ASTSerialDeclRefBaseTypeInfo -{ - typedef DeclRefBase NativeType; - struct SerialType - { - ASTSerialIndex substitutions; - ASTSerialIndex decl; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) - { - SerialType& serial = *(SerialType*)outSerial; - const NativeType& native = *(const NativeType*)inNative; - - serial.decl = writer->addPointer(native.decl); - serial.substitutions = writer->addPointer(native.substitutions.substitutions); - } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - DeclRefBase& native = *(DeclRefBase*)(outNative); - const SerialType& serial = *(const SerialType*)inSerial; - - native.decl = reader->getPointer(serial.decl).dynamicCast(); - native.substitutions.substitutions = reader->getPointer(serial.substitutions).dynamicCast(); - } - static const ASTSerialType* getType() - { - static const ASTSerialType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; - return &type; - } -}; - -template -struct ASTSerialTypeInfo> : public ASTSerialDeclRefBaseTypeInfo {}; - -// MatrixCoord can just go as is -template <> -struct ASTSerialTypeInfo : ASTSerialIdentityTypeInfo {}; - -// SourceLoc - -// Make the type exposed, so we can look for it if we want to remap. -template <> -struct ASTSerialTypeInfo -{ - typedef SourceLoc NativeType; - typedef ASTSerialSourceLoc SerialType; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialSourceLoc) }; - - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) - { - *(SerialType*)outSerial = writer->addSourceLoc(*(const NativeType*)inNative); - } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - *(NativeType*)outNative = reader->getSourceLoc(*(const SerialType*)inSerial); - } -}; - -// List -template -struct ASTSerialTypeInfo> -{ - typedef List NativeType; - typedef ASTSerialIndex SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - dst = writer->addArray(src.getBuffer(), src.getCount()); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& dst = *(NativeType*)native; - auto& src = *(const SerialType*)serial; - - reader->getArray(src, dst); - } -}; - -// Dictionary -template -struct ASTSerialTypeInfo> -{ - typedef Dictionary NativeType; - struct SerialType - { - ASTSerialIndex keys; ///< Index an array - ASTSerialIndex values; ///< Index an array - }; - - typedef typename ASTSerialTypeInfo::SerialType KeySerialType; - typedef typename ASTSerialTypeInfo::SerialType ValueSerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - List keys; - List values; - - Index count = Index(src.Count()); - keys.setCount(count); - values.setCount(count); - - Index i = 0; - for (const auto& pair : src) - { - ASTSerialTypeInfo::toSerial(writer, &pair.Key, &keys[i]); - ASTSerialTypeInfo::toSerial(writer, &pair.Value, &values[i]); - i++; - } - - dst.keys = writer->addArray(keys.getBuffer(), count); - dst.values = writer->addArray(values.getBuffer(), count); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - // Clear it - dst = NativeType(); - - List keys; - List values; - - reader->getArray(src.keys, keys); - reader->getArray(src.values, values); - - SLANG_ASSERT(keys.getCount() == values.getCount()); - - const Index count = keys.getCount(); - for (Index i = 0; i < count; ++i) - { - dst.Add(keys[i], values[i]); - } - } -}; - -// SyntaxClass -template -struct ASTSerialTypeInfo> -{ - typedef SyntaxClass NativeType; - typedef uint16_t SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - SLANG_UNUSED(writer); - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - dst = SerialType(src.classInfo->m_classId); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - SLANG_UNUSED(reader); - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - dst.classInfo = ReflectClassInfo::getInfo(ASTNodeType(src)); - } -}; - -// Handle RefPtr - just convert into * to do the conversion -template -struct ASTSerialTypeInfo> -{ - typedef RefPtr NativeType; - typedef typename ASTSerialTypeInfo::SerialType SerialType; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - T* obj = src; - ASTSerialTypeInfo::toSerial(writer, &obj, serial); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - T* obj = nullptr; - ASTSerialTypeInfo::toNative(reader, serial, &obj); - *(NativeType*)native = obj; - } -}; - -// QualType - -template <> -struct ASTSerialTypeInfo -{ - typedef QualType NativeType; - struct SerialType - { - ASTSerialIndex type; - uint8_t isLeftValue; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto dst = (SerialType*)serial; - auto src = (const NativeType*)native; - dst->isLeftValue = src->isLeftValue ? 1 : 0; - dst->type = writer->addPointer(src->type); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto src = (const SerialType*)serial; - auto dst = (NativeType*)native; - dst->type = reader->getPointer(src->type).dynamicCast(); - dst->isLeftValue = src->isLeftValue != 0; - } -}; - - -// LookupResult::Breadcrumb -template <> -struct ASTSerialTypeInfo -{ - typedef LookupResultItem::Breadcrumb NativeType; - struct SerialType - { - NativeType::Kind kind; - NativeType::ThisParameterMode thisParameterMode; - ASTSerialTypeInfo>::SerialType declRef; - ASTSerialTypeInfo> next; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - dst.kind = src.kind; - dst.thisParameterMode = src.thisParameterMode; - _toSerialValue(writer, src.declRef, dst.declRef); - _toSerialValue(writer, src.next, dst.next); - } - - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& dst = *(NativeType*)native; - auto& src = *(const SerialType*)serial; - - dst.kind = src.kind; - dst.thisParameterMode = src.thisParameterMode; - _toNativeValue(reader, src.declRef, dst.declRef); - _toNativeValue(reader, src.next, dst.next); - } -}; - -// LookupResultItem -template <> -struct ASTSerialTypeInfo -{ - typedef LookupResultItem NativeType; - struct SerialType - { - ASTSerialTypeInfo>::SerialType declRef; - ASTSerialTypeInfo> breadcrumbs; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - _toSerialValue(writer, src.declRef, dst.declRef); - _toSerialValue(writer, src.breadcrumbs, dst.breadcrumbs); - } - - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& dst = *(NativeType*)native; - auto& src = *(const SerialType*)serial; - - _toNativeValue(reader, src.declRef, dst.declRef); - _toNativeValue(reader, src.breadcrumbs, dst.breadcrumbs); - } -}; - -// LookupResult -template <> -struct ASTSerialTypeInfo -{ - typedef LookupResult NativeType; - typedef ASTSerialIndex SerialType; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - if (src.isOverloaded()) - { - // Save off as an array - dst = writer->addArray(src.items.getBuffer(), src.items.getCount()); - } - else if (src.item.declRef.getDecl()) - { - dst = writer->addArray(&src.item, 1); - } - else - { - dst = ASTSerialIndex(0); - } - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& dst = *(NativeType*)native; - auto& src = *(const SerialType*)serial; - - // Initialize - dst = NativeType(); - - List items; - reader->getArray(src, items); - - if (items.getCount() == 1) - { - dst.item = items[0]; - } - else - { - dst.items.swapWith(items); - // We have to set item such that it is valid/member of items, if items is non empty - dst.item = dst.items[0]; - } - } -}; - - -// GlobalGenericParamSubstitution::ConstraintArg -template <> -struct ASTSerialTypeInfo -{ - typedef GlobalGenericParamSubstitution::ConstraintArg NativeType; - struct SerialType - { - ASTSerialIndex decl; - ASTSerialIndex val; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& dst = *(SerialType*)serial; - auto& src = *(const NativeType*)native; - - dst.decl = writer->addPointer(src.decl); - dst.val = writer->addPointer(src.val); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - dst.decl = reader->getPointer(src.decl).dynamicCast(); - dst.val = reader->getPointer(src.val).dynamicCast(); - } -}; - -// ExpandedSpecializationArg -template <> -struct ASTSerialTypeInfo -{ - typedef ExpandedSpecializationArg NativeType; - struct SerialType - { - ASTSerialIndex val; - ASTSerialIndex witness; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& dst = *(SerialType*)serial; - auto& src = *(const NativeType*)native; - - dst.witness = writer->addPointer(src.witness); - dst.val = writer->addPointer(src.val); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - dst.witness = reader->getPointer(src.witness).dynamicCast(); - dst.val = reader->getPointer(src.val).dynamicCast(); - } -}; - -// TypeExp -template <> -struct ASTSerialTypeInfo -{ - typedef TypeExp NativeType; - struct SerialType - { - ASTSerialIndex type; - ASTSerialIndex expr; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& dst = *(SerialType*)serial; - auto& src = *(const NativeType*)native; - - dst.type = writer->addPointer(src.type); - dst.expr = writer->addPointer(src.exp); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - dst.type = reader->getPointer(src.type).dynamicCast(); - dst.exp = reader->getPointer(src.expr).dynamicCast(); - } -}; - -// DeclCheckStateExt -template <> -struct ASTSerialTypeInfo -{ - typedef DeclCheckStateExt NativeType; - typedef DeclCheckStateExt::RawType SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - SLANG_UNUSED(writer); - *(SerialType*)serial = (*(const NativeType*)native).getRaw(); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - SLANG_UNUSED(reader); - (*(NativeType*)serial).setRaw(*(const SerialType*)native); - } -}; - -// Modifiers -template <> -struct ASTSerialTypeInfo -{ - typedef Modifiers NativeType; - typedef ASTSerialIndex SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - // We need to make into an array - List modifierIndices; - for (Modifier* modifier : *(NativeType*)native) - { - modifierIndices.add(writer->addPointer(modifier)); - } - *(SerialType*)serial = writer->addArray(modifierIndices.getBuffer(), modifierIndices.getCount()); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - List modifiers; - reader->getArray(*(const SerialType*)serial, modifiers); - - Modifier* prev = nullptr; - for (Modifier* modifier : modifiers) - { - if (prev) - { - prev->next = modifier; - } - } - - NativeType& dst = *(NativeType*)native; - dst.first = modifiers.getCount() > 0 ? modifiers[0] : nullptr; - } -}; - -// ImageFormat -template <> -struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; - -// Stage -template <> -struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; - -// TokenType -template <> -struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; - -// BaseType -template <> -struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; - -// SemanticVersion -template <> -struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; - -// ASTNodeType -template <> -struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; - -// String -template <> -struct ASTSerialTypeInfo -{ - typedef String NativeType; - typedef ASTSerialIndex SerialType; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - *(SerialType*)serial = writer->addString(src); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - dst = reader->getString(src); - } -}; - -// Token -template <> -struct ASTSerialTypeInfo -{ - typedef Token NativeType; - struct SerialType - { - ASTSerialTypeInfo::SerialType type; - ASTSerialTypeInfo::SerialType loc; - ASTSerialIndex name; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - ASTSerialTypeInfo::toSerial(writer, &src.type, &dst.type); - ASTSerialTypeInfo::toSerial(writer, &src.loc, &dst.loc); - - if (src.flags & TokenFlag::Name) - { - dst.name = writer->addName(src.getName()); - } - else - { - dst.name = writer->addString(src.getContent()); - } - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - dst.flags = 0; - dst.charsNameUnion.chars = nullptr; - - ASTSerialTypeInfo::toNative(reader, &src.type, &dst.type); - ASTSerialTypeInfo::toNative(reader, &src.loc, &dst.loc); - - // At the other end all token content will appear as Names. - if (src.name != ASTSerialIndex(0)) - { - dst.charsNameUnion.name = reader->getName(src.name); - dst.flags |= TokenFlag::Name; - } - } -}; - -// NameLoc -template <> -struct ASTSerialTypeInfo -{ - typedef NameLoc NativeType; - struct SerialType - { - ASTSerialTypeInfo::SerialType loc; - ASTSerialIndex name; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - dst.name = writer->addName(src.name); - ASTSerialTypeInfo::toSerial(writer, &src.loc, &dst.loc); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - dst.name = reader->getName(src.name); - ASTSerialTypeInfo::toNative(reader, &src.loc, &dst.loc); - } -}; - -// DiagnosticInfo -template <> -struct ASTSerialTypeInfo -{ - typedef const DiagnosticInfo* NativeType; - typedef ASTSerialIndex SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - dst = src ? writer->addString(UnownedStringSlice(src->name)) : ASTSerialIndex(0); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - if (src == ASTSerialIndex(0)) - { - dst = nullptr; - } - else - { - dst = findDiagnosticByName(reader->getStringSlice(src)); - } - } -}; - -// !!!!!!!!!!!!!!!!!!!!! ASTSerialGetType !!!!!!!!!!!!!!!!!!!!!!!!!!! -// Getting the type info, let's use a static variable to hold the state to keep simple - -template -struct ASTSerialGetType -{ - static const ASTSerialType* getType() - { - typedef ASTSerialTypeInfo Info; - static const ASTSerialType type = { sizeof(typename Info::SerialType), uint8_t(Info::SerialAlignment), &Info::toSerial, &Info::toNative }; - return &type; - } -}; - -// Special case DeclRef, because it always uses the same type -template -struct ASTSerialGetType> -{ - static const ASTSerialType* getType() { return ASTSerialDeclRefBaseTypeInfo::getType(); } -}; - -// !!!!!!!!!!!!!!!!!!!!!! Generate fields for a type !!!!!!!!!!!!!!!!!!!!!!!!!!! - - -template -ASTSerialField _calcField(const char* name, T& in) -{ - uint8_t* ptr = &reinterpret_cast(in); - - ASTSerialField field; - field.name = name; - field.type = ASTSerialGetType::getType(); - // This only works because we in is an offset from 1 - field.nativeOffset = uint32_t(size_t(ptr) - 1); - field.serialOffset = 0; - return field; -} - -static ASTSerialClass _makeClass(MemoryArena* arena, ASTNodeType type, const List& fields) -{ - ASTSerialClass cls = { type, 0, 0, 0, 0 }; - cls.fieldsCount = fields.getCount(); - cls.fields = arena->allocateAndCopyArray(fields.getBuffer(), fields.getCount()); - return cls; -} - -#define SLANG_AST_SERIAL_FIELD(FIELD_NAME, TYPE, param) fields.add(_calcField(#FIELD_NAME, obj->FIELD_NAME)); - -// Note that the obj point is not nullptr, because some compilers notice this is 'indexing from null' -// and warn/error. So we offset from 1. -#define SLANG_AST_SERIAL_MAKE_CLASS(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \ -{ \ - NAME* obj = (NAME*)1; \ - SLANG_UNUSED(obj); \ - fields.clear(); \ - SLANG_FIELDS_ASTNode_##NAME(SLANG_AST_SERIAL_FIELD, param) \ - outClasses[Index(ASTNodeType::NAME)] = _makeClass(arena, ASTNodeType::NAME, fields); \ -} - -struct ASTFieldAccess -{ - static void calcClasses(MemoryArena* arena, ASTSerialClass outClasses[Index(ASTNodeType::CountOf)]) - { - List fields; - SLANG_ALL_ASTNode_NodeBase(SLANG_AST_SERIAL_MAKE_CLASS, _) - } -}; - -ASTSerialClasses::ASTSerialClasses(): - m_arena(2048) -{ - memset(m_classes, 0, sizeof(m_classes)); - ASTFieldAccess::calcClasses(&m_arena, m_classes); - - // Now work out the layout - for (Index i = 0; i < SLANG_COUNT_OF(m_classes); ++i) - { - // Set up each class in order, from lowest to highest index - // Doing so means super class is always setup - ASTSerialClass& serialClass = m_classes[i]; - - const ReflectClassInfo* info = ReflectClassInfo::getInfo(serialClass.type); - - size_t maxAlignment = 1; - size_t offset = 0; - - const ReflectClassInfo* superInfo = info->m_superClass; - if (superInfo) - { - ASTSerialClass& superSerialInfo = m_classes[superInfo->m_classId]; - - // If it's been setup, then alignment must be non zero. - // The ordering of ASTNodeType, should mean type have larger ASTNodeType greater than supers ASTNodeType. - SLANG_ASSERT(superSerialInfo.alignment != 0); - - // Must be a power of 2 - SLANG_ASSERT((superSerialInfo.alignment & (superSerialInfo.alignment - 1)) == 0); - - maxAlignment = superSerialInfo.alignment; - offset = superSerialInfo.size; - - // Check it is correctly aligned - SLANG_ASSERT((offset & (maxAlignment - 1)) == 0); - } - - // Okay, go through fields setting their offset - ASTSerialField* fields = serialClass.fields; - for (Index j = 0; j < serialClass.fieldsCount; j++) - { - ASTSerialField& field = fields[j]; - - size_t alignment = field.type->serialAlignment; - // Make sure the offset is aligned for the field requirement - offset = (offset + alignment - 1) & ~(alignment - 1); - - // Save the field offset - field.serialOffset = uint32_t(offset); - - // Move past the field - offset += field.type->serialSizeInBytes; - - // Calc the maximum alignment - maxAlignment = (alignment > maxAlignment) ? alignment : maxAlignment; - } - - // Align with maximum alignment - offset = (offset + maxAlignment - 1) & ~(maxAlignment - 1); - - serialClass.alignment = uint8_t(maxAlignment); - serialClass.size = uint32_t(offset); - } -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!! - -ASTSerialWriter::ASTSerialWriter(ASTSerialClasses* classes, ASTSerialFilter* filter) : - m_arena(2048), - m_classes(classes), - m_filter(filter) -{ - // 0 is always the null pointer - m_entries.add(nullptr); - m_ptrMap.Add(nullptr, 0); -} - -ASTSerialIndex ASTSerialWriter::writePointer(const NodeBase* node) -{ - // This pointer cannot be in the map - SLANG_ASSERT(m_ptrMap.TryGetValue(node) == nullptr); - - const ASTSerialClass* serialClass = m_classes->getSerialClass(node->astNodeType); - - typedef ASTSerialInfo::NodeEntry NodeEntry; - - NodeEntry* nodeEntry = (NodeEntry*)m_arena.allocateAligned(sizeof(NodeEntry) + serialClass->size, ASTSerialInfo::MAX_ALIGNMENT); - - nodeEntry->type = ASTSerialInfo::Type::Node; - nodeEntry->astNodeType = uint16_t(node->astNodeType); - nodeEntry->info = ASTSerialInfo::makeEntryInfo(serialClass->alignment); - - auto index = _add(node, nodeEntry); - - uint8_t* serialPayload = (uint8_t*)(nodeEntry + 1); - - while (serialClass) - { - for (Index i = 0; i < serialClass->fieldsCount; ++i) - { - auto field = serialClass->fields[i]; - - // Work out the offsets - auto srcField = ((const uint8_t*)node) + field.nativeOffset; - auto dstField = serialPayload + field.serialOffset; - - field.type->toSerialFunc(this, srcField, dstField); - } - // Get the super class - const ReflectClassInfo* reflectInfo = ReflectClassInfo::getInfo(serialClass->type); - const ReflectClassInfo* superReflectInfo = reflectInfo->m_superClass; - - serialClass = superReflectInfo ? m_classes->getSerialClass(ASTNodeType(superReflectInfo->m_classId)) : nullptr; - } - - return index; -} - -void ASTSerialWriter::setPointerIndex(const NodeBase* ptr, ASTSerialIndex index) -{ - m_ptrMap.Add(ptr, Index(index)); -} - -ASTSerialIndex ASTSerialWriter::addPointer(const NodeBase* node) -{ - // Null is always 0 - if (node == nullptr) - { - return ASTSerialIndex(0); - } - // Look up in the map - Index* indexPtr = m_ptrMap.TryGetValue(node); - if (indexPtr) - { - return ASTSerialIndex(*indexPtr); - } - - if (m_filter) - { - return m_filter->writePointer(this, node); - } - else - { - return writePointer(node); - } -} - -ASTSerialIndex ASTSerialWriter::addPointer(const RefObject* obj) -{ - // Null is always 0 - if (obj == nullptr) - { - return ASTSerialIndex(0); - } - // Look up in the map - Index* indexPtr = m_ptrMap.TryGetValue(obj); - if (indexPtr) - { - return ASTSerialIndex(*indexPtr); - } - - if (auto stringRep = dynamicCast(obj)) - { - ASTSerialIndex index = addString(StringRepresentation::asSlice(stringRep)); - m_ptrMap.Add(obj, Index(index)); - return index; - } - else if (auto breadcrumb = dynamicCast(obj)) - { - typedef ASTSerialTypeInfo TypeInfo; - typedef ASTSerialInfo::RefObjectEntry RefObjectEntry; - - size_t alignment = TypeInfo::SerialAlignment; - alignment = (alignment < SLANG_ALIGN_OF(ASTSerialInfo::RefObjectEntry)) ? SLANG_ALIGN_OF(ASTSerialInfo::RefObjectEntry) : alignment; - - RefObjectEntry* entry = (RefObjectEntry*)m_arena.allocateAligned(sizeof(RefObjectEntry) + sizeof(TypeInfo::SerialType), alignment); - - entry->type = ASTSerialInfo::Type::RefObject; - entry->info = ASTSerialInfo::makeEntryInfo(int(alignment)); - entry->subType = RefObjectEntry::SubType::Breadcrumb; - - auto index = _add(breadcrumb, entry); - - // Do any conversion - TypeInfo::toSerial(this, breadcrumb, entry + 1); - return index; - } - else if (auto name = dynamicCast(obj)) - { - return addName(name); - } - else if (auto scope = dynamicCast(obj)) - { - // We don't serialize scope - return ASTSerialIndex(0); - } - else if (auto module = dynamicCast(obj)) - { - // We don't serialize Module - return ASTSerialIndex(0); - } - - SLANG_ASSERT(!"Unhandled type"); - return ASTSerialIndex(0); -} - -ASTSerialIndex ASTSerialWriter::addString(const UnownedStringSlice& slice) -{ - typedef ByteEncodeUtil Util; - typedef ASTSerialInfo::StringEntry StringEntry; - - if (slice.getLength() == 0) - { - return ASTSerialIndex(0); - } - - Index newIndex = m_entries.getCount(); - - Index* indexPtr = m_sliceMap.TryGetValueOrAdd(slice, newIndex); - if (indexPtr) - { - return ASTSerialIndex(*indexPtr); - } - - // Okay we need to add the string - - uint8_t encodeBuf[Util::kMaxLiteEncodeUInt32]; - const int encodeCount = Util::encodeLiteUInt32(uint32_t(slice.getLength()), encodeBuf); - - StringEntry* entry = (StringEntry*)m_arena.allocateUnaligned(SLANG_OFFSET_OF(StringEntry, sizeAndChars) + encodeCount + slice.getLength()); - entry->info = ASTSerialInfo::EntryInfo::Alignment1; - entry->type = ASTSerialInfo::Type::String; - - uint8_t* dst = (uint8_t*)(entry->sizeAndChars); - for (int i = 0; i < encodeCount; ++i) - { - dst[i] = encodeBuf[i]; - } - - memcpy(dst + encodeCount, slice.begin(), slice.getLength()); - - m_entries.add(entry); - return ASTSerialIndex(newIndex); -} - - -ASTSerialIndex ASTSerialWriter::addString(const String& in) -{ - return addPointer(in.getStringRepresentation()); -} - -ASTSerialIndex ASTSerialWriter::addName(const Name* name) -{ - if (name == nullptr) - { - return ASTSerialIndex(0); - } - - // Look it up - Index* indexPtr = m_ptrMap.TryGetValue(name); - if (indexPtr) - { - return ASTSerialIndex(*indexPtr); - } - - ASTSerialIndex index = addString(name->text); - m_ptrMap.Add(name, Index(index)); - return index; -} - -ASTSerialSourceLoc ASTSerialWriter::addSourceLoc(SourceLoc sourceLoc) -{ - SLANG_UNUSED(sourceLoc); - return 0; -} - -ASTSerialIndex ASTSerialWriter::_addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount) -{ - typedef ASTSerialInfo::ArrayEntry Entry; - - if (elementCount == 0) - { - return ASTSerialIndex(0); - } - - SLANG_ASSERT(alignment >= 1 && alignment <= ASTSerialInfo::MAX_ALIGNMENT); - - // We must at a minimum have the alignment for the array prefix info - alignment = (alignment < SLANG_ALIGN_OF(Entry)) ? SLANG_ALIGN_OF(Entry) : alignment; - - size_t payloadSize = elementCount * elementSize; - - Entry* entry = (Entry*)m_arena.allocateAligned(sizeof(Entry) + payloadSize, alignment); - - entry->type = ASTSerialInfo::Type::Array; - entry->info = ASTSerialInfo::makeEntryInfo(int(alignment)); - entry->elementSize = uint16_t(elementSize); - entry->elementCount = uint32_t(elementCount); - - memcpy(entry + 1, elements, payloadSize); - - m_entries.add(entry); - return ASTSerialIndex(m_entries.getCount() - 1); -} - -static const uint8_t s_fixBuffer[ASTSerialInfo::MAX_ALIGNMENT]{ 0, }; - -SlangResult ASTSerialWriter::write(Stream* stream) -{ - const Int entriesCount = m_entries.getCount(); - - // Add a sentinal so we don't need special handling for - ASTSerialInfo::Entry sentinal; - sentinal.type = ASTSerialInfo::Type::String; - sentinal.info = ASTSerialInfo::EntryInfo::Alignment1; - - m_entries.add(&sentinal); - m_entries.removeLast(); - - ASTSerialInfo::Entry** entries = m_entries.getBuffer(); - // Note strictly required in our impl of List. But by writing this and - // knowing that removeLast cannot release memory, means the sentinal must be at the last position. - entries[entriesCount] = &sentinal; - - - { - size_t offset = 0; - - ASTSerialInfo::Entry* entry = entries[1]; - // We start on 1, because 0 is nullptr and not used for anything - for (Index i = 1; i < entriesCount; ++i) - { - ASTSerialInfo::Entry* next = entries[i + 1]; - // Before writing we need to store the next alignment - - const size_t nextAlignment = ASTSerialInfo::getAlignment(next->info); - const size_t alignment = ASTSerialInfo::getAlignment(entry->info); - - entry->info = ASTSerialInfo::combineWithNext(entry->info, next->info); - - // Check we are aligned correctly - SLANG_ASSERT((offset & (alignment - 1)) == 0); - - // When we write, we need to make sure it take into account the next alignment - const size_t entrySize = entry->calcSize(m_classes); - - // Work out the fix for next alignment - size_t nextOffset = offset + entrySize; - nextOffset = (nextOffset + nextAlignment - 1) & ~(nextAlignment - 1); - - size_t alignmentFixSize = nextOffset - (offset + entrySize); - - // The fix must be less than max alignment. We require it to be less because we aligned each Entry to - // MAX_ALIGNMENT, and so < MAX_ALIGNMENT is the most extra bytes we can write - SLANG_ASSERT( alignmentFixSize < ASTSerialInfo::MAX_ALIGNMENT); - - try - { - stream->write(entry, entrySize); - // If we needed to fix so that subsequent alignment is right, write out extra bytes here - if (alignmentFixSize) - { - stream->write(s_fixBuffer, alignmentFixSize); - } - } - catch (const IOException&) - { - return SLANG_FAIL; - } - - // Onto next - offset = nextOffset; - entry = next; - } - } - - return SLANG_OK; -} - -SlangResult ASTSerialWriter::writeIntoContainer(RiffContainer* container) -{ - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ScopeChunk ScopeChunk; - - // This is the container for the AST Data - ScopeChunk scopeModule(container, Chunk::Kind::List, ASTSerialBinary::kSlangASTModuleFourCC); - { - ScopeChunk scopeData(container, Chunk::Kind::Data, ASTSerialBinary::kSlangASTModuleDataFourCC); - - { - // Sentinal so we don't need special handling for end of list - ASTSerialInfo::Entry sentinal; - sentinal.type = ASTSerialInfo::Type::String; - sentinal.info = ASTSerialInfo::EntryInfo::Alignment1; - - size_t offset = 0; - const Int entriesCount = m_entries.getCount(); - - { - m_entries.add(&sentinal); - m_entries.removeLast(); - // Note strictly required in our impl of List. But by writing this and - // knowing that removeLast cannot release memory, means the sentinal must be at the last position. - m_entries.getBuffer()[entriesCount] = &sentinal; - } - - ASTSerialInfo::Entry*const* entries = m_entries.getBuffer(); - - ASTSerialInfo::Entry* entry = entries[1]; - // We start on 1, because 0 is nullptr and not used for anything - for (Index i = 1; i < entriesCount; ++i) - { - ASTSerialInfo::Entry* next = entries[i + 1]; - - // Before writing we need to store the next alignment - - const size_t nextAlignment = ASTSerialInfo::getAlignment(next->info); - const size_t alignment = ASTSerialInfo::getAlignment(entry->info); - - entry->info = ASTSerialInfo::combineWithNext(entry->info, next->info); - - // Check we are aligned correctly - SLANG_ASSERT((offset & (alignment - 1)) == 0); - - // When we write, we need to make sure it take into account the next alignment - const size_t entrySize = entry->calcSize(m_classes); - - // Work out the fix for next alignment - size_t nextOffset = offset + entrySize; - nextOffset = (nextOffset + nextAlignment - 1) & ~(nextAlignment - 1); - - size_t alignmentFixSize = nextOffset - (offset + entrySize); - - // The fix must be less than max alignment. We require it to be less because we aligned each Entry to - // MAX_ALIGNMENT, and so < MAX_ALIGNMENT is the most extra bytes we can write - SLANG_ASSERT(alignmentFixSize < ASTSerialInfo::MAX_ALIGNMENT); - - container->write(entry, entrySize); - if (alignmentFixSize) - { - container->write(s_fixBuffer, alignmentFixSize); - } - - // Onto next - offset = nextOffset; - entry = next; - } - } - } - - return SLANG_OK; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialInfo::Entry !!!!!!!!!!!!!!!!!!!!!!!! - -size_t ASTSerialInfo::Entry::calcSize(ASTSerialClasses* serialClasses) const -{ - switch (type) - { - case Type::String: - { - auto entry = static_cast(this); - const uint8_t* cur = (const uint8_t*)entry->sizeAndChars; - uint32_t charsSize; - int sizeSize = ByteEncodeUtil::decodeLiteUInt32(cur, &charsSize); - return SLANG_OFFSET_OF(StringEntry, sizeAndChars) + sizeSize + charsSize; - } - case Type::Node: - { - auto entry = static_cast(this); - auto serialClass = serialClasses->getSerialClass(ASTNodeType(entry->astNodeType)); - - // Align by the alignment of the entry - size_t alignment = getAlignment(entry->info); - size_t size = sizeof(NodeEntry) + serialClass->size; - - size = size + (alignment - 1) & ~(alignment - 1); - return size; - } - case Type::RefObject: - { - auto entry = static_cast(this); - - size_t payloadSize; - switch (entry->subType) - { - case RefObjectEntry::SubType::Breadcrumb: - { - payloadSize = sizeof(ASTSerialTypeInfo::SerialType); - break; - } - default: - { - SLANG_ASSERT(!"Unknown type"); - return 0; - } - } - - return sizeof(RefObjectEntry) + payloadSize; - } - case Type::Array: - { - auto entry = static_cast(this); - return sizeof(ArrayEntry) + entry->elementSize * entry->elementCount; - } - default: break; - } - - SLANG_ASSERT(!"Unknown type"); - return 0; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!! - -const void* ASTSerialReader::getArray(ASTSerialIndex index, Index& outCount) -{ - if (index == ASTSerialIndex(0)) - { - outCount = 0; - return nullptr; - } - - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - switch (entry->type) - { - case Type::Array: - { - auto arrayEntry = static_cast(entry); - outCount = Index(arrayEntry->elementCount); - return (arrayEntry + 1); - } - default: break; - } - - SLANG_ASSERT(!"Not an array"); - outCount = 0; - return nullptr; -} - -ASTSerialPointer ASTSerialReader::getPointer(ASTSerialIndex index) -{ - if (index == ASTSerialIndex(0)) - { - return ASTSerialPointer(); - } - - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - switch (entry->type) - { - case Type::String: - { - // Hmm. Tricky -> we don't know if will be cast as Name or String. Lets assume string. - String string = getString(index); - return ASTSerialPointer(string.getStringRepresentation()); - } - case Type::Node: - { - return ASTSerialPointer((NodeBase*)m_objects[Index(index)]); - } - case Type::RefObject: - { - return ASTSerialPointer((RefObject*)m_objects[Index(index)]); - } - default: break; - } - - SLANG_ASSERT(!"Cannot access as a pointer"); - return ASTSerialPointer(); -} - -String ASTSerialReader::getString(ASTSerialIndex index) -{ - if (index == ASTSerialIndex(0)) - { - return String(); - } - - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - // It has to be a string type - if (entry->type != Type::String) - { - SLANG_ASSERT(!"Not a string"); - return String(); - } - - RefObject* obj = (RefObject*)m_objects[Index(index)]; - - if (obj) - { - StringRepresentation* stringRep = dynamicCast(obj); - if (stringRep) - { - return String(stringRep); - } - // Must be a name then - Name* name = dynamicCast(obj); - SLANG_ASSERT(name); - return name->text; - } - - // Okay we need to construct as a string - UnownedStringSlice slice = getStringSlice(index); - String string(slice); - StringRepresentation* stringRep = string.getStringRepresentation(); - - m_scope.add(stringRep); - m_objects[Index(index)] = stringRep; - return string; -} - -Name* ASTSerialReader::getName(ASTSerialIndex index) -{ - if (index == ASTSerialIndex(0)) - { - return nullptr; - } - - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - // It has to be a string type - if (entry->type != Type::String) - { - SLANG_ASSERT(!"Not a string"); - return nullptr; - } - - RefObject* obj = (RefObject*)m_objects[Index(index)]; - - if (obj) - { - Name* name = dynamicCast(obj); - if (name) - { - return name; - } - // Can only be a string then - StringRepresentation* stringRep = dynamicCast(obj); - SLANG_ASSERT(stringRep); - - // I don't need to scope, as scoped in NamePool - name = m_namePool->getName(String(stringRep)); - - // Store as name, as can always access the inner string if needed - m_objects[Index(index)] = name; - return name; - } - - UnownedStringSlice slice = getStringSlice(index); - String string(slice); - Name* name = m_namePool->getName(string); - // Don't need to add to scope, because scoped on the pool - m_objects[Index(index)] = name; - return name; -} - -UnownedStringSlice ASTSerialReader::getStringSlice(ASTSerialIndex index) -{ - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - // It has to be a string type - if (entry->type != Type::String) - { - SLANG_ASSERT(!"Not a string"); - return UnownedStringSlice(); - } - - auto stringEntry = static_cast(entry); - - const uint8_t* src = (const uint8_t*)stringEntry->sizeAndChars; - - // Decode the string - uint32_t size; - int sizeSize = ByteEncodeUtil::decodeLiteUInt32(src, &size); - return UnownedStringSlice((const char*)src + sizeSize, size); -} - -SourceLoc ASTSerialReader::getSourceLoc(ASTSerialSourceLoc loc) -{ - SLANG_UNUSED(loc); - return SourceLoc(); -} - -SlangResult ASTSerialReader::loadEntries(const uint8_t* data, size_t dataCount, List& outEntries) -{ - // Check the input data is at least aligned to the max alignment (otherwise everything cannot be aligned correctly) - SLANG_ASSERT((size_t(data) & (ASTSerialInfo::MAX_ALIGNMENT - 1)) == 0); - - outEntries.setCount(1); - outEntries[0] = nullptr; - - const uint8_t*const end = data + dataCount; - - const uint8_t* cur = data; - while (cur < end) - { - const Entry* entry = (const Entry*)cur; - outEntries.add(entry); - - const size_t entrySize = entry->calcSize(m_classes); - cur += entrySize; - - // Need to get the next alignment - const size_t nextAlignment = ASTSerialInfo::getNextAlignment(entry->info); - - // Need to fix cur with the alignment - cur = (const uint8_t*)((size_t(cur) + nextAlignment - 1) & ~(nextAlignment - 1)); - } - - return SLANG_OK; -} - -SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuilder* builder, NamePool* namePool) -{ - SLANG_RETURN_ON_FAIL(loadEntries(data, dataCount, m_entries)); - - m_namePool = namePool; - - m_objects.clearAndDeallocate(); - m_objects.setCount(m_entries.getCount()); - memset(m_objects.getBuffer(), 0, m_objects.getCount() * sizeof(void*)); - - // Go through entries, constructing objects. - for (Index i = 1; i < m_entries.getCount(); ++i) - { - const Entry* entry = m_entries[i]; - - switch (entry->type) - { - case Type::String: - { - // Don't need to construct an object. This is probably a StringRepresentation, or a Name - // Will evaluate lazily. - break; - } - case Type::Node: - { - auto nodeEntry = static_cast(entry); - m_objects[i] = builder->createByNodeType(ASTNodeType(nodeEntry->astNodeType)); - break; - } - case Type::RefObject: - { - auto objEntry = static_cast(entry); - switch (objEntry->subType) - { - case ASTSerialInfo::RefObjectEntry::SubType::Breadcrumb: - { - typedef LookupResultItem::Breadcrumb Breadcrumb; - - auto breadcrumb = new LookupResultItem::Breadcrumb(Breadcrumb::Kind::Member, DeclRef(), nullptr, nullptr); - m_scope.add(breadcrumb); - m_objects[i] = breadcrumb; - break; - } - default: - { - SLANG_ASSERT(!"Unknown type"); - return SLANG_FAIL; - } - } - break; - } - case Type::Array: - { - // Don't need to construct an object, as will be accessed an interpreted by the object that holds it - break; - } - } - } - - // Deserialize - for (Index i = 1; i < m_entries.getCount(); ++i) - { - const Entry* entry = m_entries[i]; - void* native = m_objects[i]; - if (!native) - { - continue; - } - switch (entry->type) - { - case Type::Node: - { - auto nodeEntry = static_cast(entry); - auto serialClass = m_classes->getSerialClass(ASTNodeType(nodeEntry->astNodeType)); - - const uint8_t* src = (const uint8_t*)(nodeEntry + 1); - uint8_t* dst = (uint8_t*)m_objects[i]; - - // It must be constructed - SLANG_ASSERT(dst); - - while (serialClass) - { - for (Index j = 0; j < serialClass->fieldsCount; ++j) - { - auto field = serialClass->fields[j]; - auto fieldType = field.type; - fieldType->toNativeFunc(this, src + field.serialOffset, dst + field.nativeOffset); - } - - auto cls = ReflectClassInfo::getInfo(serialClass->type); - auto superCls = cls->m_superClass; - - // Get the super class - serialClass = superCls ? m_classes->getSerialClass(ASTNodeType(superCls->m_classId)) : nullptr; - } - - break; - } - case Type::RefObject: - { - auto objEntry = static_cast(entry); - switch (objEntry->subType) - { - case ASTSerialInfo::RefObjectEntry::SubType::Breadcrumb: - { - typedef LookupResultItem::Breadcrumb Breadcrumb; - auto serialType = ASTSerialGetType::getType(); - serialType->toNativeFunc(this, (entry + 1), m_objects[i]); - break; - } - default: - { - SLANG_ASSERT(!"Unknown type"); - return SLANG_FAIL; - } - } - break; - } - default: break; - } - } - - return SLANG_OK; -} - - -/* static */Result ASTSerialReader::readContainerModules(RiffContainer* container, Linkage* linkage, List>& outModules) -{ - List moduleChunks; - // First try to find a list - { - RiffContainer::ListChunk* listChunk = container->getRoot()->findListRec(SerialBinary::kSlangModuleListFourCc); - if (listChunk) - { - listChunk->findContained(ASTSerialBinary::kSlangASTModuleFourCC, moduleChunks); - } - else - { - // Maybe its just a single module - RiffContainer::ListChunk* moduleChunk = container->getRoot()->findListRec(ASTSerialBinary::kSlangASTModuleFourCC); - if (!moduleChunk) - { - // Couldn't find any modules - return SLANG_FAIL; - } - moduleChunks.add(moduleChunk); - } - } - - RefPtr serialClasses(new ASTSerialClasses); - - // Okay, deserialize the each of the module chunks - for (RiffContainer::ListChunk* listChunk : moduleChunks) - { - // Look for the module data - auto data = listChunk->findContainedData(ASTSerialBinary::kSlangASTModuleDataFourCC); - - if (!data) - { - return SLANG_FAIL; - } - - ASTSerialReader reader(serialClasses); - - RefPtr module(new Module(linkage)); - SLANG_RETURN_ON_FAIL(reader.load((uint8_t*)data->getPayload(), data->getSize(), module->getASTBuilder(), linkage->getNamePool())); - - ModuleDecl* moduleDecl = reader.getPointer(ASTSerialIndex(1)).dynamicCast(); - if (!moduleDecl) - { - return SLANG_FAIL; - } - - // Set on the module - module->setModuleDecl(moduleDecl); - - outModules.add(module); - } - - return SLANG_OK; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerializeUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!! - -/* static */SlangResult ASTSerialTestUtil::selfTest() -{ - RefPtr classes = new ASTSerialClasses; - - { - struct Thing - { - Module* node; - }; - Thing thing; - - //Pointer pointer(thing.node); - - auto field = _calcField("node", thing.node); - - - const ASTSerialType* type = ASTSerialGetType::getType(); - SLANG_UNUSED(type); - } - - { - const ASTSerialType* type = ASTSerialGetType::getType(); - SLANG_UNUSED(type); - } - - { - const ASTSerialType* type = ASTSerialGetType::getType(); - SLANG_UNUSED(type); - } - - { - const ASTSerialType* type = ASTSerialGetType::getType(); - SLANG_UNUSED(type); - } - - return SLANG_OK; -} - -/* static */SlangResult ASTSerialTestUtil::testSerialize(NodeBase* node, RootNamePool* rootNamePool, SharedASTBuilder* sharedASTBuilder, SourceManager* sourceManager) -{ - RefPtr classes = new ASTSerialClasses; - - List contents; - - { - OwnedMemoryStream stream(FileAccess::ReadWrite); - - ModuleDecl* moduleDecl = as(node); - ModuleASTSerialFilter filterStorage(moduleDecl); - - ASTSerialFilter* filter = moduleDecl ? &filterStorage : nullptr; - - ASTSerialWriter writer(classes, filter); - - // Lets serialize it all - writer.addPointer(node); - // Let's stick it all in a stream - writer.write(&stream); - - stream.swapContents(contents); - - NamePool namePool; - namePool.setRootNamePool(rootNamePool); - - ASTSerialReader reader(classes); - - ASTBuilder builder(sharedASTBuilder, "Serialize Check"); - - // We could now check that the loaded data matches - - { - const List& writtenEntries = writer.getEntries(); - List readEntries; - - SlangResult res = reader.loadEntries(contents.getBuffer(), contents.getCount(), readEntries); - SLANG_UNUSED(res); - - SLANG_ASSERT(writtenEntries.getCount() == readEntries.getCount()); - - // They should be identical up to the - for (Index i = 1; i < readEntries.getCount(); ++i) - { - auto writtenEntry = writtenEntries[i]; - auto readEntry = readEntries[i]; - - const size_t writtenSize = writtenEntry->calcSize(classes); - const size_t readSize = readEntry->calcSize(classes); - - SLANG_ASSERT(readSize == writtenSize); - // Check the payload is the same - SLANG_ASSERT(memcmp(readEntry, writtenEntry, readSize) == 0); - } - - } - - { - SlangResult res = reader.load(contents.getBuffer(), contents.getCount(), &builder, &namePool); - SLANG_UNUSED(res); - } - - // Lets see what we have - const ASTDumpUtil::Flags dumpFlags = ASTDumpUtil::Flag::HideSourceLoc | ASTDumpUtil::Flag::HideScope; - - String readDump; - { - SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None); - ASTDumpUtil::dump(reader.getPointer(ASTSerialIndex(1)).dynamicCast(), ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter); - readDump = sourceWriter.getContentAndClear(); - - } - String origDump; - { - SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None); - ASTDumpUtil::dump(node, ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter); - origDump = sourceWriter.getContentAndClear(); - } - - // Write out - File::writeAllText("ast-read.ast-dump", readDump); - File::writeAllText("ast-orig.ast-dump", origDump); - - - if (readDump != origDump) - { - return SLANG_FAIL; - } - } - - return SLANG_OK; -} - - -} // namespace Slang diff --git a/source/slang/slang-ast-serialize.h b/source/slang/slang-ast-serialize.h deleted file mode 100644 index a9fa9f605..000000000 --- a/source/slang/slang-ast-serialize.h +++ /dev/null @@ -1,598 +0,0 @@ -// slang-ast-serialize.h -#ifndef SLANG_AST_SERIALIZE_H -#define SLANG_AST_SERIALIZE_H - -#include - -#include "slang-ast-support-types.h" -#include "slang-ast-all.h" - -#include "../core/slang-riff.h" - -#include "slang-ast-builder.h" - -#include "../core/slang-byte-encode-util.h" - -#include "../core/slang-stream.h" -namespace Slang -{ - -class Linkage; - -/* -AST Serialization Overview -========================== - -The AST node types are generally types derived from the NodeBase. The C++ extractor is used to associate an ASTNodeType with -every NodeBase type, such that casting is fast and simple and we have a simple integer to uniquely identify those types. The -extractor also performs another task of associating with the type name all of the fields held in just that type. The definition -of the fields is stored in an 'x macro' which is in the slang-ast-generated-macro.h file, for example - -``` -#define SLANG_FIELDS_ASTNode_DeclRefExpr(_x_, _param_)\ - _x_(scope, (RefPtr), _param_)\ - _x_(declRef, (DeclRef), _param_)\ - _x_(name, (Name*), _param_) -`` - -For the type DeclRefExpr, this holds all of the fields held in just DeclRefExpr in this case `scope`, `declRef` and `name`. -DeclRefExpr derives from Expr and this might hold other fields and so forth. - -The implementation makes a distinction between the 'native' types, the regular C++ in memory types and 'serial' types. -Each serializable C++ type has an associated 'serial' type - with the distinction that it can be written out and (with perhaps some other data) -read back in to recreate the C++ type. The serial type can be a C++ type, but is such it can be written and read from disk and still -represent the same data. - -We need a mechanism to be able to do do a conversion between native and serial types. To make the association we use the template - -``` -template -struct ASTSerialTypeInfo; -``` - -and specialize it for each native type. The specialization holds - -SerialType - The type that will be used to represent the native type -NativeType - The native typs -SerialAlignment - A value that holds what kind of alignment the SerialType needs to be serializable (it may be different from SLANG_ALIGN_OF(SerialType)!) -toSerial - A function that with the help of ASTSerialWriter convert the NativeType into the SerialType -toNative - A function that with the help of ASTSerialReader convert the SerialType into the NativeType - -It is useful to have a structure that holds the type information, so it can be stored. That is achieved with - -``` -template -struct ASTSerialGetType; -``` - -This template can be specialized for a specific native types - but all it holds is just a function getType, which returns a ASTSerialType*, -which just holds the information held in the ASTSerialTypeInfo template, but additionally including the size of the SerialType. - -So we need to define a specialized ASTSerialTypeInfo for each type that can be a field in a NodeBase derived type. We don't need to define -anything explicitly for the NodeBase derived types, as we will just generate the layout from the fields. How do we know the fields? We just -used the macros generated from the C++ extactor. - -So first a few things to observe... - -1) Some types don't need any conversion to be serializable - int8_t, or float the bits can just be written out and read in (1) -2) Some types need a conversion but it's very simple - for example an enum without explicit size, being written as an explicit size -3) Some types can be written out but would not be directly readable or usable with different targets/processors, so need converting -4) Some types require complex conversions that require programmer code - like Dictionary/List - -For types that need no conversion (1), we can just use the template ASTSerialIdentityTypeInfo - -``` -template <> -struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; -``` - -This specialization means that SomeType can be written out and read in across targets/compilers without problems. - -For (2) we have another template that will do the conversion for us - -``` -template -struct ASTSerialConvertTypeInfo; -``` - -That we can use as above, and specify the native and serial types. - -For (3) there are a few scenarios. For any field in a serial type we must store in the serialized type such that the representation -will work across all processors/compilers. So one problematic type is `bool`. It's not specified how it's laid out in memory - and -some compiles have stored it as a word. Most recently it's been stored as a byte. To make sure bool is ok for serialization therefore -we store as a uint8_t. - -Another example would be double. It's 64 bits, but on some arches/compilers it's SLANG_ALIGN_OF is 4 and on others it's 8. On some -arches a non aligned read will lead to a fault. To work around this problem therefore we have to ensure double has the alignment that -will work across all targets - and that alignment is 8. In that specific case that issue is handled via ASTSerialBasicTypeInfo, which -makes the SerialAlignment the sizeof the type. - -For (4) there are a few things to say. First a type can always implement a custom version of how to do a conversion by specializing -`ASTSerialTypeInfo`. But there remains another nagging issue - types which allocate/use other memory that changes at runtime. Clearly -we cannot define 'any size of memory' in a fixed SerialType defined in a specialization of ASTSerialTypeInfo. The mechanism to work around -this is to allow arbitrary arrays to be stored, that can be accessed via an ASTSerialIndex. This will be discussed more once we discuss -a little more about the file system, and ASTSerialIndex. - -Serialization Format -==================== - -The serialization format used is 'stream-like' with each 'object' stored in order. Each object is given an index starting from 1. -0 is used to be in effect nullptr. The stream looks like - -``` -ASTSerialInfo::Entry (for index 1) -Payload for type in entry - -ASTSerialInfo::Entry (for index 2) -Payload for type in entry - -... -... - -That when writing we have an array that maps each index to a pointer to the associated header. We also have a map that maps native pointers -to their indices. The Payload *is* the SerialType for thing saved. The payload directly follows the Entry data. - -Each object in this list can only be a few types of things - those derived from ASTSerialInfo::Type. - -The actual Entry followed by the payloads are allocated and stored when writing in a MemoryArena. When we want to write into a stream, we -can just iterate over each entry in order and write it out. - -You may have spotted a problem here - that some Entry types can be stored without alignment (for example a string - which stores the length -VarInt encoded followed by the characters). Others require an alignment - for example an NodeBase derived type that contains a int64_t will -*require* 8 byte alignment. That as a feature of the serialization format we want to be able to just map the data into memory, and be able -to access all the SerialType as is on the CPU. For that to work we *require* that the payload for each entry has the right alignment for -the associated SerialType. - -To achieve this we store in the Entry it's alignment requirement *AND* the next entries alignment. With this when we read, as we as stepping -through the entries we can find where the next Entry starts. Because the payload comes directly after the Entry - the Entrys size must be -a modulo of the largest alignment the payload can have. - -For the code that does the conversion between native and serial types it uses either the ASTSerialWriter or ASTSerialReader. This provides -the mechanism to turn a pointer into a serializable ASTSerialIndex and vice versa. There are some special functions for turning string like -types to and forth. - -The final mechanism is that of 'Arrays'. An array allows reading or writing a chunk of data associated with a ASTSerialIndex. The chunk of -data *must* hold data that is serializable. If the array holds pointers - then the serialized array must hold ASTSerialIndices that -represent those pointers. When reading back in they are converted back. - -Arrays are the escape hatch that allows for more complex types to serialize. Dictionaries for example are saved as a serial type that is -two ASTSerialIndices one to a keys array and one to a values array. - -Note that writing has two phases, serializing out into an ASTSerialWriter, and then secondly writing out to a stream. - -NodeBase Types -============== - -The ASTSerialTypeInfo mechanism is generally for *fields* of NodeBase types. That for NodeBase derived types we use the C++ extractors -field list to work out the native fields offsets and types. With this we can then calculate the layout for NodeBase types such that they -follow the requirements for serialization - such as alignment and so forth. - -This information is held in the ASTSerialClasses, which for a given ASTNodeType gives an ASTSerialClassInfo, that specifies fields for -just that type. Super types fields need to be serialized too, and this information can be found by using the ClassReflectInfo to find the -super type. - -Reading -======= - -Due to the care in writing reading is relatively simple. We can just take the contents of the file and put in memory, as long as in memory -it has an alignment of at least MAX_ALIGNMENT. Then we can build up an entries table by stepping through the data and writing the pointer. - -The toNative functions take an ASTSerialReader - this allows the implementation to ask for pointers and arrays from other parts of the serialized -data. It also allows for types to be lazily reconstructed if necessary. - -Lazy reconstruction may be useful in the future to partially reconstruct a sub part of the serialized data. In the current implementation, lazy -evaluation is used on Strings. The m_objects array holds all of the recreated native 'objects'. Since the objects can be derived from different -base classes the associated Entry will describe what it really is. - -For the String type, we initially store the object pointer as null. If a string is requested from that index, we see if the object pointer is null, -if it is we have to construct the StringRepresentation that will be used. - -An extra wrinkle is that we allow accessing of a serialized String as a Name or a string or a UnownedSubString. Fortunately a Name just holds a string, -and a Name remains in scope as long as it's NamePool does which is passed in. -*/ - -/* Holds RIFF FourCC codes for AST types */ -struct ASTSerialBinary -{ - static const FourCC kRiffFourCC = RiffFourCC::kRiff; - - /// AST module LIST container - static const FourCC kSlangASTModuleFourCC = SLANG_FOUR_CC('S', 'A', 'm', 'l'); - /// AST module data - static const FourCC kSlangASTModuleDataFourCC = SLANG_FOUR_CC('S', 'A', 'm', 'd'); -}; - -class ASTSerialClasses; - -// Type used to implement mechanisms to convert to and from serial types. -template -struct ASTSerialTypeInfo; - -struct ASTSerialInfo -{ - enum - { - // Data held in serialized format, the maximally allowed alignment - MAX_ALIGNMENT = 8, - }; - - // We only allow up to MAX_ALIGNMENT bytes of alignment. We store alignments as shifts, so 2 bits needed for 1 - 8 - enum class EntryInfo : uint8_t - { - Alignment1 = 0, - }; - - static EntryInfo makeEntryInfo(int alignment, int nextAlignment) - { - // Make sure they are power of 2 - SLANG_ASSERT((alignment & (alignment - 1)) == 0); - SLANG_ASSERT((nextAlignment & (nextAlignment - 1)) == 0); - - const int alignmentShift = ByteEncodeUtil::calcMsb8(alignment); - const int nextAlignmentShift = ByteEncodeUtil::calcMsb8(nextAlignment); - return EntryInfo((nextAlignmentShift << 2) | alignmentShift); - } - static EntryInfo makeEntryInfo(int alignment) - { - // Make sure they are power of 2 - SLANG_ASSERT((alignment & (alignment - 1)) == 0); - return EntryInfo(ByteEncodeUtil::calcMsb8(alignment)); - } - /// Apply with the next alignment - static EntryInfo combineWithNext(EntryInfo cur, EntryInfo next) - { - return EntryInfo((int(cur) & ~0xc0) | ((int(next) & 3) << 2)); - } - - static int getAlignment(EntryInfo info) { return 1 << (int(info) & 3); } - static int getNextAlignment(EntryInfo info) { return 1 << ((int(info) >> 2) & 3); } - - enum class Type : uint8_t - { - String, ///< String - Node, ///< NodeBase derived - RefObject, ///< RefObject derived types - Array, ///< Array - }; - - - /* Alignment is a little tricky. We have a 'Entry' header before the payload. The payload alignment may change. - If we only align on the Entry header, then it's size *must* be some modulo of the maximum alignment allowed. - - We could hold Entry separate from payload. We could make the header not require the alignment of the payload - but then - we'd need payload alignment separate from entry alignment. - */ - struct Entry - { - Type type; - EntryInfo info; - - size_t calcSize(ASTSerialClasses* serialClasses) const; - }; - - struct StringEntry : Entry - { - char sizeAndChars[1]; - }; - - struct NodeEntry : Entry - { - uint16_t astNodeType; - uint32_t _pad0; ///< Necessary, because a node *can* have MAX_ALIGNEMENT - }; - - struct RefObjectEntry : Entry - { - enum class SubType : uint8_t - { - Breadcrumb, - }; - SubType subType; - uint8_t _pad0; - uint32_t _pad1; ///< Necessary because RefObjectEntry *can* have MAX_ALIGNEMENT - }; - - struct ArrayEntry : Entry - { - uint16_t elementSize; - uint32_t elementCount; - }; -}; - -typedef uint32_t ASTSerialIndexRaw; -enum class ASTSerialIndex : ASTSerialIndexRaw; -typedef uint32_t ASTSerialSourceLoc; - -/* A type to convert pointers into types such that they can be passed around to readers/writers without -having to know the specific type. If there was a base class that all the serialized types derived from, -that was dynamically castable this would not be necessary */ -struct ASTSerialPointer -{ - enum class Kind - { - Unknown, - RefObject, - NodeBase - }; - - // Helpers so we can choose what kind of pointer we have based on the (unused) type of the pointer passed in - SLANG_FORCE_INLINE RefObject* _get(const RefObject*) { return m_kind == Kind::RefObject ? reinterpret_cast(m_ptr) : nullptr; } - SLANG_FORCE_INLINE NodeBase* _get(const NodeBase*) { return m_kind == Kind::NodeBase ? reinterpret_cast(m_ptr) : nullptr; } - - template - T* dynamicCast() - { - return Slang::dynamicCast(_get((T*)nullptr)); - } - - ASTSerialPointer() : - m_kind(Kind::Unknown), - m_ptr(nullptr) - { - } - - ASTSerialPointer(RefObject* in) : - m_kind(Kind::RefObject), - m_ptr((void*)in) - { - } - ASTSerialPointer(NodeBase* in) : - m_kind(Kind::NodeBase), - m_ptr((void*)in) - { - } - - static Kind getKind(const RefObject*) { return Kind::RefObject; } - static Kind getKind(const NodeBase*) { return Kind::NodeBase; } - - Kind m_kind; - void* m_ptr; -}; - - -/* This class is the interface used by toNative implementations to recreate a type */ -class ASTSerialReader : public RefObject -{ -public: - - typedef ASTSerialInfo::Entry Entry; - typedef ASTSerialInfo::Type Type; - - template - void getArray(ASTSerialIndex index, List& out); - - const void* getArray(ASTSerialIndex index, Index& outCount); - - ASTSerialPointer getPointer(ASTSerialIndex index); - String getString(ASTSerialIndex index); - Name* getName(ASTSerialIndex index); - UnownedStringSlice getStringSlice(ASTSerialIndex index); - SourceLoc getSourceLoc(ASTSerialSourceLoc loc); - - - /// Load the entries table (without deserializing anything) - /// NOTE! data must stay ins scope for outEntries to be valid - SlangResult loadEntries(const uint8_t* data, size_t dataCount, List& outEntries); - - /// NOTE! data must stay ins scope when reading takes place - SlangResult load(const uint8_t* data, size_t dataCount, ASTBuilder* builder, NamePool* namePool); - - /// Read the modules from the container - static Result readContainerModules(RiffContainer* container, Linkage* linkage, List>& outModules); - - ASTSerialReader(ASTSerialClasses* classes): - m_classes(classes) - { - } - -protected: - List m_entries; ///< The entries - List m_objects; ///< The constructed objects - - List> m_scope; ///< Objects to keep in scope during construction - - NamePool* m_namePool; - - ASTSerialClasses* m_classes; ///< Used to deserialize -}; - -// --------------------------------------------------------------------------- -template -void ASTSerialReader::getArray(ASTSerialIndex index, List& out) -{ - typedef ASTSerialTypeInfo ElementTypeInfo; - typedef typename ElementTypeInfo::SerialType ElementSerialType; - - Index count; - auto serialElements = (const ElementSerialType*)getArray(index, count); - - if (count == 0) - { - out.clear(); - return; - } - - if (std::is_same::value) - { - // If they are the same we can just write out - out.clear(); - out.insertRange(0, (const T*)serialElements, count); - } - else - { - // Else we need to convert - out.setCount(count); - for (Index i = 0; i < count; ++i) - { - ElementTypeInfo::toNative(this, (const void*)&serialElements[i], (void*)&out[i]); - } - } -} - - -class ASTSerialClasses; -class ASTSerialWriter; - -class ASTSerialFilter -{ -public: - virtual ASTSerialIndex writePointer(ASTSerialWriter* writer, const NodeBase* ptr) = 0; -}; - -class ModuleASTSerialFilter : public ASTSerialFilter -{ -public: - virtual ASTSerialIndex writePointer(ASTSerialWriter* writer, const NodeBase* ptr) SLANG_OVERRIDE; - - ModuleASTSerialFilter(ModuleDecl* moduleDecl): - m_moduleDecl(moduleDecl) - { - } - - ModuleDecl* m_moduleDecl; -}; - -/* This is a class used tby toSerial implementations to turn native type into the serial type */ -class ASTSerialWriter : public RefObject -{ -public: - ASTSerialIndex addPointer(const NodeBase* ptr); - ASTSerialIndex addPointer(const RefObject* ptr); - - /// Write the pointer - ASTSerialIndex writePointer(const NodeBase* ptr); - - template - ASTSerialIndex addArray(const T* in, Index count); - - ASTSerialIndex addString(const UnownedStringSlice& slice); - ASTSerialIndex addString(const String& in); - ASTSerialIndex addName(const Name* name); - ASTSerialSourceLoc addSourceLoc(SourceLoc sourceLoc); - - /// Set a the index associated with an index. NOTE! That there cannot be a pre-existing setting. - void setPointerIndex(const NodeBase* ptr, ASTSerialIndex index); - - /// Get the entries table holding how each index maps to an entry - const List& getEntries() const { return m_entries; } - - /// Write to a stream - SlangResult write(Stream* stream); - - /// Write the state into the container - SlangResult writeIntoContainer(RiffContainer* container); - - ASTSerialWriter(ASTSerialClasses* classes, ASTSerialFilter* filter); - -protected: - - ASTSerialIndex _addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount); - - ASTSerialIndex _add(const void* nativePtr, ASTSerialInfo::Entry* entry) - { - m_entries.add(entry); - // Okay I need to allocate space for this - ASTSerialIndex index = ASTSerialIndex(m_entries.getCount() - 1); - // Add to the map - m_ptrMap.Add(nativePtr, Index(index)); - return index; - } - - Dictionary m_ptrMap; // Maps a pointer to an entry index - - // NOTE! Assumes the content stays in scope! - Dictionary m_sliceMap; - - List m_entries; ///< The entries - MemoryArena m_arena; ///< Holds the payloads - ASTSerialClasses* m_classes; - ASTSerialFilter* m_filter; ///< Filter to control what is serialized -}; - -// --------------------------------------------------------------------------- -template -ASTSerialIndex ASTSerialWriter::addArray(const T* in, Index count) -{ - typedef ASTSerialTypeInfo ElementTypeInfo; - typedef typename ElementTypeInfo::SerialType ElementSerialType; - - if (std::is_same::value) - { - // If they are the same we can just write out - return _addArray(sizeof(T), SLANG_ALIGN_OF(ElementSerialType), in, count); - } - else - { - // Else we need to convert - List work; - work.setCount(count); - - for (Index i = 0; i < count; ++i) - { - ElementTypeInfo::toSerial(this, &in[i], &work[i]); - } - return _addArray(sizeof(ElementSerialType), SLANG_ALIGN_OF(ElementSerialType), work.getBuffer(), count); - } -} - -struct ASTSerialType -{ - typedef void(*ToSerialFunc)(ASTSerialWriter* writer, const void* src, void* dst); - typedef void(*ToNativeFunc)(ASTSerialReader* reader, const void* src, void* dst); - - size_t serialSizeInBytes; - uint8_t serialAlignment; - ToSerialFunc toSerialFunc; - ToNativeFunc toNativeFunc; -}; - -struct ASTSerialField -{ - const char* name; ///< The name of the field - const ASTSerialType* type; ///< The type of the field - uint32_t nativeOffset; ///< Offset to field from base of type - uint32_t serialOffset; ///< Offset in serial type -}; - - -struct ASTSerialClass -{ - ASTNodeType type; - uint8_t alignment; - ASTSerialField* fields; - Index fieldsCount; - uint32_t size; -}; - -// An instance could be shared across Sessions, but for simplicity of life time -// here we don't deal with that -class ASTSerialClasses : public RefObject -{ -public: - - const ASTSerialClass* getSerialClass(ASTNodeType type) const { return &m_classes[Index(type)]; } - - /// Ctor - ASTSerialClasses(); - -protected: - MemoryArena m_arena; - - ASTSerialClass m_classes[Index(ASTNodeType::CountOf)]; -}; - - -/* None of the functions in this util should *not* be called from production code, -they exist to test features of AST Serialization */ -struct ASTSerialTestUtil -{ - static SlangResult selfTest(); - - /// Tries to serialize out, read back in and test the results are the same. - /// Will write dumped out node to files - static SlangResult testSerialize(NodeBase* node, RootNamePool* rootNamePool, SharedASTBuilder* sharedASTBuilder, SourceManager* sourceManager); -}; - -} // namespace Slang - -#endif diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp index 37a79e6c4..d8da642f4 100755 --- a/source/slang/slang-compiler.cpp +++ b/source/slang/slang-compiler.cpp @@ -23,9 +23,7 @@ #include "slang-glsl-extension-tracker.h" #include "slang-emit-cuda.h" -#include "slang-ast-serialize.h" - -#include "slang-ir-serialize.h" +#include "slang-serialize-container.h" // Enable calling through to `fxc` or `dxc` to // generate code on Windows. @@ -143,6 +141,13 @@ namespace Slang { if (downstreamResult) { + // TODO(JS): + // This seems a little questionable. As it stands downstreamResult, if it doesn't have a blob + // can try and read a file. How this currently works is that every getBlob will potentially try to read that file. + // Setting result to None would stop this, but is that reasonable as the state. + // Perhaps downstreamResult should hold some state that the read failed. + // For now we don't worry though. + SLANG_RETURN_ON_FAIL(downstreamResult->getBinary(blob)); } break; @@ -2342,104 +2347,28 @@ SlangResult dissassembleDXILUsingDXC( SlangResult EndToEndCompileRequest::writeContainerToStream(Stream* stream) { - RiffContainer container; + auto linkage = getLinkage(); - const IRSerialBinary::CompressionType compressionType = getLinkage()->irCompressionType; + // Set up options + SerialContainerUtil::WriteOptions options; + options.compressionType = linkage->serialCompressionType; + if (linkage->debugInfoLevel != DebugInfoLevel::None) { - // Module list - RiffContainer::ScopeChunk listScope(&container, RiffContainer::Chunk::Kind::List, SerialBinary::kSlangModuleListFourCc); - - auto linkage = getLinkage(); - auto sink = getSink(); - auto frontEndReq = getFrontEndReq(); - - IRSerialWriter::OptionFlags optionFlags = 0; - - if (linkage->debugInfoLevel != DebugInfoLevel::None) - { - optionFlags |= IRSerialWriter::OptionFlag::DebugInfo; - } - - RefPtr astClasses = new ASTSerialClasses; - - SourceManager* sourceManager = frontEndReq->getSourceManager(); - - for (TranslationUnitRequest* translationUnit : frontEndReq->translationUnits) - { - auto module = translationUnit->module; - auto irModule = module->getIRModule(); - - // Okay, we need to serialize this module to our container file. - // We currently don't serialize it's name..., but support for that could be added. - - // Write the IR information - { - IRSerialData serialData; - IRSerialWriter writer; - SLANG_RETURN_ON_FAIL(writer.write(irModule, sourceManager, optionFlags, &serialData)); - SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, compressionType, &container)); - } - - // Write the AST information - { - ModuleDecl* moduleDecl = translationUnit->getModuleDecl(); - - ModuleASTSerialFilter filter(moduleDecl); - ASTSerialWriter writer(astClasses, &filter); - - // Add the module and everything that isn't filtered out in the filter. - writer.addPointer(moduleDecl); - - // We can now serialize it into the riff container. - SLANG_RETURN_ON_FAIL(writer.writeIntoContainer(&container)); - } - } - - auto program = getSpecializedGlobalAndEntryPointsComponentType(); - - // TODO: in the case where we have specialization, we might need - // to serialize IR related to `program`... - - for (auto target : linkage->targets) - { - auto targetProgram = program->getTargetProgram(target); - auto irModule = targetProgram->getOrCreateIRModuleForLayout(sink); - - // Okay, we need to serialize this target program and its IR too... - IRSerialData serialData; - IRSerialWriter writer; - SLANG_RETURN_ON_FAIL(writer.write(irModule, sourceManager, optionFlags, &serialData)); - SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, compressionType, &container)); - } + options.optionFlags |= SerialOptionFlag::DebugInfo; + } - auto entryPointCount = program->getEntryPointCount(); - for( Index ii = 0; ii < entryPointCount; ++ii ) + { + RiffContainer container; { - auto entryPoint = program->getEntryPoint(ii); - auto entryPointMangledName = program->getEntryPointMangledName(ii); - - RiffContainer::ScopeChunk entryPointScope(&container, RiffContainer::Chunk::Kind::Data, SerialBinary::kEntryPointFourCc); - - auto writeString = [&](String const& str) - { - uint32_t length = (uint32_t) str.getLength(); - container.write(&length, sizeof(length)); - container.write(str.getBuffer(), length+1); - }; - - writeString(entryPoint->getName()->text); - - Profile profile = entryPoint->getProfile(); - container.write(&profile, sizeof(profile)); - - writeString(entryPointMangledName); + SerialContainerData data; + SLANG_RETURN_ON_FAIL(SerialContainerUtil::requestToData(this, options, data)); + SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container)); } + // We now write the RiffContainer to the stream + SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream)); } - // We now write the RiffContainer to the stream - SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream)); - return SLANG_OK; } diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index cbd62d2bc..f7a5d98b1 100755 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -18,7 +18,7 @@ #include "slang-include-system.h" -#include "slang-ir-serialize-types.h" +#include "slang-serialize-ir-types.h" #include "../../slang.h" @@ -50,7 +50,7 @@ namespace Slang ComputeShader, }; - enum class CodeGenTarget + enum class CodeGenTarget { Unknown = SLANG_TARGET_UNKNOWN, None = SLANG_TARGET_NONE, @@ -1375,7 +1375,7 @@ namespace Slang OptimizationLevel optimizationLevel = OptimizationLevel::Default; - IRSerialCompressionType irCompressionType = IRSerialCompressionType::VariableByteLite; + SerialCompressionType serialCompressionType = SerialCompressionType::VariableByteLite; bool m_requireCacheFileSystem = false; bool m_useFalcorCustomSharedKeywordSemantics = false; diff --git a/source/slang/slang-ir-serialize-types.cpp b/source/slang/slang-ir-serialize-types.cpp deleted file mode 100644 index b57c46b58..000000000 --- a/source/slang/slang-ir-serialize-types.cpp +++ /dev/null @@ -1,333 +0,0 @@ -// slang-ir-serialize-types.cpp -#include "slang-ir-serialize-types.h" - -#include "../core/slang-text-io.h" -#include "../core/slang-byte-encode-util.h" - -#include "slang-ir-insts.h" - -#include "../core/slang-math.h" - -namespace Slang { - -// Needed for linkage with some compilers -/* static */ const IRSerialData::StringIndex IRSerialData::kNullStringIndex; -/* static */ const IRSerialData::StringIndex IRSerialData::kEmptyStringIndex; - -/* Note that an IRInst can be derived from, but when it derived from it's new members are IRUse variables, and they in -effect alias over the operands - and reflected in the operand count. There _could_ be other members after these IRUse -variables, but only a few types include extra data, and these do not have any operands: - -* IRConstant - Needs special-case handling -* IRModuleInst - Presumably we can just set to the module pointer on reconstruction - -Note! That on an IRInst there is an IRType* variable (accessed as getFullType()). As it stands it may NOT actually point -to an IRType derived type. Its 'ok' as long as it's an instruction that can be used in the place of the type. So this code does not -bother to check if it's correct, and just casts it. -*/ - -/* static */const IRSerialData::PayloadInfo IRSerialData::s_payloadInfos[int(Inst::PayloadType::CountOf)] = -{ - { 0, 0 }, // Empty - { 1, 0 }, // Operand_1 - { 2, 0 }, // Operand_2 - { 1, 0 }, // OperandAndUInt32, - { 0, 0 }, // OperandExternal - This isn't correct, Operand has to be specially handled - { 0, 1 }, // String_1, - { 0, 2 }, // String_2, - { 0, 0 }, // UInt32, - { 0, 0 }, // Float64, - { 0, 0 } // Int64, -}; - -// Check all compressible chunk ids, start with upper case 'S' -SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kInstFourCc) == 'S'); -SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kChildRunFourCc) == 'S'); -SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kExternalOperandsFourCc) == 'S'); - -// Compressed version starts with 's' -SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(SLANG_MAKE_COMPRESSED_FOUR_CC(IRSerialBinary::kInstFourCc)) == 's'); - -struct PrefixString; - -namespace { // anonymous - -struct CharReader -{ - char operator()(int pos) const { SLANG_UNUSED(pos); return *m_pos++; } - CharReader(const char* pos) :m_pos(pos) {} - mutable const char* m_pos; -}; - -} // anonymous - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! StringRepresentationCache !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -StringRepresentationCache::StringRepresentationCache(): - m_stringTable(nullptr) -{ -} - -void StringRepresentationCache::init(const List* stringTable) -{ - m_stringTable = stringTable; - - // Decode the table - m_entries.setCount(StringSlicePool::kDefaultHandlesCount); - SLANG_COMPILE_TIME_ASSERT(StringSlicePool::kDefaultHandlesCount == 2); - - { - Entry& entry = m_entries[0]; - entry.m_numChars = 0; - entry.m_startIndex = 0; - } - { - Entry& entry = m_entries[1]; - entry.m_numChars = 0; - entry.m_startIndex = 0; - } - - { - const char* start = stringTable->begin(); - const char* cur = start; - const char* end = stringTable->end(); - - while (cur < end) - { - CharReader reader(cur); - const int len = GetUnicodePointFromUTF8(reader); - - Entry entry; - entry.m_startIndex = uint32_t(reader.m_pos - start); - entry.m_numChars = len; - - m_entries.add(entry); - - cur = reader.m_pos + len; - } - } - - m_entries.compress(); -} - -UnownedStringSlice StringRepresentationCache::getStringSlice(Handle handle) const -{ - const Entry& entry = m_entries[int(handle)]; - const char* start = m_stringTable->begin(); - - return UnownedStringSlice(start + entry.m_startIndex, int(entry.m_numChars)); -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialStringTableUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -/* static */void SerialStringTableUtil::encodeStringTable(const StringSlicePool& pool, List& stringTable) -{ - // Skip the default handles -> nothing is encoded via them - return encodeStringTable(pool.getAdded(), stringTable); -} - -/* static */void SerialStringTableUtil::encodeStringTable(const ConstArrayView& slices, List& stringTable) -{ - stringTable.clear(); - for (const auto& slice : slices) - { - const int len = int(slice.getLength()); - - // We need to write into the the string array - char prefixBytes[6]; - const int numPrefixBytes = EncodeUnicodePointToUTF8(prefixBytes, len); - const Index baseIndex = stringTable.getCount(); - - stringTable.setCount(baseIndex + numPrefixBytes + len); - - char* dst = stringTable.begin() + baseIndex; - - memcpy(dst, prefixBytes, numPrefixBytes); - memcpy(dst + numPrefixBytes, slice.begin(), len); - } -} - -/* static */void SerialStringTableUtil::appendDecodedStringTable(const List& stringTable, List& slicesOut) -{ - const char* start = stringTable.begin(); - const char* cur = start; - const char* end = stringTable.end(); - - while (cur < end) - { - CharReader reader(cur); - const int len = GetUnicodePointFromUTF8(reader); - slicesOut.add(UnownedStringSlice(reader.m_pos, len)); - cur = reader.m_pos + len; - } -} - -/* static */void SerialStringTableUtil::decodeStringTable(const List& stringTable, List& slicesOut) -{ - slicesOut.setCount(2); - slicesOut[0] = UnownedStringSlice(nullptr, size_t(0)); - slicesOut[1] = UnownedStringSlice("", size_t(0)); - - appendDecodedStringTable(stringTable, slicesOut); -} - -/* static */void SerialStringTableUtil::calcStringSlicePoolMap(const List& slices, StringSlicePool& pool, List& indexMapOut) -{ - SLANG_ASSERT(slices.getCount() >= StringSlicePool::kDefaultHandlesCount); - SLANG_ASSERT(slices[int(StringSlicePool::kNullHandle)] == "" && slices[int(StringSlicePool::kNullHandle)].begin() == nullptr); - SLANG_ASSERT(slices[int(StringSlicePool::kEmptyHandle)] == ""); - - indexMapOut.setCount(slices.getCount()); - // Set up all of the defaults - for (int i = 0; i < StringSlicePool::kDefaultHandlesCount; ++i) - { - indexMapOut[i] = StringSlicePool::Handle(i); - } - - const Index numSlices = slices.getCount(); - for (Index i = StringSlicePool::kDefaultHandlesCount; i < numSlices ; ++i) - { - indexMapOut[i] = pool.add(slices[i]); - } -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -template -static size_t _calcArraySize(const List& list) -{ - return list.getCount() * sizeof(T); -} - -size_t IRSerialData::calcSizeInBytes() const -{ - return - _calcArraySize(m_insts) + - _calcArraySize(m_childRuns) + - _calcArraySize(m_externalOperands) + - _calcArraySize(m_stringTable) + - /* Raw source locs */ - _calcArraySize(m_rawSourceLocs) + - /* Debug */ - _calcArraySize(m_debugStringTable) + - _calcArraySize(m_debugLineInfos) + - _calcArraySize(m_debugSourceInfos) + - _calcArraySize(m_debugAdjustedLineInfos) + - _calcArraySize(m_debugSourceLocRuns); -} - -IRSerialData::IRSerialData() -{ - clear(); -} - -void IRSerialData::clear() -{ - // First Instruction is null - m_insts.setCount(1); - memset(&m_insts[0], 0, sizeof(Inst)); - - m_childRuns.clear(); - m_externalOperands.clear(); - m_rawSourceLocs.clear(); - - m_stringTable.clear(); - - // Debug data - m_debugLineInfos.clear(); - m_debugAdjustedLineInfos.clear(); - m_debugSourceInfos.clear(); - m_debugSourceLocRuns.clear(); - m_debugStringTable.clear(); -} - -template -static bool _isEqual(const List& aIn, const List& bIn) -{ - if (aIn.getCount() != bIn.getCount()) - { - return false; - } - - size_t size = size_t(aIn.getCount()); - - const T* a = aIn.begin(); - const T* b = bIn.begin(); - - if (a == b) - { - return true; - } - - for (size_t i = 0; i < size; ++i) - { - if (a[i] != b[i]) - { - return false; - } - } - - return true; -} - -bool IRSerialData::operator==(const ThisType& rhs) const -{ - return (this == &rhs) || - (_isEqual(m_insts, rhs.m_insts) && - _isEqual(m_childRuns, rhs.m_childRuns) && - _isEqual(m_externalOperands, rhs.m_externalOperands) && - _isEqual(m_rawSourceLocs, rhs.m_rawSourceLocs) && - _isEqual(m_stringTable, rhs.m_stringTable) && - /* Debug */ - _isEqual(m_debugStringTable, rhs.m_debugStringTable) && - _isEqual(m_debugLineInfos, rhs.m_debugLineInfos) && - _isEqual(m_debugAdjustedLineInfos, rhs.m_debugAdjustedLineInfos) && - _isEqual(m_debugSourceInfos, rhs.m_debugSourceInfos) && - _isEqual(m_debugSourceLocRuns, rhs.m_debugSourceLocRuns)); -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialTypeUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -#define SLANG_SERIAL_BINARY_COMPRESSION_TYPE(x) \ - x(None, none) \ - x(VariableByteLite, lite) - -/* static */SlangResult IRSerialTypeUtil::parseCompressionType(const UnownedStringSlice& text, IRSerialCompressionType& outType) -{ - struct Pair - { - UnownedStringSlice name; - IRSerialCompressionType type; - }; - -#define SLANG_SERIAL_BINARY_PAIR(type, name) { UnownedStringSlice::fromLiteral(#name), IRSerialCompressionType::type}, - - static const Pair s_pairs[] = { - SLANG_SERIAL_BINARY_COMPRESSION_TYPE(SLANG_SERIAL_BINARY_PAIR) - }; - - for (const auto& pair : s_pairs) - { - if (pair.name == text) - { - outType = pair.type; - return SLANG_OK; - } - } - return SLANG_FAIL; -} - -/* static */UnownedStringSlice IRSerialTypeUtil::getText(IRSerialCompressionType type) -{ -#define SLANG_SERIAL_BINARY_CASE(type, name) case IRSerialCompressionType::type: return UnownedStringSlice::fromLiteral(#name); - switch (type) - { - SLANG_SERIAL_BINARY_COMPRESSION_TYPE(SLANG_SERIAL_BINARY_CASE) - default: break; - } - - SLANG_ASSERT(!"Unknown compression type"); - return UnownedStringSlice::fromLiteral("unknown"); -} - -} // namespace Slang diff --git a/source/slang/slang-ir-serialize-types.h b/source/slang/slang-ir-serialize-types.h deleted file mode 100644 index 3d006b86e..000000000 --- a/source/slang/slang-ir-serialize-types.h +++ /dev/null @@ -1,418 +0,0 @@ -// slang-ir-serialize-types.h -#ifndef SLANG_IR_SERIALIZE_TYPES_H_INCLUDED -#define SLANG_IR_SERIALIZE_TYPES_H_INCLUDED - -#include "../core/slang-riff.h" -#include "../core/slang-string-slice-pool.h" -#include "../core/slang-array-view.h" - -#include "slang-name.h" -#include "slang-source-loc.h" - -#include "slang-ir.h" - -namespace Slang { - -enum class IRSerialCompressionType : uint8_t -{ - None, - VariableByteLite, -}; - -class StringRepresentationCache -{ - // TODO: The name of this type is no longer accurate to its function. - // It doesn't *cache* anything, and instead just provides convenient - // access to the contents of a serialized string table. - - public: - typedef StringSlicePool::Handle Handle; - - struct Entry - { - uint32_t m_startIndex; - uint32_t m_numChars; - }; - - /// Get as a string slice - UnownedStringSlice getStringSlice(Handle handle) const; - - /// Initialize a cache to use a string table - void init(const List* stringTable); - - /// Ctor - StringRepresentationCache(); - - protected: - const List* m_stringTable; - List m_entries; -}; - -struct SerialStringTableUtil -{ - /// Convert a pool into a string table - static void encodeStringTable(const StringSlicePool& pool, List& stringTable); - static void encodeStringTable(const ConstArrayView& slices, List& stringTable); - /// Appends the decoded strings into slicesOut - static void appendDecodedStringTable(const List& stringTable, List& slicesOut); - /// Decodes a string table (and does so such that the indices are compatible with StringSlicePool) - static void decodeStringTable(const List& stringTable, List& slicesOut); - - /// Produces an index map, from slices to indices in pool - static void calcStringSlicePoolMap(const List& slices, StringSlicePool& pool, List& indexMap); -}; - -// Pre-declare -class Name; - -struct IRSerialData -{ - typedef IRSerialData ThisType; - - enum class InstIndex : uint32_t; - enum class StringIndex : uint32_t; - enum class ArrayIndex : uint32_t; - - enum class RawSourceLoc : SourceLoc::RawValue; ///< This is just to copy over source loc data (ie not strictly serialize) - enum class StringOffset : uint32_t; ///< Offset into the m_stringsBuffer - - typedef uint32_t SizeType; - - static const StringIndex kNullStringIndex = StringIndex(StringSlicePool::kNullHandle); - static const StringIndex kEmptyStringIndex = StringIndex(StringSlicePool::kEmptyHandle); - - /// A run of instructions - struct InstRun - { - typedef InstRun ThisType; - SLANG_FORCE_INLINE bool operator==(const ThisType& rhs) const - { - return m_parentIndex == rhs.m_parentIndex && - m_startInstIndex == rhs.m_startInstIndex && - m_numChildren == rhs.m_numChildren; - } - SLANG_FORCE_INLINE bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - - InstIndex m_parentIndex; ///< The parent instruction - InstIndex m_startInstIndex; ///< The index to the first instruction - SizeType m_numChildren; ///< The number of children - }; - - struct SourceLocRun - { - typedef SourceLocRun ThisType; - - bool operator==(const ThisType& rhs) const { return m_sourceLoc == rhs.m_sourceLoc && m_startInstIndex == rhs.m_startInstIndex && m_numInst == rhs.m_numInst; } - bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - bool operator<(const ThisType& rhs) const { return m_sourceLoc < rhs.m_sourceLoc; } - - uint32_t m_sourceLoc; ///< The source location - InstIndex m_startInstIndex; ///< The index to the first instruction - SizeType m_numInst; ///< The number of children - }; - - struct PayloadInfo - { - uint8_t m_numOperands; - uint8_t m_numStrings; - }; - - struct DebugSourceInfo - { - typedef DebugSourceInfo ThisType; - - bool operator==(const ThisType& rhs) const - { - return m_pathIndex == rhs.m_pathIndex && - m_startSourceLoc == rhs.m_startSourceLoc && - m_endSourceLoc == rhs.m_endSourceLoc && - m_numLineInfos == rhs.m_numLineInfos && - m_lineInfosStartIndex == rhs.m_lineInfosStartIndex && - m_numLineInfos == rhs.m_numLineInfos; - } - bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - - bool isSourceLocInRange(uint32_t sourceLoc) const { return sourceLoc >= m_startSourceLoc && sourceLoc <= m_endSourceLoc; } - - StringIndex m_pathIndex; ///< Index to the string table - uint32_t m_startSourceLoc; ///< The offset to the source - uint32_t m_endSourceLoc; ///< The number of bytes in the source - - uint32_t m_numLines; ///< Total number of lines in source file - - uint32_t m_lineInfosStartIndex; ///< Index into m_debugLineInfos - uint32_t m_numLineInfos; ///< The number of line infos - - uint32_t m_adjustedLineInfosStartIndex; ///< Adjusted start index - uint32_t m_numAdjustedLineInfos; ///< The number of line infos - }; - - struct DebugLineInfo - { - typedef DebugLineInfo ThisType; - bool operator<(const ThisType& rhs) const { return m_lineStartOffset < rhs.m_lineStartOffset; } - bool operator==(const ThisType& rhs) const - { - return m_lineStartOffset == rhs.m_lineStartOffset && - m_lineIndex == rhs.m_lineIndex; - } - bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - - uint32_t m_lineStartOffset; ///< The offset into the source file - uint32_t m_lineIndex; ///< Original line index - }; - - struct DebugAdjustedLineInfo - { - typedef DebugAdjustedLineInfo ThisType; - bool operator==(const ThisType& rhs) const - { - return m_lineInfo == rhs.m_lineInfo && - m_adjustedLineIndex == rhs.m_adjustedLineIndex && - m_pathStringIndex == rhs.m_pathStringIndex; - } - bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - bool operator<(const ThisType& rhs) const { return m_lineInfo < rhs.m_lineInfo; } - - DebugLineInfo m_lineInfo; - uint32_t m_adjustedLineIndex; ///< The line index with the adjustment (if there is any). Is 0 if m_pathStringIndex is 0. - StringIndex m_pathStringIndex; ///< The path as an index - }; - - // Instruction... - // We can store SourceLoc values separately. Just store per index information. - // Parent information is stored in m_childRuns - // Decoration information is stored in m_decorationRuns - struct Inst - { - typedef Inst ThisType; - enum - { - kMaxOperands = 2, ///< Maximum number of operands that can be held in an instruction (otherwise held 'externally') - }; - - // NOTE! Can't change order or list without changing appropriate s_payloadInfos - enum class PayloadType : uint8_t - { - // First 3 must be in this order so a cast from 0-2 is directly represented as number of operands - Empty, ///< Has no payload (or operands) - Operand_1, ///< 1 Operand - Operand_2, ///< 2 Operands - - OperandAndUInt32, ///< 1 Operand and a single UInt32 - OperandExternal, ///< Operands are held externally - String_1, ///< 1 String - String_2, ///< 2 Strings - UInt32, ///< Holds an unsigned 32 bit integral (might represent a type) - Float64, - Int64, - - CountOf, - }; - - /// Get the number of operands - SLANG_FORCE_INLINE int getNumOperands() const; - - bool operator==(const ThisType& rhs) const; - - SLANG_FORCE_INLINE bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - - uint8_t m_op; ///< For now one of IROp - PayloadType m_payloadType; ///< The type of payload - uint16_t m_pad0; ///< Not currently used - - InstIndex m_resultTypeIndex; //< 0 if has no type. The result type of this instruction - - struct ExternalOperandPayload - { - ArrayIndex m_arrayIndex; ///< Index into the m_externalOperands table - SizeType m_size; ///< The amount of entries in that table - }; - - struct OperandAndUInt32 - { - InstIndex m_operand; - uint32_t m_uint32; - }; - - union Payload - { - double m_float64; - int64_t m_int64; - uint32_t m_uint32; ///< Unsigned integral value - IRFloatingPointValue m_float; ///< Floating point value - IRIntegerValue m_int; ///< Integral value - InstIndex m_operands[kMaxOperands]; ///< For items that 2 or less operands it can use this. - StringIndex m_stringIndices[kMaxOperands]; - ExternalOperandPayload m_externalOperand; ///< Operands are stored in an an index of an operand array - OperandAndUInt32 m_operandAndUInt32; - }; - - Payload m_payload; - }; - - /// Clear to initial state - void clear(); - /// Get the operands of an instruction - SLANG_FORCE_INLINE int getOperands(const Inst& inst, const InstIndex** operandsOut) const; - - /// == - bool operator==(const ThisType& rhs) const; - SLANG_FORCE_INLINE bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - - /// Calculate the amount of memory used by this IRSerialData - size_t calcSizeInBytes() const; - - /// Ctor - IRSerialData(); - - List m_insts; ///< The instructions - - List m_childRuns; ///< Holds the information about children that belong to an instruction - - List m_externalOperands; ///< Holds external operands (for instructions with more than kNumOperands) - - List m_stringTable; ///< All strings. Indexed into by StringIndex - - List m_rawSourceLocs; ///< A source location per instruction (saved without modification from IRInst)s - - // Data only set if we have debug information - - List m_debugStringTable; ///< String table for debug use only - List m_debugLineInfos; ///< Debug line information - List m_debugAdjustedLineInfos; ///< Adjusted line infos - List m_debugSourceInfos; ///< Debug source information - List m_debugSourceLocRuns; ///< Runs of instructions that use a source loc - - static const PayloadInfo s_payloadInfos[int(Inst::PayloadType::CountOf)]; -}; - -// -------------------------------------------------------------------------- -SLANG_FORCE_INLINE int IRSerialData::Inst::getNumOperands() const -{ - return (m_payloadType == PayloadType::OperandExternal) ? m_payload.m_externalOperand.m_size : s_payloadInfos[int(m_payloadType)].m_numOperands; -} - -// -------------------------------------------------------------------------- -SLANG_FORCE_INLINE bool IRSerialData::Inst::operator==(const ThisType& rhs) const -{ - if (m_op == rhs.m_op && - m_payloadType == rhs.m_payloadType && - m_resultTypeIndex == rhs.m_resultTypeIndex) - { - switch (m_payloadType) - { - case PayloadType::Empty: - { - return true; - } - case PayloadType::Operand_1: - case PayloadType::String_1: - case PayloadType::UInt32: - { - return m_payload.m_operands[0] == rhs.m_payload.m_operands[0]; - } - case PayloadType::OperandAndUInt32: - case PayloadType::OperandExternal: - case PayloadType::Operand_2: - case PayloadType::String_2: - { - return m_payload.m_operands[0] == rhs.m_payload.m_operands[0] && - m_payload.m_operands[1] == rhs.m_payload.m_operands[1]; - } - case PayloadType::Float64: - case PayloadType::Int64: - { - return m_payload.m_int64 == rhs.m_payload.m_int64; - } - default: break; - } - } - - return false; -} -// -------------------------------------------------------------------------- -SLANG_FORCE_INLINE int IRSerialData::getOperands(const Inst& inst, const InstIndex** operandsOut) const -{ - if (inst.m_payloadType == Inst::PayloadType::OperandExternal) - { - *operandsOut = m_externalOperands.begin() + int(inst.m_payload.m_externalOperand.m_arrayIndex); - return int(inst.m_payload.m_externalOperand.m_size); - } - else - { - *operandsOut = inst.m_payload.m_operands; - return s_payloadInfos[int(inst.m_payloadType)].m_numOperands; - } -} - -// For types/FourCC that work for serializing in general (not just IR). Really this should be placed in some other header -struct SerialBinary -{ - static const FourCC kRiffFourCc = RiffFourCC::kRiff; - - static const FourCC kSlangModuleListFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'l'); - - static const FourCC kEntryPointFourCc = SLANG_FOUR_CC('E', 'P', 'n', 't'); -}; - -// Replace first char with 's' -#define SLANG_MAKE_COMPRESSED_FOUR_CC(fourCc) SLANG_FOUR_CC_REPLACE_FIRST_CHAR(fourCc, 's') - -struct IRSerialBinary -{ - static const FourCC kSlangModuleFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'd'); ///< Holds all the slang specific chunks - - static const FourCC kSlangModuleHeaderFourCc = SLANG_FOUR_CC('S', 'L', 'h', 'd'); - - /* NOTE! All FourCC that can be compressed must start with capital 'S', because compressed version is the same FourCC - with the 'S' replaced with 's' */ - - static const FourCC kInstFourCc = SLANG_FOUR_CC('S', 'L', 'i', 'n'); - static const FourCC kChildRunFourCc = SLANG_FOUR_CC('S', 'L', 'c', 'r'); - static const FourCC kExternalOperandsFourCc = SLANG_FOUR_CC('S', 'L', 'e', 'o'); - - static const FourCC kCompressedInstFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kInstFourCc); - static const FourCC kCompressedChildRunFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kChildRunFourCc); - static const FourCC kCompressedExternalOperandsFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kExternalOperandsFourCc); - - static const FourCC kStringFourCc = SLANG_FOUR_CC('S', 'L', 's', 't'); - - static const FourCC kUInt32SourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '4'); - - static const FourCC kDebugStringFourCc = SLANG_FOUR_CC('S', 'd', 's', 't'); - static const FourCC kDebugLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'l', 'n'); - static const FourCC kDebugAdjustedLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'a', 'l'); - static const FourCC kDebugSourceInfoFourCc = SLANG_FOUR_CC('S', 'd', 's', 'o'); - static const FourCC kDebugSourceLocRunFourCc = SLANG_FOUR_CC('S', 'd', 's', 'r'); - - typedef IRSerialCompressionType CompressionType; - - struct ModuleHeader - { - uint32_t compressionType; ///< Holds the compression type used (if used at all) - }; - struct ArrayHeader - { - uint32_t numEntries; - }; - struct CompressedArrayHeader - { - uint32_t numEntries; ///< The number of entries - uint32_t numCompressedEntries; ///< The amount of compressed entries - }; - -}; - -struct IRSerialTypeUtil -{ - /// Given text, finds the compression type - static SlangResult parseCompressionType(const UnownedStringSlice& text, IRSerialCompressionType& outType); - /// Given a compression type, return text - static UnownedStringSlice getText(IRSerialCompressionType type); -}; - - -} // namespace Slang - -#endif diff --git a/source/slang/slang-ir-serialize.cpp b/source/slang/slang-ir-serialize.cpp deleted file mode 100644 index 1aa6b1d85..000000000 --- a/source/slang/slang-ir-serialize.cpp +++ /dev/null @@ -1,1589 +0,0 @@ -// slang-ir-serialize.cpp -#include "slang-ir-serialize.h" - -#include "../core/slang-text-io.h" -#include "../core/slang-byte-encode-util.h" - -#include "slang-ir-insts.h" - -#include "../core/slang-math.h" - -namespace Slang { - -static bool _isTextureTypeBase(IROp opIn) -{ - const int op = (kIROpMeta_OpMask & opIn); - return op >= kIROp_FirstTextureTypeBase && op <= kIROp_LastTextureTypeBase; -} - -static bool _isConstant(IROp opIn) -{ - const int op = (kIROpMeta_OpMask & opIn); - return op >= kIROp_FirstConstant && op <= kIROp_LastConstant; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -void IRSerialWriter::_addInstruction(IRInst* inst) -{ - // It cannot already be in the map - SLANG_ASSERT(!m_instMap.ContainsKey(inst)); - - // Add to the map - m_instMap.Add(inst, Ser::InstIndex(m_insts.getCount())); - m_insts.add(inst); -} - -void IRSerialWriter::_addDebugSourceLocRun(SourceLoc sourceLoc, uint32_t startInstIndex, uint32_t numInsts) -{ - SourceView* sourceView = m_sourceManager->findSourceView(sourceLoc); - if (!sourceView) - { - return; - } - - SourceFile* sourceFile = sourceView->getSourceFile(); - DebugSourceFile* debugSourceFile; - { - RefPtr* ptrDebugSourceFile = m_debugSourceFileMap.TryGetValue(sourceFile); - if (ptrDebugSourceFile == nullptr) - { - const SourceLoc::RawValue baseSourceLoc = m_debugFreeSourceLoc; - m_debugFreeSourceLoc += SourceLoc::RawValue(sourceView->getRange().getSize() + 1); - - debugSourceFile = new DebugSourceFile(sourceFile, baseSourceLoc); - m_debugSourceFileMap.Add(sourceFile, debugSourceFile); - } - else - { - debugSourceFile = *ptrDebugSourceFile; - } - } - - // We need to work out the line index - - int offset = sourceView->getRange().getOffset(sourceLoc); - int lineIndex = sourceFile->calcLineIndexFromOffset(offset); - - IRSerialData::DebugLineInfo lineInfo; - lineInfo.m_lineStartOffset = sourceFile->getLineBreakOffsets()[lineIndex]; - lineInfo.m_lineIndex = lineIndex; - - if (!debugSourceFile->hasLineIndex(lineIndex)) - { - // Add the information about the line - int entryIndex = sourceView->findEntryIndex(sourceLoc); - if (entryIndex < 0) - { - debugSourceFile->m_lineInfos.add(lineInfo); - } - else - { - const auto& entry = sourceView->getEntries()[entryIndex]; - - IRSerialData::DebugAdjustedLineInfo adjustedLineInfo; - adjustedLineInfo.m_lineInfo = lineInfo; - adjustedLineInfo.m_pathStringIndex = Ser::kNullStringIndex; - - const auto& pool = sourceView->getSourceManager()->getStringSlicePool(); - SLANG_ASSERT(pool.getStyle() == StringSlicePool::Style::Default); - - if (!pool.isDefaultHandle(entry.m_pathHandle)) - { - UnownedStringSlice slice = pool.getSlice(entry.m_pathHandle); - SLANG_ASSERT(slice.getLength() > 0); - adjustedLineInfo.m_pathStringIndex = Ser::StringIndex(m_debugStringSlicePool.add(slice)); - } - - adjustedLineInfo.m_adjustedLineIndex = lineIndex + entry.m_lineAdjust; - - debugSourceFile->m_adjustedLineInfos.add(adjustedLineInfo); - } - - debugSourceFile->setHasLineIndex(lineIndex); - } - - // Add the run - IRSerialData::SourceLocRun sourceLocRun; - sourceLocRun.m_numInst = numInsts; - sourceLocRun.m_startInstIndex = IRSerialData::InstIndex(startInstIndex); - sourceLocRun.m_sourceLoc = uint32_t(debugSourceFile->m_baseSourceLoc + offset); - - m_serialData->m_debugSourceLocRuns.add(sourceLocRun); -} - -Result IRSerialWriter::_calcDebugInfo() -{ - // We need to find the unique source Locs - // We are not going to store SourceLocs directly, because there may be multiple views mapping down to - // the same underlying source file - - // First find all the unique locs - struct InstLoc - { - typedef InstLoc ThisType; - - SLANG_FORCE_INLINE bool operator<(const ThisType& rhs) const { return sourceLoc < rhs.sourceLoc || (sourceLoc == rhs.sourceLoc && instIndex < rhs.instIndex); } - - uint32_t instIndex; - uint32_t sourceLoc; - }; - - // Find all of the source locations and their associated instructions - List instLocs; - const Index numInsts = m_insts.getCount(); - for (Index i = 1; i < numInsts; i++) - { - IRInst* srcInst = m_insts[i]; - if (!srcInst->sourceLoc.isValid()) - { - continue; - } - InstLoc instLoc; - instLoc.instIndex = uint32_t(i); - instLoc.sourceLoc = uint32_t(srcInst->sourceLoc.getRaw()); - instLocs.add(instLoc); - } - - // Sort them - instLocs.sort(); - m_debugFreeSourceLoc = 1; - - // Look for runs - const InstLoc* startInstLoc = instLocs.begin(); - const InstLoc* endInstLoc = instLocs.end(); - - while (startInstLoc < endInstLoc) - { - const uint32_t startSourceLoc = startInstLoc->sourceLoc; - - // Find the run with the same source loc - - const InstLoc* curInstLoc = startInstLoc + 1; - uint32_t curInstIndex = startInstLoc->instIndex + 1; - - // Find the run size with same source loc and run of instruction indices - for (; curInstLoc < endInstLoc && curInstLoc->sourceLoc == startSourceLoc && curInstLoc->instIndex == curInstIndex; ++curInstLoc, ++curInstIndex) - { - } - - // Try adding the run - _addDebugSourceLocRun(SourceLoc::fromRaw(startSourceLoc), startInstLoc->instIndex, curInstIndex - startInstLoc->instIndex); - - // Next - startInstLoc = curInstLoc; - } - - // Okay we can now calculate the final source information - - for (auto& pair : m_debugSourceFileMap) - { - DebugSourceFile* debugSourceFile = pair.Value; - SourceFile* sourceFile = debugSourceFile->m_sourceFile; - - IRSerialData::DebugSourceInfo sourceInfo; - - sourceInfo.m_numLines = uint32_t(debugSourceFile->m_sourceFile->getLineBreakOffsets().getCount()); - - sourceInfo.m_startSourceLoc = uint32_t(debugSourceFile->m_baseSourceLoc); - sourceInfo.m_endSourceLoc = uint32_t(debugSourceFile->m_baseSourceLoc + sourceFile->getContentSize()); - - sourceInfo.m_pathIndex = Ser::StringIndex(m_debugStringSlicePool.add(sourceFile->getPathInfo().foundPath)); - - sourceInfo.m_lineInfosStartIndex = uint32_t(m_serialData->m_debugLineInfos.getCount()); - sourceInfo.m_adjustedLineInfosStartIndex = uint32_t(m_serialData->m_debugAdjustedLineInfos.getCount()); - - sourceInfo.m_numLineInfos = uint32_t(debugSourceFile->m_lineInfos.getCount()); - sourceInfo.m_numAdjustedLineInfos = uint32_t(debugSourceFile->m_adjustedLineInfos.getCount()); - - // Add the line infos - m_serialData->m_debugLineInfos.addRange(debugSourceFile->m_lineInfos.begin(), debugSourceFile->m_lineInfos.getCount()); - m_serialData->m_debugAdjustedLineInfos.addRange(debugSourceFile->m_adjustedLineInfos.begin(), debugSourceFile->m_adjustedLineInfos.getCount()); - - // Add the source info - m_serialData->m_debugSourceInfos.add(sourceInfo); - } - - // Convert the string pool - SerialStringTableUtil::encodeStringTable(m_debugStringSlicePool, m_serialData->m_debugStringTable); - - return SLANG_OK; -} - -Result IRSerialWriter::write(IRModule* module, SourceManager* sourceManager, OptionFlags options, IRSerialData* serialData) -{ - typedef Ser::Inst::PayloadType PayloadType; - - m_sourceManager = sourceManager; - m_serialData = serialData; - - serialData->clear(); - - // We reserve 0 for null - m_insts.clear(); - m_insts.add(nullptr); - - // Reset - m_instMap.Clear(); - m_decorations.clear(); - - // Stack for parentInst - List parentInstStack; - - IRModuleInst* moduleInst = module->getModuleInst(); - parentInstStack.add(moduleInst); - - // Add to the map - _addInstruction(moduleInst); - - // Traverse all of the instructions - while (parentInstStack.getCount()) - { - // If it's in the stack it is assumed it is already in the inst map - IRInst* parentInst = parentInstStack.getLast(); - parentInstStack.removeLast(); - SLANG_ASSERT(m_instMap.ContainsKey(parentInst)); - - // Okay we go through each of the children in order. If they are IRInstParent derived, we add to stack to process later - // cos we want breadth first so the order of children is the same as their index order, meaning we don't need to store explicit indices - const Ser::InstIndex startChildInstIndex = Ser::InstIndex(m_insts.getCount()); - - IRInstListBase childrenList = parentInst->getDecorationsAndChildren(); - for (IRInst* child : childrenList) - { - // This instruction can't be in the map... - SLANG_ASSERT(!m_instMap.ContainsKey(child)); - - _addInstruction(child); - - parentInstStack.add(child); - } - - // If it had any children, then store the information about it - if (Ser::InstIndex(m_insts.getCount()) != startChildInstIndex) - { - Ser::InstRun run; - run.m_parentIndex = m_instMap[parentInst]; - run.m_startInstIndex = startChildInstIndex; - run.m_numChildren = Ser::SizeType(m_insts.getCount() - int(startChildInstIndex)); - - m_serialData->m_childRuns.add(run); - } - } - -#if 0 - { - List workInsts; - calcInstructionList(module, workInsts); - SLANG_ASSERT(workInsts.Count() == m_insts.Count()); - for (UInt i = 0; i < workInsts.Count(); ++i) - { - SLANG_ASSERT(workInsts[i] == m_insts[i]); - } - } -#endif - - // Set to the right size - m_serialData->m_insts.setCount(m_insts.getCount()); - // Clear all instructions - memset(m_serialData->m_insts.begin(), 0, sizeof(Ser::Inst) * m_serialData->m_insts.getCount()); - - // Need to set up the actual instructions - { - const Index numInsts = m_insts.getCount(); - - for (Index i = 1; i < numInsts; ++i) - { - IRInst* srcInst = m_insts[i]; - Ser::Inst& dstInst = m_serialData->m_insts[i]; - - dstInst.m_op = uint8_t(srcInst->op & kIROpMeta_OpMask); - dstInst.m_payloadType = PayloadType::Empty; - - dstInst.m_resultTypeIndex = getInstIndex(srcInst->getFullType()); - - IRConstant* irConst = as(srcInst); - if (irConst) - { - switch (srcInst->op) - { - // Special handling for the ir const derived types - case kIROp_StringLit: - { - auto stringLit = static_cast(srcInst); - dstInst.m_payloadType = PayloadType::String_1; - dstInst.m_payload.m_stringIndices[0] = getStringIndex(stringLit->getStringSlice()); - break; - } - case kIROp_IntLit: - { - dstInst.m_payloadType = PayloadType::Int64; - dstInst.m_payload.m_int64 = irConst->value.intVal; - break; - } - case kIROp_PtrLit: - { - dstInst.m_payloadType = PayloadType::Int64; - dstInst.m_payload.m_int64 = (intptr_t) irConst->value.ptrVal; - break; - } - case kIROp_FloatLit: - { - dstInst.m_payloadType = PayloadType::Float64; - dstInst.m_payload.m_float64 = irConst->value.floatVal; - break; - } - case kIROp_BoolLit: - { - dstInst.m_payloadType = PayloadType::UInt32; - dstInst.m_payload.m_uint32 = irConst->value.intVal ? 1 : 0; - break; - } - default: - { - SLANG_RELEASE_ASSERT(!"Unhandled constant type"); - return SLANG_FAIL; - } - } - continue; - } - - IRTextureTypeBase* textureBase = as(srcInst); - if (textureBase) - { - dstInst.m_payloadType = PayloadType::OperandAndUInt32; - dstInst.m_payload.m_operandAndUInt32.m_uint32 = uint32_t(srcInst->op) >> kIROpMeta_OtherShift; - dstInst.m_payload.m_operandAndUInt32.m_operand = getInstIndex(textureBase->getElementType()); - continue; - } - - // ModuleInst is different, in so far as it holds a pointer to IRModule, but we don't need - // to save that off in a special way, so can just use regular path - - const int numOperands = int(srcInst->operandCount); - Ser::InstIndex* dstOperands = nullptr; - - if (numOperands <= Ser::Inst::kMaxOperands) - { - // Checks the compile below is valid - SLANG_COMPILE_TIME_ASSERT(PayloadType(0) == PayloadType::Empty && PayloadType(1) == PayloadType::Operand_1 && PayloadType(2) == PayloadType::Operand_2); - - dstInst.m_payloadType = PayloadType(numOperands); - dstOperands = dstInst.m_payload.m_operands; - } - else - { - dstInst.m_payloadType = PayloadType::OperandExternal; - - int operandArrayBaseIndex = int(m_serialData->m_externalOperands.getCount()); - m_serialData->m_externalOperands.setCount(operandArrayBaseIndex + numOperands); - - dstOperands = m_serialData->m_externalOperands.begin() + operandArrayBaseIndex; - - auto& externalOperands = dstInst.m_payload.m_externalOperand; - externalOperands.m_arrayIndex = Ser::ArrayIndex(operandArrayBaseIndex); - externalOperands.m_size = Ser::SizeType(numOperands); - } - - for (int j = 0; j < numOperands; ++j) - { - const Ser::InstIndex dstInstIndex = getInstIndex(srcInst->getOperand(j)); - dstOperands[j] = dstInstIndex; - } - } - } - - // Convert strings into a string table - { - SerialStringTableUtil::encodeStringTable(m_stringSlicePool, serialData->m_stringTable); - } - - // If the option to use RawSourceLocations is enabled, serialize out as is - if (options & OptionFlag::RawSourceLocation) - { - const Index numInsts = m_insts.getCount(); - serialData->m_rawSourceLocs.setCount(numInsts); - - Ser::RawSourceLoc* dstLocs = serialData->m_rawSourceLocs.begin(); - // 0 is null, just mark as no location - dstLocs[0] = Ser::RawSourceLoc(0); - for (Index i = 1; i < numInsts; ++i) - { - IRInst* srcInst = m_insts[i]; - dstLocs[i] = Ser::RawSourceLoc(srcInst->sourceLoc.getRaw()); - } - } - - if (options & OptionFlag::DebugInfo) - { - _calcDebugInfo(); - } - - m_serialData = nullptr; - return SLANG_OK; -} - -static Result _writeArrayChunk(IRSerialCompressionType compressionType, FourCC chunkId, const void* data, size_t numEntries, size_t typeSize, RiffContainer* container) -{ - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ScopeChunk ScopeChunk; - - typedef IRSerialBinary Bin; - if (numEntries == 0) - { - return SLANG_OK; - } - - // Make compressed fourCC - chunkId = (compressionType != IRSerialCompressionType::None) ? SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId) : chunkId; - - ScopeChunk scope(container, Chunk::Kind::Data, chunkId); - - switch (compressionType) - { - case IRSerialCompressionType::None: - { - Bin::ArrayHeader header; - header.numEntries = uint32_t(numEntries); - - container->write(&header, sizeof(header)); - container->write(data, typeSize * numEntries); - break; - } - case IRSerialCompressionType::VariableByteLite: - { - List compressedPayload; - - size_t numCompressedEntries = (numEntries * typeSize) / sizeof(uint32_t); - ByteEncodeUtil::encodeLiteUInt32((const uint32_t*)data, numCompressedEntries, compressedPayload); - - Bin::CompressedArrayHeader header; - header.numEntries = uint32_t(numEntries); - header.numCompressedEntries = uint32_t(numCompressedEntries); - - container->write(&header, sizeof(header)); - container->write(compressedPayload.getBuffer(), compressedPayload.getCount()); - break; - } - default: - { - return SLANG_FAIL; - } - } - return SLANG_OK; -} - -template -Result _writeArrayChunk(IRSerialCompressionType compressionType, FourCC chunkId, const List& array, RiffContainer* container) -{ - return _writeArrayChunk(compressionType, chunkId, array.begin(), size_t(array.getCount()), sizeof(T), container); -} - -Result _encodeInsts(IRSerialCompressionType compressionType, const List& instsIn, List& encodeArrayOut) -{ - typedef IRSerialBinary Bin; - typedef IRSerialData::Inst::PayloadType PayloadType; - - if (compressionType != IRSerialCompressionType::VariableByteLite) - { - return SLANG_FAIL; - } - encodeArrayOut.clear(); - - const size_t numInsts = size_t(instsIn.getCount()); - const IRSerialData::Inst* insts = instsIn.begin(); - - uint8_t* encodeOut = encodeArrayOut.begin(); - uint8_t* encodeEnd = encodeArrayOut.end(); - - // Calculate the maximum instruction size with worst case possible encoding - // 2 bytes hold the payload size, and the result type - // Note that if there were some free bits, we could encode some of this stuff into bits, but if we remove payloadType, then there are no free bits - const size_t maxInstSize = 2 + ByteEncodeUtil::kMaxLiteEncodeUInt32 + Math::Max(sizeof(insts->m_payload.m_float64), size_t(2 * ByteEncodeUtil::kMaxLiteEncodeUInt32)); - - for (size_t i = 0; i < numInsts; ++i) - { - const auto& inst = insts[i]; - - // Make sure there is space for the largest possible instruction - if (encodeOut + maxInstSize >= encodeEnd) - { - const size_t offset = size_t(encodeOut - encodeArrayOut.begin()); - - const UInt oldCapacity = encodeArrayOut.getCapacity(); - - encodeArrayOut.reserve(oldCapacity + (oldCapacity >> 1) + maxInstSize); - const UInt capacity = encodeArrayOut.getCapacity(); - encodeArrayOut.setCount(capacity); - - encodeOut = encodeArrayOut.begin() + offset; - encodeEnd = encodeArrayOut.end(); - } - - *encodeOut++ = uint8_t(inst.m_op); - *encodeOut++ = uint8_t(inst.m_payloadType); - - encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_resultTypeIndex, encodeOut); - - switch (inst.m_payloadType) - { - case PayloadType::Empty: - { - break; - } - case PayloadType::Operand_1: - case PayloadType::String_1: - case PayloadType::UInt32: - { - // 1 UInt32 - encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_payload.m_operands[0], encodeOut); - break; - } - case PayloadType::Operand_2: - case PayloadType::OperandAndUInt32: - case PayloadType::OperandExternal: - case PayloadType::String_2: - { - // 2 UInt32 - encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_payload.m_operands[0], encodeOut); - encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_payload.m_operands[1], encodeOut); - break; - } - case PayloadType::Float64: - { - memcpy(encodeOut, &inst.m_payload.m_float64, sizeof(inst.m_payload.m_float64)); - encodeOut += sizeof(inst.m_payload.m_float64); - break; - } - case PayloadType::Int64: - { - memcpy(encodeOut, &inst.m_payload.m_int64, sizeof(inst.m_payload.m_int64)); - encodeOut += sizeof(inst.m_payload.m_int64); - break; - } - } - } - - // Fix the size - encodeArrayOut.setCount(UInt(encodeOut - encodeArrayOut.begin())); - return SLANG_OK; -} - -Result _writeInstArrayChunk(IRSerialCompressionType compressionType, FourCC chunkId, const List& array, RiffContainer* container) -{ - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ScopeChunk ScopeChunk; - - typedef IRSerialBinary Bin; - if (array.getCount() == 0) - { - return SLANG_OK; - } - - switch (compressionType) - { - case IRSerialCompressionType::None: - { - return _writeArrayChunk(compressionType, chunkId, array, container); - } - case IRSerialCompressionType::VariableByteLite: - { - List compressedPayload; - SLANG_RETURN_ON_FAIL(_encodeInsts(compressionType, array, compressedPayload)); - - ScopeChunk scope(container, Chunk::Kind::Data, SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId)); - - Bin::CompressedArrayHeader header; - header.numEntries = uint32_t(array.getCount()); - header.numCompressedEntries = 0; - - container->write(&header, sizeof(header)); - container->write(compressedPayload.getBuffer(), compressedPayload.getCount()); - - return SLANG_OK; - } - default: break; - } - return SLANG_FAIL; -} - -/* static */Result IRSerialWriter::writeContainer(const IRSerialData& data, IRSerialCompressionType compressionType, RiffContainer* container) -{ - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ScopeChunk ScopeChunk; - - ScopeChunk scopeModule(container, Chunk::Kind::List, Bin::kSlangModuleFourCc); - - // Write the header - { - Bin::ModuleHeader moduleHeader; - moduleHeader.compressionType = uint32_t(IRSerialCompressionType::VariableByteLite); - - ScopeChunk scopeHeader(container, Chunk::Kind::Data, Bin::kSlangModuleHeaderFourCc); - container->write(&moduleHeader, sizeof(moduleHeader)); - } - - SLANG_RETURN_ON_FAIL(_writeInstArrayChunk(compressionType, Bin::kInstFourCc, data.m_insts, container)); - SLANG_RETURN_ON_FAIL(_writeArrayChunk(compressionType, Bin::kChildRunFourCc, data.m_childRuns, container)); - SLANG_RETURN_ON_FAIL(_writeArrayChunk(compressionType, Bin::kExternalOperandsFourCc, data.m_externalOperands, container)); - SLANG_RETURN_ON_FAIL(_writeArrayChunk(IRSerialCompressionType::None, Bin::kStringFourCc, data.m_stringTable, container)); - - SLANG_RETURN_ON_FAIL(_writeArrayChunk(IRSerialCompressionType::None, Bin::kUInt32SourceLocFourCc, data.m_rawSourceLocs, container)); - - if (data.m_debugSourceInfos.getCount()) - { - _writeArrayChunk(IRSerialCompressionType::None, Bin::kDebugStringFourCc, data.m_debugStringTable, container); - _writeArrayChunk(IRSerialCompressionType::None, Bin::kDebugLineInfoFourCc, data.m_debugLineInfos, container); - _writeArrayChunk(IRSerialCompressionType::None, Bin::kDebugAdjustedLineInfoFourCc, data.m_debugAdjustedLineInfos, container); - _writeArrayChunk(IRSerialCompressionType::None, Bin::kDebugSourceInfoFourCc, data.m_debugSourceInfos, container); - _writeArrayChunk(compressionType, Bin::kDebugSourceLocRunFourCc, data.m_debugSourceLocRuns, container); - } - - return SLANG_OK; -} - -/* static */Result IRSerialWriter::writeStream(const IRSerialData& data, IRSerialCompressionType compressionType, Stream* stream) -{ - RiffContainer container; - SLANG_RETURN_ON_FAIL(writeContainer(data, compressionType, &container)); - return RiffUtil::write(container.getRoot(), true, stream); -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -class ListResizer -{ - public: - virtual void* setSize(size_t newSize) = 0; - SLANG_FORCE_INLINE size_t getTypeSize() const { return m_typeSize; } - ListResizer(size_t typeSize):m_typeSize(typeSize) {} - - protected: - size_t m_typeSize; -}; - -template -class ListResizerForType: public ListResizer -{ - public: - typedef ListResizer Parent; - - SLANG_FORCE_INLINE ListResizerForType(List& list): - Parent(sizeof(T)), - m_list(list) - {} - - virtual void* setSize(size_t newSize) SLANG_OVERRIDE - { - m_list.setCount(UInt(newSize)); - return (void*)m_list.begin(); - } - - protected: - List& m_list; -}; - -static Result _readArrayChunk(IRSerialCompressionType compressionType, RiffContainer::DataChunk* dataChunk, ListResizer& listOut) -{ - typedef IRSerialBinary Bin; - - RiffReadHelper read = dataChunk->asReadHelper(); - const size_t typeSize = listOut.getTypeSize(); - - switch (compressionType) - { - case IRSerialCompressionType::VariableByteLite: - { - Bin::CompressedArrayHeader header; - SLANG_RETURN_ON_FAIL(read.read(header)); - - void* dst = listOut.setSize(header.numEntries); - SLANG_ASSERT(header.numCompressedEntries == uint32_t((header.numEntries * typeSize) / sizeof(uint32_t))); - - // Decode.. - ByteEncodeUtil::decodeLiteUInt32(read.getData(), header.numCompressedEntries, (uint32_t*)dst); - break; - } - case IRSerialCompressionType::None: - { - // Read uncompressed - Bin::ArrayHeader header; - SLANG_RETURN_ON_FAIL(read.read(header)); - const size_t payloadSize = header.numEntries * typeSize; - SLANG_ASSERT(payloadSize == read.getRemainingSize()); - void* dst = listOut.setSize(header.numEntries); - ::memcpy(dst, read.getData(), payloadSize); - break; - } - } - return SLANG_OK; -} - -template -static Result _readArrayChunk(const IRSerialBinary::ModuleHeader* header, RiffContainer::DataChunk* dataChunk, List& arrayOut) -{ - typedef IRSerialBinary Bin; - - Bin::CompressionType compressionType = Bin::CompressionType::None; - - if (dataChunk->m_fourCC == SLANG_MAKE_COMPRESSED_FOUR_CC(dataChunk->m_fourCC)) - { - // If it has compression, use the compression type set in the header - compressionType = Bin::CompressionType(header->compressionType); - } - ListResizerForType resizer(arrayOut); - return _readArrayChunk(compressionType, dataChunk, resizer); -} - -template -static Result _readArrayUncompressedChunk(const IRSerialBinary::ModuleHeader* header, RiffContainer::DataChunk* chunk, List& arrayOut) -{ - SLANG_UNUSED(header); - ListResizerForType resizer(arrayOut); - return _readArrayChunk(IRSerialBinary::CompressionType::None, chunk, resizer); -} - -static Result _decodeInsts(IRSerialCompressionType compressionType, const uint8_t* encodeCur, size_t encodeInSize, List& instsOut) -{ - const uint8_t* encodeEnd = encodeCur + encodeInSize; - - typedef IRSerialBinary Bin; - typedef IRSerialData::Inst::PayloadType PayloadType; - - if (compressionType != IRSerialCompressionType::VariableByteLite) - { - return SLANG_FAIL; - } - - const size_t numInsts = size_t(instsOut.getCount()); - IRSerialData::Inst* insts = instsOut.begin(); - - for (size_t i = 0; i < numInsts; ++i) - { - if (encodeCur >= encodeEnd) - { - SLANG_ASSERT(!"Invalid decode"); - return SLANG_FAIL; - } - - auto& inst = insts[i]; - - inst.m_op = *encodeCur++; - const PayloadType payloadType = PayloadType(*encodeCur++); - inst.m_payloadType = payloadType; - - // Read the result value - encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, (uint32_t*)&inst.m_resultTypeIndex); - - switch (inst.m_payloadType) - { - case PayloadType::Empty: - { - break; - } - case PayloadType::Operand_1: - case PayloadType::String_1: - case PayloadType::UInt32: - { - // 1 UInt32 - encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, (uint32_t*)&inst.m_payload.m_operands[0]); - break; - } - case PayloadType::Operand_2: - case PayloadType::OperandAndUInt32: - case PayloadType::OperandExternal: - case PayloadType::String_2: - { - // 2 UInt32 - encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, 2, (uint32_t*)&inst.m_payload.m_operands[0]); - break; - } - case PayloadType::Float64: - { - memcpy(&inst.m_payload.m_float64, encodeCur, sizeof(inst.m_payload.m_float64)); - encodeCur += sizeof(inst.m_payload.m_float64); - break; - } - case PayloadType::Int64: - { - memcpy(&inst.m_payload.m_int64, encodeCur, sizeof(inst.m_payload.m_int64)); - encodeCur += sizeof(inst.m_payload.m_int64); - break; - } - } - } - - return SLANG_OK; -} - -static Result _readInstArrayChunk(const IRSerialBinary::ModuleHeader* moduleHeader, RiffContainer::DataChunk* chunk, List& arrayOut) -{ - typedef IRSerialBinary Bin; - - IRSerialCompressionType compressionType = IRSerialCompressionType::None; - if (chunk->m_fourCC == SLANG_MAKE_COMPRESSED_FOUR_CC(chunk->m_fourCC)) - { - compressionType = IRSerialCompressionType(moduleHeader->compressionType); - } - - switch (compressionType) - { - case IRSerialCompressionType::None: - { - ListResizerForType resizer(arrayOut); - return _readArrayChunk(compressionType, chunk, resizer); - } - case IRSerialCompressionType::VariableByteLite: - { - RiffReadHelper read = chunk->asReadHelper(); - - Bin::CompressedArrayHeader header; - SLANG_RETURN_ON_FAIL(read.read(header)); - - arrayOut.setCount(header.numEntries); - - SLANG_RETURN_ON_FAIL(_decodeInsts(compressionType, read.getData(), read.getRemainingSize(), arrayOut)); - break; - } - default: - { - return SLANG_FAIL; - } - } - - return SLANG_OK; -} - -/* static */Result IRSerialReader::readContainer(RiffContainer::ListChunk* module, IRSerialData* outData) -{ - typedef IRSerialBinary Bin; - - outData->clear(); - - Bin::ModuleHeader* header = module->findContainedData(Bin::kSlangModuleHeaderFourCc); - if (!header) - { - return SLANG_FAIL; - } - - for (RiffContainer::Chunk* chunk = module->m_containedChunks; chunk; chunk = chunk->m_next) - { - RiffContainer::DataChunk* dataChunk = as(chunk); - if (!dataChunk) - { - continue; - } - - switch (dataChunk->m_fourCC) - { - case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kInstFourCc): - case Bin::kInstFourCc: - { - SLANG_RETURN_ON_FAIL(_readInstArrayChunk(header, dataChunk, outData->m_insts)); - break; - } - case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kChildRunFourCc): - case Bin::kChildRunFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayChunk(header, dataChunk, outData->m_childRuns)); - break; - } - case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kExternalOperandsFourCc): - case Bin::kExternalOperandsFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayChunk(header, dataChunk, outData->m_externalOperands)); - break; - } - case Bin::kStringFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_stringTable)); - break; - } - case Bin::kUInt32SourceLocFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_rawSourceLocs)); - break; - } - case Bin::kDebugStringFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_debugStringTable)); - break; - } - case Bin::kDebugLineInfoFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_debugLineInfos)); - break; - } - case Bin::kDebugAdjustedLineInfoFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_debugAdjustedLineInfos)); - break; - } - case Bin::kDebugSourceInfoFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayChunk(header, dataChunk, outData->m_debugSourceInfos)); - break; - } - case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kDebugSourceLocRunFourCc): - case Bin::kDebugSourceLocRunFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayChunk(header, dataChunk, outData->m_debugSourceLocRuns)); - break; - } - default: - { - break; - } - } - } - - return SLANG_OK; -} - -/* static */Result IRSerialReader::readStream(Stream* stream, IRSerialData* outData) -{ - typedef IRSerialBinary Bin; - - outData->clear(); - - // Read the riff file - RiffContainer container; - SLANG_RETURN_ON_FAIL(RiffUtil::read(stream, container)); - - // Find the riff module - RiffContainer::ListChunk* module = container.getRoot()->findListRec(Bin::kSlangModuleFourCc); - if (!module) - { - return SLANG_FAIL; - } - - SLANG_RETURN_ON_FAIL(readContainer(module, outData)); - return SLANG_OK; -} - -static SourceRange _toSourceRange(const IRSerialData::DebugSourceInfo& info) -{ - SourceRange range; - range.begin = SourceLoc::fromRaw(info.m_startSourceLoc); - range.end = SourceLoc::fromRaw(info.m_endSourceLoc); - return range; -} - -static int _findIndex(const List& infos, SourceLoc sourceLoc) -{ - const int numInfos = int(infos.getCount()); - for (int i = 0; i < numInfos; ++i) - { - if (_toSourceRange(infos[i]).contains(sourceLoc)) - { - return i; - } - } - - return -1; -} - -static int _calcFixSourceLoc(const IRSerialData::DebugSourceInfo& info, SourceView* sourceView, SourceRange& rangeOut) -{ - rangeOut = _toSourceRange(info); - return int(sourceView->getRange().begin.getRaw()) - int(info.m_startSourceLoc); -} - -/* static */Result IRSerialReader::readContainerModules(RiffContainer* container, Session* session, SourceManager* sourceManager, List>& outModules, List& outEntryPoints) -{ - List moduleChunks; - List entryPointChunks; - // First try to find a list - { - RiffContainer::ListChunk* listChunk = container->getRoot()->findListRec(SerialBinary::kSlangModuleListFourCc); - if (listChunk) - { - listChunk->findContained(IRSerialBinary::kSlangModuleFourCc, moduleChunks); - listChunk->findContained(SerialBinary::kEntryPointFourCc, entryPointChunks); - } - else - { - // Maybe its just a single module - RiffContainer::ListChunk* moduleChunk = container->getRoot()->findListRec(IRSerialBinary::kSlangModuleFourCc); - if (!moduleChunk) - { - // Couldn't find any modules - return SLANG_FAIL; - } - moduleChunks.add(moduleChunk); - } - } - - // Okay, we need to decode into ir modules - for (auto moduleChunk : moduleChunks) - { - IRSerialData serialData; - - SLANG_RETURN_ON_FAIL(IRSerialReader::readContainer(moduleChunk, &serialData)); - - // Construct into a module - RefPtr irModule; - IRSerialReader reader; - SLANG_RETURN_ON_FAIL(reader.read(serialData, session, sourceManager, irModule)); - - outModules.add(irModule); - } - - for (auto entryPointChunk : entryPointChunks) - { - auto reader = entryPointChunk->asReadHelper(); - - auto readString = [&]() - { - uint32_t length = 0; - reader.read(length); - - char* begin = (char*)reader.getData(); - reader.skip(length + 1); - - return UnownedStringSlice(begin, begin + length); - }; - - FrontEndCompileRequest::ExtraEntryPointInfo entryPointInfo; - - entryPointInfo.name = session->getNamePool()->getName(readString()); - reader.read(entryPointInfo.profile); - entryPointInfo.mangledName = readString(); - - outEntryPoints.add(entryPointInfo); - } - - return SLANG_OK; -} - - -// TODO: The following function isn't really part of the IR serialization system, but rather -// a layered "container" format, and as such probably belongs in a higher-level system that -// simply calls into the `IRSerialReader` rather than being part of it... -// -/* static */Result IRSerialReader::readStreamModules(Stream* stream, Session* session, SourceManager* sourceManager, List>& outModules, List& outEntryPoints) -{ - // Load up the module - RiffContainer container; - SLANG_RETURN_ON_FAIL(RiffUtil::read(stream, container)); - - SLANG_RETURN_ON_FAIL(readContainerModules(&container, session, sourceManager, outModules, outEntryPoints)); - return SLANG_OK; -} - -/* static */Result IRSerialReader::read(const IRSerialData& data, Session* session, SourceManager* sourceManager, RefPtr& moduleOut) -{ - typedef Ser::Inst::PayloadType PayloadType; - - m_serialData = &data; - - auto module = new IRModule(); - moduleOut = module; - m_module = module; - - module->session = session; - - // Set up the string rep cache - m_stringRepresentationCache.init(&data.m_stringTable); - - // Add all the instructions - - List insts; - - const Index numInsts = data.m_insts.getCount(); - - SLANG_ASSERT(numInsts > 0); - - insts.setCount(numInsts); - insts[0] = nullptr; - - // 0 holds null - // 1 holds the IRModuleInst - { - // Check that insts[1] is the module inst - const Ser::Inst& srcInst = data.m_insts[1]; - SLANG_RELEASE_ASSERT(srcInst.m_op == kIROp_Module); - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Empty); - - // Create the module inst - auto moduleInst = static_cast(createEmptyInstWithSize(module, kIROp_Module, sizeof(IRModuleInst))); - module->moduleInst = moduleInst; - moduleInst->module = module; - - // Set the IRModuleInst - insts[1] = moduleInst; - } - - for (Index i = 2; i < numInsts; ++i) - { - const Ser::Inst& srcInst = data.m_insts[i]; - - const IROp op((IROp)srcInst.m_op); - - if (_isConstant(op)) - { - // Handling of constants - - // Calculate the minimum object size (ie not including the payload of value) - const size_t prefixSize = SLANG_OFFSET_OF(IRConstant, value); - - IRConstant* irConst = nullptr; - switch (op) - { - case kIROp_BoolLit: - { - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::UInt32); - irConst = static_cast(createEmptyInstWithSize(module, op, prefixSize + sizeof(IRIntegerValue))); - irConst->value.intVal = srcInst.m_payload.m_uint32 != 0; - break; - } - case kIROp_IntLit: - { - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Int64); - irConst = static_cast(createEmptyInstWithSize(module, op, prefixSize + sizeof(IRIntegerValue))); - irConst->value.intVal = srcInst.m_payload.m_int64; - break; - } - case kIROp_PtrLit: - { - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Int64); - irConst = static_cast(createEmptyInstWithSize(module, op, prefixSize + sizeof(void*))); - irConst->value.ptrVal = (void*) (intptr_t) srcInst.m_payload.m_int64; - break; - } - case kIROp_FloatLit: - { - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Float64); - irConst = static_cast(createEmptyInstWithSize(module, op, prefixSize + sizeof(IRFloatingPointValue))); - irConst->value.floatVal = srcInst.m_payload.m_float64; - break; - } - case kIROp_StringLit: - { - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::String_1); - - const UnownedStringSlice slice = m_stringRepresentationCache.getStringSlice(StringHandle(srcInst.m_payload.m_stringIndices[0])); - - const size_t sliceSize = slice.getLength(); - const size_t instSize = prefixSize + SLANG_OFFSET_OF(IRConstant::StringValue, chars) + sliceSize; - - irConst = static_cast(createEmptyInstWithSize(module, op, instSize)); - - IRConstant::StringValue& dstString = irConst->value.stringVal; - - dstString.numChars = uint32_t(sliceSize); - // Turn into pointer to avoid warning of array overrun - char* dstChars = dstString.chars; - // Copy the chars - memcpy(dstChars, slice.begin(), sliceSize); - break; - } - default: - { - SLANG_ASSERT(!"Unknown constant type"); - return SLANG_FAIL; - } - } - - insts[i] = irConst; - } - else if (_isTextureTypeBase(op)) - { - IRTextureTypeBase* inst = static_cast(createEmptyInst(module, op, 1)); - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::OperandAndUInt32); - - // Reintroduce the texture type bits into the the - const uint32_t other = srcInst.m_payload.m_operandAndUInt32.m_uint32; - inst->op = IROp(uint32_t(inst->op) | (other << kIROpMeta_OtherShift)); - - insts[i] = inst; - } - else - { - int numOperands = srcInst.getNumOperands(); - insts[i] = createEmptyInst(module, op, numOperands); - } - } - - // Patch up the operands - for (Index i = 1; i < numInsts; ++i) - { - const Ser::Inst& srcInst = data.m_insts[i]; - const IROp op((IROp)srcInst.m_op); - - IRInst* dstInst = insts[i]; - - // Set the result type - if (srcInst.m_resultTypeIndex != Ser::InstIndex(0)) - { - IRInst* resultInst = insts[int(srcInst.m_resultTypeIndex)]; - // NOTE! Counter intuitively the IRType* paramter may not be IRType* derived for example - // IRGlobalGenericParam is valid, but isn't IRType* derived - - //SLANG_RELEASE_ASSERT(as(resultInst)); - dstInst->setFullType(static_cast(resultInst)); - } - - //if (!isParentDerived(op)) - { - const Ser::InstIndex* srcOperandIndices; - const int numOperands = data.getOperands(srcInst, &srcOperandIndices); - - auto dstOperands = dstInst->getOperands(); - - for (int j = 0; j < numOperands; j++) - { - dstOperands[j].init(dstInst, insts[int(srcOperandIndices[j])]); - } - } - } - - // Patch up the children - { - const Index numChildRuns = data.m_childRuns.getCount(); - for (Index i = 0; i < numChildRuns; i++) - { - const auto& run = data.m_childRuns[i]; - - IRInst* inst = insts[int(run.m_parentIndex)]; - - for (int j = 0; j < int(run.m_numChildren); ++j) - { - IRInst* child = insts[j + int(run.m_startInstIndex)]; - SLANG_ASSERT(child->parent == nullptr); - child->insertAtEnd(inst); - } - } - } - - // Re-add source locations, if they are defined - if (m_serialData->m_rawSourceLocs.getCount() == numInsts) - { - const Ser::RawSourceLoc* srcLocs = m_serialData->m_rawSourceLocs.begin(); - for (Index i = 1; i < numInsts; ++i) - { - IRInst* dstInst = insts[i]; - - dstInst->sourceLoc.setRaw(Slang::SourceLoc::RawValue(srcLocs[i])); - } - } - - if (sourceManager && m_serialData->m_debugSourceInfos.getCount()) - { - List debugStringSlices; - SerialStringTableUtil::decodeStringTable(m_serialData->m_debugStringTable, debugStringSlices); - - // All of the strings are placed in the manager (and its StringSlicePool) where the SourceView and SourceFile are constructed from - List stringMap; - SerialStringTableUtil::calcStringSlicePoolMap(debugStringSlices, sourceManager->getStringSlicePool(), stringMap); - - const List& sourceInfos = m_serialData->m_debugSourceInfos; - - // Construct the source files - Index numSourceFiles = sourceInfos.getCount(); - - // These hold the views (and SourceFile as there is only one SourceFile per view) in the same order as the sourceInfos - List sourceViews; - sourceViews.setCount(numSourceFiles); - - for (Index i = 0; i < numSourceFiles; ++i) - { - const IRSerialData::DebugSourceInfo& srcSourceInfo = sourceInfos[i]; - - PathInfo pathInfo; - pathInfo.type = PathInfo::Type::FoundPath; - pathInfo.foundPath = debugStringSlices[UInt(srcSourceInfo.m_pathIndex)]; - - SourceFile* sourceFile = sourceManager->createSourceFileWithSize(pathInfo, srcSourceInfo.m_endSourceLoc - srcSourceInfo.m_startSourceLoc); - SourceView* sourceView = sourceManager->createSourceView(sourceFile, nullptr); - - // We need to accumulate all line numbers, for this source file, both adjusted and unadjusted - List lineInfos; - // Add the adjusted lines - { - lineInfos.setCount(srcSourceInfo.m_numAdjustedLineInfos); - const IRSerialData::DebugAdjustedLineInfo* srcAdjustedLineInfos = m_serialData->m_debugAdjustedLineInfos.getBuffer() + srcSourceInfo.m_adjustedLineInfosStartIndex; - const int numAdjustedLines = int(srcSourceInfo.m_numAdjustedLineInfos); - for (int j = 0; j < numAdjustedLines; ++j) - { - lineInfos[j] = srcAdjustedLineInfos[j].m_lineInfo; - } - } - // Add regular lines - lineInfos.addRange(m_serialData->m_debugLineInfos.getBuffer() + srcSourceInfo.m_lineInfosStartIndex, srcSourceInfo.m_numLineInfos); - // Put in sourceloc order - lineInfos.sort(); - - List lineBreakOffsets; - - // We can now set up the line breaks array - const int numLines = int(srcSourceInfo.m_numLines); - lineBreakOffsets.setCount(numLines); - - { - const Index numLineInfos = lineInfos.getCount(); - Index lineIndex = 0; - - // Every line up and including should hold the same offset - for (Index lineInfoIndex = 0; lineInfoIndex < numLineInfos; ++lineInfoIndex) - { - const auto& lineInfo = lineInfos[lineInfoIndex]; - - const uint32_t offset = lineInfo.m_lineStartOffset; - SLANG_ASSERT(offset > 0); - const int finishIndex = int(lineInfo.m_lineIndex); - - SLANG_ASSERT(finishIndex < numLines); - - for (; lineIndex < finishIndex; ++lineIndex) - { - lineBreakOffsets[lineIndex] = offset - 1; - } - lineBreakOffsets[lineIndex] = offset; - lineIndex++; - } - - // Do the remaining lines - const uint32_t offset = uint32_t(srcSourceInfo.m_endSourceLoc - srcSourceInfo.m_startSourceLoc); - for (; lineIndex < numLines; ++lineIndex) - { - lineBreakOffsets[lineIndex] = offset; - } - } - - sourceFile->setLineBreakOffsets(lineBreakOffsets.getBuffer(), lineBreakOffsets.getCount()); - - if (srcSourceInfo.m_numAdjustedLineInfos) - { - List adjustedLineInfos; - - int numEntries = int(srcSourceInfo.m_numAdjustedLineInfos); - - adjustedLineInfos.addRange(m_serialData->m_debugAdjustedLineInfos.getBuffer() + srcSourceInfo.m_adjustedLineInfosStartIndex, numEntries); - adjustedLineInfos.sort(); - - // Work out the views adjustments, and place in dstEntries - List dstEntries; - dstEntries.setCount(numEntries); - - const uint32_t sourceLocOffset = uint32_t(sourceView->getRange().begin.getRaw()); - - for (int j = 0; j < numEntries; ++j) - { - const auto& srcEntry = adjustedLineInfos[j]; - auto& dstEntry = dstEntries[j]; - - dstEntry.m_pathHandle = stringMap[int(srcEntry.m_pathStringIndex)]; - dstEntry.m_startLoc = SourceLoc::fromRaw(srcEntry.m_lineInfo.m_lineStartOffset + sourceLocOffset); - dstEntry.m_lineAdjust = int32_t(srcEntry.m_adjustedLineIndex) - int32_t(srcEntry.m_lineInfo.m_lineIndex); - } - - // Set the adjustments on the view - sourceView->setEntries(dstEntries.getBuffer(), dstEntries.getCount()); - } - - sourceViews[i] = sourceView; - } - - // We now need to apply the runs - { - List sourceRuns(m_serialData->m_debugSourceLocRuns); - // They are now in source location order - sourceRuns.sort(); - - // Just guess initially 0 for the source file that contains the initial run - SourceRange range; - int fixSourceLoc = _calcFixSourceLoc(sourceInfos[0], sourceViews[0], range); - - const Index numRuns = sourceRuns.getCount(); - for (Index i = 0; i < numRuns; ++i) - { - const auto& run = sourceRuns[i]; - const SourceLoc srcSourceLoc = SourceLoc::fromRaw(run.m_sourceLoc); - - if (!range.contains(srcSourceLoc)) - { - int index = _findIndex(sourceInfos, srcSourceLoc); - if (index < 0) - { - // Didn't find the match - continue; - } - fixSourceLoc = _calcFixSourceLoc(sourceInfos[index], sourceViews[index], range); - SLANG_ASSERT(range.contains(srcSourceLoc)); - } - - // Work out the fixed source location - SourceLoc sourceLoc = SourceLoc::fromRaw(int(run.m_sourceLoc) + fixSourceLoc); - - SLANG_ASSERT(Index(uint32_t(run.m_startInstIndex) + run.m_numInst) <= insts.getCount()); - IRInst** dstInsts = insts.getBuffer() + int(run.m_startInstIndex); - - const int runSize = int(run.m_numInst); - for (int j = 0; j < runSize; ++j) - { - dstInsts[j]->sourceLoc = sourceLoc; - } - } - } - } - - return SLANG_OK; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -/* static */void IRSerialUtil::calcInstructionList(IRModule* module, List& instsOut) -{ - // We reserve 0 for null - instsOut.setCount(1); - instsOut[0] = nullptr; - - // Stack for parentInst - List parentInstStack; - - IRModuleInst* moduleInst = module->getModuleInst(); - parentInstStack.add(moduleInst); - - // Add to list - instsOut.add(moduleInst); - - // Traverse all of the instructions - while (parentInstStack.getCount()) - { - // If it's in the stack it is assumed it is already in the inst map - IRInst* parentInst = parentInstStack.getLast(); - parentInstStack.removeLast(); - - IRInstListBase childrenList = parentInst->getDecorationsAndChildren(); - for (IRInst* child : childrenList) - { - instsOut.add(child); - parentInstStack.add(child); - } - } -} - -/* static */SlangResult IRSerialUtil::verifySerialize(IRModule* module, Session* session, SourceManager* sourceManager, IRSerialCompressionType compressionType, IRSerialWriter::OptionFlags optionFlags) -{ - // Verify if we can stream out with debug information - - List originalInsts; - calcInstructionList(module, originalInsts); - - IRSerialData serialData; - { - // Write IR out to serialData - copying over SourceLoc information directly - IRSerialWriter writer; - SLANG_RETURN_ON_FAIL(writer.write(module, sourceManager, optionFlags, &serialData)); - } - - // Write the data out to stream - OwnedMemoryStream memoryStream(FileAccess::ReadWrite); - SLANG_RETURN_ON_FAIL(IRSerialWriter::writeStream(serialData, compressionType, &memoryStream)); - - // Reset stream - memoryStream.seek(SeekOrigin::Start, 0); - - IRSerialData readData; - - SLANG_RETURN_ON_FAIL(IRSerialReader::readStream(&memoryStream, &readData)); - - // Check the stream read data is the same - if (readData != serialData) - { - SLANG_ASSERT(!"Streamed in data doesn't match"); - return SLANG_FAIL; - } - - RefPtr irReadModule; - - SourceManager workSourceManager; - workSourceManager.initialize(sourceManager, nullptr); - - { - IRSerialReader reader; - SLANG_RETURN_ON_FAIL(reader.read(serialData, session, &workSourceManager, irReadModule)); - } - - List readInsts; - calcInstructionList(irReadModule, readInsts); - - if (readInsts.getCount() != originalInsts.getCount()) - { - SLANG_ASSERT(!"Instruction counts don't match"); - return SLANG_FAIL; - } - - if (optionFlags & IRSerialWriter::OptionFlag::RawSourceLocation) - { - SLANG_ASSERT(readInsts[0] == originalInsts[0]); - // All the source locs should be identical - for (Index i = 1; i < readInsts.getCount(); ++i) - { - IRInst* origInst = originalInsts[i]; - IRInst* readInst = readInsts[i]; - - if (origInst->sourceLoc.getRaw() != readInst->sourceLoc.getRaw()) - { - SLANG_ASSERT(!"Source locs don't match"); - return SLANG_FAIL; - } - } - } - else if (optionFlags & IRSerialWriter::OptionFlag::DebugInfo) - { - // They should be on the same line nos - for (Index i = 1; i < readInsts.getCount(); ++i) - { - IRInst* origInst = originalInsts[i]; - IRInst* readInst = readInsts[i]; - - if (origInst->sourceLoc.getRaw() == readInst->sourceLoc.getRaw()) - { - continue; - } - - // Work out the - SourceView* origSourceView = sourceManager->findSourceView(origInst->sourceLoc); - SourceView* readSourceView = workSourceManager.findSourceView(readInst->sourceLoc); - - // if both are null we are done - if (origSourceView == nullptr && origSourceView == readSourceView) - { - continue; - } - SLANG_ASSERT(origSourceView && readSourceView); - - { - auto origInfo = origSourceView->getHumaneLoc(origInst->sourceLoc, SourceLocType::Actual); - auto readInfo = readSourceView->getHumaneLoc(readInst->sourceLoc, SourceLocType::Actual); - - if (!(origInfo.line == readInfo.line && origInfo.column == readInfo.column && origInfo.pathInfo.foundPath == readInfo.pathInfo.foundPath)) - { - SLANG_ASSERT(!"Debug data didn't match"); - return SLANG_FAIL; - } - } - - // We may have adjusted line numbers -> but they may not match, because we only reconstruct one view - // So for now disable this test - - if (false) - { - auto origInfo = origSourceView->getHumaneLoc(origInst->sourceLoc, SourceLocType::Nominal); - auto readInfo = readSourceView->getHumaneLoc(readInst->sourceLoc, SourceLocType::Nominal); - - if (!(origInfo.line == readInfo.line && origInfo.column == readInfo.column && origInfo.pathInfo.foundPath == readInfo.pathInfo.foundPath)) - { - SLANG_ASSERT(!"Debug data didn't match"); - return SLANG_FAIL; - } - } - } - } - - return SLANG_OK; -} - -} // namespace Slang diff --git a/source/slang/slang-ir-serialize.h b/source/slang/slang-ir-serialize.h deleted file mode 100644 index ee99c08dc..000000000 --- a/source/slang/slang-ir-serialize.h +++ /dev/null @@ -1,163 +0,0 @@ -// slang-ir-serialize.h -#ifndef SLANG_IR_SERIALIZE_H_INCLUDED -#define SLANG_IR_SERIALIZE_H_INCLUDED - -#include "slang-ir-serialize-types.h" - -#include "../core/slang-riff.h" - -#include "slang-ir.h" - -#include "slang-ir-serialize-types.h" - -// For TranslationUnitRequest -// and FrontEndCompileRequest::ExtraEntryPointInfo -#include "slang-compiler.h" - -namespace Slang { - -struct IRSerialWriter -{ - typedef IRSerialData Ser; - typedef IRSerialBinary Bin; - - struct OptionFlag - { - typedef uint32_t Type; - enum Enum: Type - { - RawSourceLocation = 0x01, - DebugInfo = 0x02, - }; - }; - typedef OptionFlag::Type OptionFlags; - - Result write(IRModule* module, SourceManager* sourceManager, OptionFlags options, IRSerialData* serialData); - - static Result writeStream(const IRSerialData& data, IRSerialCompressionType compressionType, Stream* stream); - - /// Write to a container - static Result writeContainer(const IRSerialData& data, IRSerialCompressionType compressionType, RiffContainer* container); - - /// Get an instruction index from an instruction - Ser::InstIndex getInstIndex(IRInst* inst) const { return inst ? Ser::InstIndex(m_instMap[inst]) : Ser::InstIndex(0); } - - /// Get a slice from an index - UnownedStringSlice getStringSlice(Ser::StringIndex index) const { return m_stringSlicePool.getSlice(StringSlicePool::Handle(index)); } - /// Get index from string representations - Ser::StringIndex getStringIndex(StringRepresentation* string) { return Ser::StringIndex(m_stringSlicePool.add(string)); } - Ser::StringIndex getStringIndex(const UnownedStringSlice& slice) { return Ser::StringIndex(m_stringSlicePool.add(slice)); } - Ser::StringIndex getStringIndex(Name* name) { return name ? getStringIndex(name->text) : Ser::kNullStringIndex; } - Ser::StringIndex getStringIndex(const char* chars) { return Ser::StringIndex(m_stringSlicePool.add(chars)); } - Ser::StringIndex getStringIndex(const String& string) { return Ser::StringIndex(m_stringSlicePool.add(string.getUnownedSlice())); } - - StringSlicePool& getStringPool() { return m_stringSlicePool; } - StringSlicePool& getDebugStringPool() { return m_debugStringSlicePool; } - - IRSerialWriter() : - m_serialData(nullptr), - m_stringSlicePool(StringSlicePool::Style::Default), - m_debugStringSlicePool(StringSlicePool::Style::Default) - {} - -protected: - class DebugSourceFile : public RefObject - { - public: - DebugSourceFile(SourceFile* sourceFile, SourceLoc::RawValue baseSourceLoc): - m_sourceFile(sourceFile), - m_baseSourceLoc(baseSourceLoc) - { - // Need to know how many lines there are - const List& lineOffsets = sourceFile->getLineBreakOffsets(); - - const auto numLineIndices = lineOffsets.getCount(); - - // Set none as being used initially - m_lineIndexUsed.setCount(numLineIndices); - ::memset(m_lineIndexUsed.begin(), 0, numLineIndices * sizeof(uint8_t)); - } - /// True if we have information on that line index - bool hasLineIndex(int lineIndex) const { return m_lineIndexUsed[lineIndex] != 0; } - void setHasLineIndex(int lineIndex) { m_lineIndexUsed[lineIndex] = 1; } - - SourceLoc::RawValue m_baseSourceLoc; ///< The base source location - - SourceFile* m_sourceFile; ///< The source file - List m_lineIndexUsed; ///< Has 1 if the line is used - List m_usedLineIndices; ///< Holds the lines that have been hit - - List m_lineInfos; ///< The line infos - List m_adjustedLineInfos; ///< The adjusted line infos - }; - - void _addInstruction(IRInst* inst); - Result _calcDebugInfo(); - /// Returns the remapped sourceLoc, or 0 if sourceLoc couldn't be added - void _addDebugSourceLocRun(SourceLoc sourceLoc, uint32_t startInstIndex, uint32_t numInst); - - List m_insts; ///< Instructions in same order as stored in the - - List m_decorations; ///< Holds all decorations in order of the instructions as found - List m_instWithFirstDecoration; ///< All decorations are held in this order after all the regular instructions - - Dictionary m_instMap; ///< Map an instruction to an instruction index - - StringSlicePool m_stringSlicePool; - IRSerialData* m_serialData; ///< Where the data is stored - - StringSlicePool m_debugStringSlicePool; ///< Slices held just for debug usage - - SourceLoc::RawValue m_debugFreeSourceLoc; /// Locations greater than this are free - Dictionary > m_debugSourceFileMap; - - SourceManager* m_sourceManager; ///< The source manager -}; - -struct IRSerialReader -{ - typedef IRSerialData Ser; - typedef StringRepresentationCache::Handle StringHandle; - - /// Read a stream to fill in dataOut IRSerialData - static Result readStream(Stream* stream, IRSerialData* dataOut); - - /// Read potentially multiple modules from a stream - static Result readStreamModules(Stream* stream, Session* session, SourceManager* manager, List>& outModules, List& outEntryPoints); - - /// Read potentially multiple modules from a stream - static Result readContainerModules(RiffContainer* container, Session* session, SourceManager* manager, List>& outModules, List& outEntryPoints); - - /// Read a stream to fill in dataOut IRSerialData - static Result readContainer(RiffContainer::ListChunk* module, IRSerialData* outData); - - /// Read a module from serial data - Result read(const IRSerialData& data, Session* session, SourceManager* sourceManager, RefPtr& moduleOut); - - IRSerialReader(): - m_serialData(nullptr), - m_module(nullptr) - { - } - - protected: - - StringRepresentationCache m_stringRepresentationCache; - - const IRSerialData* m_serialData; - IRModule* m_module; -}; - -struct IRSerialUtil -{ - /// Produces an instruction list which is in same order as written through IRSerialWriter - static void calcInstructionList(IRModule* module, List& instsOut); - - /// Verify serialization - static SlangResult verifySerialize(IRModule* module, Session* session, SourceManager* sourceManager, IRSerialCompressionType compressionType, IRSerialWriter::OptionFlags optionFlags); -}; - - -} // namespace Slang - -#endif diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp index 1fe78977a..42003fb62 100644 --- a/source/slang/slang-options.cpp +++ b/source/slang/slang-options.cpp @@ -13,7 +13,7 @@ #include "slang-file-system.h" #include "slang-repro.h" -#include "slang-ir-serialize.h" +#include "slang-serialize-ir.h" #include "../core/slang-type-text-util.h" @@ -592,7 +592,7 @@ struct OptionsParser { String name; SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, name)); - SLANG_RETURN_ON_FAIL(IRSerialTypeUtil::parseCompressionType(name.getUnownedSlice(), requestImpl->getLinkage()->irCompressionType)); + SLANG_RETURN_ON_FAIL(SerialParseUtil::parseCompressionType(name.getUnownedSlice(), requestImpl->getLinkage()->serialCompressionType)); } else if (argStr == "-target") { diff --git a/source/slang/slang-serialize-ast.cpp b/source/slang/slang-serialize-ast.cpp new file mode 100644 index 000000000..82cd7d6b4 --- /dev/null +++ b/source/slang/slang-serialize-ast.cpp @@ -0,0 +1,2008 @@ +// slang-serialize-ast.cpp +#include "slang-serialize-ast.h" + +#include "slang-ast-generated.h" +#include "slang-ast-generated-macro.h" + +#include "slang-compiler.h" +#include "slang-type-layout.h" + +#include "slang-ast-dump.h" +#include "slang-mangle.h" + +#include "slang-ast-support-types.h" + +#include "slang-legalize-types.h" + +#include "../core/slang-byte-encode-util.h" + +namespace Slang { + + +// Things stored as references: +// +// NodeBase derived types +// Array +// +// RefObject derived types: +// +// Breadcrumb +// StringRepresentation +// Scope + + + +// Helpers to convert fields treated as values + +class ASTSerialReader; +class ASTSerialWriter; + +template +static void _toSerialValue(ASTSerialWriter* writer, const NATIVE_TYPE& src, SERIAL_TYPE& dst) +{ + ASTSerialTypeInfo::toSerial(writer, &src, &dst); +} + +template +static void _toNativeValue(ASTSerialReader* reader, const SERIAL_TYPE& src, NATIVE_TYPE& dst) +{ + ASTSerialTypeInfo::toNative(reader, &src, &dst); +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ModuleASTSerialFilter !!!!!!!!!!!!!!!!!!!!!!!! + + +ASTSerialIndex ModuleASTSerialFilter::writePointer(ASTSerialWriter* writer, const NodeBase* inPtr) +{ + NodeBase* ptr = const_cast(inPtr); + SLANG_ASSERT(ptr); + + if (Decl* decl = as(ptr)) + { + ModuleDecl* moduleDecl = findModuleForDecl(decl); + SLANG_ASSERT(moduleDecl); + if (moduleDecl && moduleDecl != m_moduleDecl) + { + ASTBuilder* astBuilder = m_moduleDecl->module->getASTBuilder(); + + // It's a reference to a declaration in another module, so create an ImportExternalDecl. + + String mangledName = getMangledName(astBuilder, decl); + + ImportExternalDecl* importDecl = astBuilder->create(); + importDecl->mangledName = mangledName; + const ASTSerialIndex index = writer->writePointer(importDecl); + + // Set as the index of this + writer->setPointerIndex(ptr, index); + + return index; + } + else + { + // Okay... we can just write it out then + return writer->writePointer(ptr); + } + } + + // TODO(JS): What we really want to do here is to ignore bodies functions. + // It's not 100% clear if this is even right though - for example does type inference + // imply the body is needed to say infer a return type? + // Also not clear if statements in other scenarios (if there are others) might need to be kept. + // + // For now we just ignore all stmts + + if (Stmt* stmt = as(ptr)) + { + // + writer->setPointerIndex(stmt, ASTSerialIndex(0)); + return ASTSerialIndex(0); + } + + // For now for everything else just write it + return writer->writePointer(ptr); +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Serial <-> Native conversion !!!!!!!!!!!!!!!!!!!!!!!! + + +// We need to have a way to map between the two. +// If no mapping is needed, (just a copy), then we don't bother with the functions +template +struct ASTSerialBasicTypeInfo +{ + typedef T NativeType; + typedef T SerialType; + + // We want the alignment to be the same as the size of the type for basic types + // NOTE! Might be different from SLANG_ALIGN_OF(SerialType) + enum { SerialAlignment = sizeof(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(T*)serial = *(const T*)native; } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(T*)native = *(const T*)serial; } + + static const ASTSerialType* getType() + { + static const ASTSerialType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; + return &type; + } +}; + +template +struct ASTSerialConvertTypeInfo +{ + typedef NATIVE_T NativeType; + typedef SERIAL_T SerialType; + + enum { SerialAlignment = ASTSerialBasicTypeInfo::SerialAlignment }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SERIAL_T*)serial = SERIAL_T(*(const NATIVE_T*)native); } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(NATIVE_T*)native = NATIVE_T(*(const SERIAL_T*)serial); } +}; + +template +struct ASTSerialIdentityTypeInfo +{ + typedef T NativeType; + typedef T SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(T*)serial = *(const T*)native; } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(T*)native = *(const T*)serial; } +}; + +// Don't need to convert the index type + +template <> +struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; + + +// Because is sized, we don't need to convert +template <> +struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; + +// Implement for Basic Types + +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; + +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; + +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; + +// SamplerStateFlavor + +template <> +struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + +// TextureFlavor + +template <> +struct ASTSerialTypeInfo +{ + typedef TextureFlavor NativeType; + typedef uint16_t SerialType; + enum { SerialAlignment = sizeof(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SerialType*)serial = ((const NativeType*)native)->flavor; } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); ((NativeType*)native)->flavor = *(const SerialType*)serial; } +}; + +// Fixed arrays + +template +struct ASTSerialTypeInfo +{ + typedef ASTSerialTypeInfo ElementASTSerialType; + typedef typename ElementASTSerialType::SerialType SerialElementType; + + typedef T NativeType[N]; + typedef SerialElementType SerialType[N]; + + enum { SerialAlignment = ASTSerialTypeInfo::SerialAlignment }; + + static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + { + SerialElementType* serial = (SerialElementType*)outSerial; + const T* native = (const T*)inNative; + for (Index i = 0; i < Index(N); ++i) + { + ElementASTSerialType::toSerial(writer, native + i, serial + i); + } + } + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + const SerialElementType* serial = (const SerialElementType*)inSerial; + T* native = (T*)outNative; + for (Index i = 0; i < Index(N); ++i) + { + ElementASTSerialType::toNative(reader, serial + i, native + i); + } + } +}; + +// Special case bool - as we can't rely on size alignment +template <> +struct ASTSerialTypeInfo +{ + typedef bool NativeType; + typedef uint8_t SerialType; + + enum { SerialAlignment = sizeof(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + { + SLANG_UNUSED(writer); + *(SerialType*)outSerial = *(const NativeType*)inNative ? 1 : 0; + } + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + SLANG_UNUSED(reader); + *(NativeType*)outNative = (*(const SerialType*)inSerial) != 0; + } +}; + + +// Pointer +// Could handle different pointer base types with some more template magic here, but instead went with Pointer type to keep +// things simpler. +template +struct ASTSerialTypeInfo +{ + typedef T* NativeType; + typedef ASTSerialIndex SerialType; + + enum + { + SerialAlignment = SLANG_ALIGN_OF(SerialType) + }; + + static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + { + *(SerialType*)outSerial = writer->addPointer(*(T**)inNative); + } + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + *(T**)outNative = reader->getPointer(*(const SerialType*)inSerial).dynamicCast(); + } +}; + +// Special case Name +template <> +struct ASTSerialTypeInfo : public ASTSerialTypeInfo +{ + // Special case + typedef Name* NativeType; + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + *(Name**)outNative = reader->getName(*(const SerialType*)inSerial); + } +}; + +template <> +struct ASTSerialTypeInfo : public ASTSerialTypeInfo +{ +}; + + +struct ASTSerialDeclRefBaseTypeInfo +{ + typedef DeclRefBase NativeType; + struct SerialType + { + ASTSerialIndex substitutions; + ASTSerialIndex decl; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + { + SerialType& serial = *(SerialType*)outSerial; + const NativeType& native = *(const NativeType*)inNative; + + serial.decl = writer->addPointer(native.decl); + serial.substitutions = writer->addPointer(native.substitutions.substitutions); + } + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + DeclRefBase& native = *(DeclRefBase*)(outNative); + const SerialType& serial = *(const SerialType*)inSerial; + + native.decl = reader->getPointer(serial.decl).dynamicCast(); + native.substitutions.substitutions = reader->getPointer(serial.substitutions).dynamicCast(); + } + static const ASTSerialType* getType() + { + static const ASTSerialType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; + return &type; + } +}; + +template +struct ASTSerialTypeInfo> : public ASTSerialDeclRefBaseTypeInfo {}; + +// MatrixCoord can just go as is +template <> +struct ASTSerialTypeInfo : ASTSerialIdentityTypeInfo {}; + +// SourceLoc + +// Make the type exposed, so we can look for it if we want to remap. +template <> +struct ASTSerialTypeInfo +{ + typedef SourceLoc NativeType; + typedef ASTSerialSourceLoc SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialSourceLoc) }; + + static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + { + *(SerialType*)outSerial = writer->addSourceLoc(*(const NativeType*)inNative); + } + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + *(NativeType*)outNative = reader->getSourceLoc(*(const SerialType*)inSerial); + } +}; + +// List +template +struct ASTSerialTypeInfo> +{ + typedef List NativeType; + typedef ASTSerialIndex SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + dst = writer->addArray(src.getBuffer(), src.getCount()); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& dst = *(NativeType*)native; + auto& src = *(const SerialType*)serial; + + reader->getArray(src, dst); + } +}; + +// Dictionary +template +struct ASTSerialTypeInfo> +{ + typedef Dictionary NativeType; + struct SerialType + { + ASTSerialIndex keys; ///< Index an array + ASTSerialIndex values; ///< Index an array + }; + + typedef typename ASTSerialTypeInfo::SerialType KeySerialType; + typedef typename ASTSerialTypeInfo::SerialType ValueSerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + List keys; + List values; + + Index count = Index(src.Count()); + keys.setCount(count); + values.setCount(count); + + Index i = 0; + for (const auto& pair : src) + { + ASTSerialTypeInfo::toSerial(writer, &pair.Key, &keys[i]); + ASTSerialTypeInfo::toSerial(writer, &pair.Value, &values[i]); + i++; + } + + dst.keys = writer->addArray(keys.getBuffer(), count); + dst.values = writer->addArray(values.getBuffer(), count); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + // Clear it + dst = NativeType(); + + List keys; + List values; + + reader->getArray(src.keys, keys); + reader->getArray(src.values, values); + + SLANG_ASSERT(keys.getCount() == values.getCount()); + + const Index count = keys.getCount(); + for (Index i = 0; i < count; ++i) + { + dst.Add(keys[i], values[i]); + } + } +}; + +// SyntaxClass +template +struct ASTSerialTypeInfo> +{ + typedef SyntaxClass NativeType; + typedef uint16_t SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + SLANG_UNUSED(writer); + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + dst = SerialType(src.classInfo->m_classId); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + SLANG_UNUSED(reader); + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + dst.classInfo = ReflectClassInfo::getInfo(ASTNodeType(src)); + } +}; + +// Handle RefPtr - just convert into * to do the conversion +template +struct ASTSerialTypeInfo> +{ + typedef RefPtr NativeType; + typedef typename ASTSerialTypeInfo::SerialType SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + T* obj = src; + ASTSerialTypeInfo::toSerial(writer, &obj, serial); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + T* obj = nullptr; + ASTSerialTypeInfo::toNative(reader, serial, &obj); + *(NativeType*)native = obj; + } +}; + +// QualType + +template <> +struct ASTSerialTypeInfo +{ + typedef QualType NativeType; + struct SerialType + { + ASTSerialIndex type; + uint8_t isLeftValue; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto dst = (SerialType*)serial; + auto src = (const NativeType*)native; + dst->isLeftValue = src->isLeftValue ? 1 : 0; + dst->type = writer->addPointer(src->type); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto src = (const SerialType*)serial; + auto dst = (NativeType*)native; + dst->type = reader->getPointer(src->type).dynamicCast(); + dst->isLeftValue = src->isLeftValue != 0; + } +}; + + +// LookupResult::Breadcrumb +template <> +struct ASTSerialTypeInfo +{ + typedef LookupResultItem::Breadcrumb NativeType; + struct SerialType + { + NativeType::Kind kind; + NativeType::ThisParameterMode thisParameterMode; + ASTSerialTypeInfo>::SerialType declRef; + ASTSerialTypeInfo> next; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + dst.kind = src.kind; + dst.thisParameterMode = src.thisParameterMode; + _toSerialValue(writer, src.declRef, dst.declRef); + _toSerialValue(writer, src.next, dst.next); + } + + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& dst = *(NativeType*)native; + auto& src = *(const SerialType*)serial; + + dst.kind = src.kind; + dst.thisParameterMode = src.thisParameterMode; + _toNativeValue(reader, src.declRef, dst.declRef); + _toNativeValue(reader, src.next, dst.next); + } +}; + +// LookupResultItem +template <> +struct ASTSerialTypeInfo +{ + typedef LookupResultItem NativeType; + struct SerialType + { + ASTSerialTypeInfo>::SerialType declRef; + ASTSerialTypeInfo> breadcrumbs; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + _toSerialValue(writer, src.declRef, dst.declRef); + _toSerialValue(writer, src.breadcrumbs, dst.breadcrumbs); + } + + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& dst = *(NativeType*)native; + auto& src = *(const SerialType*)serial; + + _toNativeValue(reader, src.declRef, dst.declRef); + _toNativeValue(reader, src.breadcrumbs, dst.breadcrumbs); + } +}; + +// LookupResult +template <> +struct ASTSerialTypeInfo +{ + typedef LookupResult NativeType; + typedef ASTSerialIndex SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + if (src.isOverloaded()) + { + // Save off as an array + dst = writer->addArray(src.items.getBuffer(), src.items.getCount()); + } + else if (src.item.declRef.getDecl()) + { + dst = writer->addArray(&src.item, 1); + } + else + { + dst = ASTSerialIndex(0); + } + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& dst = *(NativeType*)native; + auto& src = *(const SerialType*)serial; + + // Initialize + dst = NativeType(); + + List items; + reader->getArray(src, items); + + if (items.getCount() == 1) + { + dst.item = items[0]; + } + else + { + dst.items.swapWith(items); + // We have to set item such that it is valid/member of items, if items is non empty + dst.item = dst.items[0]; + } + } +}; + + +// GlobalGenericParamSubstitution::ConstraintArg +template <> +struct ASTSerialTypeInfo +{ + typedef GlobalGenericParamSubstitution::ConstraintArg NativeType; + struct SerialType + { + ASTSerialIndex decl; + ASTSerialIndex val; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& dst = *(SerialType*)serial; + auto& src = *(const NativeType*)native; + + dst.decl = writer->addPointer(src.decl); + dst.val = writer->addPointer(src.val); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + dst.decl = reader->getPointer(src.decl).dynamicCast(); + dst.val = reader->getPointer(src.val).dynamicCast(); + } +}; + +// ExpandedSpecializationArg +template <> +struct ASTSerialTypeInfo +{ + typedef ExpandedSpecializationArg NativeType; + struct SerialType + { + ASTSerialIndex val; + ASTSerialIndex witness; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& dst = *(SerialType*)serial; + auto& src = *(const NativeType*)native; + + dst.witness = writer->addPointer(src.witness); + dst.val = writer->addPointer(src.val); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + dst.witness = reader->getPointer(src.witness).dynamicCast(); + dst.val = reader->getPointer(src.val).dynamicCast(); + } +}; + +// TypeExp +template <> +struct ASTSerialTypeInfo +{ + typedef TypeExp NativeType; + struct SerialType + { + ASTSerialIndex type; + ASTSerialIndex expr; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& dst = *(SerialType*)serial; + auto& src = *(const NativeType*)native; + + dst.type = writer->addPointer(src.type); + dst.expr = writer->addPointer(src.exp); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + dst.type = reader->getPointer(src.type).dynamicCast(); + dst.exp = reader->getPointer(src.expr).dynamicCast(); + } +}; + +// DeclCheckStateExt +template <> +struct ASTSerialTypeInfo +{ + typedef DeclCheckStateExt NativeType; + typedef DeclCheckStateExt::RawType SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + SLANG_UNUSED(writer); + *(SerialType*)serial = (*(const NativeType*)native).getRaw(); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + SLANG_UNUSED(reader); + (*(NativeType*)serial).setRaw(*(const SerialType*)native); + } +}; + +// Modifiers +template <> +struct ASTSerialTypeInfo +{ + typedef Modifiers NativeType; + typedef ASTSerialIndex SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + // We need to make into an array + List modifierIndices; + for (Modifier* modifier : *(NativeType*)native) + { + modifierIndices.add(writer->addPointer(modifier)); + } + *(SerialType*)serial = writer->addArray(modifierIndices.getBuffer(), modifierIndices.getCount()); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + List modifiers; + reader->getArray(*(const SerialType*)serial, modifiers); + + Modifier* prev = nullptr; + for (Modifier* modifier : modifiers) + { + if (prev) + { + prev->next = modifier; + } + } + + NativeType& dst = *(NativeType*)native; + dst.first = modifiers.getCount() > 0 ? modifiers[0] : nullptr; + } +}; + +// ImageFormat +template <> +struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + +// Stage +template <> +struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + +// TokenType +template <> +struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + +// BaseType +template <> +struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + +// SemanticVersion +template <> +struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; + +// ASTNodeType +template <> +struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + +// String +template <> +struct ASTSerialTypeInfo +{ + typedef String NativeType; + typedef ASTSerialIndex SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + *(SerialType*)serial = writer->addString(src); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + dst = reader->getString(src); + } +}; + +// Token +template <> +struct ASTSerialTypeInfo +{ + typedef Token NativeType; + struct SerialType + { + ASTSerialTypeInfo::SerialType type; + ASTSerialTypeInfo::SerialType loc; + ASTSerialIndex name; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + ASTSerialTypeInfo::toSerial(writer, &src.type, &dst.type); + ASTSerialTypeInfo::toSerial(writer, &src.loc, &dst.loc); + + if (src.flags & TokenFlag::Name) + { + dst.name = writer->addName(src.getName()); + } + else + { + dst.name = writer->addString(src.getContent()); + } + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + dst.flags = 0; + dst.charsNameUnion.chars = nullptr; + + ASTSerialTypeInfo::toNative(reader, &src.type, &dst.type); + ASTSerialTypeInfo::toNative(reader, &src.loc, &dst.loc); + + // At the other end all token content will appear as Names. + if (src.name != ASTSerialIndex(0)) + { + dst.charsNameUnion.name = reader->getName(src.name); + dst.flags |= TokenFlag::Name; + } + } +}; + +// NameLoc +template <> +struct ASTSerialTypeInfo +{ + typedef NameLoc NativeType; + struct SerialType + { + ASTSerialTypeInfo::SerialType loc; + ASTSerialIndex name; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + dst.name = writer->addName(src.name); + ASTSerialTypeInfo::toSerial(writer, &src.loc, &dst.loc); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + dst.name = reader->getName(src.name); + ASTSerialTypeInfo::toNative(reader, &src.loc, &dst.loc); + } +}; + +// DiagnosticInfo +template <> +struct ASTSerialTypeInfo +{ + typedef const DiagnosticInfo* NativeType; + typedef ASTSerialIndex SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + dst = src ? writer->addString(UnownedStringSlice(src->name)) : ASTSerialIndex(0); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + if (src == ASTSerialIndex(0)) + { + dst = nullptr; + } + else + { + dst = findDiagnosticByName(reader->getStringSlice(src)); + } + } +}; + +// !!!!!!!!!!!!!!!!!!!!! ASTSerialGetType !!!!!!!!!!!!!!!!!!!!!!!!!!! +// Getting the type info, let's use a static variable to hold the state to keep simple + +template +struct ASTSerialGetType +{ + static const ASTSerialType* getType() + { + typedef ASTSerialTypeInfo Info; + static const ASTSerialType type = { sizeof(typename Info::SerialType), uint8_t(Info::SerialAlignment), &Info::toSerial, &Info::toNative }; + return &type; + } +}; + +// Special case DeclRef, because it always uses the same type +template +struct ASTSerialGetType> +{ + static const ASTSerialType* getType() { return ASTSerialDeclRefBaseTypeInfo::getType(); } +}; + +// !!!!!!!!!!!!!!!!!!!!!! Generate fields for a type !!!!!!!!!!!!!!!!!!!!!!!!!!! + + +template +ASTSerialField _calcField(const char* name, T& in) +{ + uint8_t* ptr = &reinterpret_cast(in); + + ASTSerialField field; + field.name = name; + field.type = ASTSerialGetType::getType(); + // This only works because we in is an offset from 1 + field.nativeOffset = uint32_t(size_t(ptr) - 1); + field.serialOffset = 0; + return field; +} + +static ASTSerialClass _makeClass(MemoryArena* arena, ASTNodeType type, const List& fields) +{ + ASTSerialClass cls = { type, 0, 0, 0, 0 }; + cls.fieldsCount = fields.getCount(); + cls.fields = arena->allocateAndCopyArray(fields.getBuffer(), fields.getCount()); + return cls; +} + +#define SLANG_AST_SERIAL_FIELD(FIELD_NAME, TYPE, param) fields.add(_calcField(#FIELD_NAME, obj->FIELD_NAME)); + +// Note that the obj point is not nullptr, because some compilers notice this is 'indexing from null' +// and warn/error. So we offset from 1. +#define SLANG_AST_SERIAL_MAKE_CLASS(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \ +{ \ + NAME* obj = (NAME*)1; \ + SLANG_UNUSED(obj); \ + fields.clear(); \ + SLANG_FIELDS_ASTNode_##NAME(SLANG_AST_SERIAL_FIELD, param) \ + outClasses[Index(ASTNodeType::NAME)] = _makeClass(arena, ASTNodeType::NAME, fields); \ +} + +struct ASTFieldAccess +{ + static void calcClasses(MemoryArena* arena, ASTSerialClass outClasses[Index(ASTNodeType::CountOf)]) + { + List fields; + SLANG_ALL_ASTNode_NodeBase(SLANG_AST_SERIAL_MAKE_CLASS, _) + } +}; + +ASTSerialClasses::ASTSerialClasses(): + m_arena(2048) +{ + memset(m_classes, 0, sizeof(m_classes)); + ASTFieldAccess::calcClasses(&m_arena, m_classes); + + // Now work out the layout + for (Index i = 0; i < SLANG_COUNT_OF(m_classes); ++i) + { + // Set up each class in order, from lowest to highest index + // Doing so means super class is always setup + ASTSerialClass& serialClass = m_classes[i]; + + const ReflectClassInfo* info = ReflectClassInfo::getInfo(serialClass.type); + + size_t maxAlignment = 1; + size_t offset = 0; + + const ReflectClassInfo* superInfo = info->m_superClass; + if (superInfo) + { + ASTSerialClass& superSerialInfo = m_classes[superInfo->m_classId]; + + // If it's been setup, then alignment must be non zero. + // The ordering of ASTNodeType, should mean type have larger ASTNodeType greater than supers ASTNodeType. + SLANG_ASSERT(superSerialInfo.alignment != 0); + + // Must be a power of 2 + SLANG_ASSERT((superSerialInfo.alignment & (superSerialInfo.alignment - 1)) == 0); + + maxAlignment = superSerialInfo.alignment; + offset = superSerialInfo.size; + + // Check it is correctly aligned + SLANG_ASSERT((offset & (maxAlignment - 1)) == 0); + } + + // Okay, go through fields setting their offset + ASTSerialField* fields = serialClass.fields; + for (Index j = 0; j < serialClass.fieldsCount; j++) + { + ASTSerialField& field = fields[j]; + + size_t alignment = field.type->serialAlignment; + // Make sure the offset is aligned for the field requirement + offset = (offset + alignment - 1) & ~(alignment - 1); + + // Save the field offset + field.serialOffset = uint32_t(offset); + + // Move past the field + offset += field.type->serialSizeInBytes; + + // Calc the maximum alignment + maxAlignment = (alignment > maxAlignment) ? alignment : maxAlignment; + } + + // Align with maximum alignment + offset = (offset + maxAlignment - 1) & ~(maxAlignment - 1); + + serialClass.alignment = uint8_t(maxAlignment); + serialClass.size = uint32_t(offset); + } +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!! + +ASTSerialWriter::ASTSerialWriter(ASTSerialClasses* classes, ASTSerialFilter* filter, DebugSerialWriter* debugWriter) : + m_arena(2048), + m_classes(classes), + m_filter(filter), + m_debugWriter(debugWriter) +{ + // 0 is always the null pointer + m_entries.add(nullptr); + m_ptrMap.Add(nullptr, 0); +} + +ASTSerialIndex ASTSerialWriter::writePointer(const NodeBase* node) +{ + // This pointer cannot be in the map + SLANG_ASSERT(m_ptrMap.TryGetValue(node) == nullptr); + + const ASTSerialClass* serialClass = m_classes->getSerialClass(node->astNodeType); + + typedef ASTSerialInfo::NodeEntry NodeEntry; + + NodeEntry* nodeEntry = (NodeEntry*)m_arena.allocateAligned(sizeof(NodeEntry) + serialClass->size, ASTSerialInfo::MAX_ALIGNMENT); + + nodeEntry->type = ASTSerialInfo::Type::Node; + nodeEntry->astNodeType = uint16_t(node->astNodeType); + nodeEntry->info = ASTSerialInfo::makeEntryInfo(serialClass->alignment); + + auto index = _add(node, nodeEntry); + + uint8_t* serialPayload = (uint8_t*)(nodeEntry + 1); + + while (serialClass) + { + for (Index i = 0; i < serialClass->fieldsCount; ++i) + { + auto field = serialClass->fields[i]; + + // Work out the offsets + auto srcField = ((const uint8_t*)node) + field.nativeOffset; + auto dstField = serialPayload + field.serialOffset; + + field.type->toSerialFunc(this, srcField, dstField); + } + // Get the super class + const ReflectClassInfo* reflectInfo = ReflectClassInfo::getInfo(serialClass->type); + const ReflectClassInfo* superReflectInfo = reflectInfo->m_superClass; + + serialClass = superReflectInfo ? m_classes->getSerialClass(ASTNodeType(superReflectInfo->m_classId)) : nullptr; + } + + return index; +} + +void ASTSerialWriter::setPointerIndex(const NodeBase* ptr, ASTSerialIndex index) +{ + m_ptrMap.Add(ptr, Index(index)); +} + +ASTSerialIndex ASTSerialWriter::addPointer(const NodeBase* node) +{ + // Null is always 0 + if (node == nullptr) + { + return ASTSerialIndex(0); + } + // Look up in the map + Index* indexPtr = m_ptrMap.TryGetValue(node); + if (indexPtr) + { + return ASTSerialIndex(*indexPtr); + } + + if (m_filter) + { + return m_filter->writePointer(this, node); + } + else + { + return writePointer(node); + } +} + +ASTSerialIndex ASTSerialWriter::addPointer(const RefObject* obj) +{ + // Null is always 0 + if (obj == nullptr) + { + return ASTSerialIndex(0); + } + // Look up in the map + Index* indexPtr = m_ptrMap.TryGetValue(obj); + if (indexPtr) + { + return ASTSerialIndex(*indexPtr); + } + + if (auto stringRep = dynamicCast(obj)) + { + ASTSerialIndex index = addString(StringRepresentation::asSlice(stringRep)); + m_ptrMap.Add(obj, Index(index)); + return index; + } + else if (auto breadcrumb = dynamicCast(obj)) + { + typedef ASTSerialTypeInfo TypeInfo; + typedef ASTSerialInfo::RefObjectEntry RefObjectEntry; + + size_t alignment = TypeInfo::SerialAlignment; + alignment = (alignment < SLANG_ALIGN_OF(ASTSerialInfo::RefObjectEntry)) ? SLANG_ALIGN_OF(ASTSerialInfo::RefObjectEntry) : alignment; + + RefObjectEntry* entry = (RefObjectEntry*)m_arena.allocateAligned(sizeof(RefObjectEntry) + sizeof(TypeInfo::SerialType), alignment); + + entry->type = ASTSerialInfo::Type::RefObject; + entry->info = ASTSerialInfo::makeEntryInfo(int(alignment)); + entry->subType = RefObjectEntry::SubType::Breadcrumb; + + auto index = _add(breadcrumb, entry); + + // Do any conversion + TypeInfo::toSerial(this, breadcrumb, entry + 1); + return index; + } + else if (auto name = dynamicCast(obj)) + { + return addName(name); + } + else if (auto scope = dynamicCast(obj)) + { + // We don't serialize scope + return ASTSerialIndex(0); + } + else if (auto module = dynamicCast(obj)) + { + // We don't serialize Module + return ASTSerialIndex(0); + } + + SLANG_ASSERT(!"Unhandled type"); + return ASTSerialIndex(0); +} + +ASTSerialIndex ASTSerialWriter::addString(const UnownedStringSlice& slice) +{ + typedef ByteEncodeUtil Util; + typedef ASTSerialInfo::StringEntry StringEntry; + + if (slice.getLength() == 0) + { + return ASTSerialIndex(0); + } + + Index newIndex = m_entries.getCount(); + + Index* indexPtr = m_sliceMap.TryGetValueOrAdd(slice, newIndex); + if (indexPtr) + { + return ASTSerialIndex(*indexPtr); + } + + // Okay we need to add the string + + uint8_t encodeBuf[Util::kMaxLiteEncodeUInt32]; + const int encodeCount = Util::encodeLiteUInt32(uint32_t(slice.getLength()), encodeBuf); + + StringEntry* entry = (StringEntry*)m_arena.allocateUnaligned(SLANG_OFFSET_OF(StringEntry, sizeAndChars) + encodeCount + slice.getLength()); + entry->info = ASTSerialInfo::EntryInfo::Alignment1; + entry->type = ASTSerialInfo::Type::String; + + uint8_t* dst = (uint8_t*)(entry->sizeAndChars); + for (int i = 0; i < encodeCount; ++i) + { + dst[i] = encodeBuf[i]; + } + + memcpy(dst + encodeCount, slice.begin(), slice.getLength()); + + m_entries.add(entry); + return ASTSerialIndex(newIndex); +} + + +ASTSerialIndex ASTSerialWriter::addString(const String& in) +{ + return addPointer(in.getStringRepresentation()); +} + +ASTSerialIndex ASTSerialWriter::addName(const Name* name) +{ + if (name == nullptr) + { + return ASTSerialIndex(0); + } + + // Look it up + Index* indexPtr = m_ptrMap.TryGetValue(name); + if (indexPtr) + { + return ASTSerialIndex(*indexPtr); + } + + ASTSerialIndex index = addString(name->text); + m_ptrMap.Add(name, Index(index)); + return index; +} + +ASTSerialSourceLoc ASTSerialWriter::addSourceLoc(SourceLoc sourceLoc) +{ + if (sourceLoc.isValid() && m_debugWriter) + { + return m_debugWriter->addSourceLoc(sourceLoc); + } + else + { + return 0; + } +} + +ASTSerialIndex ASTSerialWriter::_addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount) +{ + typedef ASTSerialInfo::ArrayEntry Entry; + + if (elementCount == 0) + { + return ASTSerialIndex(0); + } + + SLANG_ASSERT(alignment >= 1 && alignment <= ASTSerialInfo::MAX_ALIGNMENT); + + // We must at a minimum have the alignment for the array prefix info + alignment = (alignment < SLANG_ALIGN_OF(Entry)) ? SLANG_ALIGN_OF(Entry) : alignment; + + size_t payloadSize = elementCount * elementSize; + + Entry* entry = (Entry*)m_arena.allocateAligned(sizeof(Entry) + payloadSize, alignment); + + entry->type = ASTSerialInfo::Type::Array; + entry->info = ASTSerialInfo::makeEntryInfo(int(alignment)); + entry->elementSize = uint16_t(elementSize); + entry->elementCount = uint32_t(elementCount); + + memcpy(entry + 1, elements, payloadSize); + + m_entries.add(entry); + return ASTSerialIndex(m_entries.getCount() - 1); +} + +static const uint8_t s_fixBuffer[ASTSerialInfo::MAX_ALIGNMENT]{ 0, }; + +SlangResult ASTSerialWriter::write(Stream* stream) +{ + const Int entriesCount = m_entries.getCount(); + + // Add a sentinal so we don't need special handling for + ASTSerialInfo::Entry sentinal; + sentinal.type = ASTSerialInfo::Type::String; + sentinal.info = ASTSerialInfo::EntryInfo::Alignment1; + + m_entries.add(&sentinal); + m_entries.removeLast(); + + ASTSerialInfo::Entry** entries = m_entries.getBuffer(); + // Note strictly required in our impl of List. But by writing this and + // knowing that removeLast cannot release memory, means the sentinal must be at the last position. + entries[entriesCount] = &sentinal; + + + { + size_t offset = 0; + + ASTSerialInfo::Entry* entry = entries[1]; + // We start on 1, because 0 is nullptr and not used for anything + for (Index i = 1; i < entriesCount; ++i) + { + ASTSerialInfo::Entry* next = entries[i + 1]; + // Before writing we need to store the next alignment + + const size_t nextAlignment = ASTSerialInfo::getAlignment(next->info); + const size_t alignment = ASTSerialInfo::getAlignment(entry->info); + + entry->info = ASTSerialInfo::combineWithNext(entry->info, next->info); + + // Check we are aligned correctly + SLANG_ASSERT((offset & (alignment - 1)) == 0); + + // When we write, we need to make sure it take into account the next alignment + const size_t entrySize = entry->calcSize(m_classes); + + // Work out the fix for next alignment + size_t nextOffset = offset + entrySize; + nextOffset = (nextOffset + nextAlignment - 1) & ~(nextAlignment - 1); + + size_t alignmentFixSize = nextOffset - (offset + entrySize); + + // The fix must be less than max alignment. We require it to be less because we aligned each Entry to + // MAX_ALIGNMENT, and so < MAX_ALIGNMENT is the most extra bytes we can write + SLANG_ASSERT( alignmentFixSize < ASTSerialInfo::MAX_ALIGNMENT); + + try + { + stream->write(entry, entrySize); + // If we needed to fix so that subsequent alignment is right, write out extra bytes here + if (alignmentFixSize) + { + stream->write(s_fixBuffer, alignmentFixSize); + } + } + catch (const IOException&) + { + return SLANG_FAIL; + } + + // Onto next + offset = nextOffset; + entry = next; + } + } + + return SLANG_OK; +} + +SlangResult ASTSerialWriter::writeIntoContainer(RiffContainer* container) +{ + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ScopeChunk ScopeChunk; + + // This is the container for the AST Data + ScopeChunk scopeModule(container, Chunk::Kind::List, ASTSerialBinary::kSlangASTModuleFourCC); + { + ScopeChunk scopeData(container, Chunk::Kind::Data, ASTSerialBinary::kSlangASTModuleDataFourCC); + + { + // Sentinal so we don't need special handling for end of list + ASTSerialInfo::Entry sentinal; + sentinal.type = ASTSerialInfo::Type::String; + sentinal.info = ASTSerialInfo::EntryInfo::Alignment1; + + size_t offset = 0; + const Int entriesCount = m_entries.getCount(); + + { + m_entries.add(&sentinal); + m_entries.removeLast(); + // Note strictly required in our impl of List. But by writing this and + // knowing that removeLast cannot release memory, means the sentinal must be at the last position. + m_entries.getBuffer()[entriesCount] = &sentinal; + } + + ASTSerialInfo::Entry*const* entries = m_entries.getBuffer(); + + ASTSerialInfo::Entry* entry = entries[1]; + // We start on 1, because 0 is nullptr and not used for anything + for (Index i = 1; i < entriesCount; ++i) + { + ASTSerialInfo::Entry* next = entries[i + 1]; + + // Before writing we need to store the next alignment + + const size_t nextAlignment = ASTSerialInfo::getAlignment(next->info); + const size_t alignment = ASTSerialInfo::getAlignment(entry->info); + + entry->info = ASTSerialInfo::combineWithNext(entry->info, next->info); + + // Check we are aligned correctly + SLANG_ASSERT((offset & (alignment - 1)) == 0); + + // When we write, we need to make sure it take into account the next alignment + const size_t entrySize = entry->calcSize(m_classes); + + // Work out the fix for next alignment + size_t nextOffset = offset + entrySize; + nextOffset = (nextOffset + nextAlignment - 1) & ~(nextAlignment - 1); + + size_t alignmentFixSize = nextOffset - (offset + entrySize); + + // The fix must be less than max alignment. We require it to be less because we aligned each Entry to + // MAX_ALIGNMENT, and so < MAX_ALIGNMENT is the most extra bytes we can write + SLANG_ASSERT(alignmentFixSize < ASTSerialInfo::MAX_ALIGNMENT); + + container->write(entry, entrySize); + if (alignmentFixSize) + { + container->write(s_fixBuffer, alignmentFixSize); + } + + // Onto next + offset = nextOffset; + entry = next; + } + } + } + + return SLANG_OK; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialInfo::Entry !!!!!!!!!!!!!!!!!!!!!!!! + +size_t ASTSerialInfo::Entry::calcSize(ASTSerialClasses* serialClasses) const +{ + switch (type) + { + case Type::String: + { + auto entry = static_cast(this); + const uint8_t* cur = (const uint8_t*)entry->sizeAndChars; + uint32_t charsSize; + int sizeSize = ByteEncodeUtil::decodeLiteUInt32(cur, &charsSize); + return SLANG_OFFSET_OF(StringEntry, sizeAndChars) + sizeSize + charsSize; + } + case Type::Node: + { + auto entry = static_cast(this); + auto serialClass = serialClasses->getSerialClass(ASTNodeType(entry->astNodeType)); + + // Align by the alignment of the entry + size_t alignment = getAlignment(entry->info); + size_t size = sizeof(NodeEntry) + serialClass->size; + + size = size + (alignment - 1) & ~(alignment - 1); + return size; + } + case Type::RefObject: + { + auto entry = static_cast(this); + + size_t payloadSize; + switch (entry->subType) + { + case RefObjectEntry::SubType::Breadcrumb: + { + payloadSize = sizeof(ASTSerialTypeInfo::SerialType); + break; + } + default: + { + SLANG_ASSERT(!"Unknown type"); + return 0; + } + } + + return sizeof(RefObjectEntry) + payloadSize; + } + case Type::Array: + { + auto entry = static_cast(this); + return sizeof(ArrayEntry) + entry->elementSize * entry->elementCount; + } + default: break; + } + + SLANG_ASSERT(!"Unknown type"); + return 0; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!! + +const void* ASTSerialReader::getArray(ASTSerialIndex index, Index& outCount) +{ + if (index == ASTSerialIndex(0)) + { + outCount = 0; + return nullptr; + } + + SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); + const Entry* entry = m_entries[Index(index)]; + + switch (entry->type) + { + case Type::Array: + { + auto arrayEntry = static_cast(entry); + outCount = Index(arrayEntry->elementCount); + return (arrayEntry + 1); + } + default: break; + } + + SLANG_ASSERT(!"Not an array"); + outCount = 0; + return nullptr; +} + +ASTSerialPointer ASTSerialReader::getPointer(ASTSerialIndex index) +{ + if (index == ASTSerialIndex(0)) + { + return ASTSerialPointer(); + } + + SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); + const Entry* entry = m_entries[Index(index)]; + + switch (entry->type) + { + case Type::String: + { + // Hmm. Tricky -> we don't know if will be cast as Name or String. Lets assume string. + String string = getString(index); + return ASTSerialPointer(string.getStringRepresentation()); + } + case Type::Node: + { + return ASTSerialPointer((NodeBase*)m_objects[Index(index)]); + } + case Type::RefObject: + { + return ASTSerialPointer((RefObject*)m_objects[Index(index)]); + } + default: break; + } + + SLANG_ASSERT(!"Cannot access as a pointer"); + return ASTSerialPointer(); +} + +String ASTSerialReader::getString(ASTSerialIndex index) +{ + if (index == ASTSerialIndex(0)) + { + return String(); + } + + SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); + const Entry* entry = m_entries[Index(index)]; + + // It has to be a string type + if (entry->type != Type::String) + { + SLANG_ASSERT(!"Not a string"); + return String(); + } + + RefObject* obj = (RefObject*)m_objects[Index(index)]; + + if (obj) + { + StringRepresentation* stringRep = dynamicCast(obj); + if (stringRep) + { + return String(stringRep); + } + // Must be a name then + Name* name = dynamicCast(obj); + SLANG_ASSERT(name); + return name->text; + } + + // Okay we need to construct as a string + UnownedStringSlice slice = getStringSlice(index); + String string(slice); + StringRepresentation* stringRep = string.getStringRepresentation(); + + m_scope.add(stringRep); + m_objects[Index(index)] = stringRep; + return string; +} + +Name* ASTSerialReader::getName(ASTSerialIndex index) +{ + if (index == ASTSerialIndex(0)) + { + return nullptr; + } + + SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); + const Entry* entry = m_entries[Index(index)]; + + // It has to be a string type + if (entry->type != Type::String) + { + SLANG_ASSERT(!"Not a string"); + return nullptr; + } + + RefObject* obj = (RefObject*)m_objects[Index(index)]; + + if (obj) + { + Name* name = dynamicCast(obj); + if (name) + { + return name; + } + // Can only be a string then + StringRepresentation* stringRep = dynamicCast(obj); + SLANG_ASSERT(stringRep); + + // I don't need to scope, as scoped in NamePool + name = m_namePool->getName(String(stringRep)); + + // Store as name, as can always access the inner string if needed + m_objects[Index(index)] = name; + return name; + } + + UnownedStringSlice slice = getStringSlice(index); + String string(slice); + Name* name = m_namePool->getName(string); + // Don't need to add to scope, because scoped on the pool + m_objects[Index(index)] = name; + return name; +} + +UnownedStringSlice ASTSerialReader::getStringSlice(ASTSerialIndex index) +{ + SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); + const Entry* entry = m_entries[Index(index)]; + + // It has to be a string type + if (entry->type != Type::String) + { + SLANG_ASSERT(!"Not a string"); + return UnownedStringSlice(); + } + + auto stringEntry = static_cast(entry); + + const uint8_t* src = (const uint8_t*)stringEntry->sizeAndChars; + + // Decode the string + uint32_t size; + int sizeSize = ByteEncodeUtil::decodeLiteUInt32(src, &size); + return UnownedStringSlice((const char*)src + sizeSize, size); +} + +SourceLoc ASTSerialReader::getSourceLoc(ASTSerialSourceLoc loc) +{ + return (loc && m_debugReader) ? m_debugReader->getSourceLoc(loc) : SourceLoc(); +} + +SlangResult ASTSerialReader::loadEntries(const uint8_t* data, size_t dataCount, List& outEntries) +{ + // Check the input data is at least aligned to the max alignment (otherwise everything cannot be aligned correctly) + SLANG_ASSERT((size_t(data) & (ASTSerialInfo::MAX_ALIGNMENT - 1)) == 0); + + outEntries.setCount(1); + outEntries[0] = nullptr; + + const uint8_t*const end = data + dataCount; + + const uint8_t* cur = data; + while (cur < end) + { + const Entry* entry = (const Entry*)cur; + outEntries.add(entry); + + const size_t entrySize = entry->calcSize(m_classes); + cur += entrySize; + + // Need to get the next alignment + const size_t nextAlignment = ASTSerialInfo::getNextAlignment(entry->info); + + // Need to fix cur with the alignment + cur = (const uint8_t*)((size_t(cur) + nextAlignment - 1) & ~(nextAlignment - 1)); + } + + return SLANG_OK; +} + +SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuilder* builder, NamePool* namePool) +{ + SLANG_RETURN_ON_FAIL(loadEntries(data, dataCount, m_entries)); + + m_namePool = namePool; + + m_objects.clearAndDeallocate(); + m_objects.setCount(m_entries.getCount()); + memset(m_objects.getBuffer(), 0, m_objects.getCount() * sizeof(void*)); + + // Go through entries, constructing objects. + for (Index i = 1; i < m_entries.getCount(); ++i) + { + const Entry* entry = m_entries[i]; + + switch (entry->type) + { + case Type::String: + { + // Don't need to construct an object. This is probably a StringRepresentation, or a Name + // Will evaluate lazily. + break; + } + case Type::Node: + { + auto nodeEntry = static_cast(entry); + m_objects[i] = builder->createByNodeType(ASTNodeType(nodeEntry->astNodeType)); + break; + } + case Type::RefObject: + { + auto objEntry = static_cast(entry); + switch (objEntry->subType) + { + case ASTSerialInfo::RefObjectEntry::SubType::Breadcrumb: + { + typedef LookupResultItem::Breadcrumb Breadcrumb; + + auto breadcrumb = new LookupResultItem::Breadcrumb(Breadcrumb::Kind::Member, DeclRef(), nullptr, nullptr); + m_scope.add(breadcrumb); + m_objects[i] = breadcrumb; + break; + } + default: + { + SLANG_ASSERT(!"Unknown type"); + return SLANG_FAIL; + } + } + break; + } + case Type::Array: + { + // Don't need to construct an object, as will be accessed an interpreted by the object that holds it + break; + } + } + } + + // Deserialize + for (Index i = 1; i < m_entries.getCount(); ++i) + { + const Entry* entry = m_entries[i]; + void* native = m_objects[i]; + if (!native) + { + continue; + } + switch (entry->type) + { + case Type::Node: + { + auto nodeEntry = static_cast(entry); + auto serialClass = m_classes->getSerialClass(ASTNodeType(nodeEntry->astNodeType)); + + const uint8_t* src = (const uint8_t*)(nodeEntry + 1); + uint8_t* dst = (uint8_t*)m_objects[i]; + + // It must be constructed + SLANG_ASSERT(dst); + + while (serialClass) + { + for (Index j = 0; j < serialClass->fieldsCount; ++j) + { + auto field = serialClass->fields[j]; + auto fieldType = field.type; + fieldType->toNativeFunc(this, src + field.serialOffset, dst + field.nativeOffset); + } + + auto cls = ReflectClassInfo::getInfo(serialClass->type); + auto superCls = cls->m_superClass; + + // Get the super class + serialClass = superCls ? m_classes->getSerialClass(ASTNodeType(superCls->m_classId)) : nullptr; + } + + break; + } + case Type::RefObject: + { + auto objEntry = static_cast(entry); + switch (objEntry->subType) + { + case ASTSerialInfo::RefObjectEntry::SubType::Breadcrumb: + { + typedef LookupResultItem::Breadcrumb Breadcrumb; + auto serialType = ASTSerialGetType::getType(); + serialType->toNativeFunc(this, (entry + 1), m_objects[i]); + break; + } + default: + { + SLANG_ASSERT(!"Unknown type"); + return SLANG_FAIL; + } + } + break; + } + default: break; + } + } + + return SLANG_OK; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerializeUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!! + +/* static */SlangResult ASTSerialTestUtil::selfTest() +{ + RefPtr classes = new ASTSerialClasses; + + { + struct Thing + { + Module* node; + }; + Thing thing; + + //Pointer pointer(thing.node); + + auto field = _calcField("node", thing.node); + + + const ASTSerialType* type = ASTSerialGetType::getType(); + SLANG_UNUSED(type); + } + + { + const ASTSerialType* type = ASTSerialGetType::getType(); + SLANG_UNUSED(type); + } + + { + const ASTSerialType* type = ASTSerialGetType::getType(); + SLANG_UNUSED(type); + } + + { + const ASTSerialType* type = ASTSerialGetType::getType(); + SLANG_UNUSED(type); + } + + return SLANG_OK; +} + +/* static */SlangResult ASTSerialTestUtil::testSerialize(NodeBase* node, RootNamePool* rootNamePool, SharedASTBuilder* sharedASTBuilder, SourceManager* sourceManager) +{ + RefPtr classes = new ASTSerialClasses; + + List contents; + + { + OwnedMemoryStream stream(FileAccess::ReadWrite); + + ModuleDecl* moduleDecl = as(node); + ModuleASTSerialFilter filterStorage(moduleDecl); + + ASTSerialFilter* filter = moduleDecl ? &filterStorage : nullptr; + + ASTSerialWriter writer(classes, filter, nullptr); + + // Lets serialize it all + writer.addPointer(node); + // Let's stick it all in a stream + writer.write(&stream); + + stream.swapContents(contents); + + NamePool namePool; + namePool.setRootNamePool(rootNamePool); + + ASTSerialReader reader(classes, nullptr); + + ASTBuilder builder(sharedASTBuilder, "Serialize Check"); + + // We could now check that the loaded data matches + + { + const List& writtenEntries = writer.getEntries(); + List readEntries; + + SlangResult res = reader.loadEntries(contents.getBuffer(), contents.getCount(), readEntries); + SLANG_UNUSED(res); + + SLANG_ASSERT(writtenEntries.getCount() == readEntries.getCount()); + + // They should be identical up to the + for (Index i = 1; i < readEntries.getCount(); ++i) + { + auto writtenEntry = writtenEntries[i]; + auto readEntry = readEntries[i]; + + const size_t writtenSize = writtenEntry->calcSize(classes); + const size_t readSize = readEntry->calcSize(classes); + + SLANG_ASSERT(readSize == writtenSize); + // Check the payload is the same + SLANG_ASSERT(memcmp(readEntry, writtenEntry, readSize) == 0); + } + + } + + { + SlangResult res = reader.load(contents.getBuffer(), contents.getCount(), &builder, &namePool); + SLANG_UNUSED(res); + } + + // Lets see what we have + const ASTDumpUtil::Flags dumpFlags = ASTDumpUtil::Flag::HideSourceLoc | ASTDumpUtil::Flag::HideScope; + + String readDump; + { + SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None); + ASTDumpUtil::dump(reader.getPointer(ASTSerialIndex(1)).dynamicCast(), ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter); + readDump = sourceWriter.getContentAndClear(); + + } + String origDump; + { + SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None); + ASTDumpUtil::dump(node, ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter); + origDump = sourceWriter.getContentAndClear(); + } + + // Write out + File::writeAllText("ast-read.ast-dump", readDump); + File::writeAllText("ast-orig.ast-dump", origDump); + + + if (readDump != origDump) + { + return SLANG_FAIL; + } + } + + return SLANG_OK; +} + + +} // namespace Slang diff --git a/source/slang/slang-serialize-ast.h b/source/slang/slang-serialize-ast.h new file mode 100644 index 000000000..76bbc9682 --- /dev/null +++ b/source/slang/slang-serialize-ast.h @@ -0,0 +1,601 @@ +// slang-serialize-ast.h +#ifndef SLANG_SERIALIZE_AST_H +#define SLANG_SERIALIZE_AST_H + +#include + +#include "slang-ast-support-types.h" +#include "slang-ast-all.h" + +#include "slang-serialize-debug.h" + +#include "../core/slang-riff.h" + +#include "slang-ast-builder.h" + +#include "../core/slang-byte-encode-util.h" + +#include "../core/slang-stream.h" +namespace Slang +{ + +class Linkage; + +/* +AST Serialization Overview +========================== + +The AST node types are generally types derived from the NodeBase. The C++ extractor is used to associate an ASTNodeType with +every NodeBase type, such that casting is fast and simple and we have a simple integer to uniquely identify those types. The +extractor also performs another task of associating with the type name all of the fields held in just that type. The definition +of the fields is stored in an 'x macro' which is in the slang-ast-generated-macro.h file, for example + +``` +#define SLANG_FIELDS_ASTNode_DeclRefExpr(_x_, _param_)\ + _x_(scope, (RefPtr), _param_)\ + _x_(declRef, (DeclRef), _param_)\ + _x_(name, (Name*), _param_) +`` + +For the type DeclRefExpr, this holds all of the fields held in just DeclRefExpr in this case `scope`, `declRef` and `name`. +DeclRefExpr derives from Expr and this might hold other fields and so forth. + +The implementation makes a distinction between the 'native' types, the regular C++ in memory types and 'serial' types. +Each serializable C++ type has an associated 'serial' type - with the distinction that it can be written out and (with perhaps some other data) +read back in to recreate the C++ type. The serial type can be a C++ type, but is such it can be written and read from disk and still +represent the same data. + +We need a mechanism to be able to do do a conversion between native and serial types. To make the association we use the template + +``` +template +struct ASTSerialTypeInfo; +``` + +and specialize it for each native type. The specialization holds + +SerialType - The type that will be used to represent the native type +NativeType - The native type +SerialAlignment - A value that holds what kind of alignment the SerialType needs to be serializable (it may be different from SLANG_ALIGN_OF(SerialType)!) +toSerial - A function that with the help of ASTSerialWriter convert the NativeType into the SerialType +toNative - A function that with the help of ASTSerialReader convert the SerialType into the NativeType + +It is useful to have a structure that holds the type information, so it can be stored. That is achieved with + +``` +template +struct ASTSerialGetType; +``` + +This template can be specialized for a specific native types - but all it holds is just a function getType, which returns a ASTSerialType*, +which just holds the information held in the ASTSerialTypeInfo template, but additionally including the size of the SerialType. + +So we need to define a specialized ASTSerialTypeInfo for each type that can be a field in a NodeBase derived type. We don't need to define +anything explicitly for the NodeBase derived types, as we will just generate the layout from the fields. How do we know the fields? We just +used the macros generated from the C++ extractor. + +So first a few things to observe... + +1) Some types don't need any conversion to be serializable - int8_t, or float the bits can just be written out and read in (1) +2) Some types need a conversion but it's very simple - for example an enum without explicit size, being written as an explicit size +3) Some types can be written out but would not be directly readable or usable with different targets/processors, so need converting +4) Some types require complex conversions that require programmer code - like Dictionary/List + +For types that need no conversion (1), we can just use the template ASTSerialIdentityTypeInfo + +``` +template <> +struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; +``` + +This specialization means that SomeType can be written out and read in across targets/compilers without problems. + +For (2) we have another template that will do the conversion for us + +``` +template +struct ASTSerialConvertTypeInfo; +``` + +That we can use as above, and specify the native and serial types. + +For (3) there are a few scenarios. For any field in a serial type we must store in the serialized type such that the representation +will work across all processors/compilers. So one problematic type is `bool`. It's not specified how it's laid out in memory - and +some compiles have stored it as a word. Most recently it's been stored as a byte. To make sure bool is ok for serialization therefore +we store as a uint8_t. + +Another example would be double. It's 64 bits, but on some arches/compilers it's SLANG_ALIGN_OF is 4 and on others it's 8. On some +arches a non aligned read will lead to a fault. To work around this problem therefore we have to ensure double has the alignment that +will work across all targets - and that alignment is 8. In that specific case that issue is handled via ASTSerialBasicTypeInfo, which +makes the SerialAlignment the sizeof the type. + +For (4) there are a few things to say. First a type can always implement a custom version of how to do a conversion by specializing +`ASTSerialTypeInfo`. But there remains another nagging issue - types which allocate/use other memory that changes at runtime. Clearly +we cannot define 'any size of memory' in a fixed SerialType defined in a specialization of ASTSerialTypeInfo. The mechanism to work around +this is to allow arbitrary arrays to be stored, that can be accessed via an ASTSerialIndex. This will be discussed more once we discuss +a little more about the file system, and ASTSerialIndex. + +Serialization Format +==================== + +The serialization format used is 'stream-like' with each 'object' stored in order. Each object is given an index starting from 1. +0 is used to be in effect nullptr. The stream looks like + +``` +ASTSerialInfo::Entry (for index 1) +Payload for type in entry + +ASTSerialInfo::Entry (for index 2) +Payload for type in entry + +... +... + +That when writing we have an array that maps each index to a pointer to the associated header. We also have a map that maps native pointers +to their indices. The Payload *is* the SerialType for thing saved. The payload directly follows the Entry data. + +Each object in this list can only be a few types of things - those derived from ASTSerialInfo::Type. + +The actual Entry followed by the payloads are allocated and stored when writing in a MemoryArena. When we want to write into a stream, we +can just iterate over each entry in order and write it out. + +You may have spotted a problem here - that some Entry types can be stored without alignment (for example a string - which stores the length +VarInt encoded followed by the characters). Others require an alignment - for example an NodeBase derived type that contains a int64_t will +*require* 8 byte alignment. That as a feature of the serialization format we want to be able to just map the data into memory, and be able +to access all the SerialType as is on the CPU. For that to work we *require* that the payload for each entry has the right alignment for +the associated SerialType. + +To achieve this we store in the Entry it's alignment requirement *AND* the next entries alignment. With this when we read, as we as stepping +through the entries we can find where the next Entry starts. Because the payload comes directly after the Entry - the Entrys size must be +a modulo of the largest alignment the payload can have. + +For the code that does the conversion between native and serial types it uses either the ASTSerialWriter or ASTSerialReader. This provides +the mechanism to turn a pointer into a serializable ASTSerialIndex and vice versa. There are some special functions for turning string like +types to and forth. + +The final mechanism is that of 'Arrays'. An array allows reading or writing a chunk of data associated with a ASTSerialIndex. The chunk of +data *must* hold data that is serializable. If the array holds pointers - then the serialized array must hold ASTSerialIndices that +represent those pointers. When reading back in they are converted back. + +Arrays are the escape hatch that allows for more complex types to serialize. Dictionaries for example are saved as a serial type that is +two ASTSerialIndices one to a keys array and one to a values array. + +Note that writing has two phases, serializing out into an ASTSerialWriter, and then secondly writing out to a stream. + +NodeBase Types +============== + +The ASTSerialTypeInfo mechanism is generally for *fields* of NodeBase types. That for NodeBase derived types we use the C++ extractors +field list to work out the native fields offsets and types. With this we can then calculate the layout for NodeBase types such that they +follow the requirements for serialization - such as alignment and so forth. + +This information is held in the ASTSerialClasses, which for a given ASTNodeType gives an ASTSerialClassInfo, that specifies fields for +just that type. Super types fields need to be serialized too, and this information can be found by using the ClassReflectInfo to find the +super type. + +Reading +======= + +Due to the care in writing reading is relatively simple. We can just take the contents of the file and put in memory, as long as in memory +it has an alignment of at least MAX_ALIGNMENT. Then we can build up an entries table by stepping through the data and writing the pointer. + +The toNative functions take an ASTSerialReader - this allows the implementation to ask for pointers and arrays from other parts of the serialized +data. It also allows for types to be lazily reconstructed if necessary. + +Lazy reconstruction may be useful in the future to partially reconstruct a sub part of the serialized data. In the current implementation, lazy +evaluation is used on Strings. The m_objects array holds all of the recreated native 'objects'. Since the objects can be derived from different +base classes the associated Entry will describe what it really is. + +For the String type, we initially store the object pointer as null. If a string is requested from that index, we see if the object pointer is null, +if it is we have to construct the StringRepresentation that will be used. + +An extra wrinkle is that we allow accessing of a serialized String as a Name or a string or a UnownedSubString. Fortunately a Name just holds a string, +and a Name remains in scope as long as it's NamePool does which is passed in. +*/ + +/* Holds RIFF FourCC codes for AST types */ +struct ASTSerialBinary +{ + static const FourCC kRiffFourCC = RiffFourCC::kRiff; + + /// AST module LIST container + static const FourCC kSlangASTModuleFourCC = SLANG_FOUR_CC('S', 'A', 'm', 'l'); + /// AST module data + static const FourCC kSlangASTModuleDataFourCC = SLANG_FOUR_CC('S', 'A', 'm', 'd'); +}; + +class ASTSerialClasses; + +// Type used to implement mechanisms to convert to and from serial types. +template +struct ASTSerialTypeInfo; + +struct ASTSerialInfo +{ + enum + { + // Data held in serialized format, the maximally allowed alignment + MAX_ALIGNMENT = 8, + }; + + // We only allow up to MAX_ALIGNMENT bytes of alignment. We store alignments as shifts, so 2 bits needed for 1 - 8 + enum class EntryInfo : uint8_t + { + Alignment1 = 0, + }; + + static EntryInfo makeEntryInfo(int alignment, int nextAlignment) + { + // Make sure they are power of 2 + SLANG_ASSERT((alignment & (alignment - 1)) == 0); + SLANG_ASSERT((nextAlignment & (nextAlignment - 1)) == 0); + + const int alignmentShift = ByteEncodeUtil::calcMsb8(alignment); + const int nextAlignmentShift = ByteEncodeUtil::calcMsb8(nextAlignment); + return EntryInfo((nextAlignmentShift << 2) | alignmentShift); + } + static EntryInfo makeEntryInfo(int alignment) + { + // Make sure they are power of 2 + SLANG_ASSERT((alignment & (alignment - 1)) == 0); + return EntryInfo(ByteEncodeUtil::calcMsb8(alignment)); + } + /// Apply with the next alignment + static EntryInfo combineWithNext(EntryInfo cur, EntryInfo next) + { + return EntryInfo((int(cur) & ~0xc0) | ((int(next) & 3) << 2)); + } + + static int getAlignment(EntryInfo info) { return 1 << (int(info) & 3); } + static int getNextAlignment(EntryInfo info) { return 1 << ((int(info) >> 2) & 3); } + + enum class Type : uint8_t + { + String, ///< String + Node, ///< NodeBase derived + RefObject, ///< RefObject derived types + Array, ///< Array + }; + + + /* Alignment is a little tricky. We have a 'Entry' header before the payload. The payload alignment may change. + If we only align on the Entry header, then it's size *must* be some modulo of the maximum alignment allowed. + + We could hold Entry separate from payload. We could make the header not require the alignment of the payload - but then + we'd need payload alignment separate from entry alignment. + */ + struct Entry + { + Type type; + EntryInfo info; + + size_t calcSize(ASTSerialClasses* serialClasses) const; + }; + + struct StringEntry : Entry + { + char sizeAndChars[1]; + }; + + struct NodeEntry : Entry + { + uint16_t astNodeType; + uint32_t _pad0; ///< Necessary, because a node *can* have MAX_ALIGNEMENT + }; + + struct RefObjectEntry : Entry + { + enum class SubType : uint8_t + { + Breadcrumb, + }; + SubType subType; + uint8_t _pad0; + uint32_t _pad1; ///< Necessary because RefObjectEntry *can* have MAX_ALIGNEMENT + }; + + struct ArrayEntry : Entry + { + uint16_t elementSize; + uint32_t elementCount; + }; +}; + +typedef uint32_t ASTSerialIndexRaw; +enum class ASTSerialIndex : ASTSerialIndexRaw; +typedef DebugSerialData::SourceLoc ASTSerialSourceLoc; + +/* A type to convert pointers into types such that they can be passed around to readers/writers without +having to know the specific type. If there was a base class that all the serialized types derived from, +that was dynamically castable this would not be necessary */ +struct ASTSerialPointer +{ + enum class Kind + { + Unknown, + RefObject, + NodeBase + }; + + // Helpers so we can choose what kind of pointer we have based on the (unused) type of the pointer passed in + SLANG_FORCE_INLINE RefObject* _get(const RefObject*) { return m_kind == Kind::RefObject ? reinterpret_cast(m_ptr) : nullptr; } + SLANG_FORCE_INLINE NodeBase* _get(const NodeBase*) { return m_kind == Kind::NodeBase ? reinterpret_cast(m_ptr) : nullptr; } + + template + T* dynamicCast() + { + return Slang::dynamicCast(_get((T*)nullptr)); + } + + ASTSerialPointer() : + m_kind(Kind::Unknown), + m_ptr(nullptr) + { + } + + ASTSerialPointer(RefObject* in) : + m_kind(Kind::RefObject), + m_ptr((void*)in) + { + } + ASTSerialPointer(NodeBase* in) : + m_kind(Kind::NodeBase), + m_ptr((void*)in) + { + } + + static Kind getKind(const RefObject*) { return Kind::RefObject; } + static Kind getKind(const NodeBase*) { return Kind::NodeBase; } + + Kind m_kind; + void* m_ptr; +}; + + +/* This class is the interface used by toNative implementations to recreate a type */ +class ASTSerialReader : public RefObject +{ +public: + + typedef ASTSerialInfo::Entry Entry; + typedef ASTSerialInfo::Type Type; + + template + void getArray(ASTSerialIndex index, List& out); + + const void* getArray(ASTSerialIndex index, Index& outCount); + + ASTSerialPointer getPointer(ASTSerialIndex index); + String getString(ASTSerialIndex index); + Name* getName(ASTSerialIndex index); + UnownedStringSlice getStringSlice(ASTSerialIndex index); + SourceLoc getSourceLoc(ASTSerialSourceLoc loc); + + /// Load the entries table (without deserializing anything) + /// NOTE! data must stay ins scope for outEntries to be valid + SlangResult loadEntries(const uint8_t* data, size_t dataCount, List& outEntries); + + /// NOTE! data must stay ins scope when reading takes place + SlangResult load(const uint8_t* data, size_t dataCount, ASTBuilder* builder, NamePool* namePool); + + ASTSerialReader(ASTSerialClasses* classes, DebugSerialReader* debugReader): + m_classes(classes), + m_debugReader(debugReader) + { + } + +protected: + List m_entries; ///< The entries + List m_objects; ///< The constructed objects + + List> m_scope; ///< Objects to keep in scope during construction + + DebugSerialReader* m_debugReader; + + NamePool* m_namePool; + + ASTSerialClasses* m_classes; ///< Used to deserialize +}; + +// --------------------------------------------------------------------------- +template +void ASTSerialReader::getArray(ASTSerialIndex index, List& out) +{ + typedef ASTSerialTypeInfo ElementTypeInfo; + typedef typename ElementTypeInfo::SerialType ElementSerialType; + + Index count; + auto serialElements = (const ElementSerialType*)getArray(index, count); + + if (count == 0) + { + out.clear(); + return; + } + + if (std::is_same::value) + { + // If they are the same we can just write out + out.clear(); + out.insertRange(0, (const T*)serialElements, count); + } + else + { + // Else we need to convert + out.setCount(count); + for (Index i = 0; i < count; ++i) + { + ElementTypeInfo::toNative(this, (const void*)&serialElements[i], (void*)&out[i]); + } + } +} + + +class ASTSerialClasses; +class ASTSerialWriter; + +class ASTSerialFilter +{ +public: + virtual ASTSerialIndex writePointer(ASTSerialWriter* writer, const NodeBase* ptr) = 0; +}; + +class ModuleASTSerialFilter : public ASTSerialFilter +{ +public: + virtual ASTSerialIndex writePointer(ASTSerialWriter* writer, const NodeBase* ptr) SLANG_OVERRIDE; + + ModuleASTSerialFilter(ModuleDecl* moduleDecl): + m_moduleDecl(moduleDecl) + { + } + + ModuleDecl* m_moduleDecl; +}; + +/* This is a class used tby toSerial implementations to turn native type into the serial type */ +class ASTSerialWriter : public RefObject +{ +public: + ASTSerialIndex addPointer(const NodeBase* ptr); + ASTSerialIndex addPointer(const RefObject* ptr); + + /// Write the pointer + ASTSerialIndex writePointer(const NodeBase* ptr); + + template + ASTSerialIndex addArray(const T* in, Index count); + + ASTSerialIndex addString(const UnownedStringSlice& slice); + ASTSerialIndex addString(const String& in); + ASTSerialIndex addName(const Name* name); + ASTSerialSourceLoc addSourceLoc(SourceLoc sourceLoc); + + /// Set a the index associated with an index. NOTE! That there cannot be a pre-existing setting. + void setPointerIndex(const NodeBase* ptr, ASTSerialIndex index); + + /// Get the entries table holding how each index maps to an entry + const List& getEntries() const { return m_entries; } + + /// Write to a stream + SlangResult write(Stream* stream); + + /// Write the state into the container + SlangResult writeIntoContainer(RiffContainer* container); + + ASTSerialWriter(ASTSerialClasses* classes, ASTSerialFilter* filter, DebugSerialWriter* debugWriter); + +protected: + + ASTSerialIndex _addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount); + + ASTSerialIndex _add(const void* nativePtr, ASTSerialInfo::Entry* entry) + { + m_entries.add(entry); + // Okay I need to allocate space for this + ASTSerialIndex index = ASTSerialIndex(m_entries.getCount() - 1); + // Add to the map + m_ptrMap.Add(nativePtr, Index(index)); + return index; + } + + DebugSerialWriter* m_debugWriter; //< For writing/mapping serialized source locs + + Dictionary m_ptrMap; // Maps a pointer to an entry index + + // NOTE! Assumes the content stays in scope! + Dictionary m_sliceMap; + + List m_entries; ///< The entries + MemoryArena m_arena; ///< Holds the payloads + ASTSerialClasses* m_classes; + ASTSerialFilter* m_filter; ///< Filter to control what is serialized +}; + +// --------------------------------------------------------------------------- +template +ASTSerialIndex ASTSerialWriter::addArray(const T* in, Index count) +{ + typedef ASTSerialTypeInfo ElementTypeInfo; + typedef typename ElementTypeInfo::SerialType ElementSerialType; + + if (std::is_same::value) + { + // If they are the same we can just write out + return _addArray(sizeof(T), SLANG_ALIGN_OF(ElementSerialType), in, count); + } + else + { + // Else we need to convert + List work; + work.setCount(count); + + for (Index i = 0; i < count; ++i) + { + ElementTypeInfo::toSerial(this, &in[i], &work[i]); + } + return _addArray(sizeof(ElementSerialType), SLANG_ALIGN_OF(ElementSerialType), work.getBuffer(), count); + } +} + +struct ASTSerialType +{ + typedef void(*ToSerialFunc)(ASTSerialWriter* writer, const void* src, void* dst); + typedef void(*ToNativeFunc)(ASTSerialReader* reader, const void* src, void* dst); + + size_t serialSizeInBytes; + uint8_t serialAlignment; + ToSerialFunc toSerialFunc; + ToNativeFunc toNativeFunc; +}; + +struct ASTSerialField +{ + const char* name; ///< The name of the field + const ASTSerialType* type; ///< The type of the field + uint32_t nativeOffset; ///< Offset to field from base of type + uint32_t serialOffset; ///< Offset in serial type +}; + + +struct ASTSerialClass +{ + ASTNodeType type; + uint8_t alignment; + ASTSerialField* fields; + Index fieldsCount; + uint32_t size; +}; + +// An instance could be shared across Sessions, but for simplicity of life time +// here we don't deal with that +class ASTSerialClasses : public RefObject +{ +public: + + const ASTSerialClass* getSerialClass(ASTNodeType type) const { return &m_classes[Index(type)]; } + + /// Ctor + ASTSerialClasses(); + +protected: + MemoryArena m_arena; + + ASTSerialClass m_classes[Index(ASTNodeType::CountOf)]; +}; + + +/* None of the functions in this util should *not* be called from production code, +they exist to test features of AST Serialization */ +struct ASTSerialTestUtil +{ + static SlangResult selfTest(); + + /// Tries to serialize out, read back in and test the results are the same. + /// Will write dumped out node to files + static SlangResult testSerialize(NodeBase* node, RootNamePool* rootNamePool, SharedASTBuilder* sharedASTBuilder, SourceManager* sourceManager); +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-container.cpp b/source/slang/slang-serialize-container.cpp new file mode 100644 index 000000000..6f6438947 --- /dev/null +++ b/source/slang/slang-serialize-container.cpp @@ -0,0 +1,553 @@ +// slang-serialize-container.cpp +#include "slang-serialize-container.h" + +#include "../core/slang-text-io.h" +#include "../core/slang-byte-encode-util.h" + +#include "../core/slang-math.h" + +#include "slang-compiler.h" +#include "slang-serialize-ast.h" +#include "slang-serialize-ir.h" +#include "slang-serialize-debug.h" + +namespace Slang { + +/* static */SlangResult SerialContainerUtil::requestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& out) +{ + SLANG_UNUSED(options); + + out.clear(); + + auto linkage = request->getLinkage(); + auto sink = request->getSink(); + auto frontEndReq = request->getFrontEndReq(); + + for (TranslationUnitRequest* translationUnit : frontEndReq->translationUnits) + { + auto module = translationUnit->module; + auto irModule = module->getIRModule(); + + // Root AST node + auto moduleDecl = translationUnit->getModuleDecl(); + + SLANG_ASSERT(irModule || moduleDecl); + + SerialContainerData::TranslationUnit dstTranslationUnit; + dstTranslationUnit.astRootNode = moduleDecl; + dstTranslationUnit.irModule = irModule; + + out.translationUnits.add(dstTranslationUnit); + } + + auto program = request->getSpecializedGlobalAndEntryPointsComponentType(); + + // Add all the target modules + { + + for (auto target : linkage->targets) + { + auto targetProgram = program->getTargetProgram(target); + auto irModule = targetProgram->getOrCreateIRModuleForLayout(sink); + + SerialContainerData::TargetModule targetModule; + + targetModule.irModule = irModule; + + auto& dstTarget = targetModule.target; + + dstTarget.floatingPointMode = target->floatingPointMode; + dstTarget.profile = target->targetProfile; + dstTarget.flags = target->targetFlags; + dstTarget.codeGenTarget = target->target; + + out.targetModules.add(targetModule); + } + } + + // Entry points + { + auto entryPointCount = program->getEntryPointCount(); + for (Index ii = 0; ii < entryPointCount; ++ii) + { + auto entryPoint = program->getEntryPoint(ii); + auto entryPointMangledName = program->getEntryPointMangledName(ii); + + SerialContainerData::EntryPoint dstEntryPoint; + + dstEntryPoint.name = entryPoint->getName(); + dstEntryPoint.mangledName = entryPointMangledName; + dstEntryPoint.profile = entryPoint->getProfile(); + + out.entryPoints.add(dstEntryPoint); + } + } + + return SLANG_OK; +} + +/* static */SlangResult SerialContainerUtil::write(const SerialContainerData& data, const WriteOptions& options, RiffContainer* container) +{ + //auto linkage = request->getLinkage(); + //auto sink = request->getSink(); + //auto frontEndReq = request->getFrontEndReq(); + //SourceManager* sourceManager = frontEndReq->getSourceManager(); + + RefPtr debugWriter; + + // The string pool used across the whole of the container + StringSlicePool containerStringPool(StringSlicePool::Style::Default); + + RiffContainer::ScopeChunk scopeModule(container, RiffContainer::Chunk::Kind::List, SerialBinary::kContainerFourCc); + + // Write the header + { + SerialBinary::ContainerHeader containerHeader; + // Save the compression type if used - as can only serialize with a single compression type + containerHeader.compressionType = uint32_t(options.compressionType); + + RiffContainer::ScopeChunk scopeHeader(container, RiffContainer::Chunk::Kind::Data, SerialBinary::kContainerHeaderFourCc); + container->write(&containerHeader, sizeof(containerHeader)); + } + + if (data.translationUnits.getCount()) + { + // Module list + RiffContainer::ScopeChunk moduleListScope(container, RiffContainer::Chunk::Kind::List, SerialBinary::kTranslationUnitListFourCc); + + if (options.optionFlags & SerialOptionFlag::DebugInfo) + { + debugWriter = new DebugSerialWriter(options.sourceManager); + } + + RefPtr astClasses = new ASTSerialClasses; + + for (const auto& translationUnit : data.translationUnits) + { + // Okay, we need to serialize this module to our container file. + // We currently don't serialize it's name..., but support for that could be added. + + // Write the IR information + if (translationUnit.irModule) + { + IRSerialData serialData; + IRSerialWriter writer; + SLANG_RETURN_ON_FAIL(writer.write(translationUnit.irModule, debugWriter, options.optionFlags, &serialData)); + SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, options.compressionType, container)); + } + + // Write the AST information + + if (ModuleDecl* moduleDecl = as(translationUnit.astRootNode)) + { + ModuleASTSerialFilter filter(moduleDecl); + ASTSerialWriter writer(astClasses, &filter, debugWriter); + + // Add the module and everything that isn't filtered out in the filter. + writer.addPointer(moduleDecl); + + // We can now serialize it into the riff container. + SLANG_RETURN_ON_FAIL(writer.writeIntoContainer(container)); + } + } + + if (data.targetModules.getCount()) + { + // TODO: in the case where we have specialization, we might need + // to serialize IR related to `program`... + + for (const auto& targetModule : data.targetModules) + { + IRModule* irModule = targetModule.irModule; + + // Okay, we need to serialize this target program and its IR too... + IRSerialData serialData; + IRSerialWriter writer; + + SLANG_RETURN_ON_FAIL(writer.write(irModule, debugWriter, options.optionFlags, &serialData)); + SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, options.compressionType, container)); + } + } + } + + if (data.entryPoints.getCount()) + { + for (const auto& entryPoint : data.entryPoints) + { + RiffContainer::ScopeChunk entryPointScope(container, RiffContainer::Chunk::Kind::Data, SerialBinary::kEntryPointFourCc); + + SerialContainerBinary::EntryPoint dst; + + dst.name = uint32_t(containerStringPool.add(entryPoint.name->text)); + dst.profile = entryPoint.profile.raw; + dst.mangledName = uint32_t(containerStringPool.add(entryPoint.mangledName)); + + container->write(&dst, sizeof(dst)); + } + } + + // We can now output the debug information. This is for all IR and AST + if (debugWriter) + { + // Write out the debug info + DebugSerialData debugData; + debugWriter->write(&debugData); + + debugData.writeContainer(options.compressionType, container); + } + + // Write the container string table + if (containerStringPool.getAdded().getCount() > 0) + { + RiffContainer::ScopeChunk stringTableScope(container, RiffContainer::Chunk::Kind::Data, SerialBinary::kStringTableFourCc); + + List encodedTable; + SerialStringTableUtil::encodeStringTable(containerStringPool, encodedTable); + + container->write(encodedTable.getBuffer(), encodedTable.getCount()); + } + + return SLANG_OK; +} + +/* static */Result SerialContainerUtil::read(RiffContainer* container, const ReadOptions& options, SerialContainerData& out) +{ + out.clear(); + + RiffContainer::ListChunk* containerChunk = container->getRoot()->findListRec(SerialBinary::kContainerFourCc); + if (!containerChunk) + { + // Must be a container + return SLANG_FAIL; + } + + SerialBinary::ContainerHeader* containerHeader = containerChunk->findContainedData(SerialBinary::kContainerHeaderFourCc); + if (!containerHeader) + { + // Didn't find the header + return SLANG_FAIL; + } + + const SerialCompressionType containerCompressionType = SerialCompressionType(containerHeader->compressionType); + + StringSlicePool containerStringPool(StringSlicePool::Style::Default); + + if (RiffContainer::Data* stringTableData = containerChunk->findContainedData(SerialBinary::kStringTableFourCc)) + { + SerialStringTableUtil::decodeStringTable((const char*)stringTableData->getPayload(), stringTableData->getSize(), containerStringPool); + } + + RefPtr debugReader; + RefPtr astClasses; + + // Debug information + if (auto debugChunk = containerChunk->findContainedList(DebugSerialData::kDebugFourCc)) + { + // Read into data + DebugSerialData debugData; + SLANG_RETURN_ON_FAIL(debugData.readContainer(containerCompressionType, debugChunk)); + + // Turn into DebugReader + debugReader = new DebugSerialReader; + SLANG_RETURN_ON_FAIL(debugReader->read(&debugData, options.sourceManager)); + } + + // Add translation units + if (RiffContainer::ListChunk* translationUnits = containerChunk->findContainedList(SerialBinary::kTranslationUnitListFourCc)) + { + RiffContainer::Chunk* chunk = translationUnits->getFirstContainedChunk(); + while (chunk) + { + auto startChunk = chunk; + + RefPtr astBuilder; + NodeBase* astRootNode = nullptr; + RefPtr irModule; + + if (auto irChunk = as(chunk, IRSerialBinary::kIRModuleFourCc)) + { + IRSerialData serialData; + + SLANG_RETURN_ON_FAIL(IRSerialReader::readContainer(irChunk, containerCompressionType, &serialData)); + + // Read IR back from serialData + IRSerialReader reader; + SLANG_RETURN_ON_FAIL(reader.read(serialData, options.session, debugReader, irModule)); + + // Onto next chunk + chunk = chunk->m_next; + } + + if (auto astChunk = as(chunk, ASTSerialBinary::kSlangASTModuleFourCC)) + { + RiffContainer::Data* astData = astChunk->findContainedData(ASTSerialBinary::kSlangASTModuleDataFourCC); + + if (astData) + { + if (!astClasses) + { + astClasses = new ASTSerialClasses; + } + + // TODO(JS): We probably want to store off better information about each of the translation unit + // including some kind of 'name'. + // For now we just generate a name. + + StringBuilder buf; + buf << "tu" << out.translationUnits.getCount(); + + astBuilder = new ASTBuilder(options.sharedASTBuilder, buf.ProduceString()); + + ASTSerialReader reader(astClasses, debugReader); + + SLANG_RETURN_ON_FAIL(reader.load((const uint8_t*)astData->getPayload(), astData->getSize(), astBuilder, options.namePool)); + + // Get the root node. It's at index 1 (0 is the null value). + astRootNode = reader.getPointer(ASTSerialIndex(1)).dynamicCast(); + } + + // Onto next chunk + chunk = chunk->m_next; + } + + if (astBuilder || irModule) + { + SerialContainerData::TranslationUnit translationUnit; + + translationUnit.astBuilder = astBuilder; + translationUnit.astRootNode = astRootNode; + translationUnit.irModule = irModule; + + out.translationUnits.add(translationUnit); + } + + // If no progress, step to next chunk + chunk = (chunk == startChunk) ? chunk->m_next : chunk; + } + } + + // Add all the entry points + { + List entryPointChunks; + containerChunk->findContained(SerialBinary::kEntryPointFourCc, entryPointChunks); + + for (auto entryPointChunk : entryPointChunks) + { + auto reader = entryPointChunk->asReadHelper(); + + auto readString = [&]() + { + uint32_t length = 0; + reader.read(length); + + char* begin = (char*)reader.getData(); + reader.skip(length + 1); + + return UnownedStringSlice(begin, begin + length); + }; + + SerialContainerBinary::EntryPoint srcEntryPoint; + SLANG_RETURN_ON_FAIL(reader.read(srcEntryPoint)); + + SerialContainerData::EntryPoint dstEntryPoint; + + dstEntryPoint.name = options.namePool->getName(containerStringPool.getSlice(StringSlicePool::Handle(srcEntryPoint.name))); + dstEntryPoint.profile.raw = srcEntryPoint.profile; + dstEntryPoint.mangledName = containerStringPool.getSlice(StringSlicePool::Handle(srcEntryPoint.mangledName)); + + out.entryPoints.add(dstEntryPoint); + } + } + + return SLANG_OK; +} + +/* static */SlangResult SerialContainerUtil::verifyIRSerialize(IRModule* module, Session* session, const WriteOptions& options) +{ + // Verify if we can stream out with raw source locs + + List originalInsts; + IRSerialWriter::calcInstructionList(module, originalInsts); + + IRSerialData irData; + + OwnedMemoryStream memoryStream(FileAccess::ReadWrite); + + { + RiffContainer riffContainer; + + // Need to put all of this in a container + RiffContainer::ScopeChunk containerScope(&riffContainer, RiffContainer::Chunk::Kind::List, SerialBinary::kContainerFourCc); + + RefPtr debugWriter; + + if (options.optionFlags & SerialOptionFlag::DebugInfo) + { + debugWriter = new DebugSerialWriter(options.sourceManager); + } + + { + // Write IR out to serialData - copying over SourceLoc information directly + IRSerialWriter writer; + SLANG_RETURN_ON_FAIL(writer.write(module, debugWriter, options.optionFlags, &irData)); + } + SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(irData, options.compressionType, &riffContainer)); + + // Write the debug info Riff container + if (debugWriter) + { + DebugSerialData serialData; + debugWriter->write(&serialData); + + SLANG_RETURN_ON_FAIL(serialData.writeContainer(options.compressionType, &riffContainer)); + } + + SLANG_RETURN_ON_FAIL(RiffUtil::write(&riffContainer, &memoryStream)); + } + + // Reset stream + memoryStream.seek(SeekOrigin::Start, 0); + + SourceManager workSourceManager; + workSourceManager.initialize(options.sourceManager, nullptr); + + // The read ir module + RefPtr irReadModule; + { + RiffContainer riffContainer; + SLANG_RETURN_ON_FAIL(RiffUtil::read(&memoryStream, riffContainer)); + + RiffContainer::ListChunk* rootList = riffContainer.getRoot(); + + RefPtr debugReader; + + // If we have debug info then find and read it + if (options.optionFlags & SerialOptionFlag::DebugInfo) + { + RiffContainer::ListChunk* debugList = rootList->findContainedList(DebugSerialData::kDebugFourCc); + if (!debugList) + { + return SLANG_FAIL; + } + DebugSerialData debugData; + SLANG_RETURN_ON_FAIL(debugData.readContainer(options.compressionType, debugList)); + + debugReader = new DebugSerialReader; + SLANG_RETURN_ON_FAIL(debugReader->read(&debugData, &workSourceManager)); + } + + { + RiffContainer::ListChunk* irList = rootList->findContainedList(IRSerialBinary::kIRModuleFourCc); + if (!irList) + { + return SLANG_FAIL; + } + + { + IRSerialData irReadData; + IRSerialReader reader; + SLANG_RETURN_ON_FAIL(reader.readContainer(irList, options.compressionType, &irReadData)); + + // Check the stream read data is the same + if (irData != irReadData) + { + SLANG_ASSERT(!"Streamed in data doesn't match"); + return SLANG_FAIL; + } + + SLANG_RETURN_ON_FAIL(reader.read(irData, session, debugReader, irReadModule)); + } + } + } + + List readInsts; + IRSerialWriter::calcInstructionList(irReadModule, readInsts); + + if (readInsts.getCount() != originalInsts.getCount()) + { + SLANG_ASSERT(!"Instruction counts don't match"); + return SLANG_FAIL; + } + + if (options.optionFlags & SerialOptionFlag::RawSourceLocation) + { + SLANG_ASSERT(readInsts[0] == originalInsts[0]); + // All the source locs should be identical + for (Index i = 1; i < readInsts.getCount(); ++i) + { + IRInst* origInst = originalInsts[i]; + IRInst* readInst = readInsts[i]; + + if (origInst->sourceLoc.getRaw() != readInst->sourceLoc.getRaw()) + { + SLANG_ASSERT(!"Source locs don't match"); + return SLANG_FAIL; + } + } + } + else if (options.optionFlags & SerialOptionFlag::DebugInfo) + { + // They should be on the same line nos + for (Index i = 1; i < readInsts.getCount(); ++i) + { + IRInst* origInst = originalInsts[i]; + IRInst* readInst = readInsts[i]; + + if (origInst->sourceLoc.getRaw() == readInst->sourceLoc.getRaw()) + { + continue; + } + + // Work out the + SourceView* origSourceView = options.sourceManager->findSourceView(origInst->sourceLoc); + SourceView* readSourceView = workSourceManager.findSourceView(readInst->sourceLoc); + + // if both are null we are done + if (origSourceView == nullptr && origSourceView == readSourceView) + { + continue; + } + SLANG_ASSERT(origSourceView && readSourceView); + + // The offset should be the same + Index origOffset = origInst->sourceLoc.getRaw() - origSourceView->getRange().begin.getRaw(); + Index readOffset = readInst->sourceLoc.getRaw() - readSourceView->getRange().begin.getRaw(); + + if (origOffset != readOffset) + { + SLANG_ASSERT(!"SourceLoc offset debug data didn't match"); + return SLANG_FAIL; + } + + { + auto origInfo = origSourceView->getHumaneLoc(origInst->sourceLoc, SourceLocType::Actual); + auto readInfo = readSourceView->getHumaneLoc(readInst->sourceLoc, SourceLocType::Actual); + + if (!(origInfo.line == readInfo.line && origInfo.column == readInfo.column && origInfo.pathInfo.foundPath == readInfo.pathInfo.foundPath)) + { + SLANG_ASSERT(!"Debug data didn't match"); + return SLANG_FAIL; + } + } + + // We may have adjusted line numbers -> but they may not match, because we only reconstruct one view + // So for now disable this test + + if (false) + { + auto origInfo = origSourceView->getHumaneLoc(origInst->sourceLoc, SourceLocType::Nominal); + auto readInfo = readSourceView->getHumaneLoc(readInst->sourceLoc, SourceLocType::Nominal); + + if (!(origInfo.line == readInfo.line && origInfo.column == readInfo.column && origInfo.pathInfo.foundPath == readInfo.pathInfo.foundPath)) + { + SLANG_ASSERT(!"Debug data didn't match"); + return SLANG_FAIL; + } + } + } + } + + return SLANG_OK; +} + +} // namespace Slang diff --git a/source/slang/slang-serialize-container.h b/source/slang/slang-serialize-container.h new file mode 100644 index 000000000..2f2f509b8 --- /dev/null +++ b/source/slang/slang-serialize-container.h @@ -0,0 +1,107 @@ +// slang-serialize-container.h +#ifndef SLANG_SERIALIZE_CONTAINER_H +#define SLANG_SERIALIZE_CONTAINER_H + +#include "../core/slang-riff.h" +#include "slang-serialize-types.h" +#include "slang-ir-insts.h" +#include "slang-profile.h" + +namespace Slang { + +class EndToEndCompileRequest; + +/* The binary representation actually held in riff/file format*/ +struct SerialContainerBinary +{ + struct Target + { + uint32_t target; + uint32_t flags; + uint32_t profile; + uint32_t floatingPointMode; + }; + + struct EntryPoint + { + uint32_t name; + uint32_t profile; + uint32_t mangledName; + }; +}; + +/* Struct that holds all the data that can be held in a 'container' */ +struct SerialContainerData +{ + struct Target + { + CodeGenTarget codeGenTarget = CodeGenTarget::Unknown; + SlangTargetFlags flags = 0; + Profile profile; + FloatingPointMode floatingPointMode = FloatingPointMode::Default; + }; + + struct TargetModule + { + // IR module for a specific compilation target + Target target; + RefPtr irModule; + }; + + struct TranslationUnit + { + RefPtr irModule; + RefPtr astBuilder; + NodeBase* astRootNode = nullptr; + }; + + struct EntryPoint + { + Name* name = nullptr; + Profile profile; + String mangledName; + }; + + void clear() + { + entryPoints.clear(); + translationUnits.clear(); + targetModules.clear(); + } + + List translationUnits; + List targetModules; + List entryPoints; +}; + +struct SerialContainerUtil +{ + struct WriteOptions + { + SerialCompressionType compressionType = SerialCompressionType::VariableByteLite; + SerialOptionFlags optionFlags = 0; + SourceManager* sourceManager = nullptr; + }; + + struct ReadOptions + { + Session* session = nullptr; + SourceManager* sourceManager = nullptr; + NamePool* namePool = nullptr; + SharedASTBuilder* sharedASTBuilder = nullptr; + }; + + /// Get the serializable contents of the request as data + static SlangResult requestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& outData); + + static SlangResult write(const SerialContainerData& data, const WriteOptions& options, RiffContainer* container); + + static SlangResult read(RiffContainer* container, const ReadOptions& options, SerialContainerData& out); + + /// Verify IR serialization + static SlangResult verifyIRSerialize(IRModule* module, Session* session, const WriteOptions& options); +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-debug.cpp b/source/slang/slang-serialize-debug.cpp new file mode 100644 index 000000000..8f549e0f0 --- /dev/null +++ b/source/slang/slang-serialize-debug.cpp @@ -0,0 +1,404 @@ +// slang-serialize-debug.cpp +#include "slang-serialize-debug.h" + +#include "../core/slang-text-io.h" +#include "../core/slang-byte-encode-util.h" + +#include "../core/slang-math.h" + +namespace Slang { + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +size_t DebugSerialData::calcSizeInBytes() const +{ + return SerialListUtil::calcArraySize(m_debugStringTable) + + SerialListUtil::calcArraySize(m_debugLineInfos) + + SerialListUtil::calcArraySize(m_debugSourceInfos) + + SerialListUtil::calcArraySize(m_debugAdjustedLineInfos); +} + +void DebugSerialData::clear() +{ + m_debugLineInfos.clear(); + m_debugAdjustedLineInfos.clear(); + m_debugSourceInfos.clear(); + m_debugStringTable.clear(); +} + + +bool DebugSerialData::operator==(const ThisType& rhs) const +{ + return (this == &rhs) || + ( SerialListUtil::isEqual(m_debugStringTable, rhs.m_debugStringTable) && + SerialListUtil::isEqual(m_debugLineInfos, rhs.m_debugLineInfos) && + SerialListUtil::isEqual(m_debugAdjustedLineInfos, rhs.m_debugAdjustedLineInfos) && + SerialListUtil::isEqual(m_debugSourceInfos, rhs.m_debugSourceInfos)); +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +DebugSerialData::SourceLoc DebugSerialWriter::addSourceLoc(SourceLoc sourceLoc) +{ + // If it's not set we can ignore + if (!sourceLoc.isValid()) + { + return DebugSerialData::SourceLoc(0); + } + + // Look up the view it's from + SourceView* sourceView = m_sourceManager->findSourceView(sourceLoc); + if (!sourceView) + { + // If not found we just ingore + return DebugSerialData::SourceLoc(0); + } + + SourceFile* sourceFile = sourceView->getSourceFile(); + DebugSourceFile* debugSourceFile; + { + RefPtr* ptrDebugSourceFile = m_debugSourceFileMap.TryGetValue(sourceFile); + if (ptrDebugSourceFile == nullptr) + { + const SourceLoc::RawValue baseSourceLoc = m_debugFreeSourceLoc; + m_debugFreeSourceLoc += SourceLoc::RawValue(sourceView->getRange().getSize() + 1); + + debugSourceFile = new DebugSourceFile(sourceFile, baseSourceLoc); + m_debugSourceFileMap.Add(sourceFile, debugSourceFile); + } + else + { + debugSourceFile = *ptrDebugSourceFile; + } + } + + // We need to work out the line index + + int offset = sourceView->getRange().getOffset(sourceLoc); + int lineIndex = sourceFile->calcLineIndexFromOffset(offset); + + DebugSerialData::DebugLineInfo lineInfo; + lineInfo.m_lineStartOffset = sourceFile->getLineBreakOffsets()[lineIndex]; + lineInfo.m_lineIndex = lineIndex; + + if (!debugSourceFile->hasLineIndex(lineIndex)) + { + // Add the information about the line + int entryIndex = sourceView->findEntryIndex(sourceLoc); + if (entryIndex < 0) + { + debugSourceFile->m_lineInfos.add(lineInfo); + } + else + { + const auto& entry = sourceView->getEntries()[entryIndex]; + + DebugSerialData::DebugAdjustedLineInfo adjustedLineInfo; + adjustedLineInfo.m_lineInfo = lineInfo; + adjustedLineInfo.m_pathStringIndex = SerialStringData::kNullStringIndex; + + const auto& pool = sourceView->getSourceManager()->getStringSlicePool(); + SLANG_ASSERT(pool.getStyle() == StringSlicePool::Style::Default); + + if (!pool.isDefaultHandle(entry.m_pathHandle)) + { + UnownedStringSlice slice = pool.getSlice(entry.m_pathHandle); + SLANG_ASSERT(slice.getLength() > 0); + adjustedLineInfo.m_pathStringIndex = DebugSerialData::StringIndex(m_debugStringSlicePool.add(slice)); + } + + adjustedLineInfo.m_adjustedLineIndex = lineIndex + entry.m_lineAdjust; + + debugSourceFile->m_adjustedLineInfos.add(adjustedLineInfo); + } + + debugSourceFile->setHasLineIndex(lineIndex); + } + + return DebugSerialData::SourceLoc(debugSourceFile->m_baseSourceLoc + offset); +} + +void DebugSerialWriter::write(DebugSerialData* outDebugData) +{ + outDebugData->clear(); + + // Okay we can now calculate the final source information + + for (auto& pair : m_debugSourceFileMap) + { + DebugSourceFile* debugSourceFile = pair.Value; + SourceFile* sourceFile = debugSourceFile->m_sourceFile; + + DebugSerialData::DebugSourceInfo sourceInfo; + + sourceInfo.m_numLines = uint32_t(debugSourceFile->m_sourceFile->getLineBreakOffsets().getCount()); + + sourceInfo.m_range.m_start = uint32_t(debugSourceFile->m_baseSourceLoc); + sourceInfo.m_range.m_end = uint32_t(debugSourceFile->m_baseSourceLoc + sourceFile->getContentSize()); + + sourceInfo.m_pathIndex = DebugSerialData::StringIndex(m_debugStringSlicePool.add(sourceFile->getPathInfo().foundPath)); + + sourceInfo.m_lineInfosStartIndex = uint32_t(outDebugData->m_debugLineInfos.getCount()); + sourceInfo.m_adjustedLineInfosStartIndex = uint32_t(outDebugData->m_debugAdjustedLineInfos.getCount()); + + sourceInfo.m_numLineInfos = uint32_t(debugSourceFile->m_lineInfos.getCount()); + sourceInfo.m_numAdjustedLineInfos = uint32_t(debugSourceFile->m_adjustedLineInfos.getCount()); + + // Add the line infos + outDebugData->m_debugLineInfos.addRange(debugSourceFile->m_lineInfos.begin(), debugSourceFile->m_lineInfos.getCount()); + outDebugData->m_debugAdjustedLineInfos.addRange(debugSourceFile->m_adjustedLineInfos.begin(), debugSourceFile->m_adjustedLineInfos.getCount()); + + // Add the source info + outDebugData->m_debugSourceInfos.add(sourceInfo); + } + + // Convert the string pool + SerialStringTableUtil::encodeStringTable(m_debugStringSlicePool, outDebugData->m_debugStringTable); +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +Index DebugSerialReader::findViewIndex(DebugSerialData::SourceLoc loc) +{ + for (Index i = 0; i < m_views.getCount(); ++i) + { + if (m_views[i].m_range.contains(loc)) + { + return i; + } + } + return -1; +} + + +int DebugSerialReader::calcFixSourceLoc(DebugSerialData::SourceLoc loc, DebugSerialData::SourceRange& outRange) +{ + if (m_lastViewIndex < 0 || !m_views[m_lastViewIndex].m_range.contains(loc)) + { + m_lastViewIndex = findViewIndex(loc); + } + + if (m_lastViewIndex < 0) + { + // Set an invalid range, as couldn't find + outRange = DebugSerialData::SourceRange::getInvalid(); + return 0; + } + + const auto& view = m_views[m_lastViewIndex]; + + SLANG_ASSERT(view.m_range.contains(loc)); + + outRange = view.m_range; + return view.m_sourceView->getRange().begin.getRaw() - view.m_range.m_start; +} + +SourceLoc DebugSerialReader::getSourceLoc(DebugSerialData::SourceLoc loc) +{ + if (loc != 0) + { + if (m_lastViewIndex >= 0) + { + const auto& view = m_views[m_lastViewIndex]; + if (view.m_range.contains(loc)) + { + return view.m_range.getSourceLoc(loc, view.m_sourceView); + } + } + + m_lastViewIndex = findViewIndex(loc); + if (m_lastViewIndex >= 0) + { + const auto& view = m_views[m_lastViewIndex]; + return view.m_range.getSourceLoc(loc, view.m_sourceView); + } + } + return SourceLoc(); +} + +SlangResult DebugSerialReader::read(const DebugSerialData* serialData, SourceManager* sourceManager) +{ + m_views.setCount(0); + + if (!sourceManager || serialData->m_debugSourceInfos.getCount() == 0) + { + return SLANG_OK; + } + + List debugStringSlices; + SerialStringTableUtil::decodeStringTable(serialData->m_debugStringTable.getBuffer(), serialData->m_debugStringTable.getCount(), debugStringSlices); + + // All of the strings are placed in the manager (and its StringSlicePool) where the SourceView and SourceFile are constructed from + List stringMap; + SerialStringTableUtil::calcStringSlicePoolMap(debugStringSlices, sourceManager->getStringSlicePool(), stringMap); + + // Construct the source files + const Index numSourceFiles = serialData->m_debugSourceInfos.getCount(); + + // These hold the views (and SourceFile as there is only one SourceFile per view) in the same order as the sourceInfos + m_views.setCount(numSourceFiles); + + for (Index i = 0; i < numSourceFiles; ++i) + { + const auto& srcSourceInfo = serialData->m_debugSourceInfos[i]; + + PathInfo pathInfo; + pathInfo.type = PathInfo::Type::FoundPath; + pathInfo.foundPath = debugStringSlices[UInt(srcSourceInfo.m_pathIndex)]; + + SourceFile* sourceFile = sourceManager->createSourceFileWithSize(pathInfo, srcSourceInfo.m_range.getCount()); + SourceView* sourceView = sourceManager->createSourceView(sourceFile, nullptr); + + // We need to accumulate all line numbers, for this source file, both adjusted and unadjusted + List lineInfos; + // Add the adjusted lines + { + lineInfos.setCount(srcSourceInfo.m_numAdjustedLineInfos); + const DebugSerialData::DebugAdjustedLineInfo* srcAdjustedLineInfos = serialData->m_debugAdjustedLineInfos.getBuffer() + srcSourceInfo.m_adjustedLineInfosStartIndex; + const int numAdjustedLines = int(srcSourceInfo.m_numAdjustedLineInfos); + for (int j = 0; j < numAdjustedLines; ++j) + { + lineInfos[j] = srcAdjustedLineInfos[j].m_lineInfo; + } + } + // Add regular lines + lineInfos.addRange(serialData->m_debugLineInfos.getBuffer() + srcSourceInfo.m_lineInfosStartIndex, srcSourceInfo.m_numLineInfos); + + // Put in sourceloc order + lineInfos.sort(); + + List lineBreakOffsets; + + // We can now set up the line breaks array + const int numLines = int(srcSourceInfo.m_numLines); + lineBreakOffsets.setCount(numLines); + + { + const Index numLineInfos = lineInfos.getCount(); + Index lineIndex = 0; + + // Every line up and including should hold the same offset + for (Index lineInfoIndex = 0; lineInfoIndex < numLineInfos; ++lineInfoIndex) + { + const auto& lineInfo = lineInfos[lineInfoIndex]; + + const uint32_t offset = lineInfo.m_lineStartOffset; + SLANG_ASSERT(offset > 0); + const int finishIndex = int(lineInfo.m_lineIndex); + + SLANG_ASSERT(finishIndex < numLines); + + for (; lineIndex < finishIndex; ++lineIndex) + { + lineBreakOffsets[lineIndex] = offset - 1; + } + lineBreakOffsets[lineIndex] = offset; + lineIndex++; + } + + // Do the remaining lines + { + const uint32_t endOffset = srcSourceInfo.m_range.getCount(); + for (; lineIndex < numLines; ++lineIndex) + { + lineBreakOffsets[lineIndex] = endOffset; + } + } + } + + sourceFile->setLineBreakOffsets(lineBreakOffsets.getBuffer(), lineBreakOffsets.getCount()); + + if (srcSourceInfo.m_numAdjustedLineInfos) + { + List adjustedLineInfos; + + int numEntries = int(srcSourceInfo.m_numAdjustedLineInfos); + + adjustedLineInfos.addRange(serialData->m_debugAdjustedLineInfos.getBuffer() + srcSourceInfo.m_adjustedLineInfosStartIndex, numEntries); + adjustedLineInfos.sort(); + + // Work out the views adjustments, and place in dstEntries + List dstEntries; + dstEntries.setCount(numEntries); + + const uint32_t sourceLocOffset = uint32_t(sourceView->getRange().begin.getRaw()); + + for (int j = 0; j < numEntries; ++j) + { + const auto& srcEntry = adjustedLineInfos[j]; + auto& dstEntry = dstEntries[j]; + + dstEntry.m_pathHandle = stringMap[int(srcEntry.m_pathStringIndex)]; + dstEntry.m_startLoc = SourceLoc::fromRaw(srcEntry.m_lineInfo.m_lineStartOffset + sourceLocOffset); + dstEntry.m_lineAdjust = int32_t(srcEntry.m_adjustedLineIndex) - int32_t(srcEntry.m_lineInfo.m_lineIndex); + } + + // Set the adjustments on the view + sourceView->setEntries(dstEntries.getBuffer(), dstEntries.getCount()); + } + + // Set the view and the source range + View& view = m_views[i]; + view.m_sourceView = sourceView; + view.m_range = srcSourceInfo.m_range; + } + + return SLANG_OK; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +/* static */Result DebugSerialData::writeContainer(SerialCompressionType moduleCompressionType, RiffContainer* container) +{ + RiffContainer::ScopeChunk debugChunkScope(container, RiffContainer::Chunk::Kind::List, DebugSerialData::kDebugFourCc); + + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk(DebugSerialData::kDebugStringFourCc, m_debugStringTable, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk(DebugSerialData::kDebugLineInfoFourCc, m_debugLineInfos, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk(DebugSerialData::kDebugAdjustedLineInfoFourCc, m_debugAdjustedLineInfos, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk(moduleCompressionType, DebugSerialData::kDebugSourceInfoFourCc, m_debugSourceInfos, container)); + + return SLANG_OK; +} + +/* static */Result DebugSerialData::readContainer(SerialCompressionType moduleCompressionType, RiffContainer::ListChunk* listChunk) +{ + SLANG_ASSERT(listChunk->getSubType() == DebugSerialData::kDebugFourCc); + + clear(); + for (RiffContainer::Chunk* chunk = listChunk->m_containedChunks; chunk; chunk = chunk->m_next) + { + RiffContainer::DataChunk* dataChunk = as(chunk); + if (!dataChunk) + { + continue; + } + + switch (dataChunk->m_fourCC) + { + case DebugSerialData::kDebugStringFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_debugStringTable)); + break; + } + case DebugSerialData::kDebugLineInfoFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_debugLineInfos)); + break; + } + case DebugSerialData::kDebugAdjustedLineInfoFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_debugAdjustedLineInfos)); + break; + } + case DebugSerialData::kDebugSourceInfoFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(moduleCompressionType, dataChunk, m_debugSourceInfos)); + break; + } + } + } + + return SLANG_OK; +} + +} // namespace Slang diff --git a/source/slang/slang-serialize-debug.h b/source/slang/slang-serialize-debug.h new file mode 100644 index 000000000..ce39a058d --- /dev/null +++ b/source/slang/slang-serialize-debug.h @@ -0,0 +1,225 @@ +// slang-serialize-debug.h +#ifndef SLANG_SERIALIZE_DEBUG_H +#define SLANG_SERIALIZE_DEBUG_H + +#include "../core/slang-riff.h" +#include "../core/slang-string-slice-pool.h" +#include "../core/slang-array-view.h" + +#include "slang-serialize-types.h" + +#include "slang-name.h" +#include "slang-source-loc.h" + +namespace Slang { + +class DebugSerialData +{ +public: + typedef DebugSerialData ThisType; + + typedef uint32_t SourceLoc; + typedef SerialStringData::StringIndex StringIndex; + + // The list that contains all the subsequent modules + static const FourCC kDebugFourCc = SLANG_FOUR_CC('S', 'd', 'e', 'b'); + + static const FourCC kDebugStringFourCc = SLANG_FOUR_CC('S', 'd', 's', 't'); + static const FourCC kDebugLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'l', 'n'); + static const FourCC kDebugAdjustedLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'a', 'l'); + static const FourCC kDebugSourceInfoFourCc = SLANG_FOUR_CC('S', 'd', 's', 'o'); + + struct SourceRange + { + typedef SourceRange ThisType; + + bool operator==(const ThisType& rhs) const { return m_start == rhs.m_start && m_end == rhs.m_end; } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + Slang::SourceLoc getSourceLoc(SourceLoc loc, SourceView* view) const + { + return (loc && contains(loc)) ? Slang::SourceLoc::fromRaw((loc - m_start) + SourceLoc(view->getRange().begin.getRaw())) : Slang::SourceLoc(); + } + + SourceLoc getCount() const { return m_end - m_start; } + + bool contains(SourceLoc loc) const { return loc >= m_start && loc <= m_end; } + + /// Set up a range that can't occur in practice + static SourceRange getInvalid() { return SourceRange{ ~SourceLoc(0), ~SourceLoc(0) }; } + + SourceLoc m_start; ///< The offset to the source + SourceLoc m_end; ///< The number of bytes in the source + }; + + struct DebugSourceInfo + { + typedef DebugSourceInfo ThisType; + + bool operator==(const ThisType& rhs) const + { + return m_pathIndex == rhs.m_pathIndex && + m_range == rhs.m_range && + m_numLineInfos == rhs.m_numLineInfos && + m_lineInfosStartIndex == rhs.m_lineInfosStartIndex && + m_numLineInfos == rhs.m_numLineInfos; + } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + StringIndex m_pathIndex; ///< Index to the string table + + SourceRange m_range; ///< The range of locations + + uint32_t m_numLines; ///< Total number of lines in source file + + uint32_t m_lineInfosStartIndex; ///< Index into m_debugLineInfos + uint32_t m_numLineInfos; ///< The number of line infos + + uint32_t m_adjustedLineInfosStartIndex; ///< Adjusted start index + uint32_t m_numAdjustedLineInfos; ///< The number of line infos + }; + + struct DebugLineInfo + { + typedef DebugLineInfo ThisType; + bool operator<(const ThisType& rhs) const { return m_lineStartOffset < rhs.m_lineStartOffset; } + bool operator==(const ThisType& rhs) const + { + return m_lineStartOffset == rhs.m_lineStartOffset && + m_lineIndex == rhs.m_lineIndex; + } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + uint32_t m_lineStartOffset; ///< The offset into the source file + uint32_t m_lineIndex; ///< Original line index + }; + + struct DebugAdjustedLineInfo + { + typedef DebugAdjustedLineInfo ThisType; + bool operator==(const ThisType& rhs) const + { + return m_lineInfo == rhs.m_lineInfo && + m_adjustedLineIndex == rhs.m_adjustedLineIndex && + m_pathStringIndex == rhs.m_pathStringIndex; + } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + bool operator<(const ThisType& rhs) const { return m_lineInfo < rhs.m_lineInfo; } + + DebugLineInfo m_lineInfo; + uint32_t m_adjustedLineIndex; ///< The line index with the adjustment (if there is any). Is 0 if m_pathStringIndex is 0. + StringIndex m_pathStringIndex; ///< The path as an index + }; + + size_t calcSizeInBytes() const; + void clear(); + + Index findSourceInfoIndex(SourceLoc sourceLoc) const + { + const Index numInfos = m_debugSourceInfos.getCount(); + for (Index i = 0; i < numInfos; ++i) + { + if (m_debugSourceInfos[i].m_range.contains(sourceLoc)) + { + return i; + } + } + return -1; + } + + bool operator==(const ThisType& rhs) const; + + Result writeContainer(SerialCompressionType moduleCompressionType, RiffContainer* container); + Result readContainer(SerialCompressionType moduleCompressionType, RiffContainer::ListChunk* listChunk); + + // Data only set if we have debug information + + List m_debugStringTable; ///< String table for debug use only + List m_debugLineInfos; ///< Debug line information + List m_debugAdjustedLineInfos; ///< Adjusted line infos + List m_debugSourceInfos; ///< Debug source information +}; + +class DebugSerialReader : public RefObject +{ +public: + Index findViewIndex(DebugSerialData::SourceLoc loc); + + SourceLoc getSourceLoc(DebugSerialData::SourceLoc loc); + + /// Works out the amount to fix an input source loc to get a regular Slang::SourceLoc + int calcFixSourceLoc(DebugSerialData::SourceLoc loc, DebugSerialData::SourceRange& outRange); + + /// Calc the loc + static SourceLoc calcFixedLoc(DebugSerialData::SourceLoc loc, int fix, const DebugSerialData::SourceRange& range) { SLANG_ASSERT(range.contains(loc)); SLANG_UNUSED(range); return SourceLoc::fromRaw(SourceLoc::RawValue(loc + fix)); } + + SlangResult read(const DebugSerialData* serialData, SourceManager* sourceManager); + +protected: + struct View + { + DebugSerialData::SourceRange m_range; + SourceView* m_sourceView; + }; + + List m_views; ///< All the views + Index m_lastViewIndex = -1; ///< Caches last lookup +}; + +/// Used to write serialized Debug information +class DebugSerialWriter : public RefObject +{ +public: + + class DebugSourceFile : public RefObject + { + public: + DebugSourceFile(SourceFile* sourceFile, SourceLoc::RawValue baseSourceLoc) : + m_sourceFile(sourceFile), + m_baseSourceLoc(baseSourceLoc) + { + // Need to know how many lines there are + const List& lineOffsets = sourceFile->getLineBreakOffsets(); + + const auto numLineIndices = lineOffsets.getCount(); + + // Set none as being used initially + m_lineIndexUsed.setCount(numLineIndices); + ::memset(m_lineIndexUsed.begin(), 0, numLineIndices * sizeof(uint8_t)); + } + /// True if we have information on that line index + bool hasLineIndex(int lineIndex) const { return m_lineIndexUsed[lineIndex] != 0; } + void setHasLineIndex(int lineIndex) { m_lineIndexUsed[lineIndex] = 1; } + + SourceLoc::RawValue m_baseSourceLoc; ///< The base source location + + SourceFile* m_sourceFile; ///< The source file + List m_lineIndexUsed; ///< Has 1 if the line is used + List m_usedLineIndices; ///< Holds the lines that have been hit + + List m_lineInfos; ///< The line infos + List m_adjustedLineInfos; ///< The adjusted line infos + }; + + /// Add a source location. Returns the location that can be serialized. + DebugSerialData::SourceLoc addSourceLoc(SourceLoc sourceLoc); + + /// Write into outDebugData + void write(DebugSerialData* outDebugData); + + DebugSerialWriter(SourceManager* sourceManager): + m_sourceManager(sourceManager), + m_debugStringSlicePool(StringSlicePool::Style::Default), + m_debugFreeSourceLoc(1) + { + } + + SourceManager* m_sourceManager; + StringSlicePool m_debugStringSlicePool; ///< Slices held just for debug usage + SourceLoc::RawValue m_debugFreeSourceLoc; /// Locations greater than this are free + Dictionary > m_debugSourceFileMap; +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-ir-types.cpp b/source/slang/slang-serialize-ir-types.cpp new file mode 100644 index 000000000..cceb72463 --- /dev/null +++ b/source/slang/slang-serialize-ir-types.cpp @@ -0,0 +1,102 @@ +// slang-serialize-ir-types.cpp +#include "slang-serialize-ir-types.h" + +#include "../core/slang-text-io.h" +#include "../core/slang-byte-encode-util.h" + +#include "slang-ir-insts.h" + +#include "../core/slang-math.h" + +namespace Slang { + +/* Note that an IRInst can be derived from, but when it derived from it's new members are IRUse variables, and they in +effect alias over the operands - and reflected in the operand count. There _could_ be other members after these IRUse +variables, but only a few types include extra data, and these do not have any operands: + +* IRConstant - Needs special-case handling +* IRModuleInst - Presumably we can just set to the module pointer on reconstruction + +Note! That on an IRInst there is an IRType* variable (accessed as getFullType()). As it stands it may NOT actually point +to an IRType derived type. Its 'ok' as long as it's an instruction that can be used in the place of the type. So this code does not +bother to check if it's correct, and just casts it. +*/ + +/* static */const IRSerialData::PayloadInfo IRSerialData::s_payloadInfos[int(Inst::PayloadType::CountOf)] = +{ + { 0, 0 }, // Empty + { 1, 0 }, // Operand_1 + { 2, 0 }, // Operand_2 + { 1, 0 }, // OperandAndUInt32, + { 0, 0 }, // OperandExternal - This isn't correct, Operand has to be specially handled + { 0, 1 }, // String_1, + { 0, 2 }, // String_2, + { 0, 0 }, // UInt32, + { 0, 0 }, // Float64, + { 0, 0 } // Int64, +}; + +// Check all compressible chunk ids, start with upper case 'S' +SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kInstFourCc) == 'S'); +SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kChildRunFourCc) == 'S'); +SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kExternalOperandsFourCc) == 'S'); + +// Compressed version starts with 's' +SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(SLANG_MAKE_COMPRESSED_FOUR_CC(IRSerialBinary::kInstFourCc)) == 's'); + +struct PrefixString; + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +template +static size_t _calcArraySize(const List& list) +{ + return list.getCount() * sizeof(T); +} + +size_t IRSerialData::calcSizeInBytes() const +{ + return + _calcArraySize(m_insts) + + _calcArraySize(m_childRuns) + + _calcArraySize(m_externalOperands) + + _calcArraySize(m_stringTable) + + /* Raw source locs */ + _calcArraySize(m_rawSourceLocs) + + /* Debug */ + _calcArraySize(m_debugSourceLocRuns); +} + +IRSerialData::IRSerialData() +{ + clear(); +} + +void IRSerialData::clear() +{ + // First Instruction is null + m_insts.setCount(1); + memset(&m_insts[0], 0, sizeof(Inst)); + + m_childRuns.clear(); + m_externalOperands.clear(); + m_rawSourceLocs.clear(); + + m_stringTable.clear(); + + m_debugSourceLocRuns.clear(); +} + +bool IRSerialData::operator==(const ThisType& rhs) const +{ + return (this == &rhs) || + (SerialListUtil::isEqual(m_insts, rhs.m_insts) && + SerialListUtil::isEqual(m_childRuns, rhs.m_childRuns) && + SerialListUtil::isEqual(m_externalOperands, rhs.m_externalOperands) && + SerialListUtil::isEqual(m_rawSourceLocs, rhs.m_rawSourceLocs) && + SerialListUtil::isEqual(m_stringTable, rhs.m_stringTable) && + /* Debug */ + SerialListUtil::isEqual(m_debugSourceLocRuns, rhs.m_debugSourceLocRuns)); +} + +} // namespace Slang diff --git a/source/slang/slang-serialize-ir-types.h b/source/slang/slang-serialize-ir-types.h new file mode 100644 index 000000000..513212ad6 --- /dev/null +++ b/source/slang/slang-serialize-ir-types.h @@ -0,0 +1,258 @@ +// slang-serialize-ir-types.h +#ifndef SLANG_SERIALIZE_IR_TYPES_H_INCLUDED +#define SLANG_SERIALIZE_IR_TYPES_H_INCLUDED + +#include "../core/slang-riff.h" +#include "../core/slang-string-slice-pool.h" +#include "../core/slang-array-view.h" + +#include "slang-serialize-types.h" +#include "slang-serialize-debug.h" + +#include "slang-name.h" +#include "slang-source-loc.h" + +#include "slang-ir.h" + +namespace Slang { + +// Pre-declare +class Name; + +struct IRSerialBinary +{ + /// IR module list + static const FourCC kIRModuleFourCc = SLANG_FOUR_CC('S', 'i', 'm', 'd'); + + /* NOTE! All FourCC that can be compressed must start with capital 'S', because compressed version is the same FourCC + with the 'S' replaced with 's' */ + + static const FourCC kInstFourCc = SLANG_FOUR_CC('S', 'L', 'i', 'n'); + static const FourCC kChildRunFourCc = SLANG_FOUR_CC('S', 'L', 'c', 'r'); + static const FourCC kExternalOperandsFourCc = SLANG_FOUR_CC('S', 'L', 'e', 'o'); + + static const FourCC kCompressedInstFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kInstFourCc); + static const FourCC kCompressedChildRunFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kChildRunFourCc); + static const FourCC kCompressedExternalOperandsFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kExternalOperandsFourCc); + + static const FourCC kUInt32RawSourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '4'); + + /// Debug information is held elsewhere, but if this optional section exists, it maps instructions to locs + static const FourCC kDebugSourceLocRunFourCc = SLANG_FOUR_CC('S', 'd', 's', 'r'); +}; + +struct IRSerialData +{ + typedef IRSerialData ThisType; + + typedef SerialStringData::StringIndex StringIndex; + + enum class InstIndex : uint32_t; + enum class ArrayIndex : uint32_t; + + enum class RawSourceLoc : SourceLoc::RawValue; ///< This is just to copy over source loc data (ie not strictly serialize) + + typedef uint32_t SizeType; + + /// A run of instructions + struct InstRun + { + typedef InstRun ThisType; + SLANG_FORCE_INLINE bool operator==(const ThisType& rhs) const + { + return m_parentIndex == rhs.m_parentIndex && + m_startInstIndex == rhs.m_startInstIndex && + m_numChildren == rhs.m_numChildren; + } + SLANG_FORCE_INLINE bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + InstIndex m_parentIndex; ///< The parent instruction + InstIndex m_startInstIndex; ///< The index to the first instruction + SizeType m_numChildren; ///< The number of children + }; + + struct SourceLocRun + { + typedef SourceLocRun ThisType; + + bool operator==(const ThisType& rhs) const { return m_sourceLoc == rhs.m_sourceLoc && m_startInstIndex == rhs.m_startInstIndex && m_numInst == rhs.m_numInst; } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + bool operator<(const ThisType& rhs) const { return m_sourceLoc < rhs.m_sourceLoc; } + + DebugSerialData::SourceLoc m_sourceLoc; ///< The source location + InstIndex m_startInstIndex; ///< The index to the first instruction + SizeType m_numInst; ///< The number of children + }; + + struct PayloadInfo + { + uint8_t m_numOperands; + uint8_t m_numStrings; + }; + + + // Instruction... + // We can store SourceLoc values separately. Just store per index information. + // Parent information is stored in m_childRuns + // Decoration information is stored in m_decorationRuns + struct Inst + { + typedef Inst ThisType; + enum + { + kMaxOperands = 2, ///< Maximum number of operands that can be held in an instruction (otherwise held 'externally') + }; + + // NOTE! Can't change order or list without changing appropriate s_payloadInfos + enum class PayloadType : uint8_t + { + // First 3 must be in this order so a cast from 0-2 is directly represented as number of operands + Empty, ///< Has no payload (or operands) + Operand_1, ///< 1 Operand + Operand_2, ///< 2 Operands + + OperandAndUInt32, ///< 1 Operand and a single UInt32 + OperandExternal, ///< Operands are held externally + String_1, ///< 1 String + String_2, ///< 2 Strings + UInt32, ///< Holds an unsigned 32 bit integral (might represent a type) + Float64, + Int64, + + CountOf, + }; + + /// Get the number of operands + SLANG_FORCE_INLINE int getNumOperands() const; + + bool operator==(const ThisType& rhs) const; + + SLANG_FORCE_INLINE bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + uint8_t m_op; ///< For now one of IROp + PayloadType m_payloadType; ///< The type of payload + uint16_t m_pad0; ///< Not currently used + + InstIndex m_resultTypeIndex; //< 0 if has no type. The result type of this instruction + + struct ExternalOperandPayload + { + ArrayIndex m_arrayIndex; ///< Index into the m_externalOperands table + SizeType m_size; ///< The amount of entries in that table + }; + + struct OperandAndUInt32 + { + InstIndex m_operand; + uint32_t m_uint32; + }; + + union Payload + { + double m_float64; + int64_t m_int64; + uint32_t m_uint32; ///< Unsigned integral value + IRFloatingPointValue m_float; ///< Floating point value + IRIntegerValue m_int; ///< Integral value + InstIndex m_operands[kMaxOperands]; ///< For items that 2 or less operands it can use this. + StringIndex m_stringIndices[kMaxOperands]; + ExternalOperandPayload m_externalOperand; ///< Operands are stored in an an index of an operand array + OperandAndUInt32 m_operandAndUInt32; + }; + + Payload m_payload; + }; + + /// Clear to initial state + void clear(); + /// Get the operands of an instruction + SLANG_FORCE_INLINE int getOperands(const Inst& inst, const InstIndex** operandsOut) const; + + /// == + bool operator==(const ThisType& rhs) const; + SLANG_FORCE_INLINE bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + /// Calculate the amount of memory used by this IRSerialData + size_t calcSizeInBytes() const; + + /// Ctor + IRSerialData(); + + List m_insts; ///< The instructions + + List m_rawSourceLocs; ///< A source location per instruction (saved without modification from IRInst) + + List m_childRuns; ///< Holds the information about children that belong to an instruction + + List m_externalOperands; ///< Holds external operands (for instructions with more than kNumOperands) + + List m_stringTable; ///< All strings. Indexed into by StringIndex + + List m_debugSourceLocRuns; ///< Runs of instructions that use a source loc + + static const PayloadInfo s_payloadInfos[int(Inst::PayloadType::CountOf)]; +}; + +// -------------------------------------------------------------------------- +SLANG_FORCE_INLINE int IRSerialData::Inst::getNumOperands() const +{ + return (m_payloadType == PayloadType::OperandExternal) ? m_payload.m_externalOperand.m_size : s_payloadInfos[int(m_payloadType)].m_numOperands; +} + +// -------------------------------------------------------------------------- +SLANG_FORCE_INLINE bool IRSerialData::Inst::operator==(const ThisType& rhs) const +{ + if (m_op == rhs.m_op && + m_payloadType == rhs.m_payloadType && + m_resultTypeIndex == rhs.m_resultTypeIndex) + { + switch (m_payloadType) + { + case PayloadType::Empty: + { + return true; + } + case PayloadType::Operand_1: + case PayloadType::String_1: + case PayloadType::UInt32: + { + return m_payload.m_operands[0] == rhs.m_payload.m_operands[0]; + } + case PayloadType::OperandAndUInt32: + case PayloadType::OperandExternal: + case PayloadType::Operand_2: + case PayloadType::String_2: + { + return m_payload.m_operands[0] == rhs.m_payload.m_operands[0] && + m_payload.m_operands[1] == rhs.m_payload.m_operands[1]; + } + case PayloadType::Float64: + case PayloadType::Int64: + { + return m_payload.m_int64 == rhs.m_payload.m_int64; + } + default: break; + } + } + + return false; +} +// -------------------------------------------------------------------------- +SLANG_FORCE_INLINE int IRSerialData::getOperands(const Inst& inst, const InstIndex** operandsOut) const +{ + if (inst.m_payloadType == Inst::PayloadType::OperandExternal) + { + *operandsOut = m_externalOperands.begin() + int(inst.m_payload.m_externalOperand.m_arrayIndex); + return int(inst.m_payload.m_externalOperand.m_size); + } + else + { + *operandsOut = inst.m_payload.m_operands; + return s_payloadInfos[int(inst.m_payloadType)].m_numOperands; + } +} + + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-ir.cpp b/source/slang/slang-serialize-ir.cpp new file mode 100644 index 000000000..ac8085d51 --- /dev/null +++ b/source/slang/slang-serialize-ir.cpp @@ -0,0 +1,910 @@ +// slang-serialize-ir.cpp +#include "slang-serialize-ir.h" + +#include "../core/slang-text-io.h" +#include "../core/slang-byte-encode-util.h" + +#include "slang-ir-insts.h" + +#include "../core/slang-math.h" + +namespace Slang { + +static bool _isTextureTypeBase(IROp opIn) +{ + const int op = (kIROpMeta_OpMask & opIn); + return op >= kIROp_FirstTextureTypeBase && op <= kIROp_LastTextureTypeBase; +} + +static bool _isConstant(IROp opIn) +{ + const int op = (kIROpMeta_OpMask & opIn); + return op >= kIROp_FirstConstant && op <= kIROp_LastConstant; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +void IRSerialWriter::_addInstruction(IRInst* inst) +{ + // It cannot already be in the map + SLANG_ASSERT(!m_instMap.ContainsKey(inst)); + + // Add to the map + m_instMap.Add(inst, Ser::InstIndex(m_insts.getCount())); + m_insts.add(inst); +} + +Result IRSerialWriter::_calcDebugInfo(DebugSerialWriter* debugWriter) +{ + // We need to find the unique source Locs + // We are not going to store SourceLocs directly, because there may be multiple views mapping down to + // the same underlying source file + + // First find all the unique locs + struct InstLoc + { + typedef InstLoc ThisType; + + SLANG_FORCE_INLINE bool operator<(const ThisType& rhs) const { return sourceLoc < rhs.sourceLoc || (sourceLoc == rhs.sourceLoc && instIndex < rhs.instIndex); } + + uint32_t instIndex; + uint32_t sourceLoc; + }; + + // Find all of the source locations and their associated instructions + List instLocs; + const Index numInsts = m_insts.getCount(); + for (Index i = 1; i < numInsts; i++) + { + IRInst* srcInst = m_insts[i]; + if (!srcInst->sourceLoc.isValid()) + { + continue; + } + InstLoc instLoc; + instLoc.instIndex = uint32_t(i); + instLoc.sourceLoc = uint32_t(srcInst->sourceLoc.getRaw()); + instLocs.add(instLoc); + } + + // Sort them + instLocs.sort(); + + // Look for runs + const InstLoc* startInstLoc = instLocs.begin(); + const InstLoc* endInstLoc = instLocs.end(); + + while (startInstLoc < endInstLoc) + { + const uint32_t startSourceLoc = startInstLoc->sourceLoc; + + // Find the run with the same source loc + + const InstLoc* curInstLoc = startInstLoc + 1; + uint32_t curInstIndex = startInstLoc->instIndex + 1; + + // Find the run size with same source loc and run of instruction indices + for (; curInstLoc < endInstLoc && curInstLoc->sourceLoc == startSourceLoc && curInstLoc->instIndex == curInstIndex; ++curInstLoc, ++curInstIndex) + { + } + + // Add the run + + IRSerialData::SourceLocRun sourceLocRun; + sourceLocRun.m_numInst = curInstIndex - startInstLoc->instIndex;; + sourceLocRun.m_startInstIndex = IRSerialData::InstIndex(startInstLoc->instIndex); + sourceLocRun.m_sourceLoc = debugWriter->addSourceLoc(SourceLoc::fromRaw(startSourceLoc)); + + m_serialData->m_debugSourceLocRuns.add(sourceLocRun); + + // Next + startInstLoc = curInstLoc; + } + + return SLANG_OK; +} + +Result IRSerialWriter::write(IRModule* module, DebugSerialWriter* debugWriter, SerialOptionFlags options, IRSerialData* serialData) +{ + typedef Ser::Inst::PayloadType PayloadType; + + m_serialData = serialData; + + serialData->clear(); + + // We reserve 0 for null + m_insts.clear(); + m_insts.add(nullptr); + + // Reset + m_instMap.Clear(); + m_decorations.clear(); + + // Stack for parentInst + List parentInstStack; + + IRModuleInst* moduleInst = module->getModuleInst(); + parentInstStack.add(moduleInst); + + // Add to the map + _addInstruction(moduleInst); + + // Traverse all of the instructions + while (parentInstStack.getCount()) + { + // If it's in the stack it is assumed it is already in the inst map + IRInst* parentInst = parentInstStack.getLast(); + parentInstStack.removeLast(); + SLANG_ASSERT(m_instMap.ContainsKey(parentInst)); + + // Okay we go through each of the children in order. If they are IRInstParent derived, we add to stack to process later + // cos we want breadth first so the order of children is the same as their index order, meaning we don't need to store explicit indices + const Ser::InstIndex startChildInstIndex = Ser::InstIndex(m_insts.getCount()); + + IRInstListBase childrenList = parentInst->getDecorationsAndChildren(); + for (IRInst* child : childrenList) + { + // This instruction can't be in the map... + SLANG_ASSERT(!m_instMap.ContainsKey(child)); + + _addInstruction(child); + + parentInstStack.add(child); + } + + // If it had any children, then store the information about it + if (Ser::InstIndex(m_insts.getCount()) != startChildInstIndex) + { + Ser::InstRun run; + run.m_parentIndex = m_instMap[parentInst]; + run.m_startInstIndex = startChildInstIndex; + run.m_numChildren = Ser::SizeType(m_insts.getCount() - int(startChildInstIndex)); + + m_serialData->m_childRuns.add(run); + } + } + +#if 0 + { + List workInsts; + calcInstructionList(module, workInsts); + SLANG_ASSERT(workInsts.Count() == m_insts.Count()); + for (UInt i = 0; i < workInsts.Count(); ++i) + { + SLANG_ASSERT(workInsts[i] == m_insts[i]); + } + } +#endif + + // Set to the right size + m_serialData->m_insts.setCount(m_insts.getCount()); + // Clear all instructions + memset(m_serialData->m_insts.begin(), 0, sizeof(Ser::Inst) * m_serialData->m_insts.getCount()); + + // Need to set up the actual instructions + { + const Index numInsts = m_insts.getCount(); + + for (Index i = 1; i < numInsts; ++i) + { + IRInst* srcInst = m_insts[i]; + Ser::Inst& dstInst = m_serialData->m_insts[i]; + + dstInst.m_op = uint8_t(srcInst->op & kIROpMeta_OpMask); + dstInst.m_payloadType = PayloadType::Empty; + + dstInst.m_resultTypeIndex = getInstIndex(srcInst->getFullType()); + + IRConstant* irConst = as(srcInst); + if (irConst) + { + switch (srcInst->op) + { + // Special handling for the ir const derived types + case kIROp_StringLit: + { + auto stringLit = static_cast(srcInst); + dstInst.m_payloadType = PayloadType::String_1; + dstInst.m_payload.m_stringIndices[0] = getStringIndex(stringLit->getStringSlice()); + break; + } + case kIROp_IntLit: + { + dstInst.m_payloadType = PayloadType::Int64; + dstInst.m_payload.m_int64 = irConst->value.intVal; + break; + } + case kIROp_PtrLit: + { + dstInst.m_payloadType = PayloadType::Int64; + dstInst.m_payload.m_int64 = (intptr_t) irConst->value.ptrVal; + break; + } + case kIROp_FloatLit: + { + dstInst.m_payloadType = PayloadType::Float64; + dstInst.m_payload.m_float64 = irConst->value.floatVal; + break; + } + case kIROp_BoolLit: + { + dstInst.m_payloadType = PayloadType::UInt32; + dstInst.m_payload.m_uint32 = irConst->value.intVal ? 1 : 0; + break; + } + default: + { + SLANG_RELEASE_ASSERT(!"Unhandled constant type"); + return SLANG_FAIL; + } + } + continue; + } + + IRTextureTypeBase* textureBase = as(srcInst); + if (textureBase) + { + dstInst.m_payloadType = PayloadType::OperandAndUInt32; + dstInst.m_payload.m_operandAndUInt32.m_uint32 = uint32_t(srcInst->op) >> kIROpMeta_OtherShift; + dstInst.m_payload.m_operandAndUInt32.m_operand = getInstIndex(textureBase->getElementType()); + continue; + } + + // ModuleInst is different, in so far as it holds a pointer to IRModule, but we don't need + // to save that off in a special way, so can just use regular path + + const int numOperands = int(srcInst->operandCount); + Ser::InstIndex* dstOperands = nullptr; + + if (numOperands <= Ser::Inst::kMaxOperands) + { + // Checks the compile below is valid + SLANG_COMPILE_TIME_ASSERT(PayloadType(0) == PayloadType::Empty && PayloadType(1) == PayloadType::Operand_1 && PayloadType(2) == PayloadType::Operand_2); + + dstInst.m_payloadType = PayloadType(numOperands); + dstOperands = dstInst.m_payload.m_operands; + } + else + { + dstInst.m_payloadType = PayloadType::OperandExternal; + + int operandArrayBaseIndex = int(m_serialData->m_externalOperands.getCount()); + m_serialData->m_externalOperands.setCount(operandArrayBaseIndex + numOperands); + + dstOperands = m_serialData->m_externalOperands.begin() + operandArrayBaseIndex; + + auto& externalOperands = dstInst.m_payload.m_externalOperand; + externalOperands.m_arrayIndex = Ser::ArrayIndex(operandArrayBaseIndex); + externalOperands.m_size = Ser::SizeType(numOperands); + } + + for (int j = 0; j < numOperands; ++j) + { + const Ser::InstIndex dstInstIndex = getInstIndex(srcInst->getOperand(j)); + dstOperands[j] = dstInstIndex; + } + } + } + + // Convert strings into a string table + { + SerialStringTableUtil::encodeStringTable(m_stringSlicePool, serialData->m_stringTable); + } + + // If the option to use RawSourceLocations is enabled, serialize out as is + if (options & SerialOptionFlag::RawSourceLocation) + { + const Index numInsts = m_insts.getCount(); + serialData->m_rawSourceLocs.setCount(numInsts); + + Ser::RawSourceLoc* dstLocs = serialData->m_rawSourceLocs.begin(); + // 0 is null, just mark as no location + dstLocs[0] = Ser::RawSourceLoc(0); + for (Index i = 1; i < numInsts; ++i) + { + IRInst* srcInst = m_insts[i]; + dstLocs[i] = Ser::RawSourceLoc(srcInst->sourceLoc.getRaw()); + } + } + + if ((options & SerialOptionFlag::DebugInfo) && debugWriter) + { + _calcDebugInfo(debugWriter); + } + + m_serialData = nullptr; + return SLANG_OK; +} + +Result _encodeInsts(SerialCompressionType compressionType, const List& instsIn, List& encodeArrayOut) +{ + typedef IRSerialBinary Bin; + typedef IRSerialData::Inst::PayloadType PayloadType; + + if (compressionType != SerialCompressionType::VariableByteLite) + { + return SLANG_FAIL; + } + + encodeArrayOut.clear(); + + const size_t numInsts = size_t(instsIn.getCount()); + const IRSerialData::Inst* insts = instsIn.begin(); + + uint8_t* encodeOut = encodeArrayOut.begin(); + uint8_t* encodeEnd = encodeArrayOut.end(); + + // Calculate the maximum instruction size with worst case possible encoding + // 2 bytes hold the payload size, and the result type + // Note that if there were some free bits, we could encode some of this stuff into bits, but if we remove payloadType, then there are no free bits + const size_t maxInstSize = 2 + ByteEncodeUtil::kMaxLiteEncodeUInt32 + Math::Max(sizeof(insts->m_payload.m_float64), size_t(2 * ByteEncodeUtil::kMaxLiteEncodeUInt32)); + + for (size_t i = 0; i < numInsts; ++i) + { + const auto& inst = insts[i]; + + // Make sure there is space for the largest possible instruction + if (encodeOut + maxInstSize >= encodeEnd) + { + const size_t offset = size_t(encodeOut - encodeArrayOut.begin()); + + const UInt oldCapacity = encodeArrayOut.getCapacity(); + + encodeArrayOut.reserve(oldCapacity + (oldCapacity >> 1) + maxInstSize); + const UInt capacity = encodeArrayOut.getCapacity(); + encodeArrayOut.setCount(capacity); + + encodeOut = encodeArrayOut.begin() + offset; + encodeEnd = encodeArrayOut.end(); + } + + *encodeOut++ = uint8_t(inst.m_op); + *encodeOut++ = uint8_t(inst.m_payloadType); + + encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_resultTypeIndex, encodeOut); + + switch (inst.m_payloadType) + { + case PayloadType::Empty: + { + break; + } + case PayloadType::Operand_1: + case PayloadType::String_1: + case PayloadType::UInt32: + { + // 1 UInt32 + encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_payload.m_operands[0], encodeOut); + break; + } + case PayloadType::Operand_2: + case PayloadType::OperandAndUInt32: + case PayloadType::OperandExternal: + case PayloadType::String_2: + { + // 2 UInt32 + encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_payload.m_operands[0], encodeOut); + encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_payload.m_operands[1], encodeOut); + break; + } + case PayloadType::Float64: + { + memcpy(encodeOut, &inst.m_payload.m_float64, sizeof(inst.m_payload.m_float64)); + encodeOut += sizeof(inst.m_payload.m_float64); + break; + } + case PayloadType::Int64: + { + memcpy(encodeOut, &inst.m_payload.m_int64, sizeof(inst.m_payload.m_int64)); + encodeOut += sizeof(inst.m_payload.m_int64); + break; + } + } + } + + // Fix the size + encodeArrayOut.setCount(UInt(encodeOut - encodeArrayOut.begin())); + return SLANG_OK; +} + +Result _writeInstArrayChunk(SerialCompressionType compressionType, FourCC chunkId, const List& array, RiffContainer* container) +{ + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ScopeChunk ScopeChunk; + + typedef IRSerialBinary Bin; + if (array.getCount() == 0) + { + return SLANG_OK; + } + + switch (compressionType) + { + case SerialCompressionType::None: + { + return SerialRiffUtil::writeArrayChunk(compressionType, chunkId, array, container); + } + case SerialCompressionType::VariableByteLite: + { + List compressedPayload; + SLANG_RETURN_ON_FAIL(_encodeInsts(compressionType, array, compressedPayload)); + + ScopeChunk scope(container, Chunk::Kind::Data, SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId)); + + SerialBinary::CompressedArrayHeader header; + header.numEntries = uint32_t(array.getCount()); + header.numCompressedEntries = 0; + + container->write(&header, sizeof(header)); + container->write(compressedPayload.getBuffer(), compressedPayload.getCount()); + + return SLANG_OK; + } + default: break; + } + return SLANG_FAIL; +} + +/* static */Result IRSerialWriter::writeContainer(const IRSerialData& data, SerialCompressionType compressionType, RiffContainer* container) +{ + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ScopeChunk ScopeChunk; + + ScopeChunk scopeModule(container, Chunk::Kind::List, Bin::kIRModuleFourCc); + + SLANG_RETURN_ON_FAIL(_writeInstArrayChunk(compressionType, Bin::kInstFourCc, data.m_insts, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk(compressionType, Bin::kChildRunFourCc, data.m_childRuns, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk(compressionType, Bin::kExternalOperandsFourCc, data.m_externalOperands, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk(SerialCompressionType::None, SerialBinary::kStringTableFourCc, data.m_stringTable, container)); + + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk(SerialCompressionType::None, Bin::kUInt32RawSourceLocFourCc, data.m_rawSourceLocs, container)); + + if (data.m_debugSourceLocRuns.getCount()) + { + SerialRiffUtil::writeArrayChunk(compressionType, Bin::kDebugSourceLocRunFourCc, data.m_debugSourceLocRuns, container); + } + + return SLANG_OK; +} + +/* static */void IRSerialWriter::calcInstructionList(IRModule* module, List& instsOut) +{ + // We reserve 0 for null + instsOut.setCount(1); + instsOut[0] = nullptr; + + // Stack for parentInst + List parentInstStack; + + IRModuleInst* moduleInst = module->getModuleInst(); + parentInstStack.add(moduleInst); + + // Add to list + instsOut.add(moduleInst); + + // Traverse all of the instructions + while (parentInstStack.getCount()) + { + // If it's in the stack it is assumed it is already in the inst map + IRInst* parentInst = parentInstStack.getLast(); + parentInstStack.removeLast(); + + IRInstListBase childrenList = parentInst->getDecorationsAndChildren(); + for (IRInst* child : childrenList) + { + instsOut.add(child); + parentInstStack.add(child); + } + } +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +static Result _decodeInsts(SerialCompressionType compressionType, const uint8_t* encodeCur, size_t encodeInSize, List& instsOut) +{ + const uint8_t* encodeEnd = encodeCur + encodeInSize; + + typedef IRSerialBinary Bin; + typedef IRSerialData::Inst::PayloadType PayloadType; + + if (compressionType != SerialCompressionType::VariableByteLite) + { + return SLANG_FAIL; + } + + const size_t numInsts = size_t(instsOut.getCount()); + IRSerialData::Inst* insts = instsOut.begin(); + + for (size_t i = 0; i < numInsts; ++i) + { + if (encodeCur >= encodeEnd) + { + SLANG_ASSERT(!"Invalid decode"); + return SLANG_FAIL; + } + + auto& inst = insts[i]; + + inst.m_op = *encodeCur++; + const PayloadType payloadType = PayloadType(*encodeCur++); + inst.m_payloadType = payloadType; + + // Read the result value + encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, (uint32_t*)&inst.m_resultTypeIndex); + + switch (inst.m_payloadType) + { + case PayloadType::Empty: + { + break; + } + case PayloadType::Operand_1: + case PayloadType::String_1: + case PayloadType::UInt32: + { + // 1 UInt32 + encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, (uint32_t*)&inst.m_payload.m_operands[0]); + break; + } + case PayloadType::Operand_2: + case PayloadType::OperandAndUInt32: + case PayloadType::OperandExternal: + case PayloadType::String_2: + { + // 2 UInt32 + encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, 2, (uint32_t*)&inst.m_payload.m_operands[0]); + break; + } + case PayloadType::Float64: + { + memcpy(&inst.m_payload.m_float64, encodeCur, sizeof(inst.m_payload.m_float64)); + encodeCur += sizeof(inst.m_payload.m_float64); + break; + } + case PayloadType::Int64: + { + memcpy(&inst.m_payload.m_int64, encodeCur, sizeof(inst.m_payload.m_int64)); + encodeCur += sizeof(inst.m_payload.m_int64); + break; + } + } + } + + return SLANG_OK; +} + +static Result _readInstArrayChunk(SerialCompressionType containerCompressionType, RiffContainer::DataChunk* chunk, List& arrayOut) +{ + SerialCompressionType compressionType = SerialCompressionType::None; + if (chunk->m_fourCC == SLANG_MAKE_COMPRESSED_FOUR_CC(chunk->m_fourCC)) + { + compressionType = SerialCompressionType(containerCompressionType); + } + + switch (compressionType) + { + case SerialCompressionType::None: + { + SerialRiffUtil::ListResizerForType resizer(arrayOut); + return SerialRiffUtil::readArrayChunk(compressionType, chunk, resizer); + } + case SerialCompressionType::VariableByteLite: + { + RiffReadHelper read = chunk->asReadHelper(); + + SerialBinary::CompressedArrayHeader header; + SLANG_RETURN_ON_FAIL(read.read(header)); + + arrayOut.setCount(header.numEntries); + + SLANG_RETURN_ON_FAIL(_decodeInsts(compressionType, read.getData(), read.getRemainingSize(), arrayOut)); + break; + } + default: + { + return SLANG_FAIL; + } + } + + return SLANG_OK; +} + +/* static */Result IRSerialReader::readContainer(RiffContainer::ListChunk* module, SerialCompressionType containerCompressionType, IRSerialData* outData) +{ + typedef IRSerialBinary Bin; + + outData->clear(); + + for (RiffContainer::Chunk* chunk = module->m_containedChunks; chunk; chunk = chunk->m_next) + { + RiffContainer::DataChunk* dataChunk = as(chunk); + if (!dataChunk) + { + continue; + } + + switch (dataChunk->m_fourCC) + { + case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kInstFourCc): + case Bin::kInstFourCc: + { + SLANG_RETURN_ON_FAIL(_readInstArrayChunk(containerCompressionType, dataChunk, outData->m_insts)); + break; + } + case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kChildRunFourCc): + case Bin::kChildRunFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(containerCompressionType, dataChunk, outData->m_childRuns)); + break; + } + case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kExternalOperandsFourCc): + case Bin::kExternalOperandsFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(containerCompressionType, dataChunk, outData->m_externalOperands)); + break; + } + case SerialBinary::kStringTableFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, outData->m_stringTable)); + break; + } + case Bin::kUInt32RawSourceLocFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, outData->m_rawSourceLocs)); + break; + } + case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kDebugSourceLocRunFourCc): + case Bin::kDebugSourceLocRunFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(containerCompressionType, dataChunk, outData->m_debugSourceLocRuns)); + break; + } + default: + { + break; + } + } + } + + return SLANG_OK; +} + +Result IRSerialReader::read(const IRSerialData& data, Session* session, DebugSerialReader* debugReader, RefPtr& outModule) +{ + typedef Ser::Inst::PayloadType PayloadType; + + m_serialData = &data; + + auto module = new IRModule(); + outModule = module; + m_module = module; + + module->session = session; + + // Convert m_stringTable into StringSlicePool. + SerialStringTableUtil::decodeStringTable(data.m_stringTable.getBuffer(), data.m_stringTable.getCount(), m_stringTable); + + // Add all the instructions + List insts; + + const Index numInsts = data.m_insts.getCount(); + + SLANG_ASSERT(numInsts > 0); + + insts.setCount(numInsts); + insts[0] = nullptr; + + // 0 holds null + // 1 holds the IRModuleInst + { + // Check that insts[1] is the module inst + const Ser::Inst& srcInst = data.m_insts[1]; + SLANG_RELEASE_ASSERT(srcInst.m_op == kIROp_Module); + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Empty); + + // Create the module inst + auto moduleInst = static_cast(createEmptyInstWithSize(module, kIROp_Module, sizeof(IRModuleInst))); + module->moduleInst = moduleInst; + moduleInst->module = module; + + // Set the IRModuleInst + insts[1] = moduleInst; + } + + for (Index i = 2; i < numInsts; ++i) + { + const Ser::Inst& srcInst = data.m_insts[i]; + + const IROp op((IROp)srcInst.m_op); + + if (_isConstant(op)) + { + // Handling of constants + + // Calculate the minimum object size (ie not including the payload of value) + const size_t prefixSize = SLANG_OFFSET_OF(IRConstant, value); + + IRConstant* irConst = nullptr; + switch (op) + { + case kIROp_BoolLit: + { + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::UInt32); + irConst = static_cast(createEmptyInstWithSize(module, op, prefixSize + sizeof(IRIntegerValue))); + irConst->value.intVal = srcInst.m_payload.m_uint32 != 0; + break; + } + case kIROp_IntLit: + { + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Int64); + irConst = static_cast(createEmptyInstWithSize(module, op, prefixSize + sizeof(IRIntegerValue))); + irConst->value.intVal = srcInst.m_payload.m_int64; + break; + } + case kIROp_PtrLit: + { + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Int64); + irConst = static_cast(createEmptyInstWithSize(module, op, prefixSize + sizeof(void*))); + irConst->value.ptrVal = (void*) (intptr_t) srcInst.m_payload.m_int64; + break; + } + case kIROp_FloatLit: + { + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Float64); + irConst = static_cast(createEmptyInstWithSize(module, op, prefixSize + sizeof(IRFloatingPointValue))); + irConst->value.floatVal = srcInst.m_payload.m_float64; + break; + } + case kIROp_StringLit: + { + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::String_1); + + const UnownedStringSlice slice = m_stringTable.getSlice(StringSlicePool::Handle(srcInst.m_payload.m_stringIndices[0])); + + const size_t sliceSize = slice.getLength(); + const size_t instSize = prefixSize + SLANG_OFFSET_OF(IRConstant::StringValue, chars) + sliceSize; + + irConst = static_cast(createEmptyInstWithSize(module, op, instSize)); + + IRConstant::StringValue& dstString = irConst->value.stringVal; + + dstString.numChars = uint32_t(sliceSize); + // Turn into pointer to avoid warning of array overrun + char* dstChars = dstString.chars; + // Copy the chars + memcpy(dstChars, slice.begin(), sliceSize); + break; + } + default: + { + SLANG_ASSERT(!"Unknown constant type"); + return SLANG_FAIL; + } + } + + insts[i] = irConst; + } + else if (_isTextureTypeBase(op)) + { + IRTextureTypeBase* inst = static_cast(createEmptyInst(module, op, 1)); + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::OperandAndUInt32); + + // Reintroduce the texture type bits into the the + const uint32_t other = srcInst.m_payload.m_operandAndUInt32.m_uint32; + inst->op = IROp(uint32_t(inst->op) | (other << kIROpMeta_OtherShift)); + + insts[i] = inst; + } + else + { + int numOperands = srcInst.getNumOperands(); + insts[i] = createEmptyInst(module, op, numOperands); + } + } + + // Patch up the operands + for (Index i = 1; i < numInsts; ++i) + { + const Ser::Inst& srcInst = data.m_insts[i]; + const IROp op((IROp)srcInst.m_op); + + IRInst* dstInst = insts[i]; + + // Set the result type + if (srcInst.m_resultTypeIndex != Ser::InstIndex(0)) + { + IRInst* resultInst = insts[int(srcInst.m_resultTypeIndex)]; + // NOTE! Counter intuitively the IRType* paramter may not be IRType* derived for example + // IRGlobalGenericParam is valid, but isn't IRType* derived + + //SLANG_RELEASE_ASSERT(as(resultInst)); + dstInst->setFullType(static_cast(resultInst)); + } + + //if (!isParentDerived(op)) + { + const Ser::InstIndex* srcOperandIndices; + const int numOperands = data.getOperands(srcInst, &srcOperandIndices); + + auto dstOperands = dstInst->getOperands(); + + for (int j = 0; j < numOperands; j++) + { + dstOperands[j].init(dstInst, insts[int(srcOperandIndices[j])]); + } + } + } + + // Patch up the children + { + const Index numChildRuns = data.m_childRuns.getCount(); + for (Index i = 0; i < numChildRuns; i++) + { + const auto& run = data.m_childRuns[i]; + + IRInst* inst = insts[int(run.m_parentIndex)]; + + for (int j = 0; j < int(run.m_numChildren); ++j) + { + IRInst* child = insts[j + int(run.m_startInstIndex)]; + SLANG_ASSERT(child->parent == nullptr); + child->insertAtEnd(inst); + } + } + } + + // Re-add source locations, if they are defined + if (m_serialData->m_rawSourceLocs.getCount() == numInsts) + { + const Ser::RawSourceLoc* srcLocs = m_serialData->m_rawSourceLocs.begin(); + for (Index i = 1; i < numInsts; ++i) + { + IRInst* dstInst = insts[i]; + + dstInst->sourceLoc.setRaw(Slang::SourceLoc::RawValue(srcLocs[i])); + } + } + + // We now need to apply the runs + if (debugReader && m_serialData->m_debugSourceLocRuns.getCount()) + { + List sourceRuns(m_serialData->m_debugSourceLocRuns); + // They are now in source location order + sourceRuns.sort(); + + // Just guess initially 0 for the source file that contains the initial run + DebugSerialData::SourceRange range = DebugSerialData::SourceRange::getInvalid(); + int fix = 0; + + const Index numRuns = sourceRuns.getCount(); + for (Index i = 0; i < numRuns; ++i) + { + const auto& run = sourceRuns[i]; + + // Work out the fixed source location + SourceLoc sourceLoc; + if (run.m_sourceLoc) + { + if (!range.contains(run.m_sourceLoc)) + { + fix = debugReader->calcFixSourceLoc(run.m_sourceLoc, range); + } + sourceLoc = debugReader->calcFixedLoc(run.m_sourceLoc, fix, range); + } + + // Write to all the instructions + SLANG_ASSERT(Index(uint32_t(run.m_startInstIndex) + run.m_numInst) <= insts.getCount()); + IRInst** dstInsts = insts.getBuffer() + int(run.m_startInstIndex); + + const int runSize = int(run.m_numInst); + for (int j = 0; j < runSize; ++j) + { + dstInsts[j]->sourceLoc = sourceLoc; + } + } + } + + return SLANG_OK; +} + +} // namespace Slang diff --git a/source/slang/slang-serialize-ir.h b/source/slang/slang-serialize-ir.h new file mode 100644 index 000000000..368930c24 --- /dev/null +++ b/source/slang/slang-serialize-ir.h @@ -0,0 +1,97 @@ +// slang-serialize-ir.h +#ifndef SLANG_SERIALIZE_IR_H_INCLUDED +#define SLANG_SERIALIZE_IR_H_INCLUDED + +#include "slang-serialize-ir-types.h" + +#include "../core/slang-riff.h" + +#include "slang-ir.h" +#include "slang-serialize-debug.h" + +// For TranslationUnitRequest +// and FrontEndCompileRequest::ExtraEntryPointInfo +#include "slang-compiler.h" + +namespace Slang { + +struct IRSerialWriter +{ + typedef IRSerialData Ser; + typedef IRSerialBinary Bin; + + Result write(IRModule* module, DebugSerialWriter* debugWriter, SerialOptionFlags flags, IRSerialData* serialData); + + /// Write to a container + static Result writeContainer(const IRSerialData& data, SerialCompressionType compressionType, RiffContainer* container); + + /// Get an instruction index from an instruction + Ser::InstIndex getInstIndex(IRInst* inst) const { return inst ? Ser::InstIndex(m_instMap[inst]) : Ser::InstIndex(0); } + + /// Get a slice from an index + UnownedStringSlice getStringSlice(Ser::StringIndex index) const { return m_stringSlicePool.getSlice(StringSlicePool::Handle(index)); } + /// Get index from string representations + Ser::StringIndex getStringIndex(StringRepresentation* string) { return Ser::StringIndex(m_stringSlicePool.add(string)); } + Ser::StringIndex getStringIndex(const UnownedStringSlice& slice) { return Ser::StringIndex(m_stringSlicePool.add(slice)); } + Ser::StringIndex getStringIndex(Name* name) { return name ? getStringIndex(name->text) : SerialStringData::kNullStringIndex; } + Ser::StringIndex getStringIndex(const char* chars) { return Ser::StringIndex(m_stringSlicePool.add(chars)); } + Ser::StringIndex getStringIndex(const String& string) { return Ser::StringIndex(m_stringSlicePool.add(string.getUnownedSlice())); } + + StringSlicePool& getStringPool() { return m_stringSlicePool; } + + IRSerialWriter() : + m_serialData(nullptr), + m_stringSlicePool(StringSlicePool::Style::Default) + { + } + + /// Produces an instruction list which is in same order as written through IRSerialWriter + static void calcInstructionList(IRModule* module, List& instsOut); + +protected: + + void _addInstruction(IRInst* inst); + Result _calcDebugInfo(DebugSerialWriter* debugWriter); + + List m_insts; ///< Instructions in same order as stored in the + + List m_decorations; ///< Holds all decorations in order of the instructions as found + List m_instWithFirstDecoration; ///< All decorations are held in this order after all the regular instructions + + Dictionary m_instMap; ///< Map an instruction to an instruction index + + StringSlicePool m_stringSlicePool; + IRSerialData* m_serialData; ///< Where the data is stored +}; + +struct IRSerialReader +{ + typedef IRSerialData Ser; + + /// Read potentially multiple modules from a stream + static Result readStreamModules(Stream* stream, Session* session, SourceManager* manager, List>& outModules, List& outEntryPoints); + + /// Read a stream to fill in dataOut IRSerialData + static Result readContainer(RiffContainer::ListChunk* module, SerialCompressionType containerCompressionType, IRSerialData* outData); + + /// Read a module from serial data + Result read(const IRSerialData& data, Session* session, DebugSerialReader* debugReader, RefPtr& outModule); + + IRSerialReader(): + m_serialData(nullptr), + m_module(nullptr), + m_stringTable(StringSlicePool::Style::Default) + { + } + + protected: + + StringSlicePool m_stringTable; + + const IRSerialData* m_serialData; + IRModule* m_module; +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-types.cpp b/source/slang/slang-serialize-types.cpp new file mode 100644 index 000000000..fc839afc1 --- /dev/null +++ b/source/slang/slang-serialize-types.cpp @@ -0,0 +1,249 @@ +// slang-serialize-types.cpp +#include "slang-serialize-types.h" + +#include "../core/slang-text-io.h" +#include "../core/slang-byte-encode-util.h" + +#include "../core/slang-math.h" + +namespace Slang { + +// Needed for linkage with some compilers +/* static */ const SerialStringData::StringIndex SerialStringData::kNullStringIndex; +/* static */ const SerialStringData::StringIndex SerialStringData::kEmptyStringIndex; + +namespace { // anonymous + +struct CharReader +{ + char operator()(int pos) const { SLANG_UNUSED(pos); return *m_pos++; } + CharReader(const char* pos) :m_pos(pos) {} + mutable const char* m_pos; +}; + +} // anonymous + + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialStringTableUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +/* static */void SerialStringTableUtil::encodeStringTable(const StringSlicePool& pool, List& stringTable) +{ + // Skip the default handles -> nothing is encoded via them + return encodeStringTable(pool.getAdded(), stringTable); +} + +/* static */void SerialStringTableUtil::encodeStringTable(const ConstArrayView& slices, List& stringTable) +{ + stringTable.clear(); + for (const auto& slice : slices) + { + const int len = int(slice.getLength()); + + // We need to write into the the string array + char prefixBytes[6]; + const int numPrefixBytes = EncodeUnicodePointToUTF8(prefixBytes, len); + const Index baseIndex = stringTable.getCount(); + + stringTable.setCount(baseIndex + numPrefixBytes + len); + + char* dst = stringTable.begin() + baseIndex; + + memcpy(dst, prefixBytes, numPrefixBytes); + memcpy(dst + numPrefixBytes, slice.begin(), len); + } +} + +/* static */void SerialStringTableUtil::appendDecodedStringTable(const char* table, size_t tableSize, List& slicesOut) +{ + const char* start = table; + const char* cur = start; + const char* end = table + tableSize; + + while (cur < end) + { + CharReader reader(cur); + const int len = GetUnicodePointFromUTF8(reader); + slicesOut.add(UnownedStringSlice(reader.m_pos, len)); + cur = reader.m_pos + len; + } +} + +/* static */void SerialStringTableUtil::decodeStringTable(const char* table, size_t tableSize, List& slicesOut) +{ + slicesOut.setCount(2); + slicesOut[0] = UnownedStringSlice(nullptr, size_t(0)); + slicesOut[1] = UnownedStringSlice("", size_t(0)); + + appendDecodedStringTable(table, tableSize, slicesOut); +} + +/* static */void SerialStringTableUtil::decodeStringTable(const char* table, size_t tableSize, StringSlicePool& outPool) +{ + outPool.clear(); + + const char* start = table; + const char* cur = start; + const char* end = table + tableSize; + + while (cur < end) + { + CharReader reader(cur); + const int len = GetUnicodePointFromUTF8(reader); + outPool.add(UnownedStringSlice(reader.m_pos, len)); + cur = reader.m_pos + len; + } +} + +/* static */void SerialStringTableUtil::calcStringSlicePoolMap(const List& slices, StringSlicePool& pool, List& indexMapOut) +{ + SLANG_ASSERT(slices.getCount() >= StringSlicePool::kDefaultHandlesCount); + SLANG_ASSERT(slices[int(StringSlicePool::kNullHandle)] == "" && slices[int(StringSlicePool::kNullHandle)].begin() == nullptr); + SLANG_ASSERT(slices[int(StringSlicePool::kEmptyHandle)] == ""); + + indexMapOut.setCount(slices.getCount()); + // Set up all of the defaults + for (int i = 0; i < StringSlicePool::kDefaultHandlesCount; ++i) + { + indexMapOut[i] = StringSlicePool::Handle(i); + } + + const Index numSlices = slices.getCount(); + for (Index i = StringSlicePool::kDefaultHandlesCount; i < numSlices ; ++i) + { + indexMapOut[i] = pool.add(slices[i]); + } +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialRiffUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +/* static */ Result SerialRiffUtil::writeArrayChunk(SerialCompressionType compressionType, FourCC chunkId, const void* data, size_t numEntries, size_t typeSize, RiffContainer* container) +{ + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ScopeChunk ScopeChunk; + + if (numEntries == 0) + { + return SLANG_OK; + } + + // Make compressed fourCC + chunkId = (compressionType != SerialCompressionType::None) ? SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId) : chunkId; + + ScopeChunk scope(container, Chunk::Kind::Data, chunkId); + + switch (compressionType) + { + case SerialCompressionType::None: + { + SerialBinary::ArrayHeader header; + header.numEntries = uint32_t(numEntries); + + container->write(&header, sizeof(header)); + container->write(data, typeSize * numEntries); + break; + } + case SerialCompressionType::VariableByteLite: + { + List compressedPayload; + + size_t numCompressedEntries = (numEntries * typeSize) / sizeof(uint32_t); + ByteEncodeUtil::encodeLiteUInt32((const uint32_t*)data, numCompressedEntries, compressedPayload); + + SerialBinary::CompressedArrayHeader header; + header.numEntries = uint32_t(numEntries); + header.numCompressedEntries = uint32_t(numCompressedEntries); + + container->write(&header, sizeof(header)); + container->write(compressedPayload.getBuffer(), compressedPayload.getCount()); + break; + } + default: + { + return SLANG_FAIL; + } + } + return SLANG_OK; +} + +/* static */Result SerialRiffUtil::readArrayChunk(SerialCompressionType compressionType, RiffContainer::DataChunk* dataChunk, ListResizer& listOut) +{ + typedef SerialBinary Bin; + + RiffReadHelper read = dataChunk->asReadHelper(); + const size_t typeSize = listOut.getTypeSize(); + + switch (compressionType) + { + case SerialCompressionType::VariableByteLite: + { + Bin::CompressedArrayHeader header; + SLANG_RETURN_ON_FAIL(read.read(header)); + + void* dst = listOut.setSize(header.numEntries); + SLANG_ASSERT(header.numCompressedEntries == uint32_t((header.numEntries * typeSize) / sizeof(uint32_t))); + + // Decode.. + ByteEncodeUtil::decodeLiteUInt32(read.getData(), header.numCompressedEntries, (uint32_t*)dst); + break; + } + case SerialCompressionType::None: + { + // Read uncompressed + Bin::ArrayHeader header; + SLANG_RETURN_ON_FAIL(read.read(header)); + const size_t payloadSize = header.numEntries * typeSize; + SLANG_ASSERT(payloadSize == read.getRemainingSize()); + void* dst = listOut.setSize(header.numEntries); + ::memcpy(dst, read.getData(), payloadSize); + break; + } + } + return SLANG_OK; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialParseUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +#define SLANG_SERIAL_BINARY_COMPRESSION_TYPE(x) \ + x(None, none) \ + x(VariableByteLite, lite) + +/* static */SlangResult SerialParseUtil::parseCompressionType(const UnownedStringSlice& text, SerialCompressionType& outType) +{ + struct Pair + { + UnownedStringSlice name; + SerialCompressionType type; + }; + +#define SLANG_SERIAL_BINARY_PAIR(type, name) { UnownedStringSlice::fromLiteral(#name), SerialCompressionType::type}, + + static const Pair s_pairs[] = + { + SLANG_SERIAL_BINARY_COMPRESSION_TYPE(SLANG_SERIAL_BINARY_PAIR) + }; + + for (const auto& pair : s_pairs) + { + if (pair.name == text) + { + outType = pair.type; + return SLANG_OK; + } + } + return SLANG_FAIL; +} + +/* static */UnownedStringSlice SerialParseUtil::getText(SerialCompressionType type) +{ +#define SLANG_SERIAL_BINARY_CASE(type, name) case SerialCompressionType::type: return UnownedStringSlice::fromLiteral(#name); + switch (type) + { + SLANG_SERIAL_BINARY_COMPRESSION_TYPE(SLANG_SERIAL_BINARY_CASE) + default: break; + } + SLANG_ASSERT(!"Unknown compression type"); + return UnownedStringSlice::fromLiteral("unknown"); +} + + +} // namespace Slang diff --git a/source/slang/slang-serialize-types.h b/source/slang/slang-serialize-types.h new file mode 100644 index 000000000..f799dd607 --- /dev/null +++ b/source/slang/slang-serialize-types.h @@ -0,0 +1,230 @@ +// slang-serialize-types.h +#ifndef SLANG_SERIALIZE_TYPES_H +#define SLANG_SERIALIZE_TYPES_H + +#include "../core/slang-riff.h" +#include "../core/slang-string-slice-pool.h" +#include "../core/slang-array-view.h" + +#include "slang-name.h" +#include "slang-source-loc.h" + +namespace Slang { + +// Options for IR/AST/Debug serialization + +struct SerialOptionFlag +{ + typedef uint32_t Type; + enum Enum : Type + { + RawSourceLocation = 0x01, + DebugInfo = 0x02, + }; +}; +typedef SerialOptionFlag::Type SerialOptionFlags; + + +// Compression styles + +enum class SerialCompressionType : uint8_t +{ + None, + VariableByteLite, +}; + + +struct SerialStringData +{ + enum class StringIndex : uint32_t; + + ///enum class StringOffset : uint32_t; ///< Offset into the m_stringsBuffer + + typedef uint32_t SizeType; + + static const StringIndex kNullStringIndex = StringIndex(StringSlicePool::kNullHandle); + static const StringIndex kEmptyStringIndex = StringIndex(StringSlicePool::kEmptyHandle); +}; + +struct SerialStringTableUtil +{ + /// Convert a pool into a string table + static void encodeStringTable(const StringSlicePool& pool, List& stringTable); + static void encodeStringTable(const ConstArrayView& slices, List& stringTable); + + /// Appends the decoded strings into slicesOut + static void appendDecodedStringTable(const char* table, size_t tableSize, List& slicesOut); + + /// Decodes a string table (and does so such that the indices are compatible with StringSlicePool) + static void decodeStringTable(const char* table, size_t tableSize, List& slicesOut); + + /// Decodes a string table + static void decodeStringTable(const char* table, size_t tableSize, StringSlicePool& outPool); + + /// Produces an index map, from slices to indices in pool + static void calcStringSlicePoolMap(const List& slices, StringSlicePool& pool, List& indexMap); +}; + +struct SerialParseUtil +{ + /// Given text, finds the compression type + static SlangResult parseCompressionType(const UnownedStringSlice& text, SerialCompressionType& outType); + /// Given a compression type, return text + static UnownedStringSlice getText(SerialCompressionType type); +}; + +struct SerialListUtil +{ + template + static size_t calcArraySize(const List& list) + { + return list.getCount() * sizeof(T); + } + + template + static bool isEqual(const List& aIn, const List& bIn) + { + if (&aIn == &bIn) + { + return true; + } + const Index size = aIn.getCount(); + + if (size != bIn.getCount()) + { + return false; + } + + const T* a = aIn.begin(); + const T* b = bIn.begin(); + + if (a != b) + { + for (Index i = 0; i < size; ++i) + { + if (a[i] != b[i]) + { + return false; + } + } + } + + return true; + } +}; + +// For types/FourCC that work for serializing in general (not just IR). +struct SerialBinary +{ + static const FourCC kRiffFourCc = RiffFourCC::kRiff; + + /// Container + static const FourCC kContainerFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'c'); + + /// A string table + static const FourCC kStringTableFourCc = SLANG_FOUR_CC('S', 'L', 's', 't'); + + /// TranslationUnitList + static const FourCC kTranslationUnitListFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'l'); + + /// An entry point + static const FourCC kEntryPointFourCc = SLANG_FOUR_CC('E', 'P', 'n', 't'); + + /// Container + static const FourCC kContainerHeaderFourCc = SLANG_FOUR_CC('S', 'c', 'h', 'd'); + + struct ContainerHeader + { + uint32_t compressionType; ///< Holds the compression type used (if used at all) + }; + + struct ArrayHeader + { + uint32_t numEntries; + }; + struct CompressedArrayHeader + { + uint32_t numEntries; ///< The number of entries + uint32_t numCompressedEntries; ///< The amount of compressed entries + }; +}; + +// Replace first char with 's' +#define SLANG_MAKE_COMPRESSED_FOUR_CC(fourCc) SLANG_FOUR_CC_REPLACE_FIRST_CHAR(fourCc, 's') + +struct SerialRiffUtil +{ + class ListResizer + { + public: + virtual void* setSize(size_t newSize) = 0; + SLANG_FORCE_INLINE size_t getTypeSize() const { return m_typeSize; } + ListResizer(size_t typeSize) :m_typeSize(typeSize) {} + + protected: + size_t m_typeSize; + }; + + template + class ListResizerForType : public ListResizer + { + public: + typedef ListResizer Parent; + + SLANG_FORCE_INLINE ListResizerForType(List& list) : + Parent(sizeof(T)), + m_list(list) + {} + + virtual void* setSize(size_t newSize) SLANG_OVERRIDE + { + m_list.setCount(UInt(newSize)); + return (void*)m_list.begin(); + } + + protected: + List& m_list; + }; + + static Result writeArrayChunk(SerialCompressionType compressionType, FourCC chunkId, const void* data, size_t numEntries, size_t typeSize, RiffContainer* container); + + template + static Result writeArrayChunk(SerialCompressionType compressionType, FourCC chunkId, const List& array, RiffContainer* container) + { + return writeArrayChunk(compressionType, chunkId, array.begin(), size_t(array.getCount()), sizeof(T), container); + } + + template + static Result writeArrayUncompressedChunk(FourCC chunkId, const List& array, RiffContainer* container) + { + return writeArrayChunk(SerialCompressionType::None, chunkId, array.begin(), size_t(array.getCount()), sizeof(T), container); + } + + static Result readArrayChunk(SerialCompressionType compressionType, RiffContainer::DataChunk* dataChunk, ListResizer& listOut); + + template + static Result readArrayChunk(SerialCompressionType moduleCompressionType, RiffContainer::DataChunk* dataChunk, List& arrayOut) + { + SerialCompressionType compressionType = SerialCompressionType::None; + if (dataChunk->m_fourCC == SLANG_MAKE_COMPRESSED_FOUR_CC(dataChunk->m_fourCC)) + { + // If it has compression, use the compression type set in the header + compressionType = moduleCompressionType; + } + ListResizerForType resizer(arrayOut); + return readArrayChunk(compressionType, dataChunk, resizer); + } + + template + static Result readArrayUncompressedChunk(RiffContainer::DataChunk* chunk, List& arrayOut) + { + ListResizerForType resizer(arrayOut); + return readArrayChunk(SerialCompressionType::None, chunk, resizer); + } + + +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 4c83095c2..c9dccbdbf 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -14,9 +14,6 @@ #include "slang-reflection.h" #include "slang-type-layout.h" -#include "slang-ast-dump.h" -#include "slang-ast-serialize.h" - #include "slang-repro.h" #include "slang-file-system.h" @@ -25,7 +22,11 @@ #include "slang-source-loc.h" -#include "slang-ir-serialize.h" +#include "slang-ast-dump.h" + +#include "slang-serialize-ast.h" +#include "slang-serialize-ir.h" +#include "slang-serialize-container.h" #include "slang-check-impl.h" @@ -1074,20 +1075,25 @@ void FrontEndCompileRequest::generateIR() if (verifyDebugSerialization) { + SerialContainerUtil::WriteOptions options; + options.compressionType = SerialCompressionType::None; + options.sourceManager = getSourceManager(); + options.optionFlags = SerialOptionFlag::DebugInfo; + // Verify debug information - if (SLANG_FAILED(IRSerialUtil::verifySerialize(irModule, getSession(), getSourceManager(), IRSerialBinary::CompressionType::None, IRSerialWriter::OptionFlag::DebugInfo))) + if (SLANG_FAILED(SerialContainerUtil::verifyIRSerialize(irModule, getSession(), options))) { getSink()->diagnose(irModule->moduleInst->sourceLoc, Diagnostics::serialDebugVerificationFailed); } } if (useSerialIRBottleneck) - { + { IRSerialData serialData; { // Write IR out to serialData - copying over SourceLoc information directly IRSerialWriter writer; - writer.write(irModule, getSourceManager(), IRSerialWriter::OptionFlag::RawSourceLocation, &serialData); + writer.write(irModule, nullptr, SerialOptionFlag::RawSourceLocation, &serialData); // Destroy irModule such that memory can be used for newly constructed read irReadModule irModule = nullptr; @@ -3013,33 +3019,47 @@ namespace Slang SlangResult _addLibraryReference(EndToEndCompileRequest* req, Stream* stream) { // Load up the module - RiffContainer container; - SLANG_RETURN_ON_FAIL(RiffUtil::read(stream, container)); + RiffContainer riffContainer; + SLANG_RETURN_ON_FAIL(RiffUtil::read(stream, riffContainer)); + + auto linkage = req->getLinkage(); - List> modules; + // TODO(JS): May be better to have a ITypeComponent that encapsulates a collection of modules + // For now just add to the linkage - if (SLANG_FAILED(ASTSerialReader::readContainerModules(&container, req->getLinkage(), modules))) { - req->getSink()->diagnose(SourceLoc(), Diagnostics::unableToAddReferenceToModuleContainer); - return SLANG_FAIL; - } + SerialContainerData containerData; - // Read all of the contained modules - List> irModules; - List entryPointMangledNames; + SerialContainerUtil::ReadOptions options; + options.namePool = req->getNamePool(); + options.session = req->getSession(); + options.sharedASTBuilder = linkage->getASTBuilder()->getSharedASTBuilder(); + options.sourceManager = linkage->getSourceManager(); - if (SLANG_FAILED(IRSerialReader::readContainerModules(&container, req->getSession(), req->getFrontEndReq()->getSourceManager(), irModules, entryPointMangledNames))) - { - req->getSink()->diagnose(SourceLoc(), Diagnostics::unableToAddReferenceToModuleContainer); - return SLANG_FAIL; - } + SLANG_RETURN_ON_FAIL(SerialContainerUtil::read(&riffContainer, options, containerData)); - // TODO(JS): May be better to have a ITypeComponent that encapsulates a collection of modules - // For now just add to the linkage - auto linkage = req->getLinkage(); - linkage->m_libModules.addRange(irModules); + for (const auto& translationUnit : containerData.translationUnits) + { + // If the irModule is set, add it + if (translationUnit.irModule) + { + linkage->m_libModules.add(translationUnit.irModule); + } + } + + FrontEndCompileRequest* frontEndRequest = req->getFrontEndReq(); + + for (const auto& entryPoint : containerData.entryPoints) + { + FrontEndCompileRequest::ExtraEntryPointInfo dst; + dst.mangledName = entryPoint.mangledName; + dst.name = entryPoint.name; + dst.profile = entryPoint.profile; - req->getFrontEndReq()->m_extraEntryPoints.addRange(entryPointMangledNames); + // Add entry point + frontEndRequest->m_extraEntryPoints.add(dst); + } + } return SLANG_OK; } diff --git a/source/slang/slang.vcxproj b/source/slang/slang.vcxproj index 696873756..a0865d16c 100644 --- a/source/slang/slang.vcxproj +++ b/source/slang/slang.vcxproj @@ -199,7 +199,6 @@ - @@ -254,8 +253,6 @@ - - @@ -286,6 +283,12 @@ + + + + + + @@ -303,7 +306,6 @@ - @@ -366,8 +368,6 @@ - - @@ -397,6 +397,12 @@ + + + + + + diff --git a/source/slang/slang.vcxproj.filters b/source/slang/slang.vcxproj.filters index 3fd006f77..5edd4d31e 100644 --- a/source/slang/slang.vcxproj.filters +++ b/source/slang/slang.vcxproj.filters @@ -48,9 +48,6 @@ Header Files - - Header Files - Header Files @@ -213,12 +210,6 @@ Header Files - - Header Files - - - Header Files - Header Files @@ -309,6 +300,24 @@ Header Files + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + Header Files @@ -356,9 +365,6 @@ Source Files - - Source Files - Source Files @@ -545,12 +551,6 @@ Source Files - - Source Files - - - Source Files - Source Files @@ -638,6 +638,24 @@ Source Files + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + Source Files -- cgit v1.2.3