diff options
| author | jsmall-nvidia <jsmall@nvidia.com> | 2020-09-30 13:28:56 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-09-30 13:28:56 -0400 |
| commit | 274c20a5eb133779a9d890ca79120815fb92b04e (patch) | |
| tree | 50f8074917a102b25a7f34adeacffaf185d59242 | |
| parent | 94d3f2bd9c5557658751f73bc5fc443b41230d2c (diff) | |
Generalizing Serialization (#1563)
* First pass at generalizing serializer.
* Split out ReflectClassInfo
* Use the general ReflectClassInfo
* Fix some typos in debug generalized serialization.
* Add calculation of classIds.
Make distinct addCopy/add on SerialClasses.
* Write up of more generalized serialization
* WIP to transition from ASTSerialReader/Writer etc to generalized SerialReader/Writer and associated types.
* Improvements to SerialExtraObjects.
Keep RefObjects in scope in factory
* Compiles with Serial refactor - doesn't quite work yet.
* First pass serialization appears to work with refector.
* Split out type info for general slang types.
* Split out slang-serialize-misc-type-info.h
* DebugSerialData -> SerialSourecLocData
DebugSerialReader -> SerialSourceLocReader
DebugSerialWriter -> SerialSourceLocWriter
* Remove unused template that only compiles on VS.
* Fix warning around unused function on non-VS.
22 files changed, 2744 insertions, 2262 deletions
diff --git a/source/slang/slang-ast-base.h b/source/slang/slang-ast-base.h index 847dada03..275832342 100644 --- a/source/slang/slang-ast-base.h +++ b/source/slang/slang-ast-base.h @@ -7,6 +7,8 @@ #include "slang-ast-generated.h" #include "slang-ast-reflect.h" +#include "slang-serialize-reflection.h" + // This file defines the primary base classes for the hierarchy of // AST nodes and related objects. For example, this is where the // basic `Decl`, `Stmt`, `Expr`, `type`, etc. definitions come from. @@ -14,8 +16,6 @@ namespace Slang { -struct ReflectClassInfo; - class NodeBase { SLANG_ABSTRACT_CLASS(NodeBase) @@ -26,7 +26,7 @@ class NodeBase SLANG_FORCE_INLINE void init(ASTNodeType inAstNodeType, ASTBuilder* /* astBuilder*/ ) { astNodeType = inAstNodeType; } /// Get the class info - SLANG_FORCE_INLINE const ReflectClassInfo& getClassInfo() const { return *ReflectClassInfo::getInfo(astNodeType); } + SLANG_FORCE_INLINE const ReflectClassInfo& getClassInfo() const { return *ASTClassInfo::getInfo(astNodeType); } SyntaxClass<NodeBase> getClass() { return SyntaxClass<NodeBase>(&getClassInfo()); } @@ -41,25 +41,25 @@ class NodeBase template<typename T> SLANG_FORCE_INLINE T* dynamicCast(NodeBase* node) { - return (node && ReflectClassInfo::isSubClassOf(node->astNodeType, T::kReflectClassInfo)) ? static_cast<T*>(node) : nullptr; + return (node && ReflectClassInfo::isSubClassOf(uint32_t(node->astNodeType), T::kReflectClassInfo)) ? static_cast<T*>(node) : nullptr; } template<typename T> SLANG_FORCE_INLINE const T* dynamicCast(const NodeBase* node) { - return (node && ReflectClassInfo::isSubClassOf(node->astNodeType, T::kReflectClassInfo)) ? static_cast<const T*>(node) : nullptr; + return (node && ReflectClassInfo::isSubClassOf(uint32_t(node->astNodeType), T::kReflectClassInfo)) ? static_cast<const T*>(node) : nullptr; } template<typename T> SLANG_FORCE_INLINE T* as(NodeBase* node) { - return (node && ReflectClassInfo::isSubClassOf(node->astNodeType, T::kReflectClassInfo)) ? static_cast<T*>(node) : nullptr; + return (node && ReflectClassInfo::isSubClassOf(uint32_t(node->astNodeType), T::kReflectClassInfo)) ? static_cast<T*>(node) : nullptr; } template<typename T> SLANG_FORCE_INLINE const T* as(const NodeBase* node) { - return (node && ReflectClassInfo::isSubClassOf(node->astNodeType, T::kReflectClassInfo)) ? static_cast<const T*>(node) : nullptr; + return (node && ReflectClassInfo::isSubClassOf(uint32_t(node->astNodeType), T::kReflectClassInfo)) ? static_cast<const T*>(node) : nullptr; } diff --git a/source/slang/slang-ast-builder.cpp b/source/slang/slang-ast-builder.cpp index 64061549e..54fe55478 100644 --- a/source/slang/slang-ast-builder.cpp +++ b/source/slang/slang-ast-builder.cpp @@ -38,7 +38,7 @@ void SharedASTBuilder::init(Session* session) // NOTE! That this adds the names of the abstract classes too(!) for (Index i = 0; i < Index(ASTNodeType::CountOf); ++i) { - const ReflectClassInfo* info = ReflectClassInfo::getInfo(ASTNodeType(i)); + const ReflectClassInfo* info = ASTClassInfo::getInfo(ASTNodeType(i)); if (info) { m_sliceToTypeMap.Add(UnownedStringSlice(info->m_name), info); @@ -171,7 +171,7 @@ ASTBuilder::~ASTBuilder() { for (NodeBase* node : m_dtorNodes) { - const ReflectClassInfo* info = ReflectClassInfo::getInfo(node->astNodeType); + const ReflectClassInfo* info = ASTClassInfo::getInfo(node->astNodeType); SLANG_ASSERT(info->m_destructorFunc); info->m_destructorFunc(node); } @@ -179,7 +179,7 @@ ASTBuilder::~ASTBuilder() NodeBase* ASTBuilder::createByNodeType(ASTNodeType nodeType) { - const ReflectClassInfo* info = ReflectClassInfo::getInfo(nodeType); + const ReflectClassInfo* info = ASTClassInfo::getInfo(nodeType); auto createFunc = info->m_createFunc; SLANG_ASSERT(createFunc); diff --git a/source/slang/slang-ast-dump.cpp b/source/slang/slang-ast-dump.cpp index a821729a6..688e99031 100644 --- a/source/slang/slang-ast-dump.cpp +++ b/source/slang/slang-ast-dump.cpp @@ -582,7 +582,7 @@ struct ASTDumpContext void dump(ASTNodeType nodeType) { // Get the class - auto info = ReflectClassInfo::getInfo(nodeType); + auto info = ASTClassInfo::getInfo(nodeType); // Write the name m_writer->emit(info->m_name); } diff --git a/source/slang/slang-ast-reflect.cpp b/source/slang/slang-ast-reflect.cpp index 7ed5057b9..520592e73 100644 --- a/source/slang/slang-ast-reflect.cpp +++ b/source/slang/slang-ast-reflect.cpp @@ -18,27 +18,15 @@ namespace Slang #define SLANG_REFLECT_GET_REFLECT_CLASS_INFO(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) infos.infos[int(ASTNodeType::NAME)] = &NAME::kReflectClassInfo; -static ReflectClassInfo::Infos _calcInfos() +static ASTClassInfo::Infos _calcInfos() { - ReflectClassInfo::Infos infos; + ASTClassInfo::Infos infos; memset(&infos, 0, sizeof(infos)); SLANG_ALL_ASTNode_NodeBase(SLANG_REFLECT_GET_REFLECT_CLASS_INFO, _) return infos; } -/* static */const ReflectClassInfo::Infos ReflectClassInfo::kInfos = _calcInfos(); - -bool ReflectClassInfo::isSubClassOfSlow(const ThisType& super) const -{ - ReflectClassInfo const* info = this; - while (info) - { - if (info == &super) - return true; - info = info->m_superClass; - } - return false; -} +/* static */const ASTClassInfo::Infos ASTClassInfo::kInfos = _calcInfos(); // Now try and implement all of the classes // Macro generated is of the format @@ -48,8 +36,9 @@ struct ASTConstructAccess template <typename T> struct Impl { - static void* create(ASTBuilder* astBuilder) + static void* create(void* context) { + ASTBuilder* astBuilder = (ASTBuilder*)context; return astBuilder->create<T>(); } static void destroy(void* ptr) diff --git a/source/slang/slang-ast-support-types.h b/source/slang/slang-ast-support-types.h index 99ced8187..ae2f6e6be 100644 --- a/source/slang/slang-ast-support-types.h +++ b/source/slang/slang-ast-support-types.h @@ -13,6 +13,8 @@ #include "slang-ast-reflect.h" +#include "slang-serialize-reflection.h" + #include "slang-name.h" #include <assert.h> @@ -462,53 +464,13 @@ namespace Slang class ASTBuilder; - struct ReflectClassInfo + struct ASTClassInfo { - typedef ReflectClassInfo ThisType; - - typedef void* (*CreateFunc)(ASTBuilder* astBuilder); - typedef void (*DestructorFunc)(void* ptr); - - /// A constant time implementation of isSubClassOf - SLANG_FORCE_INLINE bool isSubClassOf(const ThisType& super) const - { - // We include super.m_classId, because it's a subclass of itself. - return m_classId >= super.m_classId && m_classId <= super.m_lastClassId; - } - // True if typeId derives from this type - SLANG_FORCE_INLINE bool isDerivedFrom(uint32_t typeId) const - { - return typeId >= m_classId && typeId <= m_lastClassId; - } - SLANG_FORCE_INLINE static bool isSubClassOf(ASTNodeType type, const ThisType& super) - { - // Check the type appears valid - SLANG_ASSERT(int(type) >= 0); - // We include super.m_classId, because it's a subclass of itself. - return uint32_t(type) >= super.m_classId && uint32_t(type) <= super.m_lastClassId; - } - - /// Will produce the same result as isSubClassOf, but more slowly by traversing the m_superClass - /// Works without initRange being called. - bool isSubClassOfSlow(const ThisType& super) const; - - uint32_t m_classId; - uint32_t m_lastClassId; - - const ReflectClassInfo* m_superClass; ///< The super class of this class, or nullptr if has no super class. - const char* m_name; ///< Textual class name, for debugging - CreateFunc m_createFunc; ///< Callback to use when creating instances (using an ASTBuilder for backing memory) - DestructorFunc m_destructorFunc; ///< The destructor for this type. Being just destructor, does not free backing memory for type. - uint32_t m_sizeInBytes; ///< Total size of the type - uint8_t m_alignment; ///< The required alignment of the type - struct Infos { const ReflectClassInfo* infos[int(ASTNodeType::CountOf)]; }; - SLANG_FORCE_INLINE static const ReflectClassInfo* getInfo(ASTNodeType type) { return kInfos.infos[int(type)]; } - static const Infos kInfos; }; diff --git a/source/slang/slang-serialize-ast.cpp b/source/slang/slang-serialize-ast.cpp index 82cd7d6b4..65a5488a3 100644 --- a/source/slang/slang-serialize-ast.cpp +++ b/source/slang/slang-serialize-ast.cpp @@ -4,55 +4,23 @@ #include "slang-ast-generated.h" #include "slang-ast-generated-macro.h" -#include "slang-compiler.h" -#include "slang-type-layout.h" - #include "slang-ast-dump.h" -#include "slang-mangle.h" #include "slang-ast-support-types.h" +// Needed for ModuleSerialFilter +// Needed for 'findModuleForDecl' #include "slang-legalize-types.h" +#include "slang-mangle.h" -#include "../core/slang-byte-encode-util.h" +#include "slang-serialize-type-info.h" +#include "slang-serialize-misc-type-info.h" namespace Slang { +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ModuleSerialFilter !!!!!!!!!!!!!!!!!!!!!!!! -// Things stored as references: -// -// NodeBase derived types -// Array -// -// RefObject derived types: -// -// Breadcrumb -// StringRepresentation -// Scope - - - -// Helpers to convert fields treated as values - -class ASTSerialReader; -class ASTSerialWriter; - -template <typename NATIVE_TYPE, typename SERIAL_TYPE> -static void _toSerialValue(ASTSerialWriter* writer, const NATIVE_TYPE& src, SERIAL_TYPE& dst) -{ - ASTSerialTypeInfo<NATIVE_TYPE>::toSerial(writer, &src, &dst); -} - -template <typename SERIAL_TYPE, typename NATIVE_TYPE> -static void _toNativeValue(ASTSerialReader* reader, const SERIAL_TYPE& src, NATIVE_TYPE& dst) -{ - ASTSerialTypeInfo<NATIVE_TYPE>::toNative(reader, &src, &dst); -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ModuleASTSerialFilter !!!!!!!!!!!!!!!!!!!!!!!! - - -ASTSerialIndex ModuleASTSerialFilter::writePointer(ASTSerialWriter* writer, const NodeBase* inPtr) +SerialIndex ModuleSerialFilter::writePointer(SerialWriter* writer, const NodeBase* inPtr) { NodeBase* ptr = const_cast<NodeBase*>(inPtr); SLANG_ASSERT(ptr); @@ -71,7 +39,7 @@ ASTSerialIndex ModuleASTSerialFilter::writePointer(ASTSerialWriter* writer, cons ImportExternalDecl* importDecl = astBuilder->create<ImportExternalDecl>(); importDecl->mangledName = mangledName; - const ASTSerialIndex index = writer->writePointer(importDecl); + const SerialIndex index = writer->addPointer(importDecl); // Set as the index of this writer->setPointerIndex(ptr, index); @@ -81,7 +49,7 @@ ASTSerialIndex ModuleASTSerialFilter::writePointer(ASTSerialWriter* writer, cons else { // Okay... we can just write it out then - return writer->writePointer(ptr); + return writer->writeObject(ptr); } } @@ -95,224 +63,52 @@ ASTSerialIndex ModuleASTSerialFilter::writePointer(ASTSerialWriter* writer, cons if (Stmt* stmt = as<Stmt>(ptr)) { // - writer->setPointerIndex(stmt, ASTSerialIndex(0)); - return ASTSerialIndex(0); + writer->setPointerIndex(stmt, SerialIndex(0)); + return SerialIndex(0); } // For now for everything else just write it - return writer->writePointer(ptr); + return writer->writeObject(ptr); } -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Serial <-> Native conversion !!!!!!!!!!!!!!!!!!!!!!!! - - -// We need to have a way to map between the two. -// If no mapping is needed, (just a copy), then we don't bother with the functions -template <typename T> -struct ASTSerialBasicTypeInfo -{ - typedef T NativeType; - typedef T SerialType; - - // We want the alignment to be the same as the size of the type for basic types - // NOTE! Might be different from SLANG_ALIGN_OF(SerialType) - enum { SerialAlignment = sizeof(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(T*)serial = *(const T*)native; } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(T*)native = *(const T*)serial; } - - static const ASTSerialType* getType() - { - static const ASTSerialType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; - return &type; - } -}; - -template <typename NATIVE_T, typename SERIAL_T> -struct ASTSerialConvertTypeInfo -{ - typedef NATIVE_T NativeType; - typedef SERIAL_T SerialType; - - enum { SerialAlignment = ASTSerialBasicTypeInfo<SERIAL_T>::SerialAlignment }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SERIAL_T*)serial = SERIAL_T(*(const NATIVE_T*)native); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(NATIVE_T*)native = NATIVE_T(*(const SERIAL_T*)serial); } -}; +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! AST types !!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ +// SyntaxClass<T> template <typename T> -struct ASTSerialIdentityTypeInfo -{ - typedef T NativeType; - typedef T SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(T*)serial = *(const T*)native; } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(T*)native = *(const T*)serial; } -}; - -// Don't need to convert the index type - -template <> -struct ASTSerialTypeInfo<ASTSerialIndex> : public ASTSerialIdentityTypeInfo<ASTSerialIndex> {}; - - -// Because is sized, we don't need to convert -template <> -struct ASTSerialTypeInfo<FeedbackType::Kind> : public ASTSerialIdentityTypeInfo<FeedbackType::Kind> {}; - -// Implement for Basic Types - -template <> -struct ASTSerialTypeInfo<uint8_t> : public ASTSerialBasicTypeInfo<uint8_t> {}; -template <> -struct ASTSerialTypeInfo<uint16_t> : public ASTSerialBasicTypeInfo<uint16_t> {}; -template <> -struct ASTSerialTypeInfo<uint32_t> : public ASTSerialBasicTypeInfo<uint32_t> {}; -template <> -struct ASTSerialTypeInfo<uint64_t> : public ASTSerialBasicTypeInfo<uint64_t> {}; - -template <> -struct ASTSerialTypeInfo<int8_t> : public ASTSerialBasicTypeInfo<int8_t> {}; -template <> -struct ASTSerialTypeInfo<int16_t> : public ASTSerialBasicTypeInfo<int16_t> {}; -template <> -struct ASTSerialTypeInfo<int32_t> : public ASTSerialBasicTypeInfo<int32_t> {}; -template <> -struct ASTSerialTypeInfo<int64_t> : public ASTSerialBasicTypeInfo<int64_t> {}; - -template <> -struct ASTSerialTypeInfo<float> : public ASTSerialBasicTypeInfo<float> {}; -template <> -struct ASTSerialTypeInfo<double> : public ASTSerialBasicTypeInfo<double> {}; - -// SamplerStateFlavor - -template <> -struct ASTSerialTypeInfo<SamplerStateFlavor> : public ASTSerialConvertTypeInfo<SamplerStateFlavor, uint8_t> {}; - -// TextureFlavor - -template <> -struct ASTSerialTypeInfo<TextureFlavor> +struct SerialTypeInfo<SyntaxClass<T>> { - typedef TextureFlavor NativeType; + typedef SyntaxClass<T> NativeType; typedef uint16_t SerialType; - enum { SerialAlignment = sizeof(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SerialType*)serial = ((const NativeType*)native)->flavor; } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); ((NativeType*)native)->flavor = *(const SerialType*)serial; } -}; - -// Fixed arrays - -template <typename T, size_t N> -struct ASTSerialTypeInfo<T[N]> -{ - typedef ASTSerialTypeInfo<T> ElementASTSerialType; - typedef typename ElementASTSerialType::SerialType SerialElementType; - - typedef T NativeType[N]; - typedef SerialElementType SerialType[N]; - - enum { SerialAlignment = ASTSerialTypeInfo<T>::SerialAlignment }; - - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) - { - SerialElementType* serial = (SerialElementType*)outSerial; - const T* native = (const T*)inNative; - for (Index i = 0; i < Index(N); ++i) - { - ElementASTSerialType::toSerial(writer, native + i, serial + i); - } - } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - const SerialElementType* serial = (const SerialElementType*)inSerial; - T* native = (T*)outNative; - for (Index i = 0; i < Index(N); ++i) - { - ElementASTSerialType::toNative(reader, serial + i, native + i); - } - } -}; - -// Special case bool - as we can't rely on size alignment -template <> -struct ASTSerialTypeInfo<bool> -{ - typedef bool NativeType; - typedef uint8_t SerialType; - enum { SerialAlignment = sizeof(SerialType) }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); - *(SerialType*)outSerial = *(const NativeType*)inNative ? 1 : 0; + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + dst = SerialType(src.classInfo->m_classId); } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + static void toNative(SerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); - *(NativeType*)outNative = (*(const SerialType*)inSerial) != 0; - } -}; - - -// Pointer -// Could handle different pointer base types with some more template magic here, but instead went with Pointer type to keep -// things simpler. -template <typename T> -struct ASTSerialTypeInfo<T*> -{ - typedef T* NativeType; - typedef ASTSerialIndex SerialType; - - enum - { - SerialAlignment = SLANG_ALIGN_OF(SerialType) - }; - - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) - { - *(SerialType*)outSerial = writer->addPointer(*(T**)inNative); - } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - *(T**)outNative = reader->getPointer(*(const SerialType*)inSerial).dynamicCast<T>(); - } -}; - -// Special case Name -template <> -struct ASTSerialTypeInfo<Name*> : public ASTSerialTypeInfo<RefObject*> -{ - // Special case - typedef Name* NativeType; - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - *(Name**)outNative = reader->getName(*(const SerialType*)inSerial); + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + dst.classInfo = ASTClassInfo::getInfo(ASTNodeType(src)); } }; -template <> -struct ASTSerialTypeInfo<const Name*> : public ASTSerialTypeInfo<Name*> -{ -}; - - -struct ASTSerialDeclRefBaseTypeInfo +struct SerialDeclRefBaseTypeInfo { typedef DeclRefBase NativeType; struct SerialType { - ASTSerialIndex substitutions; - ASTSerialIndex decl; + SerialIndex substitutions; + SerialIndex decl; }; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + static void toSerial(SerialWriter* writer, const void* inNative, void* outSerial) { SerialType& serial = *(SerialType*)outSerial; const NativeType& native = *(const NativeType*)inNative; @@ -320,7 +116,7 @@ struct ASTSerialDeclRefBaseTypeInfo serial.decl = writer->addPointer(native.decl); serial.substitutions = writer->addPointer(native.substitutions.substitutions); } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + static void toNative(SerialReader* reader, const void* inSerial, void* outNative) { DeclRefBase& native = *(DeclRefBase*)(outNative); const SerialType& serial = *(const SerialType*)inSerial; @@ -328,196 +124,49 @@ struct ASTSerialDeclRefBaseTypeInfo native.decl = reader->getPointer(serial.decl).dynamicCast<Decl>(); native.substitutions.substitutions = reader->getPointer(serial.substitutions).dynamicCast<Substitutions>(); } - static const ASTSerialType* getType() + static const SerialFieldType* getFieldType() { - static const ASTSerialType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; + static const SerialFieldType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; return &type; } }; - +// Special case DeclRef, because it always uses the same type template <typename T> -struct ASTSerialTypeInfo<DeclRef<T>> : public ASTSerialDeclRefBaseTypeInfo {}; - -// MatrixCoord can just go as is -template <> -struct ASTSerialTypeInfo<MatrixCoord> : ASTSerialIdentityTypeInfo<MatrixCoord> {}; - -// SourceLoc - -// Make the type exposed, so we can look for it if we want to remap. -template <> -struct ASTSerialTypeInfo<SourceLoc> -{ - typedef SourceLoc NativeType; - typedef ASTSerialSourceLoc SerialType; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialSourceLoc) }; - - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) - { - *(SerialType*)outSerial = writer->addSourceLoc(*(const NativeType*)inNative); - } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - *(NativeType*)outNative = reader->getSourceLoc(*(const SerialType*)inSerial); - } -}; - -// List -template <typename T, typename ALLOCATOR> -struct ASTSerialTypeInfo<List<T, ALLOCATOR>> +struct SerialGetFieldType<DeclRef<T>> { - typedef List<T, ALLOCATOR> NativeType; - typedef ASTSerialIndex SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - dst = writer->addArray(src.getBuffer(), src.getCount()); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& dst = *(NativeType*)native; - auto& src = *(const SerialType*)serial; - - reader->getArray(src, dst); - } + static const SerialFieldType* getFieldType() { return SerialDeclRefBaseTypeInfo::getFieldType(); } }; -// Dictionary -template <typename KEY, typename VALUE> -struct ASTSerialTypeInfo<Dictionary<KEY, VALUE>> -{ - typedef Dictionary<KEY, VALUE> NativeType; - struct SerialType - { - ASTSerialIndex keys; ///< Index an array - ASTSerialIndex values; ///< Index an array - }; - - typedef typename ASTSerialTypeInfo<KEY>::SerialType KeySerialType; - typedef typename ASTSerialTypeInfo<VALUE>::SerialType ValueSerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - List<KeySerialType> keys; - List<ValueSerialType> values; - - Index count = Index(src.Count()); - keys.setCount(count); - values.setCount(count); - - Index i = 0; - for (const auto& pair : src) - { - ASTSerialTypeInfo<KEY>::toSerial(writer, &pair.Key, &keys[i]); - ASTSerialTypeInfo<VALUE>::toSerial(writer, &pair.Value, &values[i]); - i++; - } - - dst.keys = writer->addArray(keys.getBuffer(), count); - dst.values = writer->addArray(values.getBuffer(), count); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - // Clear it - dst = NativeType(); - - List<KEY> keys; - List<VALUE> values; - - reader->getArray(src.keys, keys); - reader->getArray(src.values, values); - - SLANG_ASSERT(keys.getCount() == values.getCount()); - - const Index count = keys.getCount(); - for (Index i = 0; i < count; ++i) - { - dst.Add(keys[i], values[i]); - } - } -}; -// SyntaxClass<T> template <typename T> -struct ASTSerialTypeInfo<SyntaxClass<T>> -{ - typedef SyntaxClass<T> NativeType; - typedef uint16_t SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; +struct SerialTypeInfo<DeclRef<T>> : public SerialDeclRefBaseTypeInfo {}; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - SLANG_UNUSED(writer); - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - dst = SerialType(src.classInfo->m_classId); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - SLANG_UNUSED(reader); - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - dst.classInfo = ReflectClassInfo::getInfo(ASTNodeType(src)); - } -}; - -// Handle RefPtr - just convert into * to do the conversion -template <typename T> -struct ASTSerialTypeInfo<RefPtr<T>> -{ - typedef RefPtr<T> NativeType; - typedef typename ASTSerialTypeInfo<T*>::SerialType SerialType; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; +// MatrixCoord can just go as is +template <> +struct SerialTypeInfo<MatrixCoord> : SerialIdentityTypeInfo<MatrixCoord> {}; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - T* obj = src; - ASTSerialTypeInfo<T*>::toSerial(writer, &obj, serial); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - T* obj = nullptr; - ASTSerialTypeInfo<T*>::toNative(reader, serial, &obj); - *(NativeType*)native = obj; - } -}; // QualType template <> -struct ASTSerialTypeInfo<QualType> +struct SerialTypeInfo<QualType> { typedef QualType NativeType; struct SerialType { - ASTSerialIndex type; + SerialIndex type; uint8_t isLeftValue; }; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto dst = (SerialType*)serial; auto src = (const NativeType*)native; dst->isLeftValue = src->isLeftValue ? 1 : 0; dst->type = writer->addPointer(src->type); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto src = (const SerialType*)serial; auto dst = (NativeType*)native; @@ -529,81 +178,81 @@ struct ASTSerialTypeInfo<QualType> // LookupResult::Breadcrumb template <> -struct ASTSerialTypeInfo<LookupResultItem::Breadcrumb> +struct SerialTypeInfo<LookupResultItem::Breadcrumb> { typedef LookupResultItem::Breadcrumb NativeType; struct SerialType { NativeType::Kind kind; NativeType::ThisParameterMode thisParameterMode; - ASTSerialTypeInfo<DeclRef<Decl>>::SerialType declRef; - ASTSerialTypeInfo<RefPtr<NativeType>> next; + SerialTypeInfo<DeclRef<Decl>>::SerialType declRef; + SerialTypeInfo<RefPtr<NativeType>> next; }; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& src = *(const NativeType*)native; auto& dst = *(SerialType*)serial; dst.kind = src.kind; dst.thisParameterMode = src.thisParameterMode; - _toSerialValue(writer, src.declRef, dst.declRef); - _toSerialValue(writer, src.next, dst.next); + toSerialValue(writer, src.declRef, dst.declRef); + toSerialValue(writer, src.next, dst.next); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto& dst = *(NativeType*)native; auto& src = *(const SerialType*)serial; dst.kind = src.kind; dst.thisParameterMode = src.thisParameterMode; - _toNativeValue(reader, src.declRef, dst.declRef); - _toNativeValue(reader, src.next, dst.next); + toNativeValue(reader, src.declRef, dst.declRef); + toNativeValue(reader, src.next, dst.next); } }; // LookupResultItem template <> -struct ASTSerialTypeInfo<LookupResultItem> +struct SerialTypeInfo<LookupResultItem> { typedef LookupResultItem NativeType; struct SerialType { - ASTSerialTypeInfo<DeclRef<Decl>>::SerialType declRef; - ASTSerialTypeInfo<RefPtr<NativeType::Breadcrumb>> breadcrumbs; + SerialTypeInfo<DeclRef<Decl>>::SerialType declRef; + SerialTypeInfo<RefPtr<NativeType::Breadcrumb>> breadcrumbs; }; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& src = *(const NativeType*)native; auto& dst = *(SerialType*)serial; - _toSerialValue(writer, src.declRef, dst.declRef); - _toSerialValue(writer, src.breadcrumbs, dst.breadcrumbs); + toSerialValue(writer, src.declRef, dst.declRef); + toSerialValue(writer, src.breadcrumbs, dst.breadcrumbs); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto& dst = *(NativeType*)native; auto& src = *(const SerialType*)serial; - _toNativeValue(reader, src.declRef, dst.declRef); - _toNativeValue(reader, src.breadcrumbs, dst.breadcrumbs); + toNativeValue(reader, src.declRef, dst.declRef); + toNativeValue(reader, src.breadcrumbs, dst.breadcrumbs); } }; // LookupResult template <> -struct ASTSerialTypeInfo<LookupResult> +struct SerialTypeInfo<LookupResult> { typedef LookupResult NativeType; - typedef ASTSerialIndex SerialType; + typedef SerialIndex SerialType; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& src = *(const NativeType*)native; auto& dst = *(SerialType*)serial; @@ -619,10 +268,10 @@ struct ASTSerialTypeInfo<LookupResult> } else { - dst = ASTSerialIndex(0); + dst = SerialIndex(0); } } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto& dst = *(NativeType*)native; auto& src = *(const SerialType*)serial; @@ -646,20 +295,19 @@ struct ASTSerialTypeInfo<LookupResult> } }; - // GlobalGenericParamSubstitution::ConstraintArg template <> -struct ASTSerialTypeInfo<GlobalGenericParamSubstitution::ConstraintArg> +struct SerialTypeInfo<GlobalGenericParamSubstitution::ConstraintArg> { typedef GlobalGenericParamSubstitution::ConstraintArg NativeType; struct SerialType { - ASTSerialIndex decl; - ASTSerialIndex val; + SerialIndex decl; + SerialIndex val; }; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& dst = *(SerialType*)serial; auto& src = *(const NativeType*)native; @@ -667,7 +315,7 @@ struct ASTSerialTypeInfo<GlobalGenericParamSubstitution::ConstraintArg> dst.decl = writer->addPointer(src.decl); dst.val = writer->addPointer(src.val); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto& src = *(const SerialType*)serial; auto& dst = *(NativeType*)native; @@ -679,17 +327,17 @@ struct ASTSerialTypeInfo<GlobalGenericParamSubstitution::ConstraintArg> // ExpandedSpecializationArg template <> -struct ASTSerialTypeInfo<ExpandedSpecializationArg> +struct SerialTypeInfo<ExpandedSpecializationArg> { typedef ExpandedSpecializationArg NativeType; struct SerialType { - ASTSerialIndex val; - ASTSerialIndex witness; + SerialIndex val; + SerialIndex witness; }; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& dst = *(SerialType*)serial; auto& src = *(const NativeType*)native; @@ -697,7 +345,7 @@ struct ASTSerialTypeInfo<ExpandedSpecializationArg> dst.witness = writer->addPointer(src.witness); dst.val = writer->addPointer(src.val); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto& src = *(const SerialType*)serial; auto& dst = *(NativeType*)native; @@ -709,17 +357,17 @@ struct ASTSerialTypeInfo<ExpandedSpecializationArg> // TypeExp template <> -struct ASTSerialTypeInfo<TypeExp> +struct SerialTypeInfo<TypeExp> { typedef TypeExp NativeType; struct SerialType { - ASTSerialIndex type; - ASTSerialIndex expr; + SerialIndex type; + SerialIndex expr; }; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& dst = *(SerialType*)serial; auto& src = *(const NativeType*)native; @@ -727,7 +375,7 @@ struct ASTSerialTypeInfo<TypeExp> dst.type = writer->addPointer(src.type); dst.expr = writer->addPointer(src.exp); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto& src = *(const SerialType*)serial; auto& dst = *(NativeType*)native; @@ -739,19 +387,19 @@ struct ASTSerialTypeInfo<TypeExp> // DeclCheckStateExt template <> -struct ASTSerialTypeInfo<DeclCheckStateExt> +struct SerialTypeInfo<DeclCheckStateExt> { typedef DeclCheckStateExt NativeType; typedef DeclCheckStateExt::RawType SerialType; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SerialType*)serial = (*(const NativeType*)native).getRaw(); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); (*(NativeType*)serial).setRaw(*(const SerialType*)native); @@ -760,24 +408,24 @@ struct ASTSerialTypeInfo<DeclCheckStateExt> // Modifiers template <> -struct ASTSerialTypeInfo<Modifiers> +struct SerialTypeInfo<Modifiers> { typedef Modifiers NativeType; - typedef ASTSerialIndex SerialType; + typedef SerialIndex SerialType; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { // We need to make into an array - List<ASTSerialIndex> modifierIndices; + List<SerialIndex> modifierIndices; for (Modifier* modifier : *(NativeType*)native) { modifierIndices.add(writer->addPointer(modifier)); } *(SerialType*)serial = writer->addArray(modifierIndices.getBuffer(), modifierIndices.getCount()); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { List<Modifier*> modifiers; reader->getArray(*(const SerialType*)serial, modifiers); @@ -796,1115 +444,150 @@ struct ASTSerialTypeInfo<Modifiers> } }; -// ImageFormat -template <> -struct ASTSerialTypeInfo<ImageFormat> : public ASTSerialConvertTypeInfo<ImageFormat, uint8_t> {}; - -// Stage -template <> -struct ASTSerialTypeInfo<Stage> : public ASTSerialConvertTypeInfo<Stage, uint8_t> {}; - -// TokenType -template <> -struct ASTSerialTypeInfo<TokenType> : public ASTSerialConvertTypeInfo<TokenType, uint8_t> {}; - -// BaseType -template <> -struct ASTSerialTypeInfo<BaseType> : public ASTSerialConvertTypeInfo<BaseType, uint8_t> {}; - -// SemanticVersion -template <> -struct ASTSerialTypeInfo<SemanticVersion> : public ASTSerialIdentityTypeInfo<SemanticVersion> {}; - // ASTNodeType template <> -struct ASTSerialTypeInfo<ASTNodeType> : public ASTSerialConvertTypeInfo<ASTNodeType, uint16_t> {}; - -// String -template <> -struct ASTSerialTypeInfo<String> -{ - typedef String NativeType; - typedef ASTSerialIndex SerialType; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - *(SerialType*)serial = writer->addString(src); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - dst = reader->getString(src); - } -}; - -// Token -template <> -struct ASTSerialTypeInfo<Token> -{ - typedef Token NativeType; - struct SerialType - { - ASTSerialTypeInfo<BaseType>::SerialType type; - ASTSerialTypeInfo<SourceLoc>::SerialType loc; - ASTSerialIndex name; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - ASTSerialTypeInfo<TokenType>::toSerial(writer, &src.type, &dst.type); - ASTSerialTypeInfo<SourceLoc>::toSerial(writer, &src.loc, &dst.loc); - - if (src.flags & TokenFlag::Name) - { - dst.name = writer->addName(src.getName()); - } - else - { - dst.name = writer->addString(src.getContent()); - } - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - dst.flags = 0; - dst.charsNameUnion.chars = nullptr; - - ASTSerialTypeInfo<TokenType>::toNative(reader, &src.type, &dst.type); - ASTSerialTypeInfo<SourceLoc>::toNative(reader, &src.loc, &dst.loc); - - // At the other end all token content will appear as Names. - if (src.name != ASTSerialIndex(0)) - { - dst.charsNameUnion.name = reader->getName(src.name); - dst.flags |= TokenFlag::Name; - } - } -}; - -// NameLoc -template <> -struct ASTSerialTypeInfo<NameLoc> -{ - typedef NameLoc NativeType; - struct SerialType - { - ASTSerialTypeInfo<SourceLoc>::SerialType loc; - ASTSerialIndex name; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - dst.name = writer->addName(src.name); - ASTSerialTypeInfo<SourceLoc>::toSerial(writer, &src.loc, &dst.loc); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - dst.name = reader->getName(src.name); - ASTSerialTypeInfo<SourceLoc>::toNative(reader, &src.loc, &dst.loc); - } -}; - -// DiagnosticInfo -template <> -struct ASTSerialTypeInfo<const DiagnosticInfo*> -{ - typedef const DiagnosticInfo* NativeType; - typedef ASTSerialIndex SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - dst = src ? writer->addString(UnownedStringSlice(src->name)) : ASTSerialIndex(0); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - if (src == ASTSerialIndex(0)) - { - dst = nullptr; - } - else - { - dst = findDiagnosticByName(reader->getStringSlice(src)); - } - } -}; - -// !!!!!!!!!!!!!!!!!!!!! ASTSerialGetType<T> !!!!!!!!!!!!!!!!!!!!!!!!!!! -// Getting the type info, let's use a static variable to hold the state to keep simple - -template <typename T> -struct ASTSerialGetType -{ - static const ASTSerialType* getType() - { - typedef ASTSerialTypeInfo<T> Info; - static const ASTSerialType type = { sizeof(typename Info::SerialType), uint8_t(Info::SerialAlignment), &Info::toSerial, &Info::toNative }; - return &type; - } -}; - -// Special case DeclRef, because it always uses the same type -template <typename T> -struct ASTSerialGetType<DeclRef<T>> -{ - static const ASTSerialType* getType() { return ASTSerialDeclRefBaseTypeInfo::getType(); } -}; +struct SerialTypeInfo<ASTNodeType> : public SerialConvertTypeInfo<ASTNodeType, uint16_t> {}; // !!!!!!!!!!!!!!!!!!!!!! Generate fields for a type !!!!!!!!!!!!!!!!!!!!!!!!!!! - template <typename T> -ASTSerialField _calcField(const char* name, T& in) +SerialField _makeField(const char* name, T& in) { uint8_t* ptr = &reinterpret_cast<uint8_t&>(in); - ASTSerialField field; + SerialField field; field.name = name; - field.type = ASTSerialGetType<T>::getType(); + field.type = SerialGetFieldType<T>::getFieldType(); // This only works because we in is an offset from 1 field.nativeOffset = uint32_t(size_t(ptr) - 1); field.serialOffset = 0; return field; } -static ASTSerialClass _makeClass(MemoryArena* arena, ASTNodeType type, const List<ASTSerialField>& fields) +static const SerialClass* _addClass(SerialClasses* serialClasses, ASTNodeType type, ASTNodeType super, const List<SerialField>& fields) { - ASTSerialClass cls = { type, 0, 0, 0, 0 }; - cls.fieldsCount = fields.getCount(); - cls.fields = arena->allocateAndCopyArray(fields.getBuffer(), fields.getCount()); - return cls; + const SerialClass* superClass = serialClasses->getSerialClass(SerialTypeKind::NodeBase, SerialSubType(super)); + return serialClasses->add(SerialTypeKind::NodeBase, SerialSubType(type), fields.getBuffer(), fields.getCount(), superClass); } -#define SLANG_AST_SERIAL_FIELD(FIELD_NAME, TYPE, param) fields.add(_calcField(#FIELD_NAME, obj->FIELD_NAME)); +#define SLANG_AST_ADD_SERIAL_FIELD(FIELD_NAME, TYPE, param) fields.add(_makeField(#FIELD_NAME, obj->FIELD_NAME)); // Note that the obj point is not nullptr, because some compilers notice this is 'indexing from null' // and warn/error. So we offset from 1. -#define SLANG_AST_SERIAL_MAKE_CLASS(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \ +#define SLANG_AST_ADD_SERIAL_CLASS(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \ { \ NAME* obj = (NAME*)1; \ SLANG_UNUSED(obj); \ fields.clear(); \ - SLANG_FIELDS_ASTNode_##NAME(SLANG_AST_SERIAL_FIELD, param) \ - outClasses[Index(ASTNodeType::NAME)] = _makeClass(arena, ASTNodeType::NAME, fields); \ + SLANG_FIELDS_ASTNode_##NAME(SLANG_AST_ADD_SERIAL_FIELD, param) \ + _addClass(serialClasses, ASTNodeType::NAME, ASTNodeType::SUPER, fields); \ } struct ASTFieldAccess { - static void calcClasses(MemoryArena* arena, ASTSerialClass outClasses[Index(ASTNodeType::CountOf)]) + static void calcClasses(SerialClasses* serialClasses) { - List<ASTSerialField> fields; - SLANG_ALL_ASTNode_NodeBase(SLANG_AST_SERIAL_MAKE_CLASS, _) + // Add NodeBase first, and specially handle so that we add a null super class + serialClasses->add(SerialTypeKind::NodeBase, SerialSubType(ASTNodeType::NodeBase), nullptr, 0, nullptr); + + // Add the rest in order such that Super class is always added before its children + List<SerialField> fields; + SLANG_CHILDREN_ASTNode_NodeBase(SLANG_AST_ADD_SERIAL_CLASS, _) } }; -ASTSerialClasses::ASTSerialClasses(): - m_arena(2048) +void addASTTypes(SerialClasses* serialClasses) { - memset(m_classes, 0, sizeof(m_classes)); - ASTFieldAccess::calcClasses(&m_arena, m_classes); - - // Now work out the layout - for (Index i = 0; i < SLANG_COUNT_OF(m_classes); ++i) { - // Set up each class in order, from lowest to highest index - // Doing so means super class is always setup - ASTSerialClass& serialClass = m_classes[i]; - - const ReflectClassInfo* info = ReflectClassInfo::getInfo(serialClass.type); - - size_t maxAlignment = 1; - size_t offset = 0; - - const ReflectClassInfo* superInfo = info->m_superClass; - if (superInfo) - { - ASTSerialClass& superSerialInfo = m_classes[superInfo->m_classId]; - - // If it's been setup, then alignment must be non zero. - // The ordering of ASTNodeType, should mean type have larger ASTNodeType greater than supers ASTNodeType. - SLANG_ASSERT(superSerialInfo.alignment != 0); - - // Must be a power of 2 - SLANG_ASSERT((superSerialInfo.alignment & (superSerialInfo.alignment - 1)) == 0); - - maxAlignment = superSerialInfo.alignment; - offset = superSerialInfo.size; - - // Check it is correctly aligned - SLANG_ASSERT((offset & (maxAlignment - 1)) == 0); - } - - // Okay, go through fields setting their offset - ASTSerialField* fields = serialClass.fields; - for (Index j = 0; j < serialClass.fieldsCount; j++) - { - ASTSerialField& field = fields[j]; - - size_t alignment = field.type->serialAlignment; - // Make sure the offset is aligned for the field requirement - offset = (offset + alignment - 1) & ~(alignment - 1); - - // Save the field offset - field.serialOffset = uint32_t(offset); - - // Move past the field - offset += field.type->serialSizeInBytes; - - // Calc the maximum alignment - maxAlignment = (alignment > maxAlignment) ? alignment : maxAlignment; - } - - // Align with maximum alignment - offset = (offset + maxAlignment - 1) & ~(maxAlignment - 1); - - serialClass.alignment = uint8_t(maxAlignment); - serialClass.size = uint32_t(offset); + ASTFieldAccess::calcClasses(serialClasses); } -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!! - -ASTSerialWriter::ASTSerialWriter(ASTSerialClasses* classes, ASTSerialFilter* filter, DebugSerialWriter* debugWriter) : - m_arena(2048), - m_classes(classes), - m_filter(filter), - m_debugWriter(debugWriter) -{ - // 0 is always the null pointer - m_entries.add(nullptr); - m_ptrMap.Add(nullptr, 0); -} - -ASTSerialIndex ASTSerialWriter::writePointer(const NodeBase* node) -{ - // This pointer cannot be in the map - SLANG_ASSERT(m_ptrMap.TryGetValue(node) == nullptr); - - const ASTSerialClass* serialClass = m_classes->getSerialClass(node->astNodeType); - - typedef ASTSerialInfo::NodeEntry NodeEntry; - NodeEntry* nodeEntry = (NodeEntry*)m_arena.allocateAligned(sizeof(NodeEntry) + serialClass->size, ASTSerialInfo::MAX_ALIGNMENT); - - nodeEntry->type = ASTSerialInfo::Type::Node; - nodeEntry->astNodeType = uint16_t(node->astNodeType); - nodeEntry->info = ASTSerialInfo::makeEntryInfo(serialClass->alignment); - - auto index = _add(node, nodeEntry); - - uint8_t* serialPayload = (uint8_t*)(nodeEntry + 1); - - while (serialClass) { - for (Index i = 0; i < serialClass->fieldsCount; ++i) { - auto field = serialClass->fields[i]; - - // Work out the offsets - auto srcField = ((const uint8_t*)node) + field.nativeOffset; - auto dstField = serialPayload + field.serialOffset; + // Let's hack Breadcrumbs... - field.type->toSerialFunc(this, srcField, dstField); + typedef LookupResultItem::Breadcrumb Type; + Type* obj = (Type*)1; + SerialField field = _makeField("_", *obj); + serialClasses->add(SerialTypeKind::RefObject, SerialSubType(RefObjectSerialSubType::LookupResultItem_Breadcrumb), &field, 1, nullptr); } - // Get the super class - const ReflectClassInfo* reflectInfo = ReflectClassInfo::getInfo(serialClass->type); - const ReflectClassInfo* superReflectInfo = reflectInfo->m_superClass; - - serialClass = superReflectInfo ? m_classes->getSerialClass(ASTNodeType(superReflectInfo->m_classId)) : nullptr; - } - - return index; -} - -void ASTSerialWriter::setPointerIndex(const NodeBase* ptr, ASTSerialIndex index) -{ - m_ptrMap.Add(ptr, Index(index)); -} - -ASTSerialIndex ASTSerialWriter::addPointer(const NodeBase* node) -{ - // Null is always 0 - if (node == nullptr) - { - return ASTSerialIndex(0); - } - // Look up in the map - Index* indexPtr = m_ptrMap.TryGetValue(node); - if (indexPtr) - { - return ASTSerialIndex(*indexPtr); - } - - if (m_filter) - { - return m_filter->writePointer(this, node); - } - else - { - return writePointer(node); - } -} - -ASTSerialIndex ASTSerialWriter::addPointer(const RefObject* obj) -{ - // Null is always 0 - if (obj == nullptr) - { - return ASTSerialIndex(0); - } - // Look up in the map - Index* indexPtr = m_ptrMap.TryGetValue(obj); - if (indexPtr) - { - return ASTSerialIndex(*indexPtr); - } - - if (auto stringRep = dynamicCast<StringRepresentation>(obj)) - { - ASTSerialIndex index = addString(StringRepresentation::asSlice(stringRep)); - m_ptrMap.Add(obj, Index(index)); - return index; - } - else if (auto breadcrumb = dynamicCast<LookupResultItem::Breadcrumb>(obj)) - { - typedef ASTSerialTypeInfo<LookupResultItem::Breadcrumb> TypeInfo; - typedef ASTSerialInfo::RefObjectEntry RefObjectEntry; - - size_t alignment = TypeInfo::SerialAlignment; - alignment = (alignment < SLANG_ALIGN_OF(ASTSerialInfo::RefObjectEntry)) ? SLANG_ALIGN_OF(ASTSerialInfo::RefObjectEntry) : alignment; - - RefObjectEntry* entry = (RefObjectEntry*)m_arena.allocateAligned(sizeof(RefObjectEntry) + sizeof(TypeInfo::SerialType), alignment); - - entry->type = ASTSerialInfo::Type::RefObject; - entry->info = ASTSerialInfo::makeEntryInfo(int(alignment)); - entry->subType = RefObjectEntry::SubType::Breadcrumb; - - auto index = _add(breadcrumb, entry); - - // Do any conversion - TypeInfo::toSerial(this, breadcrumb, entry + 1); - return index; - } - else if (auto name = dynamicCast<const Name>(obj)) - { - return addName(name); - } - else if (auto scope = dynamicCast<Scope>(obj)) - { - // We don't serialize scope - return ASTSerialIndex(0); - } - else if (auto module = dynamicCast<Module>(obj)) - { - // We don't serialize Module - return ASTSerialIndex(0); - } - - SLANG_ASSERT(!"Unhandled type"); - return ASTSerialIndex(0); -} - -ASTSerialIndex ASTSerialWriter::addString(const UnownedStringSlice& slice) -{ - typedef ByteEncodeUtil Util; - typedef ASTSerialInfo::StringEntry StringEntry; - - if (slice.getLength() == 0) - { - return ASTSerialIndex(0); - } - - Index newIndex = m_entries.getCount(); - - Index* indexPtr = m_sliceMap.TryGetValueOrAdd(slice, newIndex); - if (indexPtr) - { - return ASTSerialIndex(*indexPtr); - } - - // Okay we need to add the string - - uint8_t encodeBuf[Util::kMaxLiteEncodeUInt32]; - const int encodeCount = Util::encodeLiteUInt32(uint32_t(slice.getLength()), encodeBuf); - - StringEntry* entry = (StringEntry*)m_arena.allocateUnaligned(SLANG_OFFSET_OF(StringEntry, sizeAndChars) + encodeCount + slice.getLength()); - entry->info = ASTSerialInfo::EntryInfo::Alignment1; - entry->type = ASTSerialInfo::Type::String; - uint8_t* dst = (uint8_t*)(entry->sizeAndChars); - for (int i = 0; i < encodeCount; ++i) - { - dst[i] = encodeBuf[i]; - } - - memcpy(dst + encodeCount, slice.begin(), slice.getLength()); - - m_entries.add(entry); - return ASTSerialIndex(newIndex); -} - - -ASTSerialIndex ASTSerialWriter::addString(const String& in) -{ - return addPointer(in.getStringRepresentation()); -} - -ASTSerialIndex ASTSerialWriter::addName(const Name* name) -{ - if (name == nullptr) - { - return ASTSerialIndex(0); - } - - // Look it up - Index* indexPtr = m_ptrMap.TryGetValue(name); - if (indexPtr) - { - return ASTSerialIndex(*indexPtr); + // Set these types to not serialize + serialClasses->addUnserialized(SerialTypeKind::RefObject, SerialSubType(RefObjectSerialSubType::Module)); + serialClasses->addUnserialized(SerialTypeKind::RefObject, SerialSubType(RefObjectSerialSubType::Scope)); } - - ASTSerialIndex index = addString(name->text); - m_ptrMap.Add(name, Index(index)); - return index; } -ASTSerialSourceLoc ASTSerialWriter::addSourceLoc(SourceLoc sourceLoc) +// A Hack for now to turn an RefObject* into a SubType for serialization +extern RefObjectSerialSubType getRefObjectSubType(const RefObject* obj) { - if (sourceLoc.isValid() && m_debugWriter) + if (as<LookupResultItem::Breadcrumb>(obj)) { - return m_debugWriter->addSourceLoc(sourceLoc); + return RefObjectSerialSubType::LookupResultItem_Breadcrumb; } - else + else if (as<Module>(obj)) { - return 0; + return RefObjectSerialSubType::Module; } -} - -ASTSerialIndex ASTSerialWriter::_addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount) -{ - typedef ASTSerialInfo::ArrayEntry Entry; - - if (elementCount == 0) + else if (as<Scope>(obj)) { - return ASTSerialIndex(0); + return RefObjectSerialSubType::Scope; } - - SLANG_ASSERT(alignment >= 1 && alignment <= ASTSerialInfo::MAX_ALIGNMENT); - - // We must at a minimum have the alignment for the array prefix info - alignment = (alignment < SLANG_ALIGN_OF(Entry)) ? SLANG_ALIGN_OF(Entry) : alignment; - - size_t payloadSize = elementCount * elementSize; - - Entry* entry = (Entry*)m_arena.allocateAligned(sizeof(Entry) + payloadSize, alignment); - - entry->type = ASTSerialInfo::Type::Array; - entry->info = ASTSerialInfo::makeEntryInfo(int(alignment)); - entry->elementSize = uint16_t(elementSize); - entry->elementCount = uint32_t(elementCount); - - memcpy(entry + 1, elements, payloadSize); - - m_entries.add(entry); - return ASTSerialIndex(m_entries.getCount() - 1); + return RefObjectSerialSubType::Invalid; } -static const uint8_t s_fixBuffer[ASTSerialInfo::MAX_ALIGNMENT]{ 0, }; +/* !!!!!!!!!!!!!!!!!!!!!! DefaultSerialObjectFactory !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ -SlangResult ASTSerialWriter::write(Stream* stream) +void* DefaultSerialObjectFactory::create(SerialTypeKind typeKind, SerialSubType subType) { - const Int entriesCount = m_entries.getCount(); - - // Add a sentinal so we don't need special handling for - ASTSerialInfo::Entry sentinal; - sentinal.type = ASTSerialInfo::Type::String; - sentinal.info = ASTSerialInfo::EntryInfo::Alignment1; - - m_entries.add(&sentinal); - m_entries.removeLast(); - - ASTSerialInfo::Entry** entries = m_entries.getBuffer(); - // Note strictly required in our impl of List. But by writing this and - // knowing that removeLast cannot release memory, means the sentinal must be at the last position. - entries[entriesCount] = &sentinal; - - + switch (typeKind) { - size_t offset = 0; - - ASTSerialInfo::Entry* entry = entries[1]; - // We start on 1, because 0 is nullptr and not used for anything - for (Index i = 1; i < entriesCount; ++i) + case SerialTypeKind::NodeBase: { - ASTSerialInfo::Entry* next = entries[i + 1]; - // Before writing we need to store the next alignment - - const size_t nextAlignment = ASTSerialInfo::getAlignment(next->info); - const size_t alignment = ASTSerialInfo::getAlignment(entry->info); - - entry->info = ASTSerialInfo::combineWithNext(entry->info, next->info); - - // Check we are aligned correctly - SLANG_ASSERT((offset & (alignment - 1)) == 0); - - // When we write, we need to make sure it take into account the next alignment - const size_t entrySize = entry->calcSize(m_classes); - - // Work out the fix for next alignment - size_t nextOffset = offset + entrySize; - nextOffset = (nextOffset + nextAlignment - 1) & ~(nextAlignment - 1); - - size_t alignmentFixSize = nextOffset - (offset + entrySize); - - // The fix must be less than max alignment. We require it to be less because we aligned each Entry to - // MAX_ALIGNMENT, and so < MAX_ALIGNMENT is the most extra bytes we can write - SLANG_ASSERT( alignmentFixSize < ASTSerialInfo::MAX_ALIGNMENT); - - try - { - stream->write(entry, entrySize); - // If we needed to fix so that subsequent alignment is right, write out extra bytes here - if (alignmentFixSize) - { - stream->write(s_fixBuffer, alignmentFixSize); - } - } - catch (const IOException&) - { - return SLANG_FAIL; - } - - // Onto next - offset = nextOffset; - entry = next; + return m_astBuilder->createByNodeType(ASTNodeType(subType)); } - } - - return SLANG_OK; -} - -SlangResult ASTSerialWriter::writeIntoContainer(RiffContainer* container) -{ - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ScopeChunk ScopeChunk; - - // This is the container for the AST Data - ScopeChunk scopeModule(container, Chunk::Kind::List, ASTSerialBinary::kSlangASTModuleFourCC); - { - ScopeChunk scopeData(container, Chunk::Kind::Data, ASTSerialBinary::kSlangASTModuleDataFourCC); - + case SerialTypeKind::RefObject: { - // Sentinal so we don't need special handling for end of list - ASTSerialInfo::Entry sentinal; - sentinal.type = ASTSerialInfo::Type::String; - sentinal.info = ASTSerialInfo::EntryInfo::Alignment1; - - size_t offset = 0; - const Int entriesCount = m_entries.getCount(); - - { - m_entries.add(&sentinal); - m_entries.removeLast(); - // Note strictly required in our impl of List. But by writing this and - // knowing that removeLast cannot release memory, means the sentinal must be at the last position. - m_entries.getBuffer()[entriesCount] = &sentinal; - } - - ASTSerialInfo::Entry*const* entries = m_entries.getBuffer(); - - ASTSerialInfo::Entry* entry = entries[1]; - // We start on 1, because 0 is nullptr and not used for anything - for (Index i = 1; i < entriesCount; ++i) + switch (RefObjectSerialSubType(subType)) { - ASTSerialInfo::Entry* next = entries[i + 1]; - - // Before writing we need to store the next alignment - - const size_t nextAlignment = ASTSerialInfo::getAlignment(next->info); - const size_t alignment = ASTSerialInfo::getAlignment(entry->info); - - entry->info = ASTSerialInfo::combineWithNext(entry->info, next->info); - - // Check we are aligned correctly - SLANG_ASSERT((offset & (alignment - 1)) == 0); - - // When we write, we need to make sure it take into account the next alignment - const size_t entrySize = entry->calcSize(m_classes); - - // Work out the fix for next alignment - size_t nextOffset = offset + entrySize; - nextOffset = (nextOffset + nextAlignment - 1) & ~(nextAlignment - 1); - - size_t alignmentFixSize = nextOffset - (offset + entrySize); - - // The fix must be less than max alignment. We require it to be less because we aligned each Entry to - // MAX_ALIGNMENT, and so < MAX_ALIGNMENT is the most extra bytes we can write - SLANG_ASSERT(alignmentFixSize < ASTSerialInfo::MAX_ALIGNMENT); - - container->write(entry, entrySize); - if (alignmentFixSize) + case RefObjectSerialSubType::LookupResultItem_Breadcrumb: { - container->write(s_fixBuffer, alignmentFixSize); + typedef LookupResultItem::Breadcrumb Breadcrumb; + return _add(new LookupResultItem::Breadcrumb(Breadcrumb::Kind::Member, DeclRef<Decl>(), nullptr, nullptr)); } - - // Onto next - offset = nextOffset; - entry = next; + default: break; } } - } - - return SLANG_OK; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialInfo::Entry !!!!!!!!!!!!!!!!!!!!!!!! - -size_t ASTSerialInfo::Entry::calcSize(ASTSerialClasses* serialClasses) const -{ - switch (type) - { - case Type::String: - { - auto entry = static_cast<const StringEntry*>(this); - const uint8_t* cur = (const uint8_t*)entry->sizeAndChars; - uint32_t charsSize; - int sizeSize = ByteEncodeUtil::decodeLiteUInt32(cur, &charsSize); - return SLANG_OFFSET_OF(StringEntry, sizeAndChars) + sizeSize + charsSize; - } - case Type::Node: - { - auto entry = static_cast<const NodeEntry*>(this); - auto serialClass = serialClasses->getSerialClass(ASTNodeType(entry->astNodeType)); - - // Align by the alignment of the entry - size_t alignment = getAlignment(entry->info); - size_t size = sizeof(NodeEntry) + serialClass->size; - - size = size + (alignment - 1) & ~(alignment - 1); - return size; - } - case Type::RefObject: - { - auto entry = static_cast<const RefObjectEntry*>(this); - - size_t payloadSize; - switch (entry->subType) - { - case RefObjectEntry::SubType::Breadcrumb: - { - payloadSize = sizeof(ASTSerialTypeInfo<LookupResultItem::Breadcrumb>::SerialType); - break; - } - default: - { - SLANG_ASSERT(!"Unknown type"); - return 0; - } - } - - return sizeof(RefObjectEntry) + payloadSize; - } - case Type::Array: - { - auto entry = static_cast<const ArrayEntry*>(this); - return sizeof(ArrayEntry) + entry->elementSize * entry->elementCount; - } - default: break; - } - - SLANG_ASSERT(!"Unknown type"); - return 0; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!! - -const void* ASTSerialReader::getArray(ASTSerialIndex index, Index& outCount) -{ - if (index == ASTSerialIndex(0)) - { - outCount = 0; - return nullptr; - } - - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - switch (entry->type) - { - case Type::Array: - { - auto arrayEntry = static_cast<const ASTSerialInfo::ArrayEntry*>(entry); - outCount = Index(arrayEntry->elementCount); - return (arrayEntry + 1); - } default: break; } - SLANG_ASSERT(!"Not an array"); - outCount = 0; return nullptr; } -ASTSerialPointer ASTSerialReader::getPointer(ASTSerialIndex index) -{ - if (index == ASTSerialIndex(0)) - { - return ASTSerialPointer(); - } - - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - switch (entry->type) - { - case Type::String: - { - // Hmm. Tricky -> we don't know if will be cast as Name or String. Lets assume string. - String string = getString(index); - return ASTSerialPointer(string.getStringRepresentation()); - } - case Type::Node: - { - return ASTSerialPointer((NodeBase*)m_objects[Index(index)]); - } - case Type::RefObject: - { - return ASTSerialPointer((RefObject*)m_objects[Index(index)]); - } - default: break; - } - - SLANG_ASSERT(!"Cannot access as a pointer"); - return ASTSerialPointer(); -} - -String ASTSerialReader::getString(ASTSerialIndex index) -{ - if (index == ASTSerialIndex(0)) - { - return String(); - } - - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - // It has to be a string type - if (entry->type != Type::String) - { - SLANG_ASSERT(!"Not a string"); - return String(); - } - - RefObject* obj = (RefObject*)m_objects[Index(index)]; - - if (obj) - { - StringRepresentation* stringRep = dynamicCast<StringRepresentation>(obj); - if (stringRep) - { - return String(stringRep); - } - // Must be a name then - Name* name = dynamicCast<Name>(obj); - SLANG_ASSERT(name); - return name->text; - } - - // Okay we need to construct as a string - UnownedStringSlice slice = getStringSlice(index); - String string(slice); - StringRepresentation* stringRep = string.getStringRepresentation(); - - m_scope.add(stringRep); - m_objects[Index(index)] = stringRep; - return string; -} - -Name* ASTSerialReader::getName(ASTSerialIndex index) -{ - if (index == ASTSerialIndex(0)) - { - return nullptr; - } - - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - // It has to be a string type - if (entry->type != Type::String) - { - SLANG_ASSERT(!"Not a string"); - return nullptr; - } - - RefObject* obj = (RefObject*)m_objects[Index(index)]; - - if (obj) - { - Name* name = dynamicCast<Name>(obj); - if (name) - { - return name; - } - // Can only be a string then - StringRepresentation* stringRep = dynamicCast<StringRepresentation>(obj); - SLANG_ASSERT(stringRep); - - // I don't need to scope, as scoped in NamePool - name = m_namePool->getName(String(stringRep)); - - // Store as name, as can always access the inner string if needed - m_objects[Index(index)] = name; - return name; - } - - UnownedStringSlice slice = getStringSlice(index); - String string(slice); - Name* name = m_namePool->getName(string); - // Don't need to add to scope, because scoped on the pool - m_objects[Index(index)] = name; - return name; -} - -UnownedStringSlice ASTSerialReader::getStringSlice(ASTSerialIndex index) -{ - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - // It has to be a string type - if (entry->type != Type::String) - { - SLANG_ASSERT(!"Not a string"); - return UnownedStringSlice(); - } - - auto stringEntry = static_cast<const ASTSerialInfo::StringEntry*>(entry); - - const uint8_t* src = (const uint8_t*)stringEntry->sizeAndChars; - - // Decode the string - uint32_t size; - int sizeSize = ByteEncodeUtil::decodeLiteUInt32(src, &size); - return UnownedStringSlice((const char*)src + sizeSize, size); -} - -SourceLoc ASTSerialReader::getSourceLoc(ASTSerialSourceLoc loc) -{ - return (loc && m_debugReader) ? m_debugReader->getSourceLoc(loc) : SourceLoc(); -} - -SlangResult ASTSerialReader::loadEntries(const uint8_t* data, size_t dataCount, List<const ASTSerialInfo::Entry*>& outEntries) -{ - // Check the input data is at least aligned to the max alignment (otherwise everything cannot be aligned correctly) - SLANG_ASSERT((size_t(data) & (ASTSerialInfo::MAX_ALIGNMENT - 1)) == 0); - - outEntries.setCount(1); - outEntries[0] = nullptr; - - const uint8_t*const end = data + dataCount; - - const uint8_t* cur = data; - while (cur < end) - { - const Entry* entry = (const Entry*)cur; - outEntries.add(entry); - - const size_t entrySize = entry->calcSize(m_classes); - cur += entrySize; - - // Need to get the next alignment - const size_t nextAlignment = ASTSerialInfo::getNextAlignment(entry->info); - - // Need to fix cur with the alignment - cur = (const uint8_t*)((size_t(cur) + nextAlignment - 1) & ~(nextAlignment - 1)); - } - - return SLANG_OK; -} - -SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuilder* builder, NamePool* namePool) -{ - SLANG_RETURN_ON_FAIL(loadEntries(data, dataCount, m_entries)); - - m_namePool = namePool; - - m_objects.clearAndDeallocate(); - m_objects.setCount(m_entries.getCount()); - memset(m_objects.getBuffer(), 0, m_objects.getCount() * sizeof(void*)); - - // Go through entries, constructing objects. - for (Index i = 1; i < m_entries.getCount(); ++i) - { - const Entry* entry = m_entries[i]; - - switch (entry->type) - { - case Type::String: - { - // Don't need to construct an object. This is probably a StringRepresentation, or a Name - // Will evaluate lazily. - break; - } - case Type::Node: - { - auto nodeEntry = static_cast<const ASTSerialInfo::NodeEntry*>(entry); - m_objects[i] = builder->createByNodeType(ASTNodeType(nodeEntry->astNodeType)); - break; - } - case Type::RefObject: - { - auto objEntry = static_cast<const ASTSerialInfo::RefObjectEntry*>(entry); - switch (objEntry->subType) - { - case ASTSerialInfo::RefObjectEntry::SubType::Breadcrumb: - { - typedef LookupResultItem::Breadcrumb Breadcrumb; - - auto breadcrumb = new LookupResultItem::Breadcrumb(Breadcrumb::Kind::Member, DeclRef<Decl>(), nullptr, nullptr); - m_scope.add(breadcrumb); - m_objects[i] = breadcrumb; - break; - } - default: - { - SLANG_ASSERT(!"Unknown type"); - return SLANG_FAIL; - } - } - break; - } - case Type::Array: - { - // Don't need to construct an object, as will be accessed an interpreted by the object that holds it - break; - } - } - } - - // Deserialize - for (Index i = 1; i < m_entries.getCount(); ++i) - { - const Entry* entry = m_entries[i]; - void* native = m_objects[i]; - if (!native) - { - continue; - } - switch (entry->type) - { - case Type::Node: - { - auto nodeEntry = static_cast<const ASTSerialInfo::NodeEntry*>(entry); - auto serialClass = m_classes->getSerialClass(ASTNodeType(nodeEntry->astNodeType)); - - const uint8_t* src = (const uint8_t*)(nodeEntry + 1); - uint8_t* dst = (uint8_t*)m_objects[i]; - - // It must be constructed - SLANG_ASSERT(dst); - - while (serialClass) - { - for (Index j = 0; j < serialClass->fieldsCount; ++j) - { - auto field = serialClass->fields[j]; - auto fieldType = field.type; - fieldType->toNativeFunc(this, src + field.serialOffset, dst + field.nativeOffset); - } - - auto cls = ReflectClassInfo::getInfo(serialClass->type); - auto superCls = cls->m_superClass; - - // Get the super class - serialClass = superCls ? m_classes->getSerialClass(ASTNodeType(superCls->m_classId)) : nullptr; - } - - break; - } - case Type::RefObject: - { - auto objEntry = static_cast<const ASTSerialInfo::RefObjectEntry*>(entry); - switch (objEntry->subType) - { - case ASTSerialInfo::RefObjectEntry::SubType::Breadcrumb: - { - typedef LookupResultItem::Breadcrumb Breadcrumb; - auto serialType = ASTSerialGetType<Breadcrumb>::getType(); - serialType->toNativeFunc(this, (entry + 1), m_objects[i]); - break; - } - default: - { - SLANG_ASSERT(!"Unknown type"); - return SLANG_FAIL; - } - } - break; - } - default: break; - } - } - - return SLANG_OK; -} - // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerializeUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!! /* static */SlangResult ASTSerialTestUtil::selfTest() { - RefPtr<ASTSerialClasses> classes = new ASTSerialClasses; + RefPtr<SerialClasses> classes; + SerialClasses::create(classes); { - struct Thing - { - Module* node; - }; - Thing thing; - - //Pointer pointer(thing.node); - - auto field = _calcField("node", thing.node); - - - const ASTSerialType* type = ASTSerialGetType<Type*>::getType(); + const SerialFieldType* type = SerialGetFieldType<Type*>::getFieldType(); SLANG_UNUSED(type); } { - const ASTSerialType* type = ASTSerialGetType<int[10]>::getType(); + const SerialFieldType* type = SerialGetFieldType<int[10]>::getFieldType(); SLANG_UNUSED(type); } { - const ASTSerialType* type = ASTSerialGetType<bool[3]>::getType(); + const SerialFieldType* type = SerialGetFieldType<bool[3]>::getFieldType(); SLANG_UNUSED(type); } { - const ASTSerialType* type = ASTSerialGetType<Type*[3]>::getType(); + const SerialFieldType* type = SerialGetFieldType<Type*[3]>::getFieldType(); SLANG_UNUSED(type); } @@ -1913,7 +596,8 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil /* static */SlangResult ASTSerialTestUtil::testSerialize(NodeBase* node, RootNamePool* rootNamePool, SharedASTBuilder* sharedASTBuilder, SourceManager* sourceManager) { - RefPtr<ASTSerialClasses> classes = new ASTSerialClasses; + RefPtr<SerialClasses> classes; + SerialClasses::create(classes); List<uint8_t> contents; @@ -1921,11 +605,12 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil OwnedMemoryStream stream(FileAccess::ReadWrite); ModuleDecl* moduleDecl = as<ModuleDecl>(node); - ModuleASTSerialFilter filterStorage(moduleDecl); + // Only serialize out things *in* this module + ModuleSerialFilter filterStorage(moduleDecl); - ASTSerialFilter* filter = moduleDecl ? &filterStorage : nullptr; + SerialFilter* filter = moduleDecl ? &filterStorage : nullptr; - ASTSerialWriter writer(classes, filter, nullptr); + SerialWriter writer(classes, filter); // Lets serialize it all writer.addPointer(node); @@ -1937,15 +622,17 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil NamePool namePool; namePool.setRootNamePool(rootNamePool); - ASTSerialReader reader(classes, nullptr); + SerialReader reader(classes, nullptr); ASTBuilder builder(sharedASTBuilder, "Serialize Check"); + DefaultSerialObjectFactory objectFactory(&builder); + // We could now check that the loaded data matches { - const List<ASTSerialInfo::Entry*>& writtenEntries = writer.getEntries(); - List<const ASTSerialInfo::Entry*> readEntries; + const List<SerialInfo::Entry*>& writtenEntries = writer.getEntries(); + List<const SerialInfo::Entry*> readEntries; SlangResult res = reader.loadEntries(contents.getBuffer(), contents.getCount(), readEntries); SLANG_UNUSED(res); @@ -1969,7 +656,7 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil } { - SlangResult res = reader.load(contents.getBuffer(), contents.getCount(), &builder, &namePool); + SlangResult res = reader.load(contents.getBuffer(), contents.getCount(), &namePool); SLANG_UNUSED(res); } @@ -1979,7 +666,7 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil String readDump; { SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None); - ASTDumpUtil::dump(reader.getPointer(ASTSerialIndex(1)).dynamicCast<NodeBase>(), ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter); + ASTDumpUtil::dump(reader.getPointer(SerialIndex(1)).dynamicCast<NodeBase>(), ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter); readDump = sourceWriter.getContentAndClear(); } @@ -1994,7 +681,6 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil File::writeAllText("ast-read.ast-dump", readDump); File::writeAllText("ast-orig.ast-dump", origDump); - if (readDump != origDump) { return SLANG_FAIL; @@ -2004,5 +690,4 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil return SLANG_OK; } - } // namespace Slang diff --git a/source/slang/slang-serialize-ast.h b/source/slang/slang-serialize-ast.h index 76bbc9682..e2b9a956a 100644 --- a/source/slang/slang-serialize-ast.h +++ b/source/slang/slang-serialize-ast.h @@ -2,197 +2,18 @@ #ifndef SLANG_SERIALIZE_AST_H #define SLANG_SERIALIZE_AST_H -#include <type_traits> - #include "slang-ast-support-types.h" #include "slang-ast-all.h" -#include "slang-serialize-debug.h" - #include "../core/slang-riff.h" #include "slang-ast-builder.h" -#include "../core/slang-byte-encode-util.h" +#include "slang-serialize.h" -#include "../core/slang-stream.h" namespace Slang { -class Linkage; - -/* -AST Serialization Overview -========================== - -The AST node types are generally types derived from the NodeBase. The C++ extractor is used to associate an ASTNodeType with -every NodeBase type, such that casting is fast and simple and we have a simple integer to uniquely identify those types. The -extractor also performs another task of associating with the type name all of the fields held in just that type. The definition -of the fields is stored in an 'x macro' which is in the slang-ast-generated-macro.h file, for example - -``` -#define SLANG_FIELDS_ASTNode_DeclRefExpr(_x_, _param_)\ - _x_(scope, (RefPtr<Scope>), _param_)\ - _x_(declRef, (DeclRef<Decl>), _param_)\ - _x_(name, (Name*), _param_) -`` - -For the type DeclRefExpr, this holds all of the fields held in just DeclRefExpr in this case `scope`, `declRef` and `name`. -DeclRefExpr derives from Expr and this might hold other fields and so forth. - -The implementation makes a distinction between the 'native' types, the regular C++ in memory types and 'serial' types. -Each serializable C++ type has an associated 'serial' type - with the distinction that it can be written out and (with perhaps some other data) -read back in to recreate the C++ type. The serial type can be a C++ type, but is such it can be written and read from disk and still -represent the same data. - -We need a mechanism to be able to do do a conversion between native and serial types. To make the association we use the template - -``` -template <typename T> -struct ASTSerialTypeInfo; -``` - -and specialize it for each native type. The specialization holds - -SerialType - The type that will be used to represent the native type -NativeType - The native type -SerialAlignment - A value that holds what kind of alignment the SerialType needs to be serializable (it may be different from SLANG_ALIGN_OF(SerialType)!) -toSerial - A function that with the help of ASTSerialWriter convert the NativeType into the SerialType -toNative - A function that with the help of ASTSerialReader convert the SerialType into the NativeType - -It is useful to have a structure that holds the type information, so it can be stored. That is achieved with - -``` -template <typename T> -struct ASTSerialGetType; -``` - -This template can be specialized for a specific native types - but all it holds is just a function getType, which returns a ASTSerialType*, -which just holds the information held in the ASTSerialTypeInfo template, but additionally including the size of the SerialType. - -So we need to define a specialized ASTSerialTypeInfo for each type that can be a field in a NodeBase derived type. We don't need to define -anything explicitly for the NodeBase derived types, as we will just generate the layout from the fields. How do we know the fields? We just -used the macros generated from the C++ extractor. - -So first a few things to observe... - -1) Some types don't need any conversion to be serializable - int8_t, or float the bits can just be written out and read in (1) -2) Some types need a conversion but it's very simple - for example an enum without explicit size, being written as an explicit size -3) Some types can be written out but would not be directly readable or usable with different targets/processors, so need converting -4) Some types require complex conversions that require programmer code - like Dictionary/List - -For types that need no conversion (1), we can just use the template ASTSerialIdentityTypeInfo - -``` -template <> -struct ASTSerialTypeInfo<SomeType> : public ASTSerialIdentityTypeInfo<SomeType> {}; -``` - -This specialization means that SomeType can be written out and read in across targets/compilers without problems. - -For (2) we have another template that will do the conversion for us - -``` -template <typename NATIVE_T, typename SERIAL_T> -struct ASTSerialConvertTypeInfo; -``` - -That we can use as above, and specify the native and serial types. - -For (3) there are a few scenarios. For any field in a serial type we must store in the serialized type such that the representation -will work across all processors/compilers. So one problematic type is `bool`. It's not specified how it's laid out in memory - and -some compiles have stored it as a word. Most recently it's been stored as a byte. To make sure bool is ok for serialization therefore -we store as a uint8_t. - -Another example would be double. It's 64 bits, but on some arches/compilers it's SLANG_ALIGN_OF is 4 and on others it's 8. On some -arches a non aligned read will lead to a fault. To work around this problem therefore we have to ensure double has the alignment that -will work across all targets - and that alignment is 8. In that specific case that issue is handled via ASTSerialBasicTypeInfo, which -makes the SerialAlignment the sizeof the type. - -For (4) there are a few things to say. First a type can always implement a custom version of how to do a conversion by specializing -`ASTSerialTypeInfo`. But there remains another nagging issue - types which allocate/use other memory that changes at runtime. Clearly -we cannot define 'any size of memory' in a fixed SerialType defined in a specialization of ASTSerialTypeInfo. The mechanism to work around -this is to allow arbitrary arrays to be stored, that can be accessed via an ASTSerialIndex. This will be discussed more once we discuss -a little more about the file system, and ASTSerialIndex. - -Serialization Format -==================== - -The serialization format used is 'stream-like' with each 'object' stored in order. Each object is given an index starting from 1. -0 is used to be in effect nullptr. The stream looks like - -``` -ASTSerialInfo::Entry (for index 1) -Payload for type in entry - -ASTSerialInfo::Entry (for index 2) -Payload for type in entry - -... -... - -That when writing we have an array that maps each index to a pointer to the associated header. We also have a map that maps native pointers -to their indices. The Payload *is* the SerialType for thing saved. The payload directly follows the Entry data. - -Each object in this list can only be a few types of things - those derived from ASTSerialInfo::Type. - -The actual Entry followed by the payloads are allocated and stored when writing in a MemoryArena. When we want to write into a stream, we -can just iterate over each entry in order and write it out. - -You may have spotted a problem here - that some Entry types can be stored without alignment (for example a string - which stores the length -VarInt encoded followed by the characters). Others require an alignment - for example an NodeBase derived type that contains a int64_t will -*require* 8 byte alignment. That as a feature of the serialization format we want to be able to just map the data into memory, and be able -to access all the SerialType as is on the CPU. For that to work we *require* that the payload for each entry has the right alignment for -the associated SerialType. - -To achieve this we store in the Entry it's alignment requirement *AND* the next entries alignment. With this when we read, as we as stepping -through the entries we can find where the next Entry starts. Because the payload comes directly after the Entry - the Entrys size must be -a modulo of the largest alignment the payload can have. - -For the code that does the conversion between native and serial types it uses either the ASTSerialWriter or ASTSerialReader. This provides -the mechanism to turn a pointer into a serializable ASTSerialIndex and vice versa. There are some special functions for turning string like -types to and forth. - -The final mechanism is that of 'Arrays'. An array allows reading or writing a chunk of data associated with a ASTSerialIndex. The chunk of -data *must* hold data that is serializable. If the array holds pointers - then the serialized array must hold ASTSerialIndices that -represent those pointers. When reading back in they are converted back. - -Arrays are the escape hatch that allows for more complex types to serialize. Dictionaries for example are saved as a serial type that is -two ASTSerialIndices one to a keys array and one to a values array. - -Note that writing has two phases, serializing out into an ASTSerialWriter, and then secondly writing out to a stream. - -NodeBase Types -============== - -The ASTSerialTypeInfo mechanism is generally for *fields* of NodeBase types. That for NodeBase derived types we use the C++ extractors -field list to work out the native fields offsets and types. With this we can then calculate the layout for NodeBase types such that they -follow the requirements for serialization - such as alignment and so forth. - -This information is held in the ASTSerialClasses, which for a given ASTNodeType gives an ASTSerialClassInfo, that specifies fields for -just that type. Super types fields need to be serialized too, and this information can be found by using the ClassReflectInfo to find the -super type. - -Reading -======= - -Due to the care in writing reading is relatively simple. We can just take the contents of the file and put in memory, as long as in memory -it has an alignment of at least MAX_ALIGNMENT. Then we can build up an entries table by stepping through the data and writing the pointer. - -The toNative functions take an ASTSerialReader - this allows the implementation to ask for pointers and arrays from other parts of the serialized -data. It also allows for types to be lazily reconstructed if necessary. - -Lazy reconstruction may be useful in the future to partially reconstruct a sub part of the serialized data. In the current implementation, lazy -evaluation is used on Strings. The m_objects array holds all of the recreated native 'objects'. Since the objects can be derived from different -base classes the associated Entry will describe what it really is. - -For the String type, we initially store the object pointer as null. If a string is requested from that index, we see if the object pointer is null, -if it is we have to construct the StringRepresentation that will be used. - -An extra wrinkle is that we allow accessing of a serialized String as a Name or a string or a UnownedSubString. Fortunately a Name just holds a string, -and a Name remains in scope as long as it's NamePool does which is passed in. -*/ - /* Holds RIFF FourCC codes for AST types */ struct ASTSerialBinary { @@ -204,387 +25,43 @@ struct ASTSerialBinary static const FourCC kSlangASTModuleDataFourCC = SLANG_FOUR_CC('S', 'A', 'm', 'd'); }; -class ASTSerialClasses; - -// Type used to implement mechanisms to convert to and from serial types. -template <typename T> -struct ASTSerialTypeInfo; - -struct ASTSerialInfo -{ - enum - { - // Data held in serialized format, the maximally allowed alignment - MAX_ALIGNMENT = 8, - }; - - // We only allow up to MAX_ALIGNMENT bytes of alignment. We store alignments as shifts, so 2 bits needed for 1 - 8 - enum class EntryInfo : uint8_t - { - Alignment1 = 0, - }; - - static EntryInfo makeEntryInfo(int alignment, int nextAlignment) - { - // Make sure they are power of 2 - SLANG_ASSERT((alignment & (alignment - 1)) == 0); - SLANG_ASSERT((nextAlignment & (nextAlignment - 1)) == 0); - - const int alignmentShift = ByteEncodeUtil::calcMsb8(alignment); - const int nextAlignmentShift = ByteEncodeUtil::calcMsb8(nextAlignment); - return EntryInfo((nextAlignmentShift << 2) | alignmentShift); - } - static EntryInfo makeEntryInfo(int alignment) - { - // Make sure they are power of 2 - SLANG_ASSERT((alignment & (alignment - 1)) == 0); - return EntryInfo(ByteEncodeUtil::calcMsb8(alignment)); - } - /// Apply with the next alignment - static EntryInfo combineWithNext(EntryInfo cur, EntryInfo next) - { - return EntryInfo((int(cur) & ~0xc0) | ((int(next) & 3) << 2)); - } - - static int getAlignment(EntryInfo info) { return 1 << (int(info) & 3); } - static int getNextAlignment(EntryInfo info) { return 1 << ((int(info) >> 2) & 3); } - - enum class Type : uint8_t - { - String, ///< String - Node, ///< NodeBase derived - RefObject, ///< RefObject derived types - Array, ///< Array - }; - - - /* Alignment is a little tricky. We have a 'Entry' header before the payload. The payload alignment may change. - If we only align on the Entry header, then it's size *must* be some modulo of the maximum alignment allowed. - - We could hold Entry separate from payload. We could make the header not require the alignment of the payload - but then - we'd need payload alignment separate from entry alignment. - */ - struct Entry - { - Type type; - EntryInfo info; - - size_t calcSize(ASTSerialClasses* serialClasses) const; - }; - - struct StringEntry : Entry - { - char sizeAndChars[1]; - }; - - struct NodeEntry : Entry - { - uint16_t astNodeType; - uint32_t _pad0; ///< Necessary, because a node *can* have MAX_ALIGNEMENT - }; - - struct RefObjectEntry : Entry - { - enum class SubType : uint8_t - { - Breadcrumb, - }; - SubType subType; - uint8_t _pad0; - uint32_t _pad1; ///< Necessary because RefObjectEntry *can* have MAX_ALIGNEMENT - }; - - struct ArrayEntry : Entry - { - uint16_t elementSize; - uint32_t elementCount; - }; -}; - -typedef uint32_t ASTSerialIndexRaw; -enum class ASTSerialIndex : ASTSerialIndexRaw; -typedef DebugSerialData::SourceLoc ASTSerialSourceLoc; - -/* A type to convert pointers into types such that they can be passed around to readers/writers without -having to know the specific type. If there was a base class that all the serialized types derived from, -that was dynamically castable this would not be necessary */ -struct ASTSerialPointer -{ - enum class Kind - { - Unknown, - RefObject, - NodeBase - }; - - // Helpers so we can choose what kind of pointer we have based on the (unused) type of the pointer passed in - SLANG_FORCE_INLINE RefObject* _get(const RefObject*) { return m_kind == Kind::RefObject ? reinterpret_cast<RefObject*>(m_ptr) : nullptr; } - SLANG_FORCE_INLINE NodeBase* _get(const NodeBase*) { return m_kind == Kind::NodeBase ? reinterpret_cast<NodeBase*>(m_ptr) : nullptr; } - - template <typename T> - T* dynamicCast() - { - return Slang::dynamicCast<T>(_get((T*)nullptr)); - } - - ASTSerialPointer() : - m_kind(Kind::Unknown), - m_ptr(nullptr) - { - } - - ASTSerialPointer(RefObject* in) : - m_kind(Kind::RefObject), - m_ptr((void*)in) - { - } - ASTSerialPointer(NodeBase* in) : - m_kind(Kind::NodeBase), - m_ptr((void*)in) - { - } - - static Kind getKind(const RefObject*) { return Kind::RefObject; } - static Kind getKind(const NodeBase*) { return Kind::NodeBase; } - - Kind m_kind; - void* m_ptr; -}; - - -/* This class is the interface used by toNative implementations to recreate a type */ -class ASTSerialReader : public RefObject +class ModuleSerialFilter : public SerialFilter { public: + // SerialFilter impl + virtual SerialIndex writePointer(SerialWriter* writer, const NodeBase* ptr) SLANG_OVERRIDE; - typedef ASTSerialInfo::Entry Entry; - typedef ASTSerialInfo::Type Type; - - template <typename T> - void getArray(ASTSerialIndex index, List<T>& out); - - const void* getArray(ASTSerialIndex index, Index& outCount); - - ASTSerialPointer getPointer(ASTSerialIndex index); - String getString(ASTSerialIndex index); - Name* getName(ASTSerialIndex index); - UnownedStringSlice getStringSlice(ASTSerialIndex index); - SourceLoc getSourceLoc(ASTSerialSourceLoc loc); - - /// Load the entries table (without deserializing anything) - /// NOTE! data must stay ins scope for outEntries to be valid - SlangResult loadEntries(const uint8_t* data, size_t dataCount, List<const ASTSerialInfo::Entry*>& outEntries); - - /// NOTE! data must stay ins scope when reading takes place - SlangResult load(const uint8_t* data, size_t dataCount, ASTBuilder* builder, NamePool* namePool); - - ASTSerialReader(ASTSerialClasses* classes, DebugSerialReader* debugReader): - m_classes(classes), - m_debugReader(debugReader) - { - } - -protected: - List<const Entry*> m_entries; ///< The entries - List<void*> m_objects; ///< The constructed objects - - List<RefPtr<RefObject>> m_scope; ///< Objects to keep in scope during construction - - DebugSerialReader* m_debugReader; - - NamePool* m_namePool; - - ASTSerialClasses* m_classes; ///< Used to deserialize -}; - -// --------------------------------------------------------------------------- -template <typename T> -void ASTSerialReader::getArray(ASTSerialIndex index, List<T>& out) -{ - typedef ASTSerialTypeInfo<T> ElementTypeInfo; - typedef typename ElementTypeInfo::SerialType ElementSerialType; - - Index count; - auto serialElements = (const ElementSerialType*)getArray(index, count); - - if (count == 0) - { - out.clear(); - return; - } - - if (std::is_same<T, ElementSerialType>::value) - { - // If they are the same we can just write out - out.clear(); - out.insertRange(0, (const T*)serialElements, count); - } - else - { - // Else we need to convert - out.setCount(count); - for (Index i = 0; i < count; ++i) - { - ElementTypeInfo::toNative(this, (const void*)&serialElements[i], (void*)&out[i]); - } - } -} - - -class ASTSerialClasses; -class ASTSerialWriter; - -class ASTSerialFilter -{ -public: - virtual ASTSerialIndex writePointer(ASTSerialWriter* writer, const NodeBase* ptr) = 0; -}; - -class ModuleASTSerialFilter : public ASTSerialFilter -{ -public: - virtual ASTSerialIndex writePointer(ASTSerialWriter* writer, const NodeBase* ptr) SLANG_OVERRIDE; - - ModuleASTSerialFilter(ModuleDecl* moduleDecl): + ModuleSerialFilter(ModuleDecl* moduleDecl): m_moduleDecl(moduleDecl) { } - + protected: ModuleDecl* m_moduleDecl; }; -/* This is a class used tby toSerial implementations to turn native type into the serial type */ -class ASTSerialWriter : public RefObject +class DefaultSerialObjectFactory : public SerialObjectFactory { public: - ASTSerialIndex addPointer(const NodeBase* ptr); - ASTSerialIndex addPointer(const RefObject* ptr); - - /// Write the pointer - ASTSerialIndex writePointer(const NodeBase* ptr); - - template <typename T> - ASTSerialIndex addArray(const T* in, Index count); - - ASTSerialIndex addString(const UnownedStringSlice& slice); - ASTSerialIndex addString(const String& in); - ASTSerialIndex addName(const Name* name); - ASTSerialSourceLoc addSourceLoc(SourceLoc sourceLoc); - - /// Set a the index associated with an index. NOTE! That there cannot be a pre-existing setting. - void setPointerIndex(const NodeBase* ptr, ASTSerialIndex index); - - /// Get the entries table holding how each index maps to an entry - const List<ASTSerialInfo::Entry*>& getEntries() const { return m_entries; } - - /// Write to a stream - SlangResult write(Stream* stream); - - /// Write the state into the container - SlangResult writeIntoContainer(RiffContainer* container); - ASTSerialWriter(ASTSerialClasses* classes, ASTSerialFilter* filter, DebugSerialWriter* debugWriter); + virtual void* create(SerialTypeKind typeKind, SerialSubType subType) SLANG_OVERRIDE; -protected: - - ASTSerialIndex _addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount); - - ASTSerialIndex _add(const void* nativePtr, ASTSerialInfo::Entry* entry) + DefaultSerialObjectFactory(ASTBuilder* astBuilder) : + m_astBuilder(astBuilder) { - m_entries.add(entry); - // Okay I need to allocate space for this - ASTSerialIndex index = ASTSerialIndex(m_entries.getCount() - 1); - // Add to the map - m_ptrMap.Add(nativePtr, Index(index)); - return index; } - DebugSerialWriter* m_debugWriter; //< For writing/mapping serialized source locs - - Dictionary<const void*, Index> m_ptrMap; // Maps a pointer to an entry index - - // NOTE! Assumes the content stays in scope! - Dictionary<UnownedStringSlice, Index> m_sliceMap; - - List<ASTSerialInfo::Entry*> m_entries; ///< The entries - MemoryArena m_arena; ///< Holds the payloads - ASTSerialClasses* m_classes; - ASTSerialFilter* m_filter; ///< Filter to control what is serialized -}; - -// --------------------------------------------------------------------------- -template <typename T> -ASTSerialIndex ASTSerialWriter::addArray(const T* in, Index count) -{ - typedef ASTSerialTypeInfo<T> ElementTypeInfo; - typedef typename ElementTypeInfo::SerialType ElementSerialType; - - if (std::is_same<T, ElementSerialType>::value) - { - // If they are the same we can just write out - return _addArray(sizeof(T), SLANG_ALIGN_OF(ElementSerialType), in, count); - } - else +protected: + RefObject* _add(RefObject* obj) { - // Else we need to convert - List<ElementSerialType> work; - work.setCount(count); - - for (Index i = 0; i < count; ++i) - { - ElementTypeInfo::toSerial(this, &in[i], &work[i]); - } - return _addArray(sizeof(ElementSerialType), SLANG_ALIGN_OF(ElementSerialType), work.getBuffer(), count); + m_scope.add(obj); + return obj; } -} - -struct ASTSerialType -{ - typedef void(*ToSerialFunc)(ASTSerialWriter* writer, const void* src, void* dst); - typedef void(*ToNativeFunc)(ASTSerialReader* reader, const void* src, void* dst); - - size_t serialSizeInBytes; - uint8_t serialAlignment; - ToSerialFunc toSerialFunc; - ToNativeFunc toNativeFunc; -}; - -struct ASTSerialField -{ - const char* name; ///< The name of the field - const ASTSerialType* type; ///< The type of the field - uint32_t nativeOffset; ///< Offset to field from base of type - uint32_t serialOffset; ///< Offset in serial type -}; - -struct ASTSerialClass -{ - ASTNodeType type; - uint8_t alignment; - ASTSerialField* fields; - Index fieldsCount; - uint32_t size; + // We keep RefObjects in scope + List<RefPtr<RefObject>> m_scope; + ASTBuilder* m_astBuilder; }; -// An instance could be shared across Sessions, but for simplicity of life time -// here we don't deal with that -class ASTSerialClasses : public RefObject -{ -public: - - const ASTSerialClass* getSerialClass(ASTNodeType type) const { return &m_classes[Index(type)]; } - - /// Ctor - ASTSerialClasses(); - -protected: - MemoryArena m_arena; - - ASTSerialClass m_classes[Index(ASTNodeType::CountOf)]; -}; - - /* None of the functions in this util should *not* be called from production code, they exist to test features of AST Serialization */ struct ASTSerialTestUtil diff --git a/source/slang/slang-serialize-container.cpp b/source/slang/slang-serialize-container.cpp index 2301da9d4..3d5b234ff 100644 --- a/source/slang/slang-serialize-container.cpp +++ b/source/slang/slang-serialize-container.cpp @@ -9,7 +9,7 @@ #include "slang-compiler.h" #include "slang-serialize-ast.h" #include "slang-serialize-ir.h" -#include "slang-serialize-debug.h" +#include "slang-serialize-source-loc.h" namespace Slang { @@ -92,7 +92,7 @@ namespace Slang { /* static */SlangResult SerialContainerUtil::write(const SerialContainerData& data, const WriteOptions& options, RiffContainer* container) { - RefPtr<DebugSerialWriter> debugWriter; + RefPtr<SerialSourceLocWriter> sourceLocWriter; // The string pool used across the whole of the container StringSlicePool containerStringPool(StringSlicePool::Style::Default); @@ -116,10 +116,10 @@ namespace Slang { if (options.optionFlags & SerialOptionFlag::DebugInfo) { - debugWriter = new DebugSerialWriter(options.sourceManager); + sourceLocWriter = new SerialSourceLocWriter(options.sourceManager); } - RefPtr<ASTSerialClasses> astClasses; + RefPtr<SerialClasses> serialClasses; for (const auto& module : data.modules) { @@ -131,7 +131,7 @@ namespace Slang { { IRSerialData serialData; IRSerialWriter writer; - SLANG_RETURN_ON_FAIL(writer.write(module.irModule, debugWriter, options.optionFlags, &serialData)); + SLANG_RETURN_ON_FAIL(writer.write(module.irModule, sourceLocWriter, options.optionFlags, &serialData)); SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, options.compressionType, container)); } @@ -141,19 +141,21 @@ namespace Slang { { if (ModuleDecl* moduleDecl = as<ModuleDecl>(module.astRootNode)) { - if (!astClasses) + if (!serialClasses) { - astClasses = new ASTSerialClasses; + SLANG_RETURN_ON_FAIL(SerialClasses::create(serialClasses)); } - ModuleASTSerialFilter filter(moduleDecl); - ASTSerialWriter writer(astClasses, &filter, debugWriter); + ModuleSerialFilter filter(moduleDecl); + SerialWriter writer(serialClasses, &filter); + writer.getExtraObjects().set(sourceLocWriter); + // Add the module and everything that isn't filtered out in the filter. writer.addPointer(moduleDecl); // We can now serialize it into the riff container. - SLANG_RETURN_ON_FAIL(writer.writeIntoContainer(container)); + SLANG_RETURN_ON_FAIL(writer.writeIntoContainer(ASTSerialBinary::kSlangASTModuleFourCC, container)); } } } @@ -171,7 +173,7 @@ namespace Slang { IRSerialData serialData; IRSerialWriter writer; - SLANG_RETURN_ON_FAIL(writer.write(irModule, debugWriter, options.optionFlags, &serialData)); + SLANG_RETURN_ON_FAIL(writer.write(irModule, sourceLocWriter, options.optionFlags, &serialData)); SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, options.compressionType, container)); } } @@ -194,11 +196,11 @@ namespace Slang { } // We can now output the debug information. This is for all IR and AST - if (debugWriter) + if (sourceLocWriter) { // Write out the debug info - DebugSerialData debugData; - debugWriter->write(&debugData); + SerialSourceLocData debugData; + sourceLocWriter->write(&debugData); debugData.writeContainer(options.compressionType, container); } @@ -244,19 +246,19 @@ namespace Slang { SerialStringTableUtil::decodeStringTable((const char*)stringTableData->getPayload(), stringTableData->getSize(), containerStringPool); } - RefPtr<DebugSerialReader> debugReader; - RefPtr<ASTSerialClasses> astClasses; + RefPtr<SerialSourceLocReader> sourceLocReader; + RefPtr<SerialClasses> serialClasses; // Debug information - if (auto debugChunk = containerChunk->findContainedList(DebugSerialData::kDebugFourCc)) + if (auto debugChunk = containerChunk->findContainedList(SerialSourceLocData::kDebugFourCc)) { // Read into data - DebugSerialData debugData; - SLANG_RETURN_ON_FAIL(debugData.readContainer(containerCompressionType, debugChunk)); + SerialSourceLocData sourceLocData; + SLANG_RETURN_ON_FAIL(sourceLocData.readContainer(containerCompressionType, debugChunk)); // Turn into DebugReader - debugReader = new DebugSerialReader; - SLANG_RETURN_ON_FAIL(debugReader->read(&debugData, options.sourceManager)); + sourceLocReader = new SerialSourceLocReader; + SLANG_RETURN_ON_FAIL(sourceLocReader->read(&sourceLocData, options.sourceManager)); } // Add modules @@ -279,7 +281,7 @@ namespace Slang { // Read IR back from serialData IRSerialReader reader; - SLANG_RETURN_ON_FAIL(reader.read(serialData, options.session, debugReader, irModule)); + SLANG_RETURN_ON_FAIL(reader.read(serialData, options.session, sourceLocReader, irModule)); // Onto next chunk chunk = chunk->m_next; @@ -291,9 +293,9 @@ namespace Slang { if (astData) { - if (!astClasses) + if (!serialClasses) { - astClasses = new ASTSerialClasses; + SLANG_RETURN_ON_FAIL(SerialClasses::create(serialClasses)); } // TODO(JS): We probably want to store off better information about each of the translation unit @@ -305,12 +307,16 @@ namespace Slang { astBuilder = new ASTBuilder(options.sharedASTBuilder, buf.ProduceString()); - ASTSerialReader reader(astClasses, debugReader); + DefaultSerialObjectFactory objectFactory(astBuilder); - SLANG_RETURN_ON_FAIL(reader.load((const uint8_t*)astData->getPayload(), astData->getSize(), astBuilder, options.namePool)); + SerialReader reader(serialClasses, &objectFactory); + + reader.getExtraObjects().set(sourceLocReader); + + SLANG_RETURN_ON_FAIL(reader.load((const uint8_t*)astData->getPayload(), astData->getSize(), options.namePool)); // Get the root node. It's at index 1 (0 is the null value). - astRootNode = reader.getPointer(ASTSerialIndex(1)).dynamicCast<NodeBase>(); + astRootNode = reader.getPointer(SerialIndex(1)).dynamicCast<NodeBase>(); } // Onto next chunk @@ -386,25 +392,25 @@ namespace Slang { // Need to put all of this in a container RiffContainer::ScopeChunk containerScope(&riffContainer, RiffContainer::Chunk::Kind::List, SerialBinary::kContainerFourCc); - RefPtr<DebugSerialWriter> debugWriter; + RefPtr<SerialSourceLocWriter> sourceLocWriter; if (options.optionFlags & SerialOptionFlag::DebugInfo) { - debugWriter = new DebugSerialWriter(options.sourceManager); + sourceLocWriter = new SerialSourceLocWriter(options.sourceManager); } { // Write IR out to serialData - copying over SourceLoc information directly IRSerialWriter writer; - SLANG_RETURN_ON_FAIL(writer.write(module, debugWriter, options.optionFlags, &irData)); + SLANG_RETURN_ON_FAIL(writer.write(module, sourceLocWriter, options.optionFlags, &irData)); } SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(irData, options.compressionType, &riffContainer)); // Write the debug info Riff container - if (debugWriter) + if (sourceLocWriter) { - DebugSerialData serialData; - debugWriter->write(&serialData); + SerialSourceLocData serialData; + sourceLocWriter->write(&serialData); SLANG_RETURN_ON_FAIL(serialData.writeContainer(options.compressionType, &riffContainer)); } @@ -426,21 +432,21 @@ namespace Slang { RiffContainer::ListChunk* rootList = riffContainer.getRoot(); - RefPtr<DebugSerialReader> debugReader; + RefPtr<SerialSourceLocReader> sourceLocReader; // If we have debug info then find and read it if (options.optionFlags & SerialOptionFlag::DebugInfo) { - RiffContainer::ListChunk* debugList = rootList->findContainedList(DebugSerialData::kDebugFourCc); + RiffContainer::ListChunk* debugList = rootList->findContainedList(SerialSourceLocData::kDebugFourCc); if (!debugList) { return SLANG_FAIL; } - DebugSerialData debugData; - SLANG_RETURN_ON_FAIL(debugData.readContainer(options.compressionType, debugList)); + SerialSourceLocData sourceLocData; + SLANG_RETURN_ON_FAIL(sourceLocData.readContainer(options.compressionType, debugList)); - debugReader = new DebugSerialReader; - SLANG_RETURN_ON_FAIL(debugReader->read(&debugData, &workSourceManager)); + sourceLocReader = new SerialSourceLocReader; + SLANG_RETURN_ON_FAIL(sourceLocReader->read(&sourceLocData, &workSourceManager)); } { @@ -462,7 +468,7 @@ namespace Slang { return SLANG_FAIL; } - SLANG_RETURN_ON_FAIL(reader.read(irData, session, debugReader, irReadModule)); + SLANG_RETURN_ON_FAIL(reader.read(irData, session, sourceLocReader, irReadModule)); } } } diff --git a/source/slang/slang-serialize-ir-types.h b/source/slang/slang-serialize-ir-types.h index 513212ad6..d3ffde2a2 100644 --- a/source/slang/slang-serialize-ir-types.h +++ b/source/slang/slang-serialize-ir-types.h @@ -7,7 +7,7 @@ #include "../core/slang-array-view.h" #include "slang-serialize-types.h" -#include "slang-serialize-debug.h" +#include "slang-serialize-source-loc.h" #include "slang-name.h" #include "slang-source-loc.h" @@ -79,7 +79,7 @@ struct IRSerialData bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } bool operator<(const ThisType& rhs) const { return m_sourceLoc < rhs.m_sourceLoc; } - DebugSerialData::SourceLoc m_sourceLoc; ///< The source location + SerialSourceLocData::SourceLoc m_sourceLoc; ///< The source location InstIndex m_startInstIndex; ///< The index to the first instruction SizeType m_numInst; ///< The number of children }; diff --git a/source/slang/slang-serialize-ir.cpp b/source/slang/slang-serialize-ir.cpp index ac8085d51..d923577f4 100644 --- a/source/slang/slang-serialize-ir.cpp +++ b/source/slang/slang-serialize-ir.cpp @@ -34,7 +34,7 @@ void IRSerialWriter::_addInstruction(IRInst* inst) m_insts.add(inst); } -Result IRSerialWriter::_calcDebugInfo(DebugSerialWriter* debugWriter) +Result IRSerialWriter::_calcDebugInfo(SerialSourceLocWriter* sourceLocWriter) { // We need to find the unique source Locs // We are not going to store SourceLocs directly, because there may be multiple views mapping down to @@ -93,7 +93,7 @@ Result IRSerialWriter::_calcDebugInfo(DebugSerialWriter* debugWriter) IRSerialData::SourceLocRun sourceLocRun; sourceLocRun.m_numInst = curInstIndex - startInstLoc->instIndex;; sourceLocRun.m_startInstIndex = IRSerialData::InstIndex(startInstLoc->instIndex); - sourceLocRun.m_sourceLoc = debugWriter->addSourceLoc(SourceLoc::fromRaw(startSourceLoc)); + sourceLocRun.m_sourceLoc = sourceLocWriter->addSourceLoc(SourceLoc::fromRaw(startSourceLoc)); m_serialData->m_debugSourceLocRuns.add(sourceLocRun); @@ -104,7 +104,7 @@ Result IRSerialWriter::_calcDebugInfo(DebugSerialWriter* debugWriter) return SLANG_OK; } -Result IRSerialWriter::write(IRModule* module, DebugSerialWriter* debugWriter, SerialOptionFlags options, IRSerialData* serialData) +Result IRSerialWriter::write(IRModule* module, SerialSourceLocWriter* sourceLocWriter, SerialOptionFlags options, IRSerialData* serialData) { typedef Ser::Inst::PayloadType PayloadType; @@ -307,9 +307,9 @@ Result IRSerialWriter::write(IRModule* module, DebugSerialWriter* debugWriter, S } } - if ((options & SerialOptionFlag::DebugInfo) && debugWriter) + if ((options & SerialOptionFlag::DebugInfo) && sourceLocWriter) { - _calcDebugInfo(debugWriter); + _calcDebugInfo(sourceLocWriter); } m_serialData = nullptr; @@ -669,7 +669,7 @@ static Result _readInstArrayChunk(SerialCompressionType containerCompressionType return SLANG_OK; } -Result IRSerialReader::read(const IRSerialData& data, Session* session, DebugSerialReader* debugReader, RefPtr<IRModule>& outModule) +Result IRSerialReader::read(const IRSerialData& data, Session* session, SerialSourceLocReader* sourceLocReader, RefPtr<IRModule>& outModule) { typedef Ser::Inst::PayloadType PayloadType; @@ -866,14 +866,14 @@ Result IRSerialReader::read(const IRSerialData& data, Session* session, DebugSer } // We now need to apply the runs - if (debugReader && m_serialData->m_debugSourceLocRuns.getCount()) + if (sourceLocReader && m_serialData->m_debugSourceLocRuns.getCount()) { List<IRSerialData::SourceLocRun> sourceRuns(m_serialData->m_debugSourceLocRuns); // They are now in source location order sourceRuns.sort(); // Just guess initially 0 for the source file that contains the initial run - DebugSerialData::SourceRange range = DebugSerialData::SourceRange::getInvalid(); + SerialSourceLocData::SourceRange range = SerialSourceLocData::SourceRange::getInvalid(); int fix = 0; const Index numRuns = sourceRuns.getCount(); @@ -887,9 +887,9 @@ Result IRSerialReader::read(const IRSerialData& data, Session* session, DebugSer { if (!range.contains(run.m_sourceLoc)) { - fix = debugReader->calcFixSourceLoc(run.m_sourceLoc, range); + fix = sourceLocReader->calcFixSourceLoc(run.m_sourceLoc, range); } - sourceLoc = debugReader->calcFixedLoc(run.m_sourceLoc, fix, range); + sourceLoc = sourceLocReader->calcFixedLoc(run.m_sourceLoc, fix, range); } // Write to all the instructions diff --git a/source/slang/slang-serialize-ir.h b/source/slang/slang-serialize-ir.h index 368930c24..c3c3bcf19 100644 --- a/source/slang/slang-serialize-ir.h +++ b/source/slang/slang-serialize-ir.h @@ -7,7 +7,7 @@ #include "../core/slang-riff.h" #include "slang-ir.h" -#include "slang-serialize-debug.h" +#include "slang-serialize-source-loc.h" // For TranslationUnitRequest // and FrontEndCompileRequest::ExtraEntryPointInfo @@ -20,7 +20,7 @@ struct IRSerialWriter typedef IRSerialData Ser; typedef IRSerialBinary Bin; - Result write(IRModule* module, DebugSerialWriter* debugWriter, SerialOptionFlags flags, IRSerialData* serialData); + Result write(IRModule* module, SerialSourceLocWriter* sourceLocWriter, SerialOptionFlags flags, IRSerialData* serialData); /// Write to a container static Result writeContainer(const IRSerialData& data, SerialCompressionType compressionType, RiffContainer* container); @@ -51,7 +51,7 @@ struct IRSerialWriter protected: void _addInstruction(IRInst* inst); - Result _calcDebugInfo(DebugSerialWriter* debugWriter); + Result _calcDebugInfo(SerialSourceLocWriter* sourceLocWriter); List<IRInst*> m_insts; ///< Instructions in same order as stored in the @@ -75,7 +75,7 @@ struct IRSerialReader static Result readContainer(RiffContainer::ListChunk* module, SerialCompressionType containerCompressionType, IRSerialData* outData); /// Read a module from serial data - Result read(const IRSerialData& data, Session* session, DebugSerialReader* debugReader, RefPtr<IRModule>& outModule); + Result read(const IRSerialData& data, Session* session, SerialSourceLocReader* sourceLocReader, RefPtr<IRModule>& outModule); IRSerialReader(): m_serialData(nullptr), diff --git a/source/slang/slang-serialize-misc-type-info.h b/source/slang/slang-serialize-misc-type-info.h new file mode 100644 index 000000000..bdcbf2c98 --- /dev/null +++ b/source/slang/slang-serialize-misc-type-info.h @@ -0,0 +1,190 @@ +// slang-serialize-misc-type-info.h +#ifndef SLANG_SERIALIZE_MISC_TYPE_INFO_H +#define SLANG_SERIALIZE_MISC_TYPE_INFO_H + +#include "slang-serialize-type-info.h" + +#include "slang-source-loc.h" +#include "slang-compiler.h" + +namespace Slang { + +/* Conversion for serialization for some more misc Slang types +*/ + +// SamplerStateFlavor + +template <> +struct SerialTypeInfo<SamplerStateFlavor> : public SerialConvertTypeInfo<SamplerStateFlavor, uint8_t> {}; + +// TextureFlavor + +template <> +struct SerialTypeInfo<TextureFlavor> +{ + typedef TextureFlavor NativeType; + typedef uint16_t SerialType; + enum { SerialAlignment = sizeof(SerialType) }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SerialType*)serial = ((const NativeType*)native)->flavor; } + static void toNative(SerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); ((NativeType*)native)->flavor = *(const SerialType*)serial; } +}; + +// ImageFormat +template <> +struct SerialTypeInfo<ImageFormat> : public SerialConvertTypeInfo<ImageFormat, uint8_t> {}; + +// Stage +template <> +struct SerialTypeInfo<Stage> : public SerialConvertTypeInfo<Stage, uint8_t> {}; + +// TokenType +template <> +struct SerialTypeInfo<TokenType> : public SerialConvertTypeInfo<TokenType, uint8_t> {}; + +// BaseType +template <> +struct SerialTypeInfo<BaseType> : public SerialConvertTypeInfo<BaseType, uint8_t> {}; + +// SemanticVersion +template <> +struct SerialTypeInfo<SemanticVersion> : public SerialIdentityTypeInfo<SemanticVersion> {}; + +// SourceLoc + +// Make the type exposed, so we can look for it if we want to remap. +template <> +struct SerialTypeInfo<SourceLoc> +{ + typedef SourceLoc NativeType; + typedef SerialSourceLoc SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialSourceLoc) }; + + static void toSerial(SerialWriter* writer, const void* inNative, void* outSerial) + { + SerialSourceLocWriter* sourceLocWriter = writer->getExtraObjects().get<SerialSourceLocWriter>(); + *(SerialType*)outSerial = sourceLocWriter ? sourceLocWriter->addSourceLoc(*(const NativeType*)inNative) : SerialType(0); + } + static void toNative(SerialReader* reader, const void* inSerial, void* outNative) + { + SerialSourceLocReader* sourceLocReader = reader->getExtraObjects().get<SerialSourceLocReader>(); + *(NativeType*)outNative = sourceLocReader ? sourceLocReader->getSourceLoc(*(const SerialType*)inSerial) : NativeType::fromRaw(0); + } +}; + +// Token +template <> +struct SerialTypeInfo<Token> +{ + typedef Token NativeType; + struct SerialType + { + SerialTypeInfo<BaseType>::SerialType type; + SerialTypeInfo<SourceLoc>::SerialType loc; + SerialIndex name; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + SerialTypeInfo<TokenType>::toSerial(writer, &src.type, &dst.type); + SerialTypeInfo<SourceLoc>::toSerial(writer, &src.loc, &dst.loc); + + if (src.flags & TokenFlag::Name) + { + dst.name = writer->addName(src.getName()); + } + else + { + dst.name = writer->addString(src.getContent()); + } + } + static void toNative(SerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + dst.flags = 0; + dst.charsNameUnion.chars = nullptr; + + SerialTypeInfo<TokenType>::toNative(reader, &src.type, &dst.type); + SerialTypeInfo<SourceLoc>::toNative(reader, &src.loc, &dst.loc); + + // At the other end all token content will appear as Names. + if (src.name != SerialIndex(0)) + { + dst.charsNameUnion.name = reader->getName(src.name); + dst.flags |= TokenFlag::Name; + } + } +}; + +// NameLoc +template <> +struct SerialTypeInfo<NameLoc> +{ + typedef NameLoc NativeType; + struct SerialType + { + SerialTypeInfo<SourceLoc>::SerialType loc; + SerialIndex name; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + dst.name = writer->addName(src.name); + SerialTypeInfo<SourceLoc>::toSerial(writer, &src.loc, &dst.loc); + } + static void toNative(SerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + dst.name = reader->getName(src.name); + SerialTypeInfo<SourceLoc>::toNative(reader, &src.loc, &dst.loc); + } +}; + +// DiagnosticInfo +template <> +struct SerialTypeInfo<const DiagnosticInfo*> +{ + typedef const DiagnosticInfo* NativeType; + typedef SerialIndex SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + dst = src ? writer->addString(UnownedStringSlice(src->name)) : SerialIndex(0); + } + static void toNative(SerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + if (src == SerialIndex(0)) + { + dst = nullptr; + } + else + { + dst = findDiagnosticByName(reader->getStringSlice(src)); + } + } +}; + + + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-reflection.cpp b/source/slang/slang-serialize-reflection.cpp new file mode 100644 index 000000000..6430da55e --- /dev/null +++ b/source/slang/slang-serialize-reflection.cpp @@ -0,0 +1,113 @@ +// slang-serialize-reflection.cpp +#include "slang-serialize-reflection.h" + +#include "slang-serialize.h" + +namespace Slang { + +bool ReflectClassInfo::isSubClassOfSlow(const ThisType& super) const +{ + ReflectClassInfo const* info = this; + while (info) + { + if (info == &super) + return true; + info = info->m_superClass; + } + return false; +} + +#if 0 + +// #if'd out because produces a warning->error if not used. +static bool _checkSubClassRange(ReflectClassInfo*const* typeInfos, Index typeInfosCount) +{ + for (Index i = 0; i < typeInfosCount; ++i) + { + for (Index j = 0; j < typeInfosCount; ++j) + { + auto a = typeInfos[i]; + auto b = typeInfos[j]; + if (a->isSubClassOf(*b) != a->isSubClassOfSlow(*b)) + { + return false; + } + } + } + + return true; +} + +#endif + +static uint32_t _calcRangeRec(ReflectClassInfo* classInfo, const Dictionary<const ReflectClassInfo*, List<ReflectClassInfo*> >& childMap, uint32_t index) +{ + classInfo->m_classId = index++; + // Do the calc range for all the children + auto list = childMap.TryGetValue(classInfo); + + if (list) + { + for (auto child : *list) + { + index = _calcRangeRec(child, childMap, index); + } + } + + classInfo->m_lastClassId = index; + return index; +} + +static ReflectClassInfo* _calcRoot(ReflectClassInfo* classInfo) +{ + while (classInfo->m_superClass) + { + classInfo = const_cast<ReflectClassInfo*>(classInfo->m_superClass); + } + return classInfo; +} + + +/* static */void ReflectClassInfo::calcClassIdHierachy(uint32_t baseIndex, ReflectClassInfo*const* typeInfos, Index typeInfosCount) +{ + SLANG_ASSERT(typeInfosCount > 0); + + // TODO(JS): + // Note that the calculating of the ranges could be done more efficiently by adding to an array of struct { super, class }, sorting, by super classs + // and using a dictionary to map from class it's first in list of super class use. This works for now though. + + // The root cannot be shared with another hierarchy - as doing so will mean that the range will be incorrect (it would need to span both trees) + ReflectClassInfo* root = _calcRoot(typeInfos[0]); + + // We want to produce a map from a node that holds all of it's children + Dictionary<const ThisType*, List<ThisType*> > childMap; + + const List<ThisType*> emptyList; + { + for (Index i = 0; i < typeInfosCount; ++ i) + { + auto typeInfo = typeInfos[i]; + if (typeInfo->m_superClass) + { + // Add to that item + List<ThisType*>* list = childMap.TryGetValueOrAdd(typeInfo->m_superClass, emptyList); + if (!list) + { + list = childMap.TryGetValue(typeInfo->m_superClass); + } + SLANG_ASSERT(list); + list->add(typeInfo); + } + + // The root should be the same for all types + SLANG_ASSERT(_calcRoot(typeInfo) == root); + } + } + + // We want to recursively work out a range + _calcRangeRec(root, childMap, baseIndex); + + //SLANG_ASSERT(_checkSubClassRange(typeInfos, typeInfoCount)); +} + +} // namespace Slang diff --git a/source/slang/slang-serialize-reflection.h b/source/slang/slang-serialize-reflection.h new file mode 100644 index 000000000..045b7bf97 --- /dev/null +++ b/source/slang/slang-serialize-reflection.h @@ -0,0 +1,62 @@ +// slang-serialize-reflection.h +#ifndef SLANG_SERIALIZE_REFLECTION_H +#define SLANG_SERIALIZE_REFLECTION_H + +#include "slang-name.h" + +namespace Slang +{ + +struct ReflectClassInfo +{ + typedef ReflectClassInfo ThisType; + + typedef void* (*CreateFunc)(void* context); + typedef void(*DestructorFunc)(void* ptr); + + /// A constant time implementation of isSubClassOf + SLANG_FORCE_INLINE bool isSubClassOf(const ThisType& super) const + { + // We include super.m_classId, because it's a subclass of itself. + return m_classId >= super.m_classId && m_classId <= super.m_lastClassId; + } + + SLANG_FORCE_INLINE static bool isValidTypeId(uint32_t typeId) { return int32_t(typeId) >= 0; } + + // True if typeId derives from this type + SLANG_FORCE_INLINE bool isDerivedFrom(uint32_t typeId) const + { + SLANG_ASSERT(isValidTypeId(typeId) && isValidTypeId(m_classId)); + return typeId >= m_classId && typeId <= m_lastClassId; + } + + SLANG_FORCE_INLINE static bool isSubClassOf(uint32_t type, const ThisType& super) + { + SLANG_ASSERT(isValidTypeId(type) && isValidTypeId(super.m_classId)); + // We include super.m_classId, because it's a subclass of itself. + return type >= super.m_classId && type <= super.m_lastClassId; + } + + /// Will produce the same result as isSubClassOf (if enumerated), but more slowly by traversing the m_superClass + /// Works without initRange being called. + bool isSubClassOfSlow(const ThisType& super) const; + + /// Calculate infos m_classId for all the infos specified such that they are honor the inheritance relationship + /// such that a m_classId of a child is > m_classId && <= m_lastClassId + static void calcClassIdHierachy(uint32_t baseIndex, ReflectClassInfo*const* infos, Index infosCount); + + uint32_t m_classId; ///< Not necessarily set. + uint32_t m_lastClassId; + + const ReflectClassInfo* m_superClass; ///< The super class of this class, or nullptr if has no super class. + const char* m_name; ///< Textual class name, for debugging + CreateFunc m_createFunc; ///< Callback to use when creating instances (using an ASTBuilder for backing memory) + DestructorFunc m_destructorFunc; ///< The destructor for this type. Being just destructor, does not free backing memory for type. + + uint32_t m_sizeInBytes; ///< Total size of the type + uint8_t m_alignment; ///< The required alignment of the type +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-debug.cpp b/source/slang/slang-serialize-source-loc.cpp index 8f549e0f0..1de3f9a27 100644 --- a/source/slang/slang-serialize-debug.cpp +++ b/source/slang/slang-serialize-source-loc.cpp @@ -1,5 +1,5 @@ -// slang-serialize-debug.cpp -#include "slang-serialize-debug.h" +// slang-serialize-source-loc.cpp +#include "slang-serialize-source-loc.h" #include "../core/slang-text-io.h" #include "../core/slang-byte-encode-util.h" @@ -10,40 +10,40 @@ namespace Slang { /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ -size_t DebugSerialData::calcSizeInBytes() const +size_t SerialSourceLocData::calcSizeInBytes() const { - return SerialListUtil::calcArraySize(m_debugStringTable) + - SerialListUtil::calcArraySize(m_debugLineInfos) + - SerialListUtil::calcArraySize(m_debugSourceInfos) + - SerialListUtil::calcArraySize(m_debugAdjustedLineInfos); + return SerialListUtil::calcArraySize(m_stringTable) + + SerialListUtil::calcArraySize(m_lineInfos) + + SerialListUtil::calcArraySize(m_sourceInfos) + + SerialListUtil::calcArraySize(m_adjustedLineInfos); } -void DebugSerialData::clear() +void SerialSourceLocData::clear() { - m_debugLineInfos.clear(); - m_debugAdjustedLineInfos.clear(); - m_debugSourceInfos.clear(); - m_debugStringTable.clear(); + m_lineInfos.clear(); + m_adjustedLineInfos.clear(); + m_sourceInfos.clear(); + m_stringTable.clear(); } -bool DebugSerialData::operator==(const ThisType& rhs) const +bool SerialSourceLocData::operator==(const ThisType& rhs) const { return (this == &rhs) || - ( SerialListUtil::isEqual(m_debugStringTable, rhs.m_debugStringTable) && - SerialListUtil::isEqual(m_debugLineInfos, rhs.m_debugLineInfos) && - SerialListUtil::isEqual(m_debugAdjustedLineInfos, rhs.m_debugAdjustedLineInfos) && - SerialListUtil::isEqual(m_debugSourceInfos, rhs.m_debugSourceInfos)); + ( SerialListUtil::isEqual(m_stringTable, rhs.m_stringTable) && + SerialListUtil::isEqual(m_lineInfos, rhs.m_lineInfos) && + SerialListUtil::isEqual(m_adjustedLineInfos, rhs.m_adjustedLineInfos) && + SerialListUtil::isEqual(m_sourceInfos, rhs.m_sourceInfos)); } /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ -DebugSerialData::SourceLoc DebugSerialWriter::addSourceLoc(SourceLoc sourceLoc) +SerialSourceLocData::SourceLoc SerialSourceLocWriter::addSourceLoc(SourceLoc sourceLoc) { // If it's not set we can ignore if (!sourceLoc.isValid()) { - return DebugSerialData::SourceLoc(0); + return SerialSourceLocData::SourceLoc(0); } // Look up the view it's from @@ -51,20 +51,20 @@ DebugSerialData::SourceLoc DebugSerialWriter::addSourceLoc(SourceLoc sourceLoc) if (!sourceView) { // If not found we just ingore - return DebugSerialData::SourceLoc(0); + return SerialSourceLocData::SourceLoc(0); } SourceFile* sourceFile = sourceView->getSourceFile(); - DebugSourceFile* debugSourceFile; + Source* debugSourceFile; { - RefPtr<DebugSourceFile>* ptrDebugSourceFile = m_debugSourceFileMap.TryGetValue(sourceFile); + RefPtr<Source>* ptrDebugSourceFile = m_sourceFileMap.TryGetValue(sourceFile); if (ptrDebugSourceFile == nullptr) { - const SourceLoc::RawValue baseSourceLoc = m_debugFreeSourceLoc; - m_debugFreeSourceLoc += SourceLoc::RawValue(sourceView->getRange().getSize() + 1); + const SourceLoc::RawValue baseSourceLoc = m_freeSourceLoc; + m_freeSourceLoc += SourceLoc::RawValue(sourceView->getRange().getSize() + 1); - debugSourceFile = new DebugSourceFile(sourceFile, baseSourceLoc); - m_debugSourceFileMap.Add(sourceFile, debugSourceFile); + debugSourceFile = new Source(sourceFile, baseSourceLoc); + m_sourceFileMap.Add(sourceFile, debugSourceFile); } else { @@ -77,7 +77,7 @@ DebugSerialData::SourceLoc DebugSerialWriter::addSourceLoc(SourceLoc sourceLoc) int offset = sourceView->getRange().getOffset(sourceLoc); int lineIndex = sourceFile->calcLineIndexFromOffset(offset); - DebugSerialData::DebugLineInfo lineInfo; + SerialSourceLocData::LineInfo lineInfo; lineInfo.m_lineStartOffset = sourceFile->getLineBreakOffsets()[lineIndex]; lineInfo.m_lineIndex = lineIndex; @@ -93,7 +93,7 @@ DebugSerialData::SourceLoc DebugSerialWriter::addSourceLoc(SourceLoc sourceLoc) { const auto& entry = sourceView->getEntries()[entryIndex]; - DebugSerialData::DebugAdjustedLineInfo adjustedLineInfo; + SerialSourceLocData::AdjustedLineInfo adjustedLineInfo; adjustedLineInfo.m_lineInfo = lineInfo; adjustedLineInfo.m_pathStringIndex = SerialStringData::kNullStringIndex; @@ -104,7 +104,7 @@ DebugSerialData::SourceLoc DebugSerialWriter::addSourceLoc(SourceLoc sourceLoc) { UnownedStringSlice slice = pool.getSlice(entry.m_pathHandle); SLANG_ASSERT(slice.getLength() > 0); - adjustedLineInfo.m_pathStringIndex = DebugSerialData::StringIndex(m_debugStringSlicePool.add(slice)); + adjustedLineInfo.m_pathStringIndex = SerialSourceLocData::StringIndex(m_stringSlicePool.add(slice)); } adjustedLineInfo.m_adjustedLineIndex = lineIndex + entry.m_lineAdjust; @@ -115,50 +115,50 @@ DebugSerialData::SourceLoc DebugSerialWriter::addSourceLoc(SourceLoc sourceLoc) debugSourceFile->setHasLineIndex(lineIndex); } - return DebugSerialData::SourceLoc(debugSourceFile->m_baseSourceLoc + offset); + return SerialSourceLocData::SourceLoc(debugSourceFile->m_baseSourceLoc + offset); } -void DebugSerialWriter::write(DebugSerialData* outDebugData) +void SerialSourceLocWriter::write(SerialSourceLocData* outSourceLocData) { - outDebugData->clear(); + outSourceLocData->clear(); // Okay we can now calculate the final source information - for (auto& pair : m_debugSourceFileMap) + for (auto& pair : m_sourceFileMap) { - DebugSourceFile* debugSourceFile = pair.Value; + Source* debugSourceFile = pair.Value; SourceFile* sourceFile = debugSourceFile->m_sourceFile; - DebugSerialData::DebugSourceInfo sourceInfo; + SerialSourceLocData::SourceInfo sourceInfo; sourceInfo.m_numLines = uint32_t(debugSourceFile->m_sourceFile->getLineBreakOffsets().getCount()); sourceInfo.m_range.m_start = uint32_t(debugSourceFile->m_baseSourceLoc); sourceInfo.m_range.m_end = uint32_t(debugSourceFile->m_baseSourceLoc + sourceFile->getContentSize()); - sourceInfo.m_pathIndex = DebugSerialData::StringIndex(m_debugStringSlicePool.add(sourceFile->getPathInfo().foundPath)); + sourceInfo.m_pathIndex = SerialSourceLocData::StringIndex(m_stringSlicePool.add(sourceFile->getPathInfo().foundPath)); - sourceInfo.m_lineInfosStartIndex = uint32_t(outDebugData->m_debugLineInfos.getCount()); - sourceInfo.m_adjustedLineInfosStartIndex = uint32_t(outDebugData->m_debugAdjustedLineInfos.getCount()); + sourceInfo.m_lineInfosStartIndex = uint32_t(outSourceLocData->m_lineInfos.getCount()); + sourceInfo.m_adjustedLineInfosStartIndex = uint32_t(outSourceLocData->m_adjustedLineInfos.getCount()); sourceInfo.m_numLineInfos = uint32_t(debugSourceFile->m_lineInfos.getCount()); sourceInfo.m_numAdjustedLineInfos = uint32_t(debugSourceFile->m_adjustedLineInfos.getCount()); // Add the line infos - outDebugData->m_debugLineInfos.addRange(debugSourceFile->m_lineInfos.begin(), debugSourceFile->m_lineInfos.getCount()); - outDebugData->m_debugAdjustedLineInfos.addRange(debugSourceFile->m_adjustedLineInfos.begin(), debugSourceFile->m_adjustedLineInfos.getCount()); + outSourceLocData->m_lineInfos.addRange(debugSourceFile->m_lineInfos.begin(), debugSourceFile->m_lineInfos.getCount()); + outSourceLocData->m_adjustedLineInfos.addRange(debugSourceFile->m_adjustedLineInfos.begin(), debugSourceFile->m_adjustedLineInfos.getCount()); // Add the source info - outDebugData->m_debugSourceInfos.add(sourceInfo); + outSourceLocData->m_sourceInfos.add(sourceInfo); } // Convert the string pool - SerialStringTableUtil::encodeStringTable(m_debugStringSlicePool, outDebugData->m_debugStringTable); + SerialStringTableUtil::encodeStringTable(m_stringSlicePool, outSourceLocData->m_stringTable); } /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ -Index DebugSerialReader::findViewIndex(DebugSerialData::SourceLoc loc) +Index SerialSourceLocReader::findViewIndex(SerialSourceLocData::SourceLoc loc) { for (Index i = 0; i < m_views.getCount(); ++i) { @@ -171,7 +171,7 @@ Index DebugSerialReader::findViewIndex(DebugSerialData::SourceLoc loc) } -int DebugSerialReader::calcFixSourceLoc(DebugSerialData::SourceLoc loc, DebugSerialData::SourceRange& outRange) +int SerialSourceLocReader::calcFixSourceLoc(SerialSourceLocData::SourceLoc loc, SerialSourceLocData::SourceRange& outRange) { if (m_lastViewIndex < 0 || !m_views[m_lastViewIndex].m_range.contains(loc)) { @@ -181,7 +181,7 @@ int DebugSerialReader::calcFixSourceLoc(DebugSerialData::SourceLoc loc, DebugSer if (m_lastViewIndex < 0) { // Set an invalid range, as couldn't find - outRange = DebugSerialData::SourceRange::getInvalid(); + outRange = SerialSourceLocData::SourceRange::getInvalid(); return 0; } @@ -193,7 +193,7 @@ int DebugSerialReader::calcFixSourceLoc(DebugSerialData::SourceLoc loc, DebugSer return view.m_sourceView->getRange().begin.getRaw() - view.m_range.m_start; } -SourceLoc DebugSerialReader::getSourceLoc(DebugSerialData::SourceLoc loc) +SourceLoc SerialSourceLocReader::getSourceLoc(SerialSourceLocData::SourceLoc loc) { if (loc != 0) { @@ -216,31 +216,31 @@ SourceLoc DebugSerialReader::getSourceLoc(DebugSerialData::SourceLoc loc) return SourceLoc(); } -SlangResult DebugSerialReader::read(const DebugSerialData* serialData, SourceManager* sourceManager) +SlangResult SerialSourceLocReader::read(const SerialSourceLocData* serialData, SourceManager* sourceManager) { m_views.setCount(0); - if (!sourceManager || serialData->m_debugSourceInfos.getCount() == 0) + if (!sourceManager || serialData->m_sourceInfos.getCount() == 0) { return SLANG_OK; } List<UnownedStringSlice> debugStringSlices; - SerialStringTableUtil::decodeStringTable(serialData->m_debugStringTable.getBuffer(), serialData->m_debugStringTable.getCount(), debugStringSlices); + SerialStringTableUtil::decodeStringTable(serialData->m_stringTable.getBuffer(), serialData->m_stringTable.getCount(), debugStringSlices); // All of the strings are placed in the manager (and its StringSlicePool) where the SourceView and SourceFile are constructed from List<StringSlicePool::Handle> stringMap; SerialStringTableUtil::calcStringSlicePoolMap(debugStringSlices, sourceManager->getStringSlicePool(), stringMap); // Construct the source files - const Index numSourceFiles = serialData->m_debugSourceInfos.getCount(); + const Index numSourceFiles = serialData->m_sourceInfos.getCount(); // These hold the views (and SourceFile as there is only one SourceFile per view) in the same order as the sourceInfos m_views.setCount(numSourceFiles); for (Index i = 0; i < numSourceFiles; ++i) { - const auto& srcSourceInfo = serialData->m_debugSourceInfos[i]; + const auto& srcSourceInfo = serialData->m_sourceInfos[i]; PathInfo pathInfo; pathInfo.type = PathInfo::Type::FoundPath; @@ -250,11 +250,11 @@ SlangResult DebugSerialReader::read(const DebugSerialData* serialData, SourceMan SourceView* sourceView = sourceManager->createSourceView(sourceFile, nullptr); // We need to accumulate all line numbers, for this source file, both adjusted and unadjusted - List<DebugSerialData::DebugLineInfo> lineInfos; + List<SerialSourceLocData::LineInfo> lineInfos; // Add the adjusted lines { lineInfos.setCount(srcSourceInfo.m_numAdjustedLineInfos); - const DebugSerialData::DebugAdjustedLineInfo* srcAdjustedLineInfos = serialData->m_debugAdjustedLineInfos.getBuffer() + srcSourceInfo.m_adjustedLineInfosStartIndex; + const SerialSourceLocData::AdjustedLineInfo* srcAdjustedLineInfos = serialData->m_adjustedLineInfos.getBuffer() + srcSourceInfo.m_adjustedLineInfosStartIndex; const int numAdjustedLines = int(srcSourceInfo.m_numAdjustedLineInfos); for (int j = 0; j < numAdjustedLines; ++j) { @@ -262,7 +262,7 @@ SlangResult DebugSerialReader::read(const DebugSerialData* serialData, SourceMan } } // Add regular lines - lineInfos.addRange(serialData->m_debugLineInfos.getBuffer() + srcSourceInfo.m_lineInfosStartIndex, srcSourceInfo.m_numLineInfos); + lineInfos.addRange(serialData->m_lineInfos.getBuffer() + srcSourceInfo.m_lineInfosStartIndex, srcSourceInfo.m_numLineInfos); // Put in sourceloc order lineInfos.sort(); @@ -310,11 +310,11 @@ SlangResult DebugSerialReader::read(const DebugSerialData* serialData, SourceMan if (srcSourceInfo.m_numAdjustedLineInfos) { - List<DebugSerialData::DebugAdjustedLineInfo> adjustedLineInfos; + List<SerialSourceLocData::AdjustedLineInfo> adjustedLineInfos; int numEntries = int(srcSourceInfo.m_numAdjustedLineInfos); - adjustedLineInfos.addRange(serialData->m_debugAdjustedLineInfos.getBuffer() + srcSourceInfo.m_adjustedLineInfosStartIndex, numEntries); + adjustedLineInfos.addRange(serialData->m_adjustedLineInfos.getBuffer() + srcSourceInfo.m_adjustedLineInfosStartIndex, numEntries); adjustedLineInfos.sort(); // Work out the views adjustments, and place in dstEntries @@ -348,21 +348,21 @@ SlangResult DebugSerialReader::read(const DebugSerialData* serialData, SourceMan /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ -/* static */Result DebugSerialData::writeContainer(SerialCompressionType moduleCompressionType, RiffContainer* container) +/* static */Result SerialSourceLocData::writeContainer(SerialCompressionType moduleCompressionType, RiffContainer* container) { - RiffContainer::ScopeChunk debugChunkScope(container, RiffContainer::Chunk::Kind::List, DebugSerialData::kDebugFourCc); + RiffContainer::ScopeChunk debugChunkScope(container, RiffContainer::Chunk::Kind::List, SerialSourceLocData::kDebugFourCc); - SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk(DebugSerialData::kDebugStringFourCc, m_debugStringTable, container)); - SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk(DebugSerialData::kDebugLineInfoFourCc, m_debugLineInfos, container)); - SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk(DebugSerialData::kDebugAdjustedLineInfoFourCc, m_debugAdjustedLineInfos, container)); - SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk(moduleCompressionType, DebugSerialData::kDebugSourceInfoFourCc, m_debugSourceInfos, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk(SerialSourceLocData::kDebugStringFourCc, m_stringTable, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk(SerialSourceLocData::kDebugLineInfoFourCc, m_lineInfos, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk(SerialSourceLocData::kDebugAdjustedLineInfoFourCc, m_adjustedLineInfos, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk(moduleCompressionType, SerialSourceLocData::kDebugSourceInfoFourCc, m_sourceInfos, container)); return SLANG_OK; } -/* static */Result DebugSerialData::readContainer(SerialCompressionType moduleCompressionType, RiffContainer::ListChunk* listChunk) +/* static */Result SerialSourceLocData::readContainer(SerialCompressionType moduleCompressionType, RiffContainer::ListChunk* listChunk) { - SLANG_ASSERT(listChunk->getSubType() == DebugSerialData::kDebugFourCc); + SLANG_ASSERT(listChunk->getSubType() == SerialSourceLocData::kDebugFourCc); clear(); for (RiffContainer::Chunk* chunk = listChunk->m_containedChunks; chunk; chunk = chunk->m_next) @@ -375,24 +375,24 @@ SlangResult DebugSerialReader::read(const DebugSerialData* serialData, SourceMan switch (dataChunk->m_fourCC) { - case DebugSerialData::kDebugStringFourCc: + case SerialSourceLocData::kDebugStringFourCc: { - SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_debugStringTable)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_stringTable)); break; } - case DebugSerialData::kDebugLineInfoFourCc: + case SerialSourceLocData::kDebugLineInfoFourCc: { - SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_debugLineInfos)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_lineInfos)); break; } - case DebugSerialData::kDebugAdjustedLineInfoFourCc: + case SerialSourceLocData::kDebugAdjustedLineInfoFourCc: { - SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_debugAdjustedLineInfos)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_adjustedLineInfos)); break; } - case DebugSerialData::kDebugSourceInfoFourCc: + case SerialSourceLocData::kDebugSourceInfoFourCc: { - SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(moduleCompressionType, dataChunk, m_debugSourceInfos)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(moduleCompressionType, dataChunk, m_sourceInfos)); break; } } diff --git a/source/slang/slang-serialize-debug.h b/source/slang/slang-serialize-source-loc.h index ce39a058d..c8f06d6eb 100644 --- a/source/slang/slang-serialize-debug.h +++ b/source/slang/slang-serialize-source-loc.h @@ -1,6 +1,6 @@ -// slang-serialize-debug.h -#ifndef SLANG_SERIALIZE_DEBUG_H -#define SLANG_SERIALIZE_DEBUG_H +// slang-serialize-source-loc.h +#ifndef SLANG_SERIALIZE_SOURCE_LOC_H +#define SLANG_SERIALIZE_SOURCE_LOC_H #include "../core/slang-riff.h" #include "../core/slang-string-slice-pool.h" @@ -13,10 +13,10 @@ namespace Slang { -class DebugSerialData +class SerialSourceLocData { public: - typedef DebugSerialData ThisType; + typedef SerialSourceLocData ThisType; typedef uint32_t SourceLoc; typedef SerialStringData::StringIndex StringIndex; @@ -52,9 +52,9 @@ public: SourceLoc m_end; ///< The number of bytes in the source }; - struct DebugSourceInfo + struct SourceInfo { - typedef DebugSourceInfo ThisType; + typedef SourceInfo ThisType; bool operator==(const ThisType& rhs) const { @@ -79,9 +79,9 @@ public: uint32_t m_numAdjustedLineInfos; ///< The number of line infos }; - struct DebugLineInfo + struct LineInfo { - typedef DebugLineInfo ThisType; + typedef LineInfo ThisType; bool operator<(const ThisType& rhs) const { return m_lineStartOffset < rhs.m_lineStartOffset; } bool operator==(const ThisType& rhs) const { @@ -94,9 +94,9 @@ public: uint32_t m_lineIndex; ///< Original line index }; - struct DebugAdjustedLineInfo + struct AdjustedLineInfo { - typedef DebugAdjustedLineInfo ThisType; + typedef AdjustedLineInfo ThisType; bool operator==(const ThisType& rhs) const { return m_lineInfo == rhs.m_lineInfo && @@ -106,7 +106,7 @@ public: bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } bool operator<(const ThisType& rhs) const { return m_lineInfo < rhs.m_lineInfo; } - DebugLineInfo m_lineInfo; + LineInfo m_lineInfo; uint32_t m_adjustedLineIndex; ///< The line index with the adjustment (if there is any). Is 0 if m_pathStringIndex is 0. StringIndex m_pathStringIndex; ///< The path as an index }; @@ -116,10 +116,10 @@ public: Index findSourceInfoIndex(SourceLoc sourceLoc) const { - const Index numInfos = m_debugSourceInfos.getCount(); + const Index numInfos = m_sourceInfos.getCount(); for (Index i = 0; i < numInfos; ++i) { - if (m_debugSourceInfos[i].m_range.contains(sourceLoc)) + if (m_sourceInfos[i].m_range.contains(sourceLoc)) { return i; } @@ -132,33 +132,33 @@ public: Result writeContainer(SerialCompressionType moduleCompressionType, RiffContainer* container); Result readContainer(SerialCompressionType moduleCompressionType, RiffContainer::ListChunk* listChunk); - // Data only set if we have debug information - - List<char> m_debugStringTable; ///< String table for debug use only - List<DebugLineInfo> m_debugLineInfos; ///< Debug line information - List<DebugAdjustedLineInfo> m_debugAdjustedLineInfos; ///< Adjusted line infos - List<DebugSourceInfo> m_debugSourceInfos; ///< Debug source information + List<char> m_stringTable; ///< String table for debug use only + List<LineInfo> m_lineInfos; ///< Line information + List<AdjustedLineInfo> m_adjustedLineInfos; ///< Adjusted line infos + List<SourceInfo> m_sourceInfos; ///< Source infos }; -class DebugSerialReader : public RefObject +class SerialSourceLocReader : public RefObject { public: - Index findViewIndex(DebugSerialData::SourceLoc loc); + static const SerialExtraType kExtraType = SerialExtraType::DebugReader; + + Index findViewIndex(SerialSourceLocData::SourceLoc loc); - SourceLoc getSourceLoc(DebugSerialData::SourceLoc loc); + SourceLoc getSourceLoc(SerialSourceLocData::SourceLoc loc); /// Works out the amount to fix an input source loc to get a regular Slang::SourceLoc - int calcFixSourceLoc(DebugSerialData::SourceLoc loc, DebugSerialData::SourceRange& outRange); + int calcFixSourceLoc(SerialSourceLocData::SourceLoc loc, SerialSourceLocData::SourceRange& outRange); /// Calc the loc - static SourceLoc calcFixedLoc(DebugSerialData::SourceLoc loc, int fix, const DebugSerialData::SourceRange& range) { SLANG_ASSERT(range.contains(loc)); SLANG_UNUSED(range); return SourceLoc::fromRaw(SourceLoc::RawValue(loc + fix)); } + static SourceLoc calcFixedLoc(SerialSourceLocData::SourceLoc loc, int fix, const SerialSourceLocData::SourceRange& range) { SLANG_ASSERT(range.contains(loc)); SLANG_UNUSED(range); return SourceLoc::fromRaw(SourceLoc::RawValue(loc + fix)); } - SlangResult read(const DebugSerialData* serialData, SourceManager* sourceManager); + SlangResult read(const SerialSourceLocData* serialData, SourceManager* sourceManager); protected: struct View { - DebugSerialData::SourceRange m_range; + SerialSourceLocData::SourceRange m_range; SourceView* m_sourceView; }; @@ -166,15 +166,16 @@ protected: Index m_lastViewIndex = -1; ///< Caches last lookup }; -/// Used to write serialized Debug information -class DebugSerialWriter : public RefObject +/// Used to write serialized SourceLoc information +class SerialSourceLocWriter : public RefObject { public: + static const SerialExtraType kExtraType = SerialExtraType::DebugWriter; - class DebugSourceFile : public RefObject + class Source : public RefObject { public: - DebugSourceFile(SourceFile* sourceFile, SourceLoc::RawValue baseSourceLoc) : + Source(SourceFile* sourceFile, SourceLoc::RawValue baseSourceLoc) : m_sourceFile(sourceFile), m_baseSourceLoc(baseSourceLoc) { @@ -197,27 +198,27 @@ public: List<uint8_t> m_lineIndexUsed; ///< Has 1 if the line is used List<uint32_t> m_usedLineIndices; ///< Holds the lines that have been hit - List<DebugSerialData::DebugLineInfo> m_lineInfos; ///< The line infos - List<DebugSerialData::DebugAdjustedLineInfo> m_adjustedLineInfos; ///< The adjusted line infos + List<SerialSourceLocData::LineInfo> m_lineInfos; ///< The line infos + List<SerialSourceLocData::AdjustedLineInfo> m_adjustedLineInfos; ///< The adjusted line infos }; /// Add a source location. Returns the location that can be serialized. - DebugSerialData::SourceLoc addSourceLoc(SourceLoc sourceLoc); + SerialSourceLocData::SourceLoc addSourceLoc(SourceLoc sourceLoc); /// Write into outDebugData - void write(DebugSerialData* outDebugData); + void write(SerialSourceLocData* outSourceLocData); - DebugSerialWriter(SourceManager* sourceManager): + SerialSourceLocWriter(SourceManager* sourceManager): m_sourceManager(sourceManager), - m_debugStringSlicePool(StringSlicePool::Style::Default), - m_debugFreeSourceLoc(1) + m_stringSlicePool(StringSlicePool::Style::Default), + m_freeSourceLoc(1) { } SourceManager* m_sourceManager; - StringSlicePool m_debugStringSlicePool; ///< Slices held just for debug usage - SourceLoc::RawValue m_debugFreeSourceLoc; /// Locations greater than this are free - Dictionary<SourceFile*, RefPtr<DebugSourceFile> > m_debugSourceFileMap; + StringSlicePool m_stringSlicePool; ///< Slices held just for debug usage + SourceLoc::RawValue m_freeSourceLoc; ///< Locations greater than this are free + Dictionary<SourceFile*, RefPtr<Source> > m_sourceFileMap; }; } // namespace Slang diff --git a/source/slang/slang-serialize-type-info.h b/source/slang/slang-serialize-type-info.h new file mode 100644 index 000000000..5440fc201 --- /dev/null +++ b/source/slang/slang-serialize-type-info.h @@ -0,0 +1,324 @@ +// slang-serialize-type-info.h +#ifndef SLANG_SERIALIZE_TYPE_INFO_H +#define SLANG_SERIALIZE_TYPE_INFO_H + +#include "slang-serialize.h" +namespace Slang { + +/* For the serialization system to work we need to defined how native types are represented in the serialized format. +This information is defined by specializing SerialTypeInfo with the native type to be converted +This header provides conversion for common Slang types. +*/ + + +// We need to have a way to map between the two. +// If no mapping is needed, (just a copy), then we don't bother with the functions +template <typename T> +struct SerialBasicTypeInfo +{ + typedef T NativeType; + typedef T SerialType; + + // We want the alignment to be the same as the size of the type for basic types + // NOTE! Might be different from SLANG_ALIGN_OF(SerialType) + enum { SerialAlignment = sizeof(SerialType) }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(T*)serial = *(const T*)native; } + static void toNative(SerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(T*)native = *(const T*)serial; } + + static const SerialType* getType() + { + static const SerialType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; + return &type; + } +}; + +template <typename NATIVE_T, typename SERIAL_T> +struct SerialConvertTypeInfo +{ + typedef NATIVE_T NativeType; + typedef SERIAL_T SerialType; + + enum { SerialAlignment = SerialBasicTypeInfo<SERIAL_T>::SerialAlignment }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SERIAL_T*)serial = SERIAL_T(*(const NATIVE_T*)native); } + static void toNative(SerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(NATIVE_T*)native = NATIVE_T(*(const SERIAL_T*)serial); } +}; + +template <typename T> +struct SerialIdentityTypeInfo +{ + typedef T NativeType; + typedef T SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(T*)serial = *(const T*)native; } + static void toNative(SerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(T*)native = *(const T*)serial; } +}; + +// Don't need to convert the index type + +template <> +struct SerialTypeInfo<SerialIndex> : public SerialIdentityTypeInfo<SerialIndex> {}; + + +// Because is sized, we don't need to convert +template <> +struct SerialTypeInfo<FeedbackType::Kind> : public SerialIdentityTypeInfo<FeedbackType::Kind> {}; + +// Implement for Basic Types + +template <> +struct SerialTypeInfo<uint8_t> : public SerialBasicTypeInfo<uint8_t> {}; +template <> +struct SerialTypeInfo<uint16_t> : public SerialBasicTypeInfo<uint16_t> {}; +template <> +struct SerialTypeInfo<uint32_t> : public SerialBasicTypeInfo<uint32_t> {}; +template <> +struct SerialTypeInfo<uint64_t> : public SerialBasicTypeInfo<uint64_t> {}; + +template <> +struct SerialTypeInfo<int8_t> : public SerialBasicTypeInfo<int8_t> {}; +template <> +struct SerialTypeInfo<int16_t> : public SerialBasicTypeInfo<int16_t> {}; +template <> +struct SerialTypeInfo<int32_t> : public SerialBasicTypeInfo<int32_t> {}; +template <> +struct SerialTypeInfo<int64_t> : public SerialBasicTypeInfo<int64_t> {}; + +template <> +struct SerialTypeInfo<float> : public SerialBasicTypeInfo<float> {}; +template <> +struct SerialTypeInfo<double> : public SerialBasicTypeInfo<double> {}; + + +// Fixed arrays + +template <typename T, size_t N> +struct SerialTypeInfo<T[N]> +{ + typedef SerialTypeInfo<T> ElementASTSerialType; + typedef typename ElementASTSerialType::SerialType SerialElementType; + + typedef T NativeType[N]; + typedef SerialElementType SerialType[N]; + + enum { SerialAlignment = SerialTypeInfo<T>::SerialAlignment }; + + static void toSerial(SerialWriter* writer, const void* inNative, void* outSerial) + { + SerialElementType* serial = (SerialElementType*)outSerial; + const T* native = (const T*)inNative; + for (Index i = 0; i < Index(N); ++i) + { + ElementASTSerialType::toSerial(writer, native + i, serial + i); + } + } + static void toNative(SerialReader* reader, const void* inSerial, void* outNative) + { + const SerialElementType* serial = (const SerialElementType*)inSerial; + T* native = (T*)outNative; + for (Index i = 0; i < Index(N); ++i) + { + ElementASTSerialType::toNative(reader, serial + i, native + i); + } + } +}; + +// Special case bool - as we can't rely on size alignment +template <> +struct SerialTypeInfo<bool> +{ + typedef bool NativeType; + typedef uint8_t SerialType; + + enum { SerialAlignment = sizeof(SerialType) }; + + static void toSerial(SerialWriter* writer, const void* inNative, void* outSerial) + { + SLANG_UNUSED(writer); + *(SerialType*)outSerial = *(const NativeType*)inNative ? 1 : 0; + } + static void toNative(SerialReader* reader, const void* inSerial, void* outNative) + { + SLANG_UNUSED(reader); + *(NativeType*)outNative = (*(const SerialType*)inSerial) != 0; + } +}; + +// Pointer +// Could handle different pointer base types with some more template magic here, but instead went with Pointer type to keep +// things simpler. +template <typename T> +struct SerialTypeInfo<T*> +{ + typedef T* NativeType; + typedef SerialIndex SerialType; + + enum + { + SerialAlignment = SLANG_ALIGN_OF(SerialType) + }; + + static void toSerial(SerialWriter* writer, const void* inNative, void* outSerial) + { + *(SerialType*)outSerial = writer->addPointer(*(T**)inNative); + } + static void toNative(SerialReader* reader, const void* inSerial, void* outNative) + { + *(T**)outNative = reader->getPointer(*(const SerialType*)inSerial).dynamicCast<T>(); + } +}; + +// Special case Name +template <> +struct SerialTypeInfo<Name*> : public SerialTypeInfo<RefObject*> +{ + // Special case + typedef Name* NativeType; + static void toNative(SerialReader* reader, const void* inSerial, void* outNative) + { + *(Name**)outNative = reader->getName(*(const SerialType*)inSerial); + } +}; + +template <> +struct SerialTypeInfo<const Name*> : public SerialTypeInfo<Name*> +{ +}; + +// List +template <typename T, typename ALLOCATOR> +struct SerialTypeInfo<List<T, ALLOCATOR>> +{ + typedef List<T, ALLOCATOR> NativeType; + typedef SerialIndex SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + dst = writer->addArray(src.getBuffer(), src.getCount()); + } + static void toNative(SerialReader* reader, const void* serial, void* native) + { + auto& dst = *(NativeType*)native; + auto& src = *(const SerialType*)serial; + + reader->getArray(src, dst); + } +}; + +// Dictionary +template <typename KEY, typename VALUE> +struct SerialTypeInfo<Dictionary<KEY, VALUE>> +{ + typedef Dictionary<KEY, VALUE> NativeType; + struct SerialType + { + SerialIndex keys; ///< Index an array + SerialIndex values; ///< Index an array + }; + + typedef typename SerialTypeInfo<KEY>::SerialType KeySerialType; + typedef typename SerialTypeInfo<VALUE>::SerialType ValueSerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + List<KeySerialType> keys; + List<ValueSerialType> values; + + Index count = Index(src.Count()); + keys.setCount(count); + values.setCount(count); + + Index i = 0; + for (const auto& pair : src) + { + SerialTypeInfo<KEY>::toSerial(writer, &pair.Key, &keys[i]); + SerialTypeInfo<VALUE>::toSerial(writer, &pair.Value, &values[i]); + i++; + } + + dst.keys = writer->addArray(keys.getBuffer(), count); + dst.values = writer->addArray(values.getBuffer(), count); + } + static void toNative(SerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + // Clear it + dst = NativeType(); + + List<KEY> keys; + List<VALUE> values; + + reader->getArray(src.keys, keys); + reader->getArray(src.values, values); + + SLANG_ASSERT(keys.getCount() == values.getCount()); + + const Index count = keys.getCount(); + for (Index i = 0; i < count; ++i) + { + dst.Add(keys[i], values[i]); + } + } +}; + +// Handle RefPtr - just convert into * to do the conversion +template <typename T> +struct SerialTypeInfo<RefPtr<T>> +{ + typedef RefPtr<T> NativeType; + typedef typename SerialTypeInfo<T*>::SerialType SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + T* obj = src; + SerialTypeInfo<T*>::toSerial(writer, &obj, serial); + } + static void toNative(SerialReader* reader, const void* serial, void* native) + { + T* obj = nullptr; + SerialTypeInfo<T*>::toNative(reader, serial, &obj); + *(NativeType*)native = obj; + } +}; + +// String +template <> +struct SerialTypeInfo<String> +{ + typedef String NativeType; + typedef SerialIndex SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + *(SerialType*)serial = writer->addString(src); + } + static void toNative(SerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + dst = reader->getString(src); + } +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-types.h b/source/slang/slang-serialize-types.h index b0e28efcb..8df2f362f 100644 --- a/source/slang/slang-serialize-types.h +++ b/source/slang/slang-serialize-types.h @@ -6,11 +6,19 @@ #include "../core/slang-string-slice-pool.h" #include "../core/slang-array-view.h" -#include "slang-name.h" -#include "slang-source-loc.h" +//#include "slang-name.h" +//#include "slang-source-loc.h" namespace Slang { +// An enumeration of types that can be set +enum class SerialExtraType +{ + DebugReader, + DebugWriter, + CountOf, +}; + // Options for IR/AST/Debug serialization struct SerialOptionFlag diff --git a/source/slang/slang-serialize.cpp b/source/slang/slang-serialize.cpp new file mode 100644 index 000000000..06ddbdde0 --- /dev/null +++ b/source/slang/slang-serialize.cpp @@ -0,0 +1,892 @@ +// slang-serialize.cpp +#include "slang-serialize.h" + +#include "slang-ast-base.h" + +namespace Slang { + +const SerialClass* SerialClasses::add(const SerialClass* cls) +{ + List<const SerialClass*>& classes = m_classesByTypeKind[Index(cls->typeKind)]; + + if (cls->subType >= classes.getCount()) + { + classes.setCount(cls->subType + 1); + } + else + { + if (classes[cls->subType]) + { + SLANG_ASSERT(!"Type is already set"); + return nullptr; + } + } + + SerialClass* copy = _createSerialClass(cls); + classes[cls->subType] = copy; + + return copy; +} + +const SerialClass* SerialClasses::add(SerialTypeKind kind, SerialSubType subType, const SerialField* fields, Index fieldsCount, const SerialClass* superCls) +{ + SerialClass cls; + cls.typeKind = kind; + cls.subType = subType; + + cls.fields = fields; + cls.fieldsCount = fieldsCount; + + // If the superCls is set it must be owned + SLANG_ASSERT(superCls == nullptr || isOwned(superCls)); + + cls.super = superCls; + + // Set to invalid values for now + cls.alignment = 0; + cls.size = 0; + cls.flags = 0; + + return add(&cls); +} + +const SerialClass* SerialClasses::addUnserialized(SerialTypeKind kind, SerialSubType subType) +{ + List<const SerialClass*>& classes = m_classesByTypeKind[Index(kind)]; + + if (subType >= classes.getCount()) + { + classes.setCount(subType + 1); + } + else + { + if (classes[subType]) + { + SLANG_ASSERT(!"Type is already set"); + return nullptr; + } + } + + SerialClass* dst = m_arena.allocate<SerialClass>(); + + dst->typeKind = kind; + dst->subType = subType; + + dst->size = 0; + dst->alignment = 0; + + dst->fields = nullptr; + dst->fieldsCount = 0; + dst->flags = SerialClassFlag::DontSerialize; + dst->super = nullptr; + + classes[subType] = dst; + return dst; +} + +bool SerialClasses::isOwned(const SerialClass* cls) const +{ + const List<const SerialClass*>& classes = m_classesByTypeKind[Index(cls->typeKind)]; + return cls->subType < classes.getCount() && classes[cls->subType] == cls; +} + +SerialClass* SerialClasses::_createSerialClass(const SerialClass* cls) +{ + uint32_t maxAlignment = 1; + uint32_t offset = 0; + + if (cls->super) + { + SLANG_ASSERT(isOwned(cls->super)); + + maxAlignment = cls->super->alignment; + offset = cls->super->size; + } + + // Can't be 0 + SLANG_ASSERT(maxAlignment != 0); + // Must be a power of 2 + SLANG_ASSERT((maxAlignment & (maxAlignment - 1)) == 0); + + // Check it is correctly aligned + SLANG_ASSERT((offset & (maxAlignment - 1)) == 0); + + SerialField* dstFields = m_arena.allocateArray<SerialField>(cls->fieldsCount); + + // Okay, go through fields setting their offset + const SerialField* srcFields = cls->fields; + for (Index j = 0; j < cls->fieldsCount; j++) + { + const SerialField& srcField = srcFields[j]; + SerialField& dstField = dstFields[j]; + + // Copy the field + dstField = srcField; + + uint32_t alignment = srcField.type->serialAlignment; + // Make sure the offset is aligned for the field requirement + offset = (offset + alignment - 1) & ~(alignment - 1); + + // Save the field offset + dstField.serialOffset = uint32_t(offset); + + // Move past the field + offset += uint32_t(srcField.type->serialSizeInBytes); + + // Calc the maximum alignment + maxAlignment = (alignment > maxAlignment) ? alignment : maxAlignment; + } + + // Align with maximum alignment + offset = (offset + maxAlignment - 1) & ~(maxAlignment - 1); + + SerialClass* dst = m_arena.allocate<SerialClass>(); + *dst = *cls; + + dst->alignment = uint8_t(maxAlignment); + dst->size = uint32_t(offset); + + dst->fields = dstFields; + + return dst; +} + +SerialClasses::SerialClasses(): + m_arena(2048) +{ +} + +// For now just use an extern so we don't need to include AST serialize +extern void addASTTypes(SerialClasses* serialClasses); +extern RefObjectSerialSubType getRefObjectSubType(const RefObject* obj); + +/* static */SlangResult SerialClasses::create(RefPtr<SerialClasses>& out) +{ + RefPtr<SerialClasses> classes(new SerialClasses); + addASTTypes(classes); + + out = classes; + return SLANG_OK; +} + +/* static */RefObjectSerialSubType SerialClasses::getSubType(const RefObject* obj) +{ + return getRefObjectSubType(obj); +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!! + +SerialWriter::SerialWriter(SerialClasses* classes, SerialFilter* filter) : + m_arena(2048), + m_classes(classes), + m_filter(filter) +{ + // 0 is always the null pointer + m_entries.add(nullptr); + m_ptrMap.Add(nullptr, 0); +} + +SerialIndex SerialWriter::writeObject(const SerialClass* serialCls, const void* ptr) +{ + if (serialCls->flags & SerialClassFlag::DontSerialize) + { + return SerialIndex(0); + } + + // This pointer cannot be in the map + SLANG_ASSERT(m_ptrMap.TryGetValue(ptr) == nullptr); + + typedef SerialInfo::ObjectEntry ObjectEntry; + + ObjectEntry* nodeEntry = (ObjectEntry*)m_arena.allocateAligned(sizeof(ObjectEntry) + serialCls->size, SerialInfo::MAX_ALIGNMENT); + + nodeEntry->typeKind = serialCls->typeKind; + nodeEntry->subType = serialCls->subType; + + nodeEntry->info = SerialInfo::makeEntryInfo(serialCls->alignment); + + // We add before adding fields, so if the fields point to this, the entry will be set + auto index = _add(ptr, nodeEntry); + + // Point to start of payload + uint8_t* serialPayload = (uint8_t*)(nodeEntry + 1); + while (serialCls) + { + for (Index i = 0; i < serialCls->fieldsCount; ++i) + { + auto field = serialCls->fields[i]; + + // Work out the offsets + auto srcField = ((const uint8_t*)ptr) + field.nativeOffset; + auto dstField = serialPayload + field.serialOffset; + + field.type->toSerialFunc(this, srcField, dstField); + } + + // Get the super class + serialCls = serialCls->super; + } + + return index; +} + +SerialIndex SerialWriter::writeObject(const NodeBase* node) +{ + const SerialClass* serialClass = m_classes->getSerialClass(SerialTypeKind::NodeBase, SerialSubType(node->astNodeType)); + return writeObject(serialClass, (const void*)node); +} + +SerialIndex SerialWriter::writeObject(const RefObject* obj) +{ + const RefObjectSerialSubType subType = SerialClasses::getSubType(obj); + if (subType == RefObjectSerialSubType::Invalid) + { + SLANG_ASSERT(!"Unhandled type"); + return SerialIndex(0); + } + + const SerialClass* serialClass = m_classes->getSerialClass(SerialTypeKind::RefObject, SerialSubType(subType)); + return writeObject(serialClass, (const void*)obj); +} + +void SerialWriter::setPointerIndex(const NodeBase* ptr, SerialIndex index) +{ + m_ptrMap.Add(ptr, Index(index)); +} + +SerialIndex SerialWriter::addPointer(const NodeBase* node) +{ + // Null is always 0 + if (node == nullptr) + { + return SerialIndex(0); + } + // Look up in the map + Index* indexPtr = m_ptrMap.TryGetValue(node); + if (indexPtr) + { + return SerialIndex(*indexPtr); + } + + if (m_filter) + { + return m_filter->writePointer(this, node); + } + else + { + return writeObject(node); + } +} + +SerialIndex SerialWriter::addPointer(const RefObject* obj) +{ + // Null is always 0 + if (obj == nullptr) + { + return SerialIndex(0); + } + // Look up in the map + Index* indexPtr = m_ptrMap.TryGetValue(obj); + if (indexPtr) + { + return SerialIndex(*indexPtr); + } + + // TODO(JS): + // Arguably the lookup for these types should be done the same way as arbitrary RefObject types + // and have a enum for them, such we can use a switch instead of all this casting + + if (auto stringRep = dynamicCast<StringRepresentation>(obj)) + { + SerialIndex index = addString(StringRepresentation::asSlice(stringRep)); + m_ptrMap.Add(obj, Index(index)); + return index; + } + else if (auto name = dynamicCast<const Name>(obj)) + { + return addName(name); + } + + return writeObject(obj); +} + +SerialIndex SerialWriter::addString(const UnownedStringSlice& slice) +{ + typedef ByteEncodeUtil Util; + typedef SerialInfo::StringEntry StringEntry; + + if (slice.getLength() == 0) + { + return SerialIndex(0); + } + + Index newIndex = m_entries.getCount(); + + Index* indexPtr = m_sliceMap.TryGetValueOrAdd(slice, newIndex); + if (indexPtr) + { + return SerialIndex(*indexPtr); + } + + // Okay we need to add the string + + uint8_t encodeBuf[Util::kMaxLiteEncodeUInt32]; + const int encodeCount = Util::encodeLiteUInt32(uint32_t(slice.getLength()), encodeBuf); + + StringEntry* entry = (StringEntry*)m_arena.allocateUnaligned(SLANG_OFFSET_OF(StringEntry, sizeAndChars) + encodeCount + slice.getLength()); + entry->info = SerialInfo::EntryInfo::Alignment1; + entry->typeKind = SerialTypeKind::String; + + uint8_t* dst = (uint8_t*)(entry->sizeAndChars); + for (int i = 0; i < encodeCount; ++i) + { + dst[i] = encodeBuf[i]; + } + + memcpy(dst + encodeCount, slice.begin(), slice.getLength()); + + m_entries.add(entry); + return SerialIndex(newIndex); +} + +SerialIndex SerialWriter::addString(const String& in) +{ + return addPointer(in.getStringRepresentation()); +} + +SerialIndex SerialWriter::addName(const Name* name) +{ + if (name == nullptr) + { + return SerialIndex(0); + } + + // Look it up + Index* indexPtr = m_ptrMap.TryGetValue(name); + if (indexPtr) + { + return SerialIndex(*indexPtr); + } + + SerialIndex index = addString(name->text); + m_ptrMap.Add(name, Index(index)); + return index; +} + +SerialIndex SerialWriter::_addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount) +{ + typedef SerialInfo::ArrayEntry Entry; + + if (elementCount == 0) + { + return SerialIndex(0); + } + + SLANG_ASSERT(alignment >= 1 && alignment <= SerialInfo::MAX_ALIGNMENT); + + // We must at a minimum have the alignment for the array prefix info + alignment = (alignment < SLANG_ALIGN_OF(Entry)) ? SLANG_ALIGN_OF(Entry) : alignment; + + size_t payloadSize = elementCount * elementSize; + + Entry* entry = (Entry*)m_arena.allocateAligned(sizeof(Entry) + payloadSize, alignment); + + entry->typeKind = SerialTypeKind::Array; + entry->info = SerialInfo::makeEntryInfo(int(alignment)); + entry->elementSize = uint16_t(elementSize); + entry->elementCount = uint32_t(elementCount); + + memcpy(entry + 1, elements, payloadSize); + + m_entries.add(entry); + return SerialIndex(m_entries.getCount() - 1); +} + +static const uint8_t s_fixBuffer[SerialInfo::MAX_ALIGNMENT]{ 0, }; + +SlangResult SerialWriter::write(Stream* stream) +{ + const Int entriesCount = m_entries.getCount(); + + // Add a sentinal so we don't need special handling for + SerialInfo::Entry sentinal; + sentinal.typeKind = SerialTypeKind::String; + sentinal.info = SerialInfo::EntryInfo::Alignment1; + + m_entries.add(&sentinal); + m_entries.removeLast(); + + SerialInfo::Entry** entries = m_entries.getBuffer(); + // Note strictly required in our impl of List. But by writing this and + // knowing that removeLast cannot release memory, means the sentinal must be at the last position. + entries[entriesCount] = &sentinal; + + { + size_t offset = 0; + + SerialInfo::Entry* entry = entries[1]; + // We start on 1, because 0 is nullptr and not used for anything + for (Index i = 1; i < entriesCount; ++i) + { + SerialInfo::Entry* next = entries[i + 1]; + // Before writing we need to store the next alignment + + const size_t nextAlignment = SerialInfo::getAlignment(next->info); + const size_t alignment = SerialInfo::getAlignment(entry->info); + + entry->info = SerialInfo::combineWithNext(entry->info, next->info); + + // Check we are aligned correctly + SLANG_ASSERT((offset & (alignment - 1)) == 0); + + // When we write, we need to make sure it take into account the next alignment + const size_t entrySize = entry->calcSize(m_classes); + + // Work out the fix for next alignment + size_t nextOffset = offset + entrySize; + nextOffset = (nextOffset + nextAlignment - 1) & ~(nextAlignment - 1); + + size_t alignmentFixSize = nextOffset - (offset + entrySize); + + // The fix must be less than max alignment. We require it to be less because we aligned each Entry to + // MAX_ALIGNMENT, and so < MAX_ALIGNMENT is the most extra bytes we can write + SLANG_ASSERT( alignmentFixSize < SerialInfo::MAX_ALIGNMENT); + + try + { + stream->write(entry, entrySize); + // If we needed to fix so that subsequent alignment is right, write out extra bytes here + if (alignmentFixSize) + { + stream->write(s_fixBuffer, alignmentFixSize); + } + } + catch (const IOException&) + { + return SLANG_FAIL; + } + + // Onto next + offset = nextOffset; + entry = next; + } + } + + return SLANG_OK; +} + +SlangResult SerialWriter::writeIntoContainer(FourCC fourCc, RiffContainer* container) +{ + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ScopeChunk ScopeChunk; + + { + ScopeChunk scopeData(container, Chunk::Kind::Data, fourCc); + + { + // Sentinel so we don't need special handling for end of list + SerialInfo::Entry sentinal; + sentinal.typeKind = SerialTypeKind::String; + sentinal.info = SerialInfo::EntryInfo::Alignment1; + + size_t offset = 0; + const Int entriesCount = m_entries.getCount(); + + { + m_entries.add(&sentinal); + m_entries.removeLast(); + // Note strictly required in our impl of List. But by writing this and + // knowing that removeLast cannot release memory, means the sentinal must be at the last position. + m_entries.getBuffer()[entriesCount] = &sentinal; + } + + SerialInfo::Entry*const* entries = m_entries.getBuffer(); + + SerialInfo::Entry* entry = entries[1]; + // We start on 1, because 0 is nullptr and not used for anything + for (Index i = 1; i < entriesCount; ++i) + { + SerialInfo::Entry* next = entries[i + 1]; + + // Before writing we need to store the next alignment + + const size_t nextAlignment = SerialInfo::getAlignment(next->info); + const size_t alignment = SerialInfo::getAlignment(entry->info); + + entry->info = SerialInfo::combineWithNext(entry->info, next->info); + + // Check we are aligned correctly + SLANG_ASSERT((offset & (alignment - 1)) == 0); + + // When we write, we need to make sure it take into account the next alignment + const size_t entrySize = entry->calcSize(m_classes); + + // Work out the fix for next alignment + size_t nextOffset = offset + entrySize; + nextOffset = (nextOffset + nextAlignment - 1) & ~(nextAlignment - 1); + + size_t alignmentFixSize = nextOffset - (offset + entrySize); + + // The fix must be less than max alignment. We require it to be less because we aligned each Entry to + // MAX_ALIGNMENT, and so < MAX_ALIGNMENT is the most extra bytes we can write + SLANG_ASSERT(alignmentFixSize < SerialInfo::MAX_ALIGNMENT); + + container->write(entry, entrySize); + if (alignmentFixSize) + { + container->write(s_fixBuffer, alignmentFixSize); + } + + // Onto next + offset = nextOffset; + entry = next; + } + } + } + + return SLANG_OK; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialInfo::Entry !!!!!!!!!!!!!!!!!!!!!!!! + +size_t SerialInfo::Entry::calcSize(SerialClasses* serialClasses) const +{ + switch (typeKind) + { + case SerialTypeKind::String: + { + auto entry = static_cast<const StringEntry*>(this); + const uint8_t* cur = (const uint8_t*)entry->sizeAndChars; + uint32_t charsSize; + int sizeSize = ByteEncodeUtil::decodeLiteUInt32(cur, &charsSize); + return SLANG_OFFSET_OF(StringEntry, sizeAndChars) + sizeSize + charsSize; + } + case SerialTypeKind::Array: + { + auto entry = static_cast<const ArrayEntry*>(this); + return sizeof(ArrayEntry) + entry->elementSize * entry->elementCount; + } + case SerialTypeKind::RefObject: + case SerialTypeKind::NodeBase: + { + auto entry = static_cast<const ObjectEntry*>(this); + + auto serialClass = serialClasses->getSerialClass(typeKind, entry->subType); + + // Align by the alignment of the entry + size_t alignment = getAlignment(entry->info); + size_t size = sizeof(ObjectEntry) + serialClass->size; + + size = size + (alignment - 1) & ~(alignment - 1); + return size; + } + + default: break; + } + + SLANG_ASSERT(!"Unknown type"); + return 0; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!! + +SerialReader::~SerialReader() +{ + for (const RefObject* obj : m_scope) + { + const_cast<RefObject*>(obj)->releaseReference(); + } +} + +const void* SerialReader::getArray(SerialIndex index, Index& outCount) +{ + if (index == SerialIndex(0)) + { + outCount = 0; + return nullptr; + } + + SLANG_ASSERT(SerialIndexRaw(index) < SerialIndexRaw(m_entries.getCount())); + const Entry* entry = m_entries[Index(index)]; + + switch (entry->typeKind) + { + case SerialTypeKind::Array: + { + auto arrayEntry = static_cast<const SerialInfo::ArrayEntry*>(entry); + outCount = Index(arrayEntry->elementCount); + return (arrayEntry + 1); + } + default: break; + } + + SLANG_ASSERT(!"Not an array"); + outCount = 0; + return nullptr; +} + +SerialPointer SerialReader::getPointer(SerialIndex index) +{ + if (index == SerialIndex(0)) + { + return SerialPointer(); + } + + SLANG_ASSERT(SerialIndexRaw(index) < SerialIndexRaw(m_entries.getCount())); + const Entry* entry = m_entries[Index(index)]; + + switch (entry->typeKind) + { + case SerialTypeKind::String: + { + // Hmm. Tricky -> we don't know if will be cast as Name or String. Lets assume string. + String string = getString(index); + return SerialPointer(string.getStringRepresentation()); + } + case SerialTypeKind::NodeBase: + { + return SerialPointer((NodeBase*)m_objects[Index(index)]); + } + case SerialTypeKind::RefObject: + { + return SerialPointer((RefObject*)m_objects[Index(index)]); + } + default: break; + } + + SLANG_ASSERT(!"Cannot access as a pointer"); + return SerialPointer(); +} + +String SerialReader::getString(SerialIndex index) +{ + if (index == SerialIndex(0)) + { + return String(); + } + + SLANG_ASSERT(SerialIndexRaw(index) < SerialIndexRaw(m_entries.getCount())); + const Entry* entry = m_entries[Index(index)]; + + // It has to be a string type + if (entry->typeKind != SerialTypeKind::String) + { + SLANG_ASSERT(!"Not a string"); + return String(); + } + + RefObject* obj = (RefObject*)m_objects[Index(index)]; + + if (obj) + { + StringRepresentation* stringRep = dynamicCast<StringRepresentation>(obj); + if (stringRep) + { + return String(stringRep); + } + // Must be a name then + Name* name = dynamicCast<Name>(obj); + SLANG_ASSERT(name); + return name->text; + } + + // Okay we need to construct as a string + UnownedStringSlice slice = getStringSlice(index); + String string(slice); + StringRepresentation* stringRep = string.getStringRepresentation(); + + m_scope.add(stringRep); + m_objects[Index(index)] = stringRep; + return string; +} + +Name* SerialReader::getName(SerialIndex index) +{ + if (index == SerialIndex(0)) + { + return nullptr; + } + + SLANG_ASSERT(SerialIndexRaw(index) < SerialIndexRaw(m_entries.getCount())); + const Entry* entry = m_entries[Index(index)]; + + // It has to be a string type + if (entry->typeKind != SerialTypeKind::String) + { + SLANG_ASSERT(!"Not a string"); + return nullptr; + } + + RefObject* obj = (RefObject*)m_objects[Index(index)]; + + if (obj) + { + Name* name = dynamicCast<Name>(obj); + if (name) + { + return name; + } + // Can only be a string then + StringRepresentation* stringRep = dynamicCast<StringRepresentation>(obj); + SLANG_ASSERT(stringRep); + + // I don't need to scope, as scoped in NamePool + name = m_namePool->getName(String(stringRep)); + + // Store as name, as can always access the inner string if needed + m_objects[Index(index)] = name; + return name; + } + + UnownedStringSlice slice = getStringSlice(index); + String string(slice); + Name* name = m_namePool->getName(string); + // Don't need to add to scope, because scoped on the pool + m_objects[Index(index)] = name; + return name; +} + +UnownedStringSlice SerialReader::getStringSlice(SerialIndex index) +{ + SLANG_ASSERT(SerialIndexRaw(index) < SerialIndexRaw(m_entries.getCount())); + const Entry* entry = m_entries[Index(index)]; + + // It has to be a string type + if (entry->typeKind != SerialTypeKind::String) + { + SLANG_ASSERT(!"Not a string"); + return UnownedStringSlice(); + } + + auto stringEntry = static_cast<const SerialInfo::StringEntry*>(entry); + + const uint8_t* src = (const uint8_t*)stringEntry->sizeAndChars; + + // Decode the string + uint32_t size; + int sizeSize = ByteEncodeUtil::decodeLiteUInt32(src, &size); + return UnownedStringSlice((const char*)src + sizeSize, size); +} + +SlangResult SerialReader::loadEntries(const uint8_t* data, size_t dataCount, List<const SerialInfo::Entry*>& outEntries) +{ + // Check the input data is at least aligned to the max alignment (otherwise everything cannot be aligned correctly) + SLANG_ASSERT((size_t(data) & (SerialInfo::MAX_ALIGNMENT - 1)) == 0); + + outEntries.setCount(1); + outEntries[0] = nullptr; + + const uint8_t*const end = data + dataCount; + + const uint8_t* cur = data; + while (cur < end) + { + const Entry* entry = (const Entry*)cur; + outEntries.add(entry); + + const size_t entrySize = entry->calcSize(m_classes); + cur += entrySize; + + // Need to get the next alignment + const size_t nextAlignment = SerialInfo::getNextAlignment(entry->info); + + // Need to fix cur with the alignment + cur = (const uint8_t*)((size_t(cur) + nextAlignment - 1) & ~(nextAlignment - 1)); + } + + return SLANG_OK; +} + +SlangResult SerialReader::load(const uint8_t* data, size_t dataCount, NamePool* namePool) +{ + SLANG_RETURN_ON_FAIL(loadEntries(data, dataCount, m_entries)); + + m_namePool = namePool; + + m_objects.clearAndDeallocate(); + m_objects.setCount(m_entries.getCount()); + memset(m_objects.getBuffer(), 0, m_objects.getCount() * sizeof(void*)); + + // Go through entries, constructing objects. + for (Index i = 1; i < m_entries.getCount(); ++i) + { + const Entry* entry = m_entries[i]; + + switch (entry->typeKind) + { + case SerialTypeKind::String: + { + // Don't need to construct an object. This is probably a StringRepresentation, or a Name + // Will evaluate lazily. + break; + } + case SerialTypeKind::RefObject: + case SerialTypeKind::NodeBase: + { + auto objectEntry = static_cast<const SerialInfo::ObjectEntry*>(entry); + void* obj = m_objectFactory->create(objectEntry->typeKind, objectEntry->subType); + if (!obj) + { + return SLANG_FAIL; + } + m_objects[i] = obj; + break; + } + case SerialTypeKind::Array: + { + // Don't need to construct an object, as will be accessed an interpreted by the object that holds it + break; + } + } + } + + // Deserialize + for (Index i = 1; i < m_entries.getCount(); ++i) + { + const Entry* entry = m_entries[i]; + void* native = m_objects[i]; + if (!native) + { + continue; + } + switch (entry->typeKind) + { + case SerialTypeKind::NodeBase: + case SerialTypeKind::RefObject: + { + auto objectEntry = static_cast<const SerialInfo::ObjectEntry*>(entry); + auto serialClass = m_classes->getSerialClass(objectEntry->typeKind, objectEntry->subType); + if (!serialClass) + { + return SLANG_FAIL; + } + + const uint8_t* src = (const uint8_t*)(objectEntry + 1); + uint8_t* dst = (uint8_t*)m_objects[i]; + + // It must be constructed + SLANG_ASSERT(dst); + + while (serialClass) + { + for (Index j = 0; j < serialClass->fieldsCount; ++j) + { + auto field = serialClass->fields[j]; + auto fieldType = field.type; + fieldType->toNativeFunc(this, src + field.serialOffset, dst + field.nativeOffset); + } + + // Get the super class + serialClass = serialClass->super; + } + + break; + } + default: break; + } + } + + return SLANG_OK; +} + +} // namespace Slang diff --git a/source/slang/slang-serialize.h b/source/slang/slang-serialize.h new file mode 100644 index 000000000..d43bb7960 --- /dev/null +++ b/source/slang/slang-serialize.h @@ -0,0 +1,749 @@ +// slang-serialize.h +#ifndef SLANG_SERIALIZE_H +#define SLANG_SERIALIZE_H + +#include <type_traits> + +#include "../core/slang-riff.h" +#include "../core/slang-byte-encode-util.h" + +#include "../core/slang-stream.h" + +#include "slang-serialize-types.h" + +#include "slang-name.h" + +namespace Slang +{ + +class Linkage; + +/* +General Serialization Overview +============================== + +The AST node types are generally types derived from the NodeBase. The C++ extractor is used to associate an ASTNodeType with +every NodeBase type, such that casting is fast and simple and we have a simple integer to uniquely identify those types. The +extractor also performs another task of associating with the type name all of the fields held in just that type. The definition +of the fields is stored in an 'x macro' which is in the slang-ast-generated-macro.h file, for example + +``` +#define SLANG_FIELDS_ASTNode_DeclRefExpr(_x_, _param_)\ + _x_(scope, (RefPtr<Scope>), _param_)\ + _x_(declRef, (DeclRef<Decl>), _param_)\ + _x_(name, (Name*), _param_) +`` + +For the type DeclRefExpr, this holds all of the fields held in just DeclRefExpr in this case `scope`, `declRef` and `name`. +DeclRefExpr derives from Expr and this might hold other fields and so forth. + +The implementation makes a distinction between the 'native' types, the regular C++ in memory types and 'serial' types. +Each serializable C++ type has an associated 'serial' type - with the distinction that it can be written out and (with perhaps some other data) +read back in to recreate the C++ type. The serial type can be a C++ type, but is such it can be written and read from disk and still +represent the same data. + +We need a mechanism to be able to do do a conversion between native and serial types. To make the association we use the template + +``` +template <typename T> +struct SerialTypeInfo; +``` + +and specialize it for each native type. The specialization holds + +SerialType - The type that will be used to represent the native type +NativeType - The native type +SerialAlignment - A value that holds what kind of alignment the SerialType needs to be serializable (it may be different from SLANG_ALIGN_OF(SerialType)!) +toSerial - A function that with the help of ASTSerialWriter convert the NativeType into the SerialType +toNative - A function that with the help of ASTSerialReader convert the SerialType into the NativeType + +It is useful to have a structure that holds the type information, so it can be stored. That is achieved with + +``` +template <typename T> +struct SerialGetType; +``` + +This template can be specialized for a specific native types - but all it holds is just a function getType, which returns a SerialType*, +which just holds the information held in the SerialTypeInfo template, but additionally including the size of the SerialType. + +So we need to define a specialized SerialTypeInfo for each type that can be a field in a NodeBase/RefObject derived type. We don't need to define +anything explicitly for the NodeBase derived types, as we will just generate the layout from the fields. How do we know the fields? We just +used the macros generated from the C++ extractor. + +So first a few things to observe... + +1) Some types don't need any conversion to be serializable - int8_t, or float the bits can just be written out and read in (1) +2) Some types need a conversion but it's very simple - for example an enum without explicit size, being written as an explicit size +3) Some types can be written out but would not be directly readable or usable with different targets/processors, so need converting +4) Some types require complex conversions that require programmer code - like Dictionary/List + +For types that need no conversion (1), we can just use the template SerialIdentityTypeInfo + +``` +template <> +struct SerialTypeInfo<SomeType> : public SerialIdentityTypeInfo<SomeType> {}; +``` + +This specialization means that SomeType can be written out and read in across targets/compilers without problems. + +For (2) we have another template that will do the conversion for us + +``` +template <typename NATIVE_T, typename SERIAL_T> +struct SerialConvertTypeInfo; +``` + +That we can use as above, and specify the native and serial types. + +For (3) there are a few scenarios. For any field in a serial type we must store in the serialized type such that the representation +will work across all processors/compilers. So one problematic type is `bool`. It's not specified how it's laid out in memory - and +some compiles have stored it as a word. Most recently it's been stored as a byte. To make sure bool is ok for serialization therefore +we store as a uint8_t. + +Another example would be double. It's 64 bits, but on some arches/compilers it's SLANG_ALIGN_OF is 4 and on others it's 8. On some +arches a non aligned read will lead to a fault. To work around this problem therefore we have to ensure double has the alignment that +will work across all targets - and that alignment is 8. In that specific case that issue is handled via SerialBasicTypeInfo, which +makes the SerialAlignment the sizeof the type. + +For (4) there are a few things to say. First a type can always implement a custom version of how to do a conversion by specializing +`SerialTypeInfo`. But there remains another nagging issue - types which allocate/use other memory that changes at runtime. Clearly +we cannot define 'any size of memory' in a fixed SerialType defined in a specialization of SerialTypeInfo. The mechanism to work around +this is to allow arbitrary arrays to be stored, that can be accessed via an SerialIndex. This will be discussed more once we discuss +a little more about the file system, and SerialIndex. + +Serialization Format +==================== + +The serialization format used is 'stream-like' with each 'object' stored in order. Each object is given an index starting from 1. +0 is used to be in effect nullptr. The stream looks like + +``` +SerialInfo::Entry (for index 1) +Payload for type in entry + +SerialInfo::Entry (for index 2) +Payload for type in entry + +... +... + +That when writing we have an array that maps each index to a pointer to the associated header. We also have a map that maps native pointers +to their indices. The Payload *is* the SerialType for thing saved. The payload directly follows the Entry data. + +Each object in this list can only be a few types of things + +* NodeBase derived type +* RefObject derived type +* String +* Array + +The actual Entry followed by the payloads are allocated and stored when writing in a MemoryArena. When we want to write into a stream, we +can just iterate over each entry in order and write it out. + +You may have spotted a problem here - that some Entry types can be stored without alignment (for example a string - which stores the length +VarInt encoded followed by the characters). Others require an alignment - for example an NodeBase derived type that contains a int64_t will +*require* 8 byte alignment. That as a feature of the serialization format we want to be able to just map the data into memory, and be able +to access all the SerialType as is on the CPU. For that to work we *require* that the payload for each entry has the right alignment for +the associated SerialType. + +To achieve this we store in the Entry it's alignment requirement *AND* the next entries alignment. With this when we read, as we as stepping +through the entries we can find where the next Entry starts. Because the payload comes directly after the Entry - the Entrys size must be +a modulo of the largest alignment the payload can have. + +For the code that does the conversion between native and serial types it uses either the SerialWriter or SerialReader. This provides +the mechanism to turn a pointer into a serializable ASTSerialIndex and vice versa. There are some special functions for turning string like +types to and forth. + +The final mechanism is that of 'Arrays'. An array allows reading or writing a chunk of data associated with a ASTSerialIndex. The chunk of +data *must* hold data that is serializable. If the array holds pointers - then the serialized array must hold SerialIndices that +represent those pointers. When reading back in they are converted back. + +Arrays are the escape hatch that allows for more complex types to serialize. Dictionaries for example are saved as a serial type that is +two SerialIndices one to a keys array and one to a values array. + +Note that writing has two phases, serializing out into an SerialWriter, and then secondly writing out to a stream. + +Object/Reference Types +====================== + +When talking about Object/Reference types this means types that can be referenced natively as pointers. Currently that means NodeBase and +some RefObject derived types. + +The SerialTypeInfo mechanism is generally for *fields* of object types. That for derived types we use the C++ extractors +field list to work out the native fields offsets and types. With this we can then calculate the layout for NodeBase types such that they +follow the requirements for serialization - such as alignment and so forth. + +This information is held in the SerialClasses, which for a given TypeKind/SubType gives a SerialClassInfo, that specifies fields for +just that type. + +Reading +======= + +Due to the care in writing reading is relatively simple. We can just take the contents of the file and put in memory, as long as in memory +it has an alignment of at least MAX_ALIGNMENT. Then we can build up an entries table by stepping through the data and writing the pointer. + +The toNative functions take an SerialReader - this allows the implementation to ask for pointers and arrays from other parts of the serialized +data. It also allows for types to be lazily reconstructed if necessary. + +Lazy reconstruction may be useful in the future to partially reconstruct a sub part of the serialized data. In the current implementation, lazy +evaluation is used on Strings. The m_objects array holds all of the recreated native 'objects'. Since the objects can be derived from different +base classes the associated Entry will describe what it really is. + +For the String type, we initially store the object pointer as null. If a string is requested from that index, we see if the object pointer is null, +if it is we have to construct the StringRepresentation that will be used. + +An extra wrinkle is that we allow accessing of a serialized String as a Name or a string or a UnownedSubString. Fortunately a Name just holds a string, +and a Name remains in scope as long as it's NamePool does which is passed in. + +Other Reading issues +==================== + +## SourceLoc + +SourceLoc present a problem. If we follow the simple mechanism described above, then we require two things + +1) That the SourceLoc information is blossomed before anything that defines a SourceLoc +2) That the structure for accessing SourceLoc information is conveniently available. + +This was sidestepped previously because the SourceLoc information was held in a different structure, and a separate Riff section. It was deserialized +before anything else took place. + +That *is* a strategy we could use here. That we could make the SourceLoc information generally serialized. On loading locate it in a Riff section +deserialize it (perhaps with general serialization), then deserialize the rest using this structure. + +## IRModule + +In this case we may want to have IRModule serialized in someway unlike the generalized serialization (for example supporting compression). In other +frameworks this aspect might be handing by 'read/writeReplacing'. Doing so would significantly complicate the simple reading mechanism - because instead +of just constructing and referencing we would have to care about construction order. That this could perhaps be achieved by having any reference access +be handled lazily. Note that SourceLoc would still require being handled specially because it requires construction before any SourceLoc is referenced, +and SourceLocs *aren't* pointers. + +## Modified reading + +We could modify reading as follows. + +1) Don't construct anything at the start +2) Find 'root's they must be created and deserialized first + . Any read/writeReplace is a root + . Any marked (like SourceLocData) is a root. (When deconstructed it also needs to add information to the Reader) + . The root of the objects (note we could just deserialize first to last if not already constructed) +3) During deserialization pointer references and constructed on demand +4) Extra code is needed to make sure there aren't cycles. Any object is either Pre/Created/Deserialized. + +For now we might want to just do this with Riff sections for simplicity + +Other Issues +============ + +A final issue is around the special extra types needed for serializing or deserializing. SourceLoc information (on reading and writing), +but it could be other types in the future. + +We probably don't want to have them as specific types on the SerialReader/SerialWriter, as doing so requires exposing the types to this interface. +What we really want is a mechanism for the Reader/Writer where it's possible to get a pointer based on some type. We want this to be fairly fast +because every SourceLoc reference will have to do this lookup. + +We could use an enum, and just have an array of pointers on the reader and writer. How that pointer is interpreted is dependent on the Reader/Writer. +This would be very fast, extendable without making types specific. On debug builds we could do a dynamic cast to make sure it is the expected type. + +Rich Information +================ + +Nothing is done here about versioning, patching, backward or forward compatibility. +*/ + +// Predeclare +typedef uint32_t SerialSourceLoc; +class NodeBase; + +// Pre-declare +class SerialClasses; +class SerialWriter; +class SerialReader; + +struct SerialClass; +struct SerialField; + +// Type used to implement mechanisms to convert to and from serial types. +template <typename T> +struct SerialTypeInfo; + +enum class SerialTypeKind : uint8_t +{ + Unknown, + + String, ///< String + Array, ///< Array + + NodeBase, ///< NodeBase derived + RefObject, ///< RefObject derived types + + CountOf, +}; +typedef uint16_t SerialSubType; + +enum class RefObjectSerialSubType +{ + Invalid, ///< Invalid (ie not a known RefObject type) + LookupResultItem_Breadcrumb, + + // TODO(JS): + // ! We don't want to serialize these types. + // We could set a nullptr SerialClass in classes ? + // Perhaps we need some special SerialClass that indicates we want to always write as 0? + + Scope, + Module, + + CountOf, +}; + +struct SerialInfo +{ + enum + { + // Data held in serialized format, the maximally allowed alignment + MAX_ALIGNMENT = 8, + }; + + // We only allow up to MAX_ALIGNMENT bytes of alignment. We store alignments as shifts, so 2 bits needed for 1 - 8 + enum class EntryInfo : uint8_t + { + Alignment1 = 0, + }; + + static EntryInfo makeEntryInfo(int alignment, int nextAlignment) + { + // Make sure they are power of 2 + SLANG_ASSERT((alignment & (alignment - 1)) == 0); + SLANG_ASSERT((nextAlignment & (nextAlignment - 1)) == 0); + + const int alignmentShift = ByteEncodeUtil::calcMsb8(alignment); + const int nextAlignmentShift = ByteEncodeUtil::calcMsb8(nextAlignment); + return EntryInfo((nextAlignmentShift << 2) | alignmentShift); + } + static EntryInfo makeEntryInfo(int alignment) + { + // Make sure they are power of 2 + SLANG_ASSERT((alignment & (alignment - 1)) == 0); + return EntryInfo(ByteEncodeUtil::calcMsb8(alignment)); + } + /// Apply with the next alignment + static EntryInfo combineWithNext(EntryInfo cur, EntryInfo next) + { + return EntryInfo((int(cur) & ~0xc0) | ((int(next) & 3) << 2)); + } + + static int getAlignment(EntryInfo info) { return 1 << (int(info) & 3); } + static int getNextAlignment(EntryInfo info) { return 1 << ((int(info) >> 2) & 3); } + + /* Alignment is a little tricky. We have a 'Entry' header before the payload. The payload alignment may change. + If we only align on the Entry header, then it's size *must* be some modulo of the maximum alignment allowed. + + We could hold Entry separate from payload. We could make the header not require the alignment of the payload - but then + we'd need payload alignment separate from entry alignment. + */ + struct Entry + { + SerialTypeKind typeKind; + EntryInfo info; + + size_t calcSize(SerialClasses* serialClasses) const; + }; + + struct StringEntry : Entry + { + char sizeAndChars[1]; + }; + + struct ObjectEntry : Entry + { + SerialSubType subType; ///< Can be ASTType or other subtypes (as used for RefObjects for example) + uint32_t _pad0; ///< Necessary, because a node *can* have MAX_ALIGNEMENT + }; + + struct ArrayEntry : Entry + { + uint16_t elementSize; + uint32_t elementCount; + }; +}; + +typedef uint32_t SerialIndexRaw; +enum class SerialIndex : SerialIndexRaw; + +/* A type to convert pointers into types such that they can be passed around to readers/writers without +having to know the specific type. If there was a base class that all the serialized types derived from, +that was dynamically castable this would not be necessary */ +struct SerialPointer +{ + // Helpers so we can choose what kind of pointer we have based on the (unused) type of the pointer passed in + SLANG_FORCE_INLINE RefObject* _get(const RefObject*) { return m_kind == SerialTypeKind::RefObject ? reinterpret_cast<RefObject*>(m_ptr) : nullptr; } + SLANG_FORCE_INLINE NodeBase* _get(const NodeBase*) { return m_kind == SerialTypeKind::NodeBase ? reinterpret_cast<NodeBase*>(m_ptr) : nullptr; } + + template <typename T> + T* dynamicCast() + { + return Slang::dynamicCast<T>(_get((T*)nullptr)); + } + + SerialPointer() : + m_kind(SerialTypeKind::Unknown), + m_ptr(nullptr) + { + } + + SerialPointer(RefObject* in) : + m_kind(SerialTypeKind::RefObject), + m_ptr((void*)in) + { + } + SerialPointer(NodeBase* in) : + m_kind(SerialTypeKind::NodeBase), + m_ptr((void*)in) + { + } + + static SerialTypeKind getKind(const RefObject*) { return SerialTypeKind::RefObject; } + static SerialTypeKind getKind(const NodeBase*) { return SerialTypeKind::NodeBase; } + + SerialTypeKind m_kind; + void* m_ptr; +}; + +class SerialFilter +{ +public: + virtual SerialIndex writePointer(SerialWriter* writer, const NodeBase* ptr) = 0; +}; + +class SerialObjectFactory +{ +public: + virtual void* create(SerialTypeKind typeKind, SerialSubType subType) = 0; +}; + +class SerialExtraObjects +{ +public: + template <typename T> + void set(T* obj) { m_objects[Index(T::kExtraType)] = obj; } + template <typename T> + void set(const RefPtr<T>& obj) + { + m_objects[Index(T::kExtraType)] = obj.Ptr(); + } + + /// Get the extra type + template <typename T> + T* get() { return reinterpret_cast<T*>(m_objects[Index(T::kExtraType)]); } + + SerialExtraObjects() + { + for (auto& obj : m_objects) obj = nullptr; + } + +protected: + void* m_objects[Index(SerialExtraType::CountOf)]; +}; + +/* This class is the interface used by toNative implementations to recreate a type. */ +class SerialReader : public RefObject +{ +public: + + typedef SerialInfo::Entry Entry; + + template <typename T> + void getArray(SerialIndex index, List<T>& out); + + const void* getArray(SerialIndex index, Index& outCount); + + SerialPointer getPointer(SerialIndex index); + String getString(SerialIndex index); + Name* getName(SerialIndex index); + UnownedStringSlice getStringSlice(SerialIndex index); + + /// Load the entries table (without deserializing anything) + /// NOTE! data must stay ins scope for outEntries to be valid + SlangResult loadEntries(const uint8_t* data, size_t dataCount, List<const SerialInfo::Entry*>& outEntries); + + /// NOTE! data must stay ins scope when reading takes place + SlangResult load(const uint8_t* data, size_t dataCount, NamePool* namePool); + + /// Add an object to be kept in scope + void addScope(const RefObject* obj) { m_scope.add(obj); } + + /// Used for attaching extra objects necessary for serializing + SerialExtraObjects& getExtraObjects() { return m_extraObjects; } + + /// Ctor + SerialReader(SerialClasses* classes, SerialObjectFactory* objectFactory): + m_classes(classes), + m_objectFactory(objectFactory) + { + } + ~SerialReader(); + +protected: + List<const Entry*> m_entries; ///< The entries + List<void*> m_objects; ///< The constructed objects + NamePool* m_namePool; ///< Pool names are added to + + List<const RefObject*> m_scope; ///< Keeping objects in scope + + SerialExtraObjects m_extraObjects; + + SerialObjectFactory* m_objectFactory; + SerialClasses* m_classes; ///< Information used to deserialize +}; + +// --------------------------------------------------------------------------- +template <typename T> +void SerialReader::getArray(SerialIndex index, List<T>& out) +{ + typedef SerialTypeInfo<T> ElementTypeInfo; + typedef typename ElementTypeInfo::SerialType ElementSerialType; + + Index count; + auto serialElements = (const ElementSerialType*)getArray(index, count); + + if (count == 0) + { + out.clear(); + return; + } + + if (std::is_same<T, ElementSerialType>::value) + { + // If they are the same we can just write out + out.clear(); + out.insertRange(0, (const T*)serialElements, count); + } + else + { + // Else we need to convert + out.setCount(count); + for (Index i = 0; i < count; ++i) + { + ElementTypeInfo::toNative(this, (const void*)&serialElements[i], (void*)&out[i]); + } + } +} + +/* This is a class used tby toSerial implementations to turn native type into the serial type */ +class SerialWriter : public RefObject +{ +public: + SerialIndex addPointer(const NodeBase* ptr); + SerialIndex addPointer(const RefObject* ptr); + + /// Write the object at ptr of type serialCls + SerialIndex writeObject(const SerialClass* serialCls, const void* ptr); + + /// Write the object at the pointer + SerialIndex writeObject(const NodeBase* ptr); + SerialIndex writeObject(const RefObject* ptr); + + template <typename T> + SerialIndex addArray(const T* in, Index count); + + SerialIndex addString(const UnownedStringSlice& slice); + SerialIndex addString(const String& in); + SerialIndex addName(const Name* name); + + /// Set a the ptr associated with an index. + /// NOTE! That there cannot be a pre-existing setting. + void setPointerIndex(const NodeBase* ptr, SerialIndex index); + + /// Get the entries table holding how each index maps to an entry + const List<SerialInfo::Entry*>& getEntries() const { return m_entries; } + + /// Write to a stream + SlangResult write(Stream* stream); + + /// Write a data chunk with fourCC + SlangResult writeIntoContainer(FourCC fourCC, RiffContainer* container); + + /// Used for attaching extra objects necessary for serializing + SerialExtraObjects& getExtraObjects() { return m_extraObjects; } + + /// Ctor + SerialWriter(SerialClasses* classes, SerialFilter* filter); + +protected: + + SerialIndex _addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount); + + SerialIndex _add(const void* nativePtr, SerialInfo::Entry* entry) + { + m_entries.add(entry); + // Okay I need to allocate space for this + SerialIndex index = SerialIndex(m_entries.getCount() - 1); + // Add to the map + m_ptrMap.Add(nativePtr, Index(index)); + return index; + } + + Dictionary<const void*, Index> m_ptrMap; // Maps a pointer to an entry index + + // NOTE! Assumes the content stays in scope! + Dictionary<UnownedStringSlice, Index> m_sliceMap; + + SerialExtraObjects m_extraObjects; ///< Extra objects + + List<SerialInfo::Entry*> m_entries; ///< The entries + MemoryArena m_arena; ///< Holds the payloads + SerialClasses* m_classes; + SerialFilter* m_filter; ///< Filter to control what is serialized +}; + +// --------------------------------------------------------------------------- +template <typename T> +SerialIndex SerialWriter::addArray(const T* in, Index count) +{ + typedef SerialTypeInfo<T> ElementTypeInfo; + typedef typename ElementTypeInfo::SerialType ElementSerialType; + + if (std::is_same<T, ElementSerialType>::value) + { + // If they are the same we can just write out + return _addArray(sizeof(T), SLANG_ALIGN_OF(ElementSerialType), in, count); + } + else + { + // Else we need to convert + List<ElementSerialType> work; + work.setCount(count); + + for (Index i = 0; i < count; ++i) + { + ElementTypeInfo::toSerial(this, &in[i], &work[i]); + } + return _addArray(sizeof(ElementSerialType), SLANG_ALIGN_OF(ElementSerialType), work.getBuffer(), count); + } +} + +/* A SerialFieldType describes the size of field, it's alignment, and contains the +functions that convert between serial and native data */ +struct SerialFieldType +{ + typedef void(*ToSerialFunc)(SerialWriter* writer, const void* src, void* dst); + typedef void(*ToNativeFunc)(SerialReader* reader, const void* src, void* dst); + + size_t serialSizeInBytes; + uint8_t serialAlignment; + ToSerialFunc toSerialFunc; + ToNativeFunc toNativeFunc; +}; + +/* Describes a field in a SerialClass. */ +struct SerialField +{ + const char* name; ///< The name of the field + const SerialFieldType* type; ///< The type of the field + uint32_t nativeOffset; ///< Offset to field from base of type + uint32_t serialOffset; ///< Offset in serial type +}; + +typedef uint8_t SerialClassFlags; + +struct SerialClassFlag +{ + enum Enum : SerialClassFlags + { + DontSerialize = 0x01, ///< If set the type is not serialized, so can turn into SerialIndex(0) + }; +}; + +/* SerialClass defines the type (typeKind/subType) and the fields in just this class definition (ie not it's super class). +Also contains a pointer to the super type if there is one */ +struct SerialClass +{ + SerialTypeKind typeKind; ///< The type kind + SerialSubType subType; ///< Subtype - meaning depends on typeKind + + uint8_t alignment; ///< Alignment of this type + SerialClassFlags flags; ///< Flags + + uint32_t size; ///< Size of the field in bytes + + Index fieldsCount; + const SerialField* fields; + + const SerialClass* super; ///< The super class +}; + +// An instance could be shared across Sessions, but for simplicity of life time +// here we don't deal with that +class SerialClasses : public RefObject +{ +public: + + /// Will add it's own copy into m_classesByType + /// In process will calculate alignment, offset etc for fields + /// NOTE! the super set, *must* be an already added to this SerialClasses + const SerialClass* add(const SerialClass* cls); + + const SerialClass* add(SerialTypeKind kind, SerialSubType subType, const SerialField* fields, Index fieldsCount, const SerialClass* superCls); + + /// Add a type which will not serialize + const SerialClass* addUnserialized(SerialTypeKind kind, SerialSubType subType); + + /// Returns true if this cls is *owned* by this SerialClasses + bool isOwned(const SerialClass* cls) const; + + /// Get a serial class based on its type/subType + const SerialClass* getSerialClass(SerialTypeKind typeKind, SerialSubType subType) const + { + const auto& classes = m_classesByTypeKind[Index(typeKind)]; + return (subType < classes.getCount()) ? classes[subType] : nullptr; + } + + /// Ctor + SerialClasses(); + + static SlangResult create(RefPtr<SerialClasses>& out); + + static RefObjectSerialSubType getSubType(const RefObject* obj); + +protected: + SerialClass* _createSerialClass(const SerialClass* cls); + + MemoryArena m_arena; + + List<const SerialClass*> m_classesByTypeKind[Index(SerialTypeKind::CountOf)]; +}; + +// !!!!!!!!!!!!!!!!!!!!! SerialGetFieldType<T> !!!!!!!!!!!!!!!!!!!!!!!!!!! +// Getting the type info, let's use a static variable to hold the state to keep simple + +template <typename T> +struct SerialGetFieldType +{ + static const SerialFieldType* getFieldType() + { + typedef SerialTypeInfo<T> Info; + static const SerialFieldType type = { sizeof(typename Info::SerialType), uint8_t(Info::SerialAlignment), &Info::toSerial, &Info::toNative }; + return &type; + } +}; + +// !!!!!!!!!!!!!!!!!!!!! Convenience functions !!!!!!!!!!!!!!!!!!!!!!!!!!! + +template <typename NATIVE_TYPE, typename SERIAL_TYPE> +SLANG_FORCE_INLINE void toSerialValue(SerialWriter* writer, const NATIVE_TYPE& src, SERIAL_TYPE& dst) +{ + SerialTypeInfo<NATIVE_TYPE>::toSerial(writer, &src, &dst); +} + +template <typename SERIAL_TYPE, typename NATIVE_TYPE> +SLANG_FORCE_INLINE void toNativeValue(SerialReader* reader, const SERIAL_TYPE& src, NATIVE_TYPE& dst) +{ + SerialTypeInfo<NATIVE_TYPE>::toNative(reader, &src, &dst); +} + +} // namespace Slang + +#endif diff --git a/source/slang/slang.vcxproj b/source/slang/slang.vcxproj index d98a29118..7dafd3823 100644 --- a/source/slang/slang.vcxproj +++ b/source/slang/slang.vcxproj @@ -285,10 +285,14 @@ <ClInclude Include="slang-repro.h" /> <ClInclude Include="slang-serialize-ast.h" /> <ClInclude Include="slang-serialize-container.h" /> - <ClInclude Include="slang-serialize-debug.h" /> <ClInclude Include="slang-serialize-ir-types.h" /> <ClInclude Include="slang-serialize-ir.h" /> + <ClInclude Include="slang-serialize-misc-type-info.h" /> + <ClInclude Include="slang-serialize-reflection.h" /> + <ClInclude Include="slang-serialize-source-loc.h" /> + <ClInclude Include="slang-serialize-type-info.h" /> <ClInclude Include="slang-serialize-types.h" /> + <ClInclude Include="slang-serialize.h" /> <ClInclude Include="slang-source-loc.h" /> <ClInclude Include="slang-syntax.h" /> <ClInclude Include="slang-token-defs.h" /> @@ -398,10 +402,12 @@ <ClCompile Include="slang-repro.cpp" /> <ClCompile Include="slang-serialize-ast.cpp" /> <ClCompile Include="slang-serialize-container.cpp" /> - <ClCompile Include="slang-serialize-debug.cpp" /> <ClCompile Include="slang-serialize-ir-types.cpp" /> <ClCompile Include="slang-serialize-ir.cpp" /> + <ClCompile Include="slang-serialize-reflection.cpp" /> + <ClCompile Include="slang-serialize-source-loc.cpp" /> <ClCompile Include="slang-serialize-types.cpp" /> + <ClCompile Include="slang-serialize.cpp" /> <ClCompile Include="slang-source-loc.cpp" /> <ClCompile Include="slang-stdlib.cpp" /> <ClCompile Include="slang-syntax.cpp" /> diff --git a/source/slang/slang.vcxproj.filters b/source/slang/slang.vcxproj.filters index b9f9d69c0..be81a48c2 100644 --- a/source/slang/slang.vcxproj.filters +++ b/source/slang/slang.vcxproj.filters @@ -306,18 +306,30 @@ <ClInclude Include="slang-serialize-container.h"> <Filter>Header Files</Filter> </ClInclude> - <ClInclude Include="slang-serialize-debug.h"> - <Filter>Header Files</Filter> - </ClInclude> <ClInclude Include="slang-serialize-ir-types.h"> <Filter>Header Files</Filter> </ClInclude> <ClInclude Include="slang-serialize-ir.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="slang-serialize-misc-type-info.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="slang-serialize-reflection.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="slang-serialize-source-loc.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="slang-serialize-type-info.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="slang-serialize-types.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="slang-serialize.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="slang-source-loc.h"> <Filter>Header Files</Filter> </ClInclude> @@ -641,18 +653,24 @@ <ClCompile Include="slang-serialize-container.cpp"> <Filter>Source Files</Filter> </ClCompile> - <ClCompile Include="slang-serialize-debug.cpp"> - <Filter>Source Files</Filter> - </ClCompile> <ClCompile Include="slang-serialize-ir-types.cpp"> <Filter>Source Files</Filter> </ClCompile> <ClCompile Include="slang-serialize-ir.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="slang-serialize-reflection.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="slang-serialize-source-loc.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="slang-serialize-types.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="slang-serialize.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="slang-source-loc.cpp"> <Filter>Source Files</Filter> </ClCompile> |
