From 274c20a5eb133779a9d890ca79120815fb92b04e Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Wed, 30 Sep 2020 13:28:56 -0400 Subject: Generalizing Serialization (#1563) * First pass at generalizing serializer. * Split out ReflectClassInfo * Use the general ReflectClassInfo * Fix some typos in debug generalized serialization. * Add calculation of classIds. Make distinct addCopy/add on SerialClasses. * Write up of more generalized serialization * WIP to transition from ASTSerialReader/Writer etc to generalized SerialReader/Writer and associated types. * Improvements to SerialExtraObjects. Keep RefObjects in scope in factory * Compiles with Serial refactor - doesn't quite work yet. * First pass serialization appears to work with refector. * Split out type info for general slang types. * Split out slang-serialize-misc-type-info.h * DebugSerialData -> SerialSourecLocData DebugSerialReader -> SerialSourceLocReader DebugSerialWriter -> SerialSourceLocWriter * Remove unused template that only compiles on VS. * Fix warning around unused function on non-VS. --- source/slang/slang-serialize-ast.cpp | 1685 ++++------------------------------ 1 file changed, 185 insertions(+), 1500 deletions(-) (limited to 'source/slang/slang-serialize-ast.cpp') diff --git a/source/slang/slang-serialize-ast.cpp b/source/slang/slang-serialize-ast.cpp index 82cd7d6b4..65a5488a3 100644 --- a/source/slang/slang-serialize-ast.cpp +++ b/source/slang/slang-serialize-ast.cpp @@ -4,55 +4,23 @@ #include "slang-ast-generated.h" #include "slang-ast-generated-macro.h" -#include "slang-compiler.h" -#include "slang-type-layout.h" - #include "slang-ast-dump.h" -#include "slang-mangle.h" #include "slang-ast-support-types.h" +// Needed for ModuleSerialFilter +// Needed for 'findModuleForDecl' #include "slang-legalize-types.h" +#include "slang-mangle.h" -#include "../core/slang-byte-encode-util.h" +#include "slang-serialize-type-info.h" +#include "slang-serialize-misc-type-info.h" namespace Slang { +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ModuleSerialFilter !!!!!!!!!!!!!!!!!!!!!!!! -// Things stored as references: -// -// NodeBase derived types -// Array -// -// RefObject derived types: -// -// Breadcrumb -// StringRepresentation -// Scope - - - -// Helpers to convert fields treated as values - -class ASTSerialReader; -class ASTSerialWriter; - -template -static void _toSerialValue(ASTSerialWriter* writer, const NATIVE_TYPE& src, SERIAL_TYPE& dst) -{ - ASTSerialTypeInfo::toSerial(writer, &src, &dst); -} - -template -static void _toNativeValue(ASTSerialReader* reader, const SERIAL_TYPE& src, NATIVE_TYPE& dst) -{ - ASTSerialTypeInfo::toNative(reader, &src, &dst); -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ModuleASTSerialFilter !!!!!!!!!!!!!!!!!!!!!!!! - - -ASTSerialIndex ModuleASTSerialFilter::writePointer(ASTSerialWriter* writer, const NodeBase* inPtr) +SerialIndex ModuleSerialFilter::writePointer(SerialWriter* writer, const NodeBase* inPtr) { NodeBase* ptr = const_cast(inPtr); SLANG_ASSERT(ptr); @@ -71,7 +39,7 @@ ASTSerialIndex ModuleASTSerialFilter::writePointer(ASTSerialWriter* writer, cons ImportExternalDecl* importDecl = astBuilder->create(); importDecl->mangledName = mangledName; - const ASTSerialIndex index = writer->writePointer(importDecl); + const SerialIndex index = writer->addPointer(importDecl); // Set as the index of this writer->setPointerIndex(ptr, index); @@ -81,7 +49,7 @@ ASTSerialIndex ModuleASTSerialFilter::writePointer(ASTSerialWriter* writer, cons else { // Okay... we can just write it out then - return writer->writePointer(ptr); + return writer->writeObject(ptr); } } @@ -95,224 +63,52 @@ ASTSerialIndex ModuleASTSerialFilter::writePointer(ASTSerialWriter* writer, cons if (Stmt* stmt = as(ptr)) { // - writer->setPointerIndex(stmt, ASTSerialIndex(0)); - return ASTSerialIndex(0); + writer->setPointerIndex(stmt, SerialIndex(0)); + return SerialIndex(0); } // For now for everything else just write it - return writer->writePointer(ptr); + return writer->writeObject(ptr); } -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Serial <-> Native conversion !!!!!!!!!!!!!!!!!!!!!!!! - - -// We need to have a way to map between the two. -// If no mapping is needed, (just a copy), then we don't bother with the functions -template -struct ASTSerialBasicTypeInfo -{ - typedef T NativeType; - typedef T SerialType; - - // We want the alignment to be the same as the size of the type for basic types - // NOTE! Might be different from SLANG_ALIGN_OF(SerialType) - enum { SerialAlignment = sizeof(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(T*)serial = *(const T*)native; } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(T*)native = *(const T*)serial; } - - static const ASTSerialType* getType() - { - static const ASTSerialType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; - return &type; - } -}; - -template -struct ASTSerialConvertTypeInfo -{ - typedef NATIVE_T NativeType; - typedef SERIAL_T SerialType; - - enum { SerialAlignment = ASTSerialBasicTypeInfo::SerialAlignment }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SERIAL_T*)serial = SERIAL_T(*(const NATIVE_T*)native); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(NATIVE_T*)native = NATIVE_T(*(const SERIAL_T*)serial); } -}; +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! AST types !!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ +// SyntaxClass template -struct ASTSerialIdentityTypeInfo -{ - typedef T NativeType; - typedef T SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(T*)serial = *(const T*)native; } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(T*)native = *(const T*)serial; } -}; - -// Don't need to convert the index type - -template <> -struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; - - -// Because is sized, we don't need to convert -template <> -struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; - -// Implement for Basic Types - -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; - -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; - -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; -template <> -struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; - -// SamplerStateFlavor - -template <> -struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; - -// TextureFlavor - -template <> -struct ASTSerialTypeInfo +struct SerialTypeInfo> { - typedef TextureFlavor NativeType; + typedef SyntaxClass NativeType; typedef uint16_t SerialType; - enum { SerialAlignment = sizeof(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SerialType*)serial = ((const NativeType*)native)->flavor; } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); ((NativeType*)native)->flavor = *(const SerialType*)serial; } -}; - -// Fixed arrays - -template -struct ASTSerialTypeInfo -{ - typedef ASTSerialTypeInfo ElementASTSerialType; - typedef typename ElementASTSerialType::SerialType SerialElementType; - - typedef T NativeType[N]; - typedef SerialElementType SerialType[N]; - - enum { SerialAlignment = ASTSerialTypeInfo::SerialAlignment }; - - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) - { - SerialElementType* serial = (SerialElementType*)outSerial; - const T* native = (const T*)inNative; - for (Index i = 0; i < Index(N); ++i) - { - ElementASTSerialType::toSerial(writer, native + i, serial + i); - } - } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - const SerialElementType* serial = (const SerialElementType*)inSerial; - T* native = (T*)outNative; - for (Index i = 0; i < Index(N); ++i) - { - ElementASTSerialType::toNative(reader, serial + i, native + i); - } - } -}; - -// Special case bool - as we can't rely on size alignment -template <> -struct ASTSerialTypeInfo -{ - typedef bool NativeType; - typedef uint8_t SerialType; - enum { SerialAlignment = sizeof(SerialType) }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); - *(SerialType*)outSerial = *(const NativeType*)inNative ? 1 : 0; + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + dst = SerialType(src.classInfo->m_classId); } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + static void toNative(SerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); - *(NativeType*)outNative = (*(const SerialType*)inSerial) != 0; - } -}; - - -// Pointer -// Could handle different pointer base types with some more template magic here, but instead went with Pointer type to keep -// things simpler. -template -struct ASTSerialTypeInfo -{ - typedef T* NativeType; - typedef ASTSerialIndex SerialType; - - enum - { - SerialAlignment = SLANG_ALIGN_OF(SerialType) - }; - - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) - { - *(SerialType*)outSerial = writer->addPointer(*(T**)inNative); - } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - *(T**)outNative = reader->getPointer(*(const SerialType*)inSerial).dynamicCast(); - } -}; - -// Special case Name -template <> -struct ASTSerialTypeInfo : public ASTSerialTypeInfo -{ - // Special case - typedef Name* NativeType; - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - *(Name**)outNative = reader->getName(*(const SerialType*)inSerial); + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + dst.classInfo = ASTClassInfo::getInfo(ASTNodeType(src)); } }; -template <> -struct ASTSerialTypeInfo : public ASTSerialTypeInfo -{ -}; - - -struct ASTSerialDeclRefBaseTypeInfo +struct SerialDeclRefBaseTypeInfo { typedef DeclRefBase NativeType; struct SerialType { - ASTSerialIndex substitutions; - ASTSerialIndex decl; + SerialIndex substitutions; + SerialIndex decl; }; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + static void toSerial(SerialWriter* writer, const void* inNative, void* outSerial) { SerialType& serial = *(SerialType*)outSerial; const NativeType& native = *(const NativeType*)inNative; @@ -320,7 +116,7 @@ struct ASTSerialDeclRefBaseTypeInfo serial.decl = writer->addPointer(native.decl); serial.substitutions = writer->addPointer(native.substitutions.substitutions); } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + static void toNative(SerialReader* reader, const void* inSerial, void* outNative) { DeclRefBase& native = *(DeclRefBase*)(outNative); const SerialType& serial = *(const SerialType*)inSerial; @@ -328,196 +124,49 @@ struct ASTSerialDeclRefBaseTypeInfo native.decl = reader->getPointer(serial.decl).dynamicCast(); native.substitutions.substitutions = reader->getPointer(serial.substitutions).dynamicCast(); } - static const ASTSerialType* getType() + static const SerialFieldType* getFieldType() { - static const ASTSerialType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; + static const SerialFieldType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; return &type; } }; - +// Special case DeclRef, because it always uses the same type template -struct ASTSerialTypeInfo> : public ASTSerialDeclRefBaseTypeInfo {}; - -// MatrixCoord can just go as is -template <> -struct ASTSerialTypeInfo : ASTSerialIdentityTypeInfo {}; - -// SourceLoc - -// Make the type exposed, so we can look for it if we want to remap. -template <> -struct ASTSerialTypeInfo -{ - typedef SourceLoc NativeType; - typedef ASTSerialSourceLoc SerialType; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialSourceLoc) }; - - static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) - { - *(SerialType*)outSerial = writer->addSourceLoc(*(const NativeType*)inNative); - } - static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) - { - *(NativeType*)outNative = reader->getSourceLoc(*(const SerialType*)inSerial); - } -}; - -// List -template -struct ASTSerialTypeInfo> +struct SerialGetFieldType> { - typedef List NativeType; - typedef ASTSerialIndex SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - dst = writer->addArray(src.getBuffer(), src.getCount()); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& dst = *(NativeType*)native; - auto& src = *(const SerialType*)serial; - - reader->getArray(src, dst); - } + static const SerialFieldType* getFieldType() { return SerialDeclRefBaseTypeInfo::getFieldType(); } }; -// Dictionary -template -struct ASTSerialTypeInfo> -{ - typedef Dictionary NativeType; - struct SerialType - { - ASTSerialIndex keys; ///< Index an array - ASTSerialIndex values; ///< Index an array - }; - - typedef typename ASTSerialTypeInfo::SerialType KeySerialType; - typedef typename ASTSerialTypeInfo::SerialType ValueSerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - List keys; - List values; - - Index count = Index(src.Count()); - keys.setCount(count); - values.setCount(count); - - Index i = 0; - for (const auto& pair : src) - { - ASTSerialTypeInfo::toSerial(writer, &pair.Key, &keys[i]); - ASTSerialTypeInfo::toSerial(writer, &pair.Value, &values[i]); - i++; - } - - dst.keys = writer->addArray(keys.getBuffer(), count); - dst.values = writer->addArray(values.getBuffer(), count); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - // Clear it - dst = NativeType(); - - List keys; - List values; - - reader->getArray(src.keys, keys); - reader->getArray(src.values, values); - SLANG_ASSERT(keys.getCount() == values.getCount()); - - const Index count = keys.getCount(); - for (Index i = 0; i < count; ++i) - { - dst.Add(keys[i], values[i]); - } - } -}; - -// SyntaxClass template -struct ASTSerialTypeInfo> -{ - typedef SyntaxClass NativeType; - typedef uint16_t SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - SLANG_UNUSED(writer); - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - dst = SerialType(src.classInfo->m_classId); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - SLANG_UNUSED(reader); - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - dst.classInfo = ReflectClassInfo::getInfo(ASTNodeType(src)); - } -}; +struct SerialTypeInfo> : public SerialDeclRefBaseTypeInfo {}; -// Handle RefPtr - just convert into * to do the conversion -template -struct ASTSerialTypeInfo> -{ - typedef RefPtr NativeType; - typedef typename ASTSerialTypeInfo::SerialType SerialType; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; +// MatrixCoord can just go as is +template <> +struct SerialTypeInfo : SerialIdentityTypeInfo {}; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - T* obj = src; - ASTSerialTypeInfo::toSerial(writer, &obj, serial); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - T* obj = nullptr; - ASTSerialTypeInfo::toNative(reader, serial, &obj); - *(NativeType*)native = obj; - } -}; // QualType template <> -struct ASTSerialTypeInfo +struct SerialTypeInfo { typedef QualType NativeType; struct SerialType { - ASTSerialIndex type; + SerialIndex type; uint8_t isLeftValue; }; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto dst = (SerialType*)serial; auto src = (const NativeType*)native; dst->isLeftValue = src->isLeftValue ? 1 : 0; dst->type = writer->addPointer(src->type); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto src = (const SerialType*)serial; auto dst = (NativeType*)native; @@ -529,81 +178,81 @@ struct ASTSerialTypeInfo // LookupResult::Breadcrumb template <> -struct ASTSerialTypeInfo +struct SerialTypeInfo { typedef LookupResultItem::Breadcrumb NativeType; struct SerialType { NativeType::Kind kind; NativeType::ThisParameterMode thisParameterMode; - ASTSerialTypeInfo>::SerialType declRef; - ASTSerialTypeInfo> next; + SerialTypeInfo>::SerialType declRef; + SerialTypeInfo> next; }; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& src = *(const NativeType*)native; auto& dst = *(SerialType*)serial; dst.kind = src.kind; dst.thisParameterMode = src.thisParameterMode; - _toSerialValue(writer, src.declRef, dst.declRef); - _toSerialValue(writer, src.next, dst.next); + toSerialValue(writer, src.declRef, dst.declRef); + toSerialValue(writer, src.next, dst.next); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto& dst = *(NativeType*)native; auto& src = *(const SerialType*)serial; dst.kind = src.kind; dst.thisParameterMode = src.thisParameterMode; - _toNativeValue(reader, src.declRef, dst.declRef); - _toNativeValue(reader, src.next, dst.next); + toNativeValue(reader, src.declRef, dst.declRef); + toNativeValue(reader, src.next, dst.next); } }; // LookupResultItem template <> -struct ASTSerialTypeInfo +struct SerialTypeInfo { typedef LookupResultItem NativeType; struct SerialType { - ASTSerialTypeInfo>::SerialType declRef; - ASTSerialTypeInfo> breadcrumbs; + SerialTypeInfo>::SerialType declRef; + SerialTypeInfo> breadcrumbs; }; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& src = *(const NativeType*)native; auto& dst = *(SerialType*)serial; - _toSerialValue(writer, src.declRef, dst.declRef); - _toSerialValue(writer, src.breadcrumbs, dst.breadcrumbs); + toSerialValue(writer, src.declRef, dst.declRef); + toSerialValue(writer, src.breadcrumbs, dst.breadcrumbs); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto& dst = *(NativeType*)native; auto& src = *(const SerialType*)serial; - _toNativeValue(reader, src.declRef, dst.declRef); - _toNativeValue(reader, src.breadcrumbs, dst.breadcrumbs); + toNativeValue(reader, src.declRef, dst.declRef); + toNativeValue(reader, src.breadcrumbs, dst.breadcrumbs); } }; // LookupResult template <> -struct ASTSerialTypeInfo +struct SerialTypeInfo { typedef LookupResult NativeType; - typedef ASTSerialIndex SerialType; + typedef SerialIndex SerialType; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& src = *(const NativeType*)native; auto& dst = *(SerialType*)serial; @@ -619,10 +268,10 @@ struct ASTSerialTypeInfo } else { - dst = ASTSerialIndex(0); + dst = SerialIndex(0); } } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto& dst = *(NativeType*)native; auto& src = *(const SerialType*)serial; @@ -646,20 +295,19 @@ struct ASTSerialTypeInfo } }; - // GlobalGenericParamSubstitution::ConstraintArg template <> -struct ASTSerialTypeInfo +struct SerialTypeInfo { typedef GlobalGenericParamSubstitution::ConstraintArg NativeType; struct SerialType { - ASTSerialIndex decl; - ASTSerialIndex val; + SerialIndex decl; + SerialIndex val; }; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& dst = *(SerialType*)serial; auto& src = *(const NativeType*)native; @@ -667,7 +315,7 @@ struct ASTSerialTypeInfo dst.decl = writer->addPointer(src.decl); dst.val = writer->addPointer(src.val); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto& src = *(const SerialType*)serial; auto& dst = *(NativeType*)native; @@ -679,17 +327,17 @@ struct ASTSerialTypeInfo // ExpandedSpecializationArg template <> -struct ASTSerialTypeInfo +struct SerialTypeInfo { typedef ExpandedSpecializationArg NativeType; struct SerialType { - ASTSerialIndex val; - ASTSerialIndex witness; + SerialIndex val; + SerialIndex witness; }; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& dst = *(SerialType*)serial; auto& src = *(const NativeType*)native; @@ -697,7 +345,7 @@ struct ASTSerialTypeInfo dst.witness = writer->addPointer(src.witness); dst.val = writer->addPointer(src.val); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto& src = *(const SerialType*)serial; auto& dst = *(NativeType*)native; @@ -709,17 +357,17 @@ struct ASTSerialTypeInfo // TypeExp template <> -struct ASTSerialTypeInfo +struct SerialTypeInfo { typedef TypeExp NativeType; struct SerialType { - ASTSerialIndex type; - ASTSerialIndex expr; + SerialIndex type; + SerialIndex expr; }; - enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& dst = *(SerialType*)serial; auto& src = *(const NativeType*)native; @@ -727,7 +375,7 @@ struct ASTSerialTypeInfo dst.type = writer->addPointer(src.type); dst.expr = writer->addPointer(src.exp); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { auto& src = *(const SerialType*)serial; auto& dst = *(NativeType*)native; @@ -739,19 +387,19 @@ struct ASTSerialTypeInfo // DeclCheckStateExt template <> -struct ASTSerialTypeInfo +struct SerialTypeInfo { typedef DeclCheckStateExt NativeType; typedef DeclCheckStateExt::RawType SerialType; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SerialType*)serial = (*(const NativeType*)native).getRaw(); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); (*(NativeType*)serial).setRaw(*(const SerialType*)native); @@ -760,24 +408,24 @@ struct ASTSerialTypeInfo // Modifiers template <> -struct ASTSerialTypeInfo +struct SerialTypeInfo { typedef Modifiers NativeType; - typedef ASTSerialIndex SerialType; + typedef SerialIndex SerialType; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + static void toSerial(SerialWriter* writer, const void* native, void* serial) { // We need to make into an array - List modifierIndices; + List modifierIndices; for (Modifier* modifier : *(NativeType*)native) { modifierIndices.add(writer->addPointer(modifier)); } *(SerialType*)serial = writer->addArray(modifierIndices.getBuffer(), modifierIndices.getCount()); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) + static void toNative(SerialReader* reader, const void* serial, void* native) { List modifiers; reader->getArray(*(const SerialType*)serial, modifiers); @@ -796,1115 +444,150 @@ struct ASTSerialTypeInfo } }; -// ImageFormat +// ASTNodeType template <> -struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; +struct SerialTypeInfo : public SerialConvertTypeInfo {}; -// Stage -template <> -struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; +// !!!!!!!!!!!!!!!!!!!!!! Generate fields for a type !!!!!!!!!!!!!!!!!!!!!!!!!!! -// TokenType -template <> -struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; +template +SerialField _makeField(const char* name, T& in) +{ + uint8_t* ptr = &reinterpret_cast(in); -// BaseType -template <> -struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + SerialField field; + field.name = name; + field.type = SerialGetFieldType::getFieldType(); + // This only works because we in is an offset from 1 + field.nativeOffset = uint32_t(size_t(ptr) - 1); + field.serialOffset = 0; + return field; +} -// SemanticVersion -template <> -struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; +static const SerialClass* _addClass(SerialClasses* serialClasses, ASTNodeType type, ASTNodeType super, const List& fields) +{ + const SerialClass* superClass = serialClasses->getSerialClass(SerialTypeKind::NodeBase, SerialSubType(super)); + return serialClasses->add(SerialTypeKind::NodeBase, SerialSubType(type), fields.getBuffer(), fields.getCount(), superClass); +} -// ASTNodeType -template <> -struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; +#define SLANG_AST_ADD_SERIAL_FIELD(FIELD_NAME, TYPE, param) fields.add(_makeField(#FIELD_NAME, obj->FIELD_NAME)); -// String -template <> -struct ASTSerialTypeInfo -{ - typedef String NativeType; - typedef ASTSerialIndex SerialType; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; +// Note that the obj point is not nullptr, because some compilers notice this is 'indexing from null' +// and warn/error. So we offset from 1. +#define SLANG_AST_ADD_SERIAL_CLASS(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \ +{ \ + NAME* obj = (NAME*)1; \ + SLANG_UNUSED(obj); \ + fields.clear(); \ + SLANG_FIELDS_ASTNode_##NAME(SLANG_AST_ADD_SERIAL_FIELD, param) \ + _addClass(serialClasses, ASTNodeType::NAME, ASTNodeType::SUPER, fields); \ +} - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - *(SerialType*)serial = writer->addString(src); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) +struct ASTFieldAccess +{ + static void calcClasses(SerialClasses* serialClasses) { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - dst = reader->getString(src); + // Add NodeBase first, and specially handle so that we add a null super class + serialClasses->add(SerialTypeKind::NodeBase, SerialSubType(ASTNodeType::NodeBase), nullptr, 0, nullptr); + + // Add the rest in order such that Super class is always added before its children + List fields; + SLANG_CHILDREN_ASTNode_NodeBase(SLANG_AST_ADD_SERIAL_CLASS, _) } }; -// Token -template <> -struct ASTSerialTypeInfo +void addASTTypes(SerialClasses* serialClasses) { - typedef Token NativeType; - struct SerialType { - ASTSerialTypeInfo::SerialType type; - ASTSerialTypeInfo::SerialType loc; - ASTSerialIndex name; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + ASTFieldAccess::calcClasses(serialClasses); + } - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - ASTSerialTypeInfo::toSerial(writer, &src.type, &dst.type); - ASTSerialTypeInfo::toSerial(writer, &src.loc, &dst.loc); - - if (src.flags & TokenFlag::Name) - { - dst.name = writer->addName(src.getName()); - } - else { - dst.name = writer->addString(src.getContent()); + // Let's hack Breadcrumbs... + + typedef LookupResultItem::Breadcrumb Type; + Type* obj = (Type*)1; + SerialField field = _makeField("_", *obj); + serialClasses->add(SerialTypeKind::RefObject, SerialSubType(RefObjectSerialSubType::LookupResultItem_Breadcrumb), &field, 1, nullptr); } + + // Set these types to not serialize + serialClasses->addUnserialized(SerialTypeKind::RefObject, SerialSubType(RefObjectSerialSubType::Module)); + serialClasses->addUnserialized(SerialTypeKind::RefObject, SerialSubType(RefObjectSerialSubType::Scope)); } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - dst.flags = 0; - dst.charsNameUnion.chars = nullptr; - - ASTSerialTypeInfo::toNative(reader, &src.type, &dst.type); - ASTSerialTypeInfo::toNative(reader, &src.loc, &dst.loc); - - // At the other end all token content will appear as Names. - if (src.name != ASTSerialIndex(0)) - { - dst.charsNameUnion.name = reader->getName(src.name); - dst.flags |= TokenFlag::Name; - } - } -}; - -// NameLoc -template <> -struct ASTSerialTypeInfo -{ - typedef NameLoc NativeType; - struct SerialType - { - ASTSerialTypeInfo::SerialType loc; - ASTSerialIndex name; - }; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - - dst.name = writer->addName(src.name); - ASTSerialTypeInfo::toSerial(writer, &src.loc, &dst.loc); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - dst.name = reader->getName(src.name); - ASTSerialTypeInfo::toNative(reader, &src.loc, &dst.loc); - } -}; - -// DiagnosticInfo -template <> -struct ASTSerialTypeInfo -{ - typedef const DiagnosticInfo* NativeType; - typedef ASTSerialIndex SerialType; - - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; - - static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - auto& dst = *(SerialType*)serial; - dst = src ? writer->addString(UnownedStringSlice(src->name)) : ASTSerialIndex(0); - } - static void toNative(ASTSerialReader* reader, const void* serial, void* native) - { - auto& src = *(const SerialType*)serial; - auto& dst = *(NativeType*)native; - - if (src == ASTSerialIndex(0)) - { - dst = nullptr; - } - else - { - dst = findDiagnosticByName(reader->getStringSlice(src)); - } - } -}; - -// !!!!!!!!!!!!!!!!!!!!! ASTSerialGetType !!!!!!!!!!!!!!!!!!!!!!!!!!! -// Getting the type info, let's use a static variable to hold the state to keep simple - -template -struct ASTSerialGetType -{ - static const ASTSerialType* getType() - { - typedef ASTSerialTypeInfo Info; - static const ASTSerialType type = { sizeof(typename Info::SerialType), uint8_t(Info::SerialAlignment), &Info::toSerial, &Info::toNative }; - return &type; - } -}; - -// Special case DeclRef, because it always uses the same type -template -struct ASTSerialGetType> -{ - static const ASTSerialType* getType() { return ASTSerialDeclRefBaseTypeInfo::getType(); } -}; - -// !!!!!!!!!!!!!!!!!!!!!! Generate fields for a type !!!!!!!!!!!!!!!!!!!!!!!!!!! - - -template -ASTSerialField _calcField(const char* name, T& in) -{ - uint8_t* ptr = &reinterpret_cast(in); - - ASTSerialField field; - field.name = name; - field.type = ASTSerialGetType::getType(); - // This only works because we in is an offset from 1 - field.nativeOffset = uint32_t(size_t(ptr) - 1); - field.serialOffset = 0; - return field; -} - -static ASTSerialClass _makeClass(MemoryArena* arena, ASTNodeType type, const List& fields) -{ - ASTSerialClass cls = { type, 0, 0, 0, 0 }; - cls.fieldsCount = fields.getCount(); - cls.fields = arena->allocateAndCopyArray(fields.getBuffer(), fields.getCount()); - return cls; -} - -#define SLANG_AST_SERIAL_FIELD(FIELD_NAME, TYPE, param) fields.add(_calcField(#FIELD_NAME, obj->FIELD_NAME)); - -// Note that the obj point is not nullptr, because some compilers notice this is 'indexing from null' -// and warn/error. So we offset from 1. -#define SLANG_AST_SERIAL_MAKE_CLASS(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \ -{ \ - NAME* obj = (NAME*)1; \ - SLANG_UNUSED(obj); \ - fields.clear(); \ - SLANG_FIELDS_ASTNode_##NAME(SLANG_AST_SERIAL_FIELD, param) \ - outClasses[Index(ASTNodeType::NAME)] = _makeClass(arena, ASTNodeType::NAME, fields); \ } -struct ASTFieldAccess +// A Hack for now to turn an RefObject* into a SubType for serialization +extern RefObjectSerialSubType getRefObjectSubType(const RefObject* obj) { - static void calcClasses(MemoryArena* arena, ASTSerialClass outClasses[Index(ASTNodeType::CountOf)]) + if (as(obj)) { - List fields; - SLANG_ALL_ASTNode_NodeBase(SLANG_AST_SERIAL_MAKE_CLASS, _) + return RefObjectSerialSubType::LookupResultItem_Breadcrumb; } -}; - -ASTSerialClasses::ASTSerialClasses(): - m_arena(2048) -{ - memset(m_classes, 0, sizeof(m_classes)); - ASTFieldAccess::calcClasses(&m_arena, m_classes); - - // Now work out the layout - for (Index i = 0; i < SLANG_COUNT_OF(m_classes); ++i) + else if (as(obj)) { - // Set up each class in order, from lowest to highest index - // Doing so means super class is always setup - ASTSerialClass& serialClass = m_classes[i]; - - const ReflectClassInfo* info = ReflectClassInfo::getInfo(serialClass.type); - - size_t maxAlignment = 1; - size_t offset = 0; - - const ReflectClassInfo* superInfo = info->m_superClass; - if (superInfo) - { - ASTSerialClass& superSerialInfo = m_classes[superInfo->m_classId]; - - // If it's been setup, then alignment must be non zero. - // The ordering of ASTNodeType, should mean type have larger ASTNodeType greater than supers ASTNodeType. - SLANG_ASSERT(superSerialInfo.alignment != 0); - - // Must be a power of 2 - SLANG_ASSERT((superSerialInfo.alignment & (superSerialInfo.alignment - 1)) == 0); - - maxAlignment = superSerialInfo.alignment; - offset = superSerialInfo.size; - - // Check it is correctly aligned - SLANG_ASSERT((offset & (maxAlignment - 1)) == 0); - } - - // Okay, go through fields setting their offset - ASTSerialField* fields = serialClass.fields; - for (Index j = 0; j < serialClass.fieldsCount; j++) - { - ASTSerialField& field = fields[j]; - - size_t alignment = field.type->serialAlignment; - // Make sure the offset is aligned for the field requirement - offset = (offset + alignment - 1) & ~(alignment - 1); - - // Save the field offset - field.serialOffset = uint32_t(offset); - - // Move past the field - offset += field.type->serialSizeInBytes; - - // Calc the maximum alignment - maxAlignment = (alignment > maxAlignment) ? alignment : maxAlignment; - } - - // Align with maximum alignment - offset = (offset + maxAlignment - 1) & ~(maxAlignment - 1); - - serialClass.alignment = uint8_t(maxAlignment); - serialClass.size = uint32_t(offset); - } -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!! - -ASTSerialWriter::ASTSerialWriter(ASTSerialClasses* classes, ASTSerialFilter* filter, DebugSerialWriter* debugWriter) : - m_arena(2048), - m_classes(classes), - m_filter(filter), - m_debugWriter(debugWriter) -{ - // 0 is always the null pointer - m_entries.add(nullptr); - m_ptrMap.Add(nullptr, 0); -} - -ASTSerialIndex ASTSerialWriter::writePointer(const NodeBase* node) -{ - // This pointer cannot be in the map - SLANG_ASSERT(m_ptrMap.TryGetValue(node) == nullptr); - - const ASTSerialClass* serialClass = m_classes->getSerialClass(node->astNodeType); - - typedef ASTSerialInfo::NodeEntry NodeEntry; - - NodeEntry* nodeEntry = (NodeEntry*)m_arena.allocateAligned(sizeof(NodeEntry) + serialClass->size, ASTSerialInfo::MAX_ALIGNMENT); - - nodeEntry->type = ASTSerialInfo::Type::Node; - nodeEntry->astNodeType = uint16_t(node->astNodeType); - nodeEntry->info = ASTSerialInfo::makeEntryInfo(serialClass->alignment); - - auto index = _add(node, nodeEntry); - - uint8_t* serialPayload = (uint8_t*)(nodeEntry + 1); - - while (serialClass) - { - for (Index i = 0; i < serialClass->fieldsCount; ++i) - { - auto field = serialClass->fields[i]; - - // Work out the offsets - auto srcField = ((const uint8_t*)node) + field.nativeOffset; - auto dstField = serialPayload + field.serialOffset; - - field.type->toSerialFunc(this, srcField, dstField); - } - // Get the super class - const ReflectClassInfo* reflectInfo = ReflectClassInfo::getInfo(serialClass->type); - const ReflectClassInfo* superReflectInfo = reflectInfo->m_superClass; - - serialClass = superReflectInfo ? m_classes->getSerialClass(ASTNodeType(superReflectInfo->m_classId)) : nullptr; - } - - return index; -} - -void ASTSerialWriter::setPointerIndex(const NodeBase* ptr, ASTSerialIndex index) -{ - m_ptrMap.Add(ptr, Index(index)); -} - -ASTSerialIndex ASTSerialWriter::addPointer(const NodeBase* node) -{ - // Null is always 0 - if (node == nullptr) - { - return ASTSerialIndex(0); - } - // Look up in the map - Index* indexPtr = m_ptrMap.TryGetValue(node); - if (indexPtr) - { - return ASTSerialIndex(*indexPtr); - } - - if (m_filter) - { - return m_filter->writePointer(this, node); - } - else - { - return writePointer(node); - } -} - -ASTSerialIndex ASTSerialWriter::addPointer(const RefObject* obj) -{ - // Null is always 0 - if (obj == nullptr) - { - return ASTSerialIndex(0); - } - // Look up in the map - Index* indexPtr = m_ptrMap.TryGetValue(obj); - if (indexPtr) - { - return ASTSerialIndex(*indexPtr); - } - - if (auto stringRep = dynamicCast(obj)) - { - ASTSerialIndex index = addString(StringRepresentation::asSlice(stringRep)); - m_ptrMap.Add(obj, Index(index)); - return index; - } - else if (auto breadcrumb = dynamicCast(obj)) - { - typedef ASTSerialTypeInfo TypeInfo; - typedef ASTSerialInfo::RefObjectEntry RefObjectEntry; - - size_t alignment = TypeInfo::SerialAlignment; - alignment = (alignment < SLANG_ALIGN_OF(ASTSerialInfo::RefObjectEntry)) ? SLANG_ALIGN_OF(ASTSerialInfo::RefObjectEntry) : alignment; - - RefObjectEntry* entry = (RefObjectEntry*)m_arena.allocateAligned(sizeof(RefObjectEntry) + sizeof(TypeInfo::SerialType), alignment); - - entry->type = ASTSerialInfo::Type::RefObject; - entry->info = ASTSerialInfo::makeEntryInfo(int(alignment)); - entry->subType = RefObjectEntry::SubType::Breadcrumb; - - auto index = _add(breadcrumb, entry); - - // Do any conversion - TypeInfo::toSerial(this, breadcrumb, entry + 1); - return index; - } - else if (auto name = dynamicCast(obj)) - { - return addName(name); - } - else if (auto scope = dynamicCast(obj)) - { - // We don't serialize scope - return ASTSerialIndex(0); - } - else if (auto module = dynamicCast(obj)) - { - // We don't serialize Module - return ASTSerialIndex(0); - } - - SLANG_ASSERT(!"Unhandled type"); - return ASTSerialIndex(0); -} - -ASTSerialIndex ASTSerialWriter::addString(const UnownedStringSlice& slice) -{ - typedef ByteEncodeUtil Util; - typedef ASTSerialInfo::StringEntry StringEntry; - - if (slice.getLength() == 0) - { - return ASTSerialIndex(0); - } - - Index newIndex = m_entries.getCount(); - - Index* indexPtr = m_sliceMap.TryGetValueOrAdd(slice, newIndex); - if (indexPtr) - { - return ASTSerialIndex(*indexPtr); - } - - // Okay we need to add the string - - uint8_t encodeBuf[Util::kMaxLiteEncodeUInt32]; - const int encodeCount = Util::encodeLiteUInt32(uint32_t(slice.getLength()), encodeBuf); - - StringEntry* entry = (StringEntry*)m_arena.allocateUnaligned(SLANG_OFFSET_OF(StringEntry, sizeAndChars) + encodeCount + slice.getLength()); - entry->info = ASTSerialInfo::EntryInfo::Alignment1; - entry->type = ASTSerialInfo::Type::String; - - uint8_t* dst = (uint8_t*)(entry->sizeAndChars); - for (int i = 0; i < encodeCount; ++i) - { - dst[i] = encodeBuf[i]; - } - - memcpy(dst + encodeCount, slice.begin(), slice.getLength()); - - m_entries.add(entry); - return ASTSerialIndex(newIndex); -} - - -ASTSerialIndex ASTSerialWriter::addString(const String& in) -{ - return addPointer(in.getStringRepresentation()); -} - -ASTSerialIndex ASTSerialWriter::addName(const Name* name) -{ - if (name == nullptr) - { - return ASTSerialIndex(0); + return RefObjectSerialSubType::Module; } - - // Look it up - Index* indexPtr = m_ptrMap.TryGetValue(name); - if (indexPtr) + else if (as(obj)) { - return ASTSerialIndex(*indexPtr); + return RefObjectSerialSubType::Scope; } - - ASTSerialIndex index = addString(name->text); - m_ptrMap.Add(name, Index(index)); - return index; + return RefObjectSerialSubType::Invalid; } -ASTSerialSourceLoc ASTSerialWriter::addSourceLoc(SourceLoc sourceLoc) -{ - if (sourceLoc.isValid() && m_debugWriter) - { - return m_debugWriter->addSourceLoc(sourceLoc); - } - else - { - return 0; - } -} +/* !!!!!!!!!!!!!!!!!!!!!! DefaultSerialObjectFactory !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ -ASTSerialIndex ASTSerialWriter::_addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount) +void* DefaultSerialObjectFactory::create(SerialTypeKind typeKind, SerialSubType subType) { - typedef ASTSerialInfo::ArrayEntry Entry; - - if (elementCount == 0) + switch (typeKind) { - return ASTSerialIndex(0); - } - - SLANG_ASSERT(alignment >= 1 && alignment <= ASTSerialInfo::MAX_ALIGNMENT); - - // We must at a minimum have the alignment for the array prefix info - alignment = (alignment < SLANG_ALIGN_OF(Entry)) ? SLANG_ALIGN_OF(Entry) : alignment; - - size_t payloadSize = elementCount * elementSize; - - Entry* entry = (Entry*)m_arena.allocateAligned(sizeof(Entry) + payloadSize, alignment); - - entry->type = ASTSerialInfo::Type::Array; - entry->info = ASTSerialInfo::makeEntryInfo(int(alignment)); - entry->elementSize = uint16_t(elementSize); - entry->elementCount = uint32_t(elementCount); - - memcpy(entry + 1, elements, payloadSize); - - m_entries.add(entry); - return ASTSerialIndex(m_entries.getCount() - 1); -} - -static const uint8_t s_fixBuffer[ASTSerialInfo::MAX_ALIGNMENT]{ 0, }; - -SlangResult ASTSerialWriter::write(Stream* stream) -{ - const Int entriesCount = m_entries.getCount(); - - // Add a sentinal so we don't need special handling for - ASTSerialInfo::Entry sentinal; - sentinal.type = ASTSerialInfo::Type::String; - sentinal.info = ASTSerialInfo::EntryInfo::Alignment1; - - m_entries.add(&sentinal); - m_entries.removeLast(); - - ASTSerialInfo::Entry** entries = m_entries.getBuffer(); - // Note strictly required in our impl of List. But by writing this and - // knowing that removeLast cannot release memory, means the sentinal must be at the last position. - entries[entriesCount] = &sentinal; - - - { - size_t offset = 0; - - ASTSerialInfo::Entry* entry = entries[1]; - // We start on 1, because 0 is nullptr and not used for anything - for (Index i = 1; i < entriesCount; ++i) - { - ASTSerialInfo::Entry* next = entries[i + 1]; - // Before writing we need to store the next alignment - - const size_t nextAlignment = ASTSerialInfo::getAlignment(next->info); - const size_t alignment = ASTSerialInfo::getAlignment(entry->info); - - entry->info = ASTSerialInfo::combineWithNext(entry->info, next->info); - - // Check we are aligned correctly - SLANG_ASSERT((offset & (alignment - 1)) == 0); - - // When we write, we need to make sure it take into account the next alignment - const size_t entrySize = entry->calcSize(m_classes); - - // Work out the fix for next alignment - size_t nextOffset = offset + entrySize; - nextOffset = (nextOffset + nextAlignment - 1) & ~(nextAlignment - 1); - - size_t alignmentFixSize = nextOffset - (offset + entrySize); - - // The fix must be less than max alignment. We require it to be less because we aligned each Entry to - // MAX_ALIGNMENT, and so < MAX_ALIGNMENT is the most extra bytes we can write - SLANG_ASSERT( alignmentFixSize < ASTSerialInfo::MAX_ALIGNMENT); - - try - { - stream->write(entry, entrySize); - // If we needed to fix so that subsequent alignment is right, write out extra bytes here - if (alignmentFixSize) - { - stream->write(s_fixBuffer, alignmentFixSize); - } - } - catch (const IOException&) - { - return SLANG_FAIL; - } - - // Onto next - offset = nextOffset; - entry = next; - } - } - - return SLANG_OK; -} - -SlangResult ASTSerialWriter::writeIntoContainer(RiffContainer* container) -{ - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ScopeChunk ScopeChunk; - - // This is the container for the AST Data - ScopeChunk scopeModule(container, Chunk::Kind::List, ASTSerialBinary::kSlangASTModuleFourCC); - { - ScopeChunk scopeData(container, Chunk::Kind::Data, ASTSerialBinary::kSlangASTModuleDataFourCC); - - { - // Sentinal so we don't need special handling for end of list - ASTSerialInfo::Entry sentinal; - sentinal.type = ASTSerialInfo::Type::String; - sentinal.info = ASTSerialInfo::EntryInfo::Alignment1; - - size_t offset = 0; - const Int entriesCount = m_entries.getCount(); - - { - m_entries.add(&sentinal); - m_entries.removeLast(); - // Note strictly required in our impl of List. But by writing this and - // knowing that removeLast cannot release memory, means the sentinal must be at the last position. - m_entries.getBuffer()[entriesCount] = &sentinal; - } - - ASTSerialInfo::Entry*const* entries = m_entries.getBuffer(); - - ASTSerialInfo::Entry* entry = entries[1]; - // We start on 1, because 0 is nullptr and not used for anything - for (Index i = 1; i < entriesCount; ++i) - { - ASTSerialInfo::Entry* next = entries[i + 1]; - - // Before writing we need to store the next alignment - - const size_t nextAlignment = ASTSerialInfo::getAlignment(next->info); - const size_t alignment = ASTSerialInfo::getAlignment(entry->info); - - entry->info = ASTSerialInfo::combineWithNext(entry->info, next->info); - - // Check we are aligned correctly - SLANG_ASSERT((offset & (alignment - 1)) == 0); - - // When we write, we need to make sure it take into account the next alignment - const size_t entrySize = entry->calcSize(m_classes); - - // Work out the fix for next alignment - size_t nextOffset = offset + entrySize; - nextOffset = (nextOffset + nextAlignment - 1) & ~(nextAlignment - 1); - - size_t alignmentFixSize = nextOffset - (offset + entrySize); - - // The fix must be less than max alignment. We require it to be less because we aligned each Entry to - // MAX_ALIGNMENT, and so < MAX_ALIGNMENT is the most extra bytes we can write - SLANG_ASSERT(alignmentFixSize < ASTSerialInfo::MAX_ALIGNMENT); - - container->write(entry, entrySize); - if (alignmentFixSize) - { - container->write(s_fixBuffer, alignmentFixSize); - } - - // Onto next - offset = nextOffset; - entry = next; - } - } - } - - return SLANG_OK; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialInfo::Entry !!!!!!!!!!!!!!!!!!!!!!!! - -size_t ASTSerialInfo::Entry::calcSize(ASTSerialClasses* serialClasses) const -{ - switch (type) - { - case Type::String: + case SerialTypeKind::NodeBase: { - auto entry = static_cast(this); - const uint8_t* cur = (const uint8_t*)entry->sizeAndChars; - uint32_t charsSize; - int sizeSize = ByteEncodeUtil::decodeLiteUInt32(cur, &charsSize); - return SLANG_OFFSET_OF(StringEntry, sizeAndChars) + sizeSize + charsSize; + return m_astBuilder->createByNodeType(ASTNodeType(subType)); } - case Type::Node: + case SerialTypeKind::RefObject: { - auto entry = static_cast(this); - auto serialClass = serialClasses->getSerialClass(ASTNodeType(entry->astNodeType)); - - // Align by the alignment of the entry - size_t alignment = getAlignment(entry->info); - size_t size = sizeof(NodeEntry) + serialClass->size; - - size = size + (alignment - 1) & ~(alignment - 1); - return size; - } - case Type::RefObject: - { - auto entry = static_cast(this); - - size_t payloadSize; - switch (entry->subType) + switch (RefObjectSerialSubType(subType)) { - case RefObjectEntry::SubType::Breadcrumb: - { - payloadSize = sizeof(ASTSerialTypeInfo::SerialType); - break; - } - default: + case RefObjectSerialSubType::LookupResultItem_Breadcrumb: { - SLANG_ASSERT(!"Unknown type"); - return 0; + typedef LookupResultItem::Breadcrumb Breadcrumb; + return _add(new LookupResultItem::Breadcrumb(Breadcrumb::Kind::Member, DeclRef(), nullptr, nullptr)); } + default: break; } - - return sizeof(RefObjectEntry) + payloadSize; - } - case Type::Array: - { - auto entry = static_cast(this); - return sizeof(ArrayEntry) + entry->elementSize * entry->elementCount; - } - default: break; - } - - SLANG_ASSERT(!"Unknown type"); - return 0; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!! - -const void* ASTSerialReader::getArray(ASTSerialIndex index, Index& outCount) -{ - if (index == ASTSerialIndex(0)) - { - outCount = 0; - return nullptr; - } - - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - switch (entry->type) - { - case Type::Array: - { - auto arrayEntry = static_cast(entry); - outCount = Index(arrayEntry->elementCount); - return (arrayEntry + 1); } default: break; } - SLANG_ASSERT(!"Not an array"); - outCount = 0; return nullptr; } -ASTSerialPointer ASTSerialReader::getPointer(ASTSerialIndex index) -{ - if (index == ASTSerialIndex(0)) - { - return ASTSerialPointer(); - } - - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - switch (entry->type) - { - case Type::String: - { - // Hmm. Tricky -> we don't know if will be cast as Name or String. Lets assume string. - String string = getString(index); - return ASTSerialPointer(string.getStringRepresentation()); - } - case Type::Node: - { - return ASTSerialPointer((NodeBase*)m_objects[Index(index)]); - } - case Type::RefObject: - { - return ASTSerialPointer((RefObject*)m_objects[Index(index)]); - } - default: break; - } - - SLANG_ASSERT(!"Cannot access as a pointer"); - return ASTSerialPointer(); -} - -String ASTSerialReader::getString(ASTSerialIndex index) -{ - if (index == ASTSerialIndex(0)) - { - return String(); - } - - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - // It has to be a string type - if (entry->type != Type::String) - { - SLANG_ASSERT(!"Not a string"); - return String(); - } - - RefObject* obj = (RefObject*)m_objects[Index(index)]; - - if (obj) - { - StringRepresentation* stringRep = dynamicCast(obj); - if (stringRep) - { - return String(stringRep); - } - // Must be a name then - Name* name = dynamicCast(obj); - SLANG_ASSERT(name); - return name->text; - } - - // Okay we need to construct as a string - UnownedStringSlice slice = getStringSlice(index); - String string(slice); - StringRepresentation* stringRep = string.getStringRepresentation(); - - m_scope.add(stringRep); - m_objects[Index(index)] = stringRep; - return string; -} - -Name* ASTSerialReader::getName(ASTSerialIndex index) -{ - if (index == ASTSerialIndex(0)) - { - return nullptr; - } - - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - // It has to be a string type - if (entry->type != Type::String) - { - SLANG_ASSERT(!"Not a string"); - return nullptr; - } - - RefObject* obj = (RefObject*)m_objects[Index(index)]; - - if (obj) - { - Name* name = dynamicCast(obj); - if (name) - { - return name; - } - // Can only be a string then - StringRepresentation* stringRep = dynamicCast(obj); - SLANG_ASSERT(stringRep); - - // I don't need to scope, as scoped in NamePool - name = m_namePool->getName(String(stringRep)); - - // Store as name, as can always access the inner string if needed - m_objects[Index(index)] = name; - return name; - } - - UnownedStringSlice slice = getStringSlice(index); - String string(slice); - Name* name = m_namePool->getName(string); - // Don't need to add to scope, because scoped on the pool - m_objects[Index(index)] = name; - return name; -} - -UnownedStringSlice ASTSerialReader::getStringSlice(ASTSerialIndex index) -{ - SLANG_ASSERT(ASTSerialIndexRaw(index) < ASTSerialIndexRaw(m_entries.getCount())); - const Entry* entry = m_entries[Index(index)]; - - // It has to be a string type - if (entry->type != Type::String) - { - SLANG_ASSERT(!"Not a string"); - return UnownedStringSlice(); - } - - auto stringEntry = static_cast(entry); - - const uint8_t* src = (const uint8_t*)stringEntry->sizeAndChars; - - // Decode the string - uint32_t size; - int sizeSize = ByteEncodeUtil::decodeLiteUInt32(src, &size); - return UnownedStringSlice((const char*)src + sizeSize, size); -} - -SourceLoc ASTSerialReader::getSourceLoc(ASTSerialSourceLoc loc) -{ - return (loc && m_debugReader) ? m_debugReader->getSourceLoc(loc) : SourceLoc(); -} - -SlangResult ASTSerialReader::loadEntries(const uint8_t* data, size_t dataCount, List& outEntries) -{ - // Check the input data is at least aligned to the max alignment (otherwise everything cannot be aligned correctly) - SLANG_ASSERT((size_t(data) & (ASTSerialInfo::MAX_ALIGNMENT - 1)) == 0); - - outEntries.setCount(1); - outEntries[0] = nullptr; - - const uint8_t*const end = data + dataCount; - - const uint8_t* cur = data; - while (cur < end) - { - const Entry* entry = (const Entry*)cur; - outEntries.add(entry); - - const size_t entrySize = entry->calcSize(m_classes); - cur += entrySize; - - // Need to get the next alignment - const size_t nextAlignment = ASTSerialInfo::getNextAlignment(entry->info); - - // Need to fix cur with the alignment - cur = (const uint8_t*)((size_t(cur) + nextAlignment - 1) & ~(nextAlignment - 1)); - } - - return SLANG_OK; -} - -SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuilder* builder, NamePool* namePool) -{ - SLANG_RETURN_ON_FAIL(loadEntries(data, dataCount, m_entries)); - - m_namePool = namePool; - - m_objects.clearAndDeallocate(); - m_objects.setCount(m_entries.getCount()); - memset(m_objects.getBuffer(), 0, m_objects.getCount() * sizeof(void*)); - - // Go through entries, constructing objects. - for (Index i = 1; i < m_entries.getCount(); ++i) - { - const Entry* entry = m_entries[i]; - - switch (entry->type) - { - case Type::String: - { - // Don't need to construct an object. This is probably a StringRepresentation, or a Name - // Will evaluate lazily. - break; - } - case Type::Node: - { - auto nodeEntry = static_cast(entry); - m_objects[i] = builder->createByNodeType(ASTNodeType(nodeEntry->astNodeType)); - break; - } - case Type::RefObject: - { - auto objEntry = static_cast(entry); - switch (objEntry->subType) - { - case ASTSerialInfo::RefObjectEntry::SubType::Breadcrumb: - { - typedef LookupResultItem::Breadcrumb Breadcrumb; - - auto breadcrumb = new LookupResultItem::Breadcrumb(Breadcrumb::Kind::Member, DeclRef(), nullptr, nullptr); - m_scope.add(breadcrumb); - m_objects[i] = breadcrumb; - break; - } - default: - { - SLANG_ASSERT(!"Unknown type"); - return SLANG_FAIL; - } - } - break; - } - case Type::Array: - { - // Don't need to construct an object, as will be accessed an interpreted by the object that holds it - break; - } - } - } - - // Deserialize - for (Index i = 1; i < m_entries.getCount(); ++i) - { - const Entry* entry = m_entries[i]; - void* native = m_objects[i]; - if (!native) - { - continue; - } - switch (entry->type) - { - case Type::Node: - { - auto nodeEntry = static_cast(entry); - auto serialClass = m_classes->getSerialClass(ASTNodeType(nodeEntry->astNodeType)); - - const uint8_t* src = (const uint8_t*)(nodeEntry + 1); - uint8_t* dst = (uint8_t*)m_objects[i]; - - // It must be constructed - SLANG_ASSERT(dst); - - while (serialClass) - { - for (Index j = 0; j < serialClass->fieldsCount; ++j) - { - auto field = serialClass->fields[j]; - auto fieldType = field.type; - fieldType->toNativeFunc(this, src + field.serialOffset, dst + field.nativeOffset); - } - - auto cls = ReflectClassInfo::getInfo(serialClass->type); - auto superCls = cls->m_superClass; - - // Get the super class - serialClass = superCls ? m_classes->getSerialClass(ASTNodeType(superCls->m_classId)) : nullptr; - } - - break; - } - case Type::RefObject: - { - auto objEntry = static_cast(entry); - switch (objEntry->subType) - { - case ASTSerialInfo::RefObjectEntry::SubType::Breadcrumb: - { - typedef LookupResultItem::Breadcrumb Breadcrumb; - auto serialType = ASTSerialGetType::getType(); - serialType->toNativeFunc(this, (entry + 1), m_objects[i]); - break; - } - default: - { - SLANG_ASSERT(!"Unknown type"); - return SLANG_FAIL; - } - } - break; - } - default: break; - } - } - - return SLANG_OK; -} - // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerializeUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!! /* static */SlangResult ASTSerialTestUtil::selfTest() { - RefPtr classes = new ASTSerialClasses; + RefPtr classes; + SerialClasses::create(classes); { - struct Thing - { - Module* node; - }; - Thing thing; - - //Pointer pointer(thing.node); - - auto field = _calcField("node", thing.node); - - - const ASTSerialType* type = ASTSerialGetType::getType(); + const SerialFieldType* type = SerialGetFieldType::getFieldType(); SLANG_UNUSED(type); } { - const ASTSerialType* type = ASTSerialGetType::getType(); + const SerialFieldType* type = SerialGetFieldType::getFieldType(); SLANG_UNUSED(type); } { - const ASTSerialType* type = ASTSerialGetType::getType(); + const SerialFieldType* type = SerialGetFieldType::getFieldType(); SLANG_UNUSED(type); } { - const ASTSerialType* type = ASTSerialGetType::getType(); + const SerialFieldType* type = SerialGetFieldType::getFieldType(); SLANG_UNUSED(type); } @@ -1913,7 +596,8 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil /* static */SlangResult ASTSerialTestUtil::testSerialize(NodeBase* node, RootNamePool* rootNamePool, SharedASTBuilder* sharedASTBuilder, SourceManager* sourceManager) { - RefPtr classes = new ASTSerialClasses; + RefPtr classes; + SerialClasses::create(classes); List contents; @@ -1921,11 +605,12 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil OwnedMemoryStream stream(FileAccess::ReadWrite); ModuleDecl* moduleDecl = as(node); - ModuleASTSerialFilter filterStorage(moduleDecl); + // Only serialize out things *in* this module + ModuleSerialFilter filterStorage(moduleDecl); - ASTSerialFilter* filter = moduleDecl ? &filterStorage : nullptr; + SerialFilter* filter = moduleDecl ? &filterStorage : nullptr; - ASTSerialWriter writer(classes, filter, nullptr); + SerialWriter writer(classes, filter); // Lets serialize it all writer.addPointer(node); @@ -1937,15 +622,17 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil NamePool namePool; namePool.setRootNamePool(rootNamePool); - ASTSerialReader reader(classes, nullptr); + SerialReader reader(classes, nullptr); ASTBuilder builder(sharedASTBuilder, "Serialize Check"); + DefaultSerialObjectFactory objectFactory(&builder); + // We could now check that the loaded data matches { - const List& writtenEntries = writer.getEntries(); - List readEntries; + const List& writtenEntries = writer.getEntries(); + List readEntries; SlangResult res = reader.loadEntries(contents.getBuffer(), contents.getCount(), readEntries); SLANG_UNUSED(res); @@ -1969,7 +656,7 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil } { - SlangResult res = reader.load(contents.getBuffer(), contents.getCount(), &builder, &namePool); + SlangResult res = reader.load(contents.getBuffer(), contents.getCount(), &namePool); SLANG_UNUSED(res); } @@ -1979,7 +666,7 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil String readDump; { SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None); - ASTDumpUtil::dump(reader.getPointer(ASTSerialIndex(1)).dynamicCast(), ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter); + ASTDumpUtil::dump(reader.getPointer(SerialIndex(1)).dynamicCast(), ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter); readDump = sourceWriter.getContentAndClear(); } @@ -1994,7 +681,6 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil File::writeAllText("ast-read.ast-dump", readDump); File::writeAllText("ast-orig.ast-dump", origDump); - if (readDump != origDump) { return SLANG_FAIL; @@ -2004,5 +690,4 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil return SLANG_OK; } - } // namespace Slang -- cgit v1.2.3