From ae41db80aa95ee7243d91a3ae4f56e6deb17f7f4 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Wed, 24 Jun 2020 13:56:06 -0400 Subject: AST Serialization writing (#1407) * Try to fix problem with C++ extractor concating tokens producing an erroneous result. * Improve naming/comments around C++ extractor fix. * Another small improvement around space concating when outputing token list. * Handle some more special cases for consecutive tokens for C++ extractor concat of tokens. * WIP AST serialization. * Comment out so compile works. * More work on AST serialization. * WIP AST serialize. * WIP AST Serialization - handling more types. * WIP: Compiles but not all types are converted, as not all List element types are handled. * Compiles with array types. * Finish off AST serialization of remaining types. * Remove ComputedLayoutModifier and TupleVarModifier. * Add fields to ASTSerialClass type. * Construct AST type layout. * AST Serialization working for writing to ASTSerialWriter. * Removed call to ASTSerialization::selfTest in session creation. * Fixes for gcc. * Diagnostics handling - better handling of dashify. * Improve comment around DiagnosticLookup. * Updated VS project. --- source/slang/slang-ast-serialize.cpp | 1289 ++++++++++++++++++++++++++++++++++ 1 file changed, 1289 insertions(+) create mode 100644 source/slang/slang-ast-serialize.cpp (limited to 'source/slang/slang-ast-serialize.cpp') diff --git a/source/slang/slang-ast-serialize.cpp b/source/slang/slang-ast-serialize.cpp new file mode 100644 index 000000000..411e7a7ad --- /dev/null +++ b/source/slang/slang-ast-serialize.cpp @@ -0,0 +1,1289 @@ +// slang-ast-serialize.cpp +#include "slang-ast-serialize.h" + +#include "slang-ast-generated.h" +#include "slang-ast-generated-macro.h" + +#include "slang-compiler.h" +#include "slang-type-layout.h" + +#include "slang-ast-support-types.h" + +#include "../core/slang-byte-encode-util.h" + +namespace Slang { + + +// Things stored as references: +// +// NodeBase derived types +// Array +// +// RefObject derived types: +// +// Breadcrumb +// StringRepresentation +// Scope + + + +// Helpers to convert fields treated as values + +class ASTSerialReader; +class ASTSerialWriter; + +template +static void _toSerialValue(ASTSerialWriter* writer, const NATIVE_TYPE& src, SERIAL_TYPE& dst) +{ + ASTSerialTypeInfo::toSerial(writer, &src, &dst); +} + +template +static void _toNativeValue(ASTSerialReader* reader, const SERIAL_TYPE& src, NATIVE_TYPE& dst) +{ + ASTSerialTypeInfo::toNative(reader, &src, &dst); +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Serial <-> Native conversion !!!!!!!!!!!!!!!!!!!!!!!! + + +// We need to have a way to map between the two. +// If no mapping is needed, (just a copy), then we don't bother with the functions +template +struct ASTSerialBasicTypeInfo +{ + typedef T NativeType; + typedef T SerialType; + + // We want the alignment to be the same as the size of the type for basic types + // NOTE! Might be different from SLANG_ALIGN_OF(SerialType) + enum { SerialAlignment = sizeof(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(T*)serial = *(const T*)native; } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(T*)native = *(const T*)serial; } + + static const ASTSerialType* getType() + { + static const ASTSerialType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; + return &type; + } +}; + +template +struct ASTSerialConvertTypeInfo +{ + typedef NATIVE_T NativeType; + typedef SERIAL_T SerialType; + + enum { SerialAlignment = ASTSerialBasicTypeInfo::SerialAlignment }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SERIAL_T*)serial = SERIAL_T(*(const NATIVE_T*)native); } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(NATIVE_T*)native = NATIVE_T(*(const SERIAL_T*)serial); } +}; + +template +struct ASTSerialIdentityTypeInfo +{ + typedef T NativeType; + typedef T SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(T*)serial = *(const T*)native; } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); *(T*)native = *(const T*)serial; } +}; + +// Don't need to convert the index type + +template <> +struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; + +// Implement for Basic Types + +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; + +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; + +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; +template <> +struct ASTSerialTypeInfo : public ASTSerialBasicTypeInfo {}; + +// SamplerStateFlavor + +template <> +struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + +// TextureFlavor + +template <> +struct ASTSerialTypeInfo +{ + typedef TextureFlavor NativeType; + typedef uint16_t SerialType; + enum { SerialAlignment = sizeof(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) { SLANG_UNUSED(writer); *(SerialType*)serial = ((const NativeType*)native)->flavor; } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) { SLANG_UNUSED(reader); ((NativeType*)native)->flavor = *(const SerialType*)serial; } +}; + +// Fixed arrays + +template +struct ASTSerialTypeInfo +{ + typedef ASTSerialTypeInfo ElementASTSerialType; + typedef typename ElementASTSerialType::SerialType SerialElementType; + + typedef T NativeType[N]; + typedef SerialElementType SerialType[N]; + + enum { SerialAlignment = ASTSerialTypeInfo::SerialAlignment }; + + static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + { + SerialElementType* serial = (SerialElementType*)outSerial; + const T* native = (const T*)inNative; + for (Index i = 0; i < Index(N); ++i) + { + ElementASTSerialType::toSerial(writer, native + i, serial + i); + } + } + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + const SerialElementType* serial = (const SerialElementType*)inSerial; + T* native = (T*)outNative; + for (Index i = 0; i < Index(N); ++i) + { + ElementASTSerialType::toNative(reader, serial + i, native + i); + } + } +}; + +// Special case bool - as we can't rely on size alignment +template <> +struct ASTSerialTypeInfo +{ + typedef bool NativeType; + typedef uint8_t SerialType; + + enum { SerialAlignment = sizeof(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + { + SLANG_UNUSED(writer); + *(SerialType*)outSerial = *(const NativeType*)inNative ? 1 : 0; + } + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + SLANG_UNUSED(reader); + *(NativeType*)outNative = (*(const SerialType*)inSerial) != 0; + } +}; + + +// Pointer +// Could handle different pointer base types with some more template magic here, but instead went with Pointer type to keep +// things simpler. +template +struct ASTSerialTypeInfo +{ + typedef T* NativeType; + typedef ASTSerialIndex SerialType; + + enum + { + SerialAlignment = SLANG_ALIGN_OF(SerialType) + }; + + static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + { + *(SerialType*)outSerial = writer->addPointer(*(T**)inNative); + } + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + *(T**)outNative = reader->getPointer(*(const SerialType*)inSerial).dynamicCast(); + } +}; + +struct ASTSerialDeclRefBaseTypeInfo +{ + typedef DeclRefBase NativeType; + struct SerialType + { + ASTSerialIndex substitutions; + ASTSerialIndex decl; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + { + SerialType& serial = *(SerialType*)outSerial; + const NativeType& native = *(const NativeType*)inNative; + + serial.decl = writer->addPointer(native.decl); + serial.substitutions = writer->addPointer(native.substitutions.substitutions); + } + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + DeclRefBase& native = *(DeclRefBase*)(outNative); + const SerialType& serial = *(const SerialType*)inSerial; + + native.decl = reader->getPointer(serial.decl).dynamicCast(); + native.substitutions.substitutions = reader->getPointer(serial.substitutions).dynamicCast(); + } + static const ASTSerialType* getType() + { + static const ASTSerialType type = { sizeof(SerialType), uint8_t(SerialAlignment), &toSerial, &toNative }; + return &type; + } +}; + +template +struct ASTSerialTypeInfo> : public ASTSerialDeclRefBaseTypeInfo {}; + +// MatrixCoord can just go as is +template <> +struct ASTSerialTypeInfo : ASTSerialIdentityTypeInfo {}; + +// SourceLoc + +// Make the type exposed, so we can look for it if we want to remap. +template <> +struct ASTSerialTypeInfo +{ + typedef SourceLoc NativeType; + typedef ASTSerialSourceLoc SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialSourceLoc) }; + + static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + { + *(SerialType*)outSerial = writer->addSourceLoc(*(const NativeType*)inNative); + } + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + *(NativeType*)outNative = reader->getSourceLoc(*(const SerialType*)inSerial); + } +}; + +// List +template +struct ASTSerialTypeInfo> +{ + typedef List NativeType; + typedef ASTSerialIndex SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + dst = writer->addArray(src.getBuffer(), src.getCount()); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& dst = *(NativeType*)native; + auto& src = *(const SerialType*)serial; + + reader->getArray(src, dst); + } +}; + +// Dictionary +template +struct ASTSerialTypeInfo> +{ + typedef Dictionary NativeType; + struct SerialType + { + ASTSerialIndex keys; ///< Index an array + ASTSerialIndex values; ///< Index an array + }; + + typedef typename ASTSerialTypeInfo::SerialType KeySerialType; + typedef typename ASTSerialTypeInfo::SerialType ValueSerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + List keys; + List values; + + Index count = Index(src.Count()); + keys.setCount(count); + values.setCount(count); + + Index i = 0; + for (const auto& pair : src) + { + ASTSerialTypeInfo::toSerial(writer, &pair.Key, &keys[i]); + ASTSerialTypeInfo::toSerial(writer, &pair.Value, &values[i]); + i++; + } + + dst.keys = writer->addArray(keys.getBuffer(), count); + dst.values = writer->addArray(values.getBuffer(), count); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + // Clear it + dst = NativeType(); + + List keys; + List values; + + reader->getArray(src.keys, keys); + reader->getArray(src.values, values); + + SLANG_ASSERT(keys.getCount() == values.getCount()); + + const Index count = keys.getCount(); + for (Index i = 0; i < count; ++i) + { + dst.Add(keys[i], values[i]); + } + } +}; + +// SyntaxClass +template +struct ASTSerialTypeInfo> +{ + typedef SyntaxClass NativeType; + typedef uint16_t SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + SLANG_UNUSED(writer); + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + dst = SerialType(src.classInfo->m_classId); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + SLANG_UNUSED(reader); + auto& src = *(const SerialType*)native; + auto& dst = *(NativeType*)serial; + dst.classInfo = ReflectClassInfo::getInfo(ASTNodeType(src)); + } +}; + +// Handle RefPtr - just convert into * to do the conversion +template +struct ASTSerialTypeInfo> +{ + typedef RefPtr NativeType; + typedef typename ASTSerialTypeInfo::SerialType SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + T* obj = src; + ASTSerialTypeInfo::toSerial(writer, &obj, serial); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + T* obj = nullptr; + ASTSerialTypeInfo::toNative(reader, serial, &obj); + *(NativeType*)native = obj; + } +}; + +// QualType + +template <> +struct ASTSerialTypeInfo +{ + typedef QualType NativeType; + struct SerialType + { + ASTSerialIndex type; + uint8_t isLeftValue; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + + static void toSerial(ASTSerialWriter* writer, const void* inNative, void* outSerial) + { + auto dst = (SerialType*)outSerial; + auto src = (const NativeType*)inNative; + dst->isLeftValue = src->isLeftValue ? 1 : 0; + dst->type = writer->addPointer(src->type); + } + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + auto src = (const SerialType*)inSerial; + auto dst = (NativeType*)outNative; + dst->type = reader->getPointer(src->type).dynamicCast(); + dst->isLeftValue = src->isLeftValue != 0; + } +}; + + +// LookupResult::Breadcrumb +template <> +struct ASTSerialTypeInfo +{ + typedef LookupResultItem::Breadcrumb NativeType; + struct SerialType + { + NativeType::Kind kind; + NativeType::ThisParameterMode thisParameterMode; + ASTSerialTypeInfo>::SerialType declRef; + ASTSerialTypeInfo> next; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + dst.kind = src.kind; + dst.thisParameterMode = src.thisParameterMode; + _toSerialValue(writer, src.declRef, dst.declRef); + _toSerialValue(writer, src.next, dst.next); + } + + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& dst = *(NativeType*)native; + auto& src = *(const SerialType*)serial; + + dst.kind = src.kind; + dst.thisParameterMode = src.thisParameterMode; + _toNativeValue(reader, src.declRef, dst.declRef); + _toNativeValue(reader, src.next, dst.next); + } +}; + +// LookupResultItem +template <> +struct ASTSerialTypeInfo +{ + typedef LookupResultItem NativeType; + struct SerialType + { + ASTSerialTypeInfo>::SerialType declRef; + ASTSerialTypeInfo> breadcrumbs; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + _toSerialValue(writer, src.declRef, dst.declRef); + _toSerialValue(writer, src.breadcrumbs, dst.breadcrumbs); + } + + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& dst = *(NativeType*)native; + auto& src = *(const SerialType*)serial; + + _toNativeValue(reader, src.declRef, dst.declRef); + _toNativeValue(reader, src.breadcrumbs, dst.breadcrumbs); + } +}; + +// LookupResult +template <> +struct ASTSerialTypeInfo +{ + typedef LookupResult NativeType; + typedef ASTSerialIndex SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + if (src.isOverloaded()) + { + // Save off as an array + dst = writer->addArray(src.items.getBuffer(), src.items.getCount()); + } + else if (src.item.declRef.getDecl()) + { + dst = writer->addArray(&src.item, 1); + } + else + { + dst = ASTSerialIndex(0); + } + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& dst = *(NativeType*)native; + auto& src = *(const SerialType*)serial; + + // Initialize + dst = NativeType(); + + List items; + reader->getArray(src, items); + + if (items.getCount() == 1) + { + dst.item = items[0]; + } + else + { + dst.items.swapWith(items); + // We have to set item such that it is valid/member of items, if items is non empty + dst.item = dst.items[0]; + } + } +}; + + +// GlobalGenericParamSubstitution::ConstraintArg +template <> +struct ASTSerialTypeInfo +{ + typedef GlobalGenericParamSubstitution::ConstraintArg NativeType; + struct SerialType + { + ASTSerialIndex decl; + ASTSerialIndex val; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& dst = *(SerialType*)serial; + auto& src = *(const NativeType*)native; + + dst.decl = writer->addPointer(src.decl); + dst.val = writer->addPointer(src.val); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + dst.decl = reader->getPointer(src.decl).dynamicCast(); + dst.val = reader->getPointer(src.val).dynamicCast(); + } +}; + +// ExpandedSpecializationArg +template <> +struct ASTSerialTypeInfo +{ + typedef ExpandedSpecializationArg NativeType; + struct SerialType + { + ASTSerialIndex val; + ASTSerialIndex witness; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& dst = *(SerialType*)serial; + auto& src = *(const NativeType*)native; + + dst.witness = writer->addPointer(src.witness); + dst.val = writer->addPointer(src.val); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + dst.witness = reader->getPointer(src.witness).dynamicCast(); + dst.val = reader->getPointer(src.val).dynamicCast(); + } +}; + +// TypeExp +template <> +struct ASTSerialTypeInfo +{ + typedef TypeExp NativeType; + struct SerialType + { + ASTSerialIndex type; + ASTSerialIndex expr; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(ASTSerialIndex) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& dst = *(SerialType*)serial; + auto& src = *(const NativeType*)native; + + dst.type = writer->addPointer(src.type); + dst.expr = writer->addPointer(src.exp); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + dst.type = reader->getPointer(src.type).dynamicCast(); + dst.exp = reader->getPointer(src.type).dynamicCast(); + } +}; + +// DeclCheckStateExt +template <> +struct ASTSerialTypeInfo +{ + typedef DeclCheckStateExt NativeType; + typedef DeclCheckStateExt::RawType SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + SLANG_UNUSED(writer); + *(SerialType*)serial = (*(const NativeType*)native).getRaw(); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + SLANG_UNUSED(reader); + (*(NativeType*)serial).setRaw(*(const SerialType*)native); + } +}; + +// Modifiers +template <> +struct ASTSerialTypeInfo +{ + typedef Modifiers NativeType; + typedef ASTSerialIndex SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + // We need to make into an array + List modifierIndices; + for (Modifier* modifier : *(NativeType*)native) + { + modifierIndices.add(writer->addPointer(modifier)); + } + *(SerialType*)serial = writer->addArray(modifierIndices.getBuffer(), modifierIndices.getCount()); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + List modifiers; + reader->getArray(*(const SerialType*)serial, modifiers); + + Modifier* prev = nullptr; + for (Modifier* modifier : modifiers) + { + if (prev) + { + prev->next = modifier; + } + } + + NativeType& dst = *(NativeType*)native; + dst.first = modifiers.getCount() > 0 ? modifiers[0] : nullptr; + } +}; + +// ImageFormat +template <> +struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + +// Stage +template <> +struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + +// TokenType +template <> +struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + +// BaseType +template <> +struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + +// SemanticVersion +template <> +struct ASTSerialTypeInfo : public ASTSerialIdentityTypeInfo {}; + +// ASTNodeType +template <> +struct ASTSerialTypeInfo : public ASTSerialConvertTypeInfo {}; + +// String +template <> +struct ASTSerialTypeInfo +{ + typedef String NativeType; + typedef ASTSerialIndex SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + *(SerialType*)serial = writer->addString(src); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + dst = reader->getString(src); + } +}; + +// Token +template <> +struct ASTSerialTypeInfo +{ + typedef Token NativeType; + struct SerialType + { + ASTSerialTypeInfo::SerialType type; + ASTSerialTypeInfo::SerialType loc; + ASTSerialIndex name; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + ASTSerialTypeInfo::toSerial(writer, &src.type, &dst.type); + ASTSerialTypeInfo::toSerial(writer, &src.loc, &dst.loc); + dst.name = writer->addName(src.getName()); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + dst.flags = 0; + dst.charsNameUnion.chars = nullptr; + + ASTSerialTypeInfo::toNative(reader, &src.type, &dst.type); + ASTSerialTypeInfo::toNative(reader, &src.loc, &dst.loc); + + if (src.name != ASTSerialIndex(0)) + { + dst.charsNameUnion.name = reader->getName(src.name); + dst.flags |= TokenFlag::Name; + } + } +}; + +// NameLoc +template <> +struct ASTSerialTypeInfo +{ + typedef NameLoc NativeType; + struct SerialType + { + ASTSerialTypeInfo::SerialType loc; + ASTSerialIndex name; + }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + + dst.name = writer->addName(src.name); + ASTSerialTypeInfo::toSerial(writer, &src.loc, &dst.loc); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + dst.name = reader->getName(src.name); + ASTSerialTypeInfo::toNative(reader, &src.loc, &dst.loc); + } +}; + +// DiagnosticInfo +template <> +struct ASTSerialTypeInfo +{ + typedef const DiagnosticInfo* NativeType; + typedef ASTSerialIndex SerialType; + + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(ASTSerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + auto& dst = *(SerialType*)serial; + dst = src ? writer->addString(UnownedStringSlice(src->name)) : ASTSerialIndex(0); + } + static void toNative(ASTSerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + + if (src == ASTSerialIndex(0)) + { + dst = nullptr; + } + else + { + dst = findDiagnosticByName(reader->getStringSlice(src)); + } + } +}; + +// !!!!!!!!!!!!!!!!!!!!! ASTSerialGetType !!!!!!!!!!!!!!!!!!!!!!!!!!! +// Getting the type info, let's use a static variable to hold the state to keep simple + +template +struct ASTSerialGetType +{ + static const ASTSerialType* getType() + { + typedef ASTSerialTypeInfo Info; + static const ASTSerialType type = { sizeof(typename Info::SerialType), uint8_t(Info::SerialAlignment), &Info::toSerial, &Info::toNative }; + return &type; + } +}; + +// Special case DeclRef, because it always uses the same type +template +struct ASTSerialGetType> +{ + static const ASTSerialType* getType() { return ASTSerialDeclRefBaseTypeInfo::getType(); } +}; + +// !!!!!!!!!!!!!!!!!!!!!! Generate fields for a type !!!!!!!!!!!!!!!!!!!!!!!!!!! + + +template +ASTSerialField _calcField(const char* name, T& in) +{ + uint8_t* ptr = &reinterpret_cast(in); + + ASTSerialField field; + field.name = name; + field.type = ASTSerialGetType::getType(); + // This only works because we in is an offset from 1 + field.nativeOffset = uint32_t(size_t(ptr) - 1); + field.serialOffset = 0; + return field; +} + +static ASTSerialClass _makeClass(MemoryArena* arena, ASTNodeType type, const List& fields) +{ + ASTSerialClass cls = {}; + cls.type = type; + cls.fieldsCount = fields.getCount(); + cls.fields = arena->allocateAndCopyArray(fields.getBuffer(), fields.getCount()); + return cls; +} + +#define SLANG_AST_SERIAL_FIELD(FIELD_NAME, TYPE, param) fields.add(_calcField(#FIELD_NAME, obj->FIELD_NAME)); + +// Note that the obj point is not nullptr, because some compilers notice this is 'indexing from null' +// and warn/error. So we offset from 1. +#define SLANG_AST_SERIAL_MAKE_CLASS(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \ +{ \ + NAME* obj = (NAME*)1; \ + SLANG_UNUSED(obj); \ + fields.clear(); \ + SLANG_FIELDS_ASTNode_##NAME(SLANG_AST_SERIAL_FIELD, param) \ + outClasses[Index(ASTNodeType::NAME)] = _makeClass(arena, ASTNodeType::NAME, fields); \ +} + +struct ASTFieldAccess +{ + static void calcClasses(MemoryArena* arena, ASTSerialClass outClasses[Index(ASTNodeType::CountOf)]) + { + List fields; + SLANG_ALL_ASTNode_NodeBase(SLANG_AST_SERIAL_MAKE_CLASS, _) + } +}; + +ASTSerialClasses::ASTSerialClasses(): + m_arena(2048) +{ + memset(m_classes, 0, sizeof(m_classes)); + ASTFieldAccess::calcClasses(&m_arena, m_classes); + + // Now work out the layout + for (Index i = 0; i < SLANG_COUNT_OF(m_classes); ++i) + { + // Set up each class in order, from lowest to highest index + // Doing so means super class is always setup + ASTSerialClass& serialClass = m_classes[i]; + + const ReflectClassInfo* info = ReflectClassInfo::getInfo(serialClass.type); + + size_t maxAlignment = 1; + size_t offset = 0; + + const ReflectClassInfo* superInfo = info->m_superClass; + if (superInfo) + { + ASTSerialClass& superSerialInfo = m_classes[superInfo->m_classId]; + + // If it's been setup, then alignment must be non zero. + // The ordering of ASTNodeType, should mean type have larger ASTNodeType greater than supers ASTNodeType. + SLANG_ASSERT(superSerialInfo.alignment != 0); + + // Must be a power of 2 + SLANG_ASSERT((superSerialInfo.alignment & (superSerialInfo.alignment - 1)) == 0); + + maxAlignment = superSerialInfo.alignment; + offset = superSerialInfo.size; + + // Check it is correctly aligned + SLANG_ASSERT((offset & (maxAlignment - 1)) == 0); + } + + // Okay, go through fields setting their offset + ASTSerialField* field = serialClass.fields; + for (Index j = 0; j < serialClass.fieldsCount; j++) + { + size_t alignment = field->type->serialAlignment; + // Make sure the offset is aligned for the field requirement + offset = (offset + alignment - 1) & ~(alignment - 1); + + // Save the field offset + field->serialOffset = uint32_t(offset); + + // Move past the field + offset += field->type->serialSizeInBytes; + + // Calc the maximum alignment + maxAlignment = (alignment > maxAlignment) ? alignment : maxAlignment; + } + + // Align with maximum alignment + offset += (offset + maxAlignment - 1) & ~(maxAlignment - 1); + + serialClass.alignment = uint8_t(maxAlignment); + serialClass.size = uint32_t(offset); + } +} + + + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!! + +ASTSerialWriter::ASTSerialWriter(ASTSerialClasses* classes) : + m_arena(2048), + m_classes(classes) +{ + // 0 is always the null pointer + m_entries.add(nullptr); + m_ptrMap.Add(nullptr, 0); +} + +ASTSerialIndex ASTSerialWriter::addPointer(const NodeBase* node) +{ + // Null is always 0 + if (node == nullptr) + { + return ASTSerialIndex(0); + } + // Look up in the map + Index* indexPtr = m_ptrMap.TryGetValue(node); + if (indexPtr) + { + return ASTSerialIndex(*indexPtr); + } + + const ASTSerialClass* serialClass = m_classes->getSerialClass(node->astNodeType); + + typedef ASTSerialInfo::NodeEntry NodeEntry; + + NodeEntry* nodeEntry = (NodeEntry*)m_arena.allocateAligned(sizeof(NodeEntry) + serialClass->size, 8); + + nodeEntry->type = ASTSerialInfo::Type::Node; + nodeEntry->astNodeType = uint16_t(node->astNodeType); + nodeEntry->nextAlignment = 0; + + auto index = _add(node, nodeEntry); + + uint8_t* serialPayload = (uint8_t*)(nodeEntry + 1); + + while (serialClass) + { + for (Index i = 0; i < serialClass->fieldsCount; ++i) + { + auto field = serialClass->fields[i]; + // Work out the offsets + auto srcField = ((const uint8_t*)node) + field.nativeOffset; + auto dstField = serialPayload + field.serialOffset; + + field.type->toSerialFunc(this, srcField, dstField); + } + // Get the super class + const ReflectClassInfo* reflectInfo = ReflectClassInfo::getInfo(serialClass->type); + const ReflectClassInfo* superReflectInfo = reflectInfo->m_superClass; + + serialClass = superReflectInfo ? m_classes->getSerialClass(ASTNodeType(superReflectInfo->m_classId)) : nullptr; + } + + return index; +} + +ASTSerialIndex ASTSerialWriter::addPointer(const RefObject* obj) +{ + // Null is always 0 + if (obj == nullptr) + { + return ASTSerialIndex(0); + } + // Look up in the map + Index* indexPtr = m_ptrMap.TryGetValue(obj); + if (indexPtr) + { + return ASTSerialIndex(*indexPtr); + } + + if (auto stringRep = dynamicCast(obj)) + { + ASTSerialIndex index = addString(StringRepresentation::asSlice(stringRep)); + m_ptrMap.Add(obj, Index(index)); + return index; + } + else if (auto breadcrumb = dynamicCast(obj)) + { + typedef ASTSerialTypeInfo TypeInfo; + typedef ASTSerialInfo::RefObjectEntry RefObjectEntry; + + RefObjectEntry* refEntry = (RefObjectEntry*)m_arena.allocateAligned(sizeof(RefObjectEntry) + sizeof(TypeInfo::SerialType), 8); + + refEntry->type = ASTSerialInfo::Type::RefObject; + refEntry->subType = RefObjectEntry::SubType::Breadcrumb; + + auto index = _add(breadcrumb, refEntry); + + // Do any conversion + TypeInfo::toSerial(this, breadcrumb, refEntry + 1); + return index; + } + else if (auto name = dynamicCast(obj)) + { + return addName(name); + } + else if (auto scope = dynamicCast(obj)) + { + // We don't serialize scope + return ASTSerialIndex(0); + } + else if (auto module = dynamicCast(obj)) + { + // We don't serialize Module + return ASTSerialIndex(0); + } + + SLANG_ASSERT(!"Unhandled type"); + return ASTSerialIndex(0); +} + +ASTSerialIndex ASTSerialWriter::addString(const UnownedStringSlice& slice) +{ + typedef ByteEncodeUtil Util; + typedef ASTSerialInfo::StringEntry StringEntry; + + if (slice.getLength() == 0) + { + return ASTSerialIndex(0); + } + + Index newIndex = m_entries.getCount(); + + Index* indexPtr = m_sliceMap.TryGetValueOrAdd(slice, newIndex); + if (indexPtr) + { + return ASTSerialIndex(*indexPtr); + } + + // Okay we need to add the string + + uint8_t encodeBuf[Util::kMaxLiteEncodeUInt32]; + const int encodeCount = Util::encodeLiteUInt32(uint32_t(slice.getLength()), encodeBuf); + + StringEntry* entry = (StringEntry*)m_arena.allocateUnaligned(sizeof(StringEntry) + encodeCount + slice.getLength()); + entry->nextAlignment = 0; + entry->type = ASTSerialInfo::Type::String; + + uint8_t* dst = (uint8_t*)(entry + 1); + for (int i = 0; i < encodeCount; ++i) + { + dst[i] = encodeBuf[i]; + } + + memcpy(dst + encodeCount, slice.begin(), slice.getLength()); + + m_entries.add(entry); + return ASTSerialIndex(newIndex); +} + + +ASTSerialIndex ASTSerialWriter::addString(const String& in) +{ + return addPointer(in.getStringRepresentation()); +} + +ASTSerialIndex ASTSerialWriter::addName(const Name* name) +{ + if (name == nullptr) + { + return ASTSerialIndex(0); + } + + // Look it up + Index* indexPtr = m_ptrMap.TryGetValue(name); + if (indexPtr) + { + return ASTSerialIndex(*indexPtr); + } + + ASTSerialIndex index = addString(name->text); + m_ptrMap.Add(name, Index(index)); + return index; +} + +ASTSerialSourceLoc ASTSerialWriter::addSourceLoc(SourceLoc sourceLoc) +{ + SLANG_UNUSED(sourceLoc); + return 0; +} + +ASTSerialIndex ASTSerialWriter::_addArray(size_t elementSize, const void* elements, Index elementCount) +{ + typedef ASTSerialInfo::ArrayEntry Entry; + + if (elementCount == 0) + { + return ASTSerialIndex(0); + } + + size_t payloadSize = elementCount * elementSize; + + Entry* entry = (Entry*)m_arena.allocateAligned(sizeof(Entry) + payloadSize, 8); + + entry->type = ASTSerialInfo::Type::Array; + entry->nextAlignment = 0; + entry->elementSize = uint16_t(elementSize); + entry->elementCount = uint32_t(elementCount); + + memcpy(entry + 1, elements, payloadSize); + + m_entries.add(entry); + ASTSerialIndex index = ASTSerialIndex(m_entries.getCount() - 1); + + // We don't add to a pointer map, because arrays are not shared + + // Do the conversion + + return index; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!! + +ASTSerialPointer ASTSerialReader::getPointer(ASTSerialIndex index) +{ + SLANG_UNUSED(index); + return ASTSerialPointer(); +} + +String ASTSerialReader::getString(ASTSerialIndex index) +{ + SLANG_UNUSED(index); + return String(); +} + +Name* ASTSerialReader::getName(ASTSerialIndex index) +{ + if (index == ASTSerialIndex(0)) + { + return nullptr; + } + return nullptr; +} + +UnownedStringSlice ASTSerialReader::getStringSlice(ASTSerialIndex index) +{ + SLANG_UNUSED(index); + return UnownedStringSlice(); +} + +SourceLoc ASTSerialReader::getSourceLoc(ASTSerialSourceLoc loc) +{ + SLANG_UNUSED(loc); + return SourceLoc(); +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerializeUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!! + +/* static */SlangResult ASTSerializeUtil::selfTest() +{ + RefPtr classes = new ASTSerialClasses; + + { + struct Thing + { + Module* node; + }; + Thing thing; + + //Pointer pointer(thing.node); + + auto field = _calcField("node", thing.node); + + + const ASTSerialType* type = ASTSerialGetType::getType(); + SLANG_UNUSED(type); + } + + { + const ASTSerialType* type = ASTSerialGetType::getType(); + SLANG_UNUSED(type); + } + + { + const ASTSerialType* type = ASTSerialGetType::getType(); + SLANG_UNUSED(type); + } + + { + const ASTSerialType* type = ASTSerialGetType::getType(); + SLANG_UNUSED(type); + } + + return SLANG_OK; +} + + +} // namespace Slang -- cgit v1.2.3