From d084f632a136354dd12952183994240b459240ee Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Fri, 26 Jun 2020 12:40:31 -0400 Subject: AST serialize improvements (#1412) * Try to fix problem with C++ extractor concating tokens producing an erroneous result. * Improve naming/comments around C++ extractor fix. * Another small improvement around space concating when outputing token list. * Handle some more special cases for consecutive tokens for C++ extractor concat of tokens. * WIP AST serialization. * Comment out so compile works. * More work on AST serialization. * WIP AST serialize. * WIP AST Serialization - handling more types. * WIP: Compiles but not all types are converted, as not all List element types are handled. * Compiles with array types. * Finish off AST serialization of remaining types. * Remove ComputedLayoutModifier and TupleVarModifier. * Add fields to ASTSerialClass type. * Construct AST type layout. * AST Serialization working for writing to ASTSerialWriter. * Removed call to ASTSerialization::selfTest in session creation. * Fixes for gcc. * Diagnostics handling - better handling of dashify. * Improve comment around DiagnosticLookup. * Updated VS project. * Write out as a Stream, taking into account alignment. * First pass at serializing in AST. * Added support for deserializing arrays. * Small bug fixes. * Fix problem calculating layout. Split out loading on entries. * Fix typo in AST conversion. * Add some flags to control AST dumping. * Fix bug from a typo. * Special case handling of Name* in AST serialization. * Special case handling of Token lexemes, make Names on read. * Documentation on AST serialization. * ASTSerialTestUtil - put AST testing functions. Fix typo that broke compilation. * Fix typo. --- source/slang/slang-ast-serialize.cpp | 122 ++++++++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 3 deletions(-) (limited to 'source/slang/slang-ast-serialize.cpp') diff --git a/source/slang/slang-ast-serialize.cpp b/source/slang/slang-ast-serialize.cpp index f8364e777..bc58ebebf 100644 --- a/source/slang/slang-ast-serialize.cpp +++ b/source/slang/slang-ast-serialize.cpp @@ -7,6 +7,8 @@ #include "slang-compiler.h" #include "slang-type-layout.h" +#include "slang-ast-dump.h" + #include "slang-ast-support-types.h" #include "../core/slang-byte-encode-util.h" @@ -220,6 +222,24 @@ struct ASTSerialTypeInfo } }; +// Special case Name +template <> +struct ASTSerialTypeInfo : public ASTSerialTypeInfo +{ + // Special case + typedef Name* NativeType; + static void toNative(ASTSerialReader* reader, const void* inSerial, void* outNative) + { + *(Name**)outNative = reader->getName(*(const SerialType*)inSerial); + } +}; + +template <> +struct ASTSerialTypeInfo : public ASTSerialTypeInfo +{ +}; + + struct ASTSerialDeclRefBaseTypeInfo { typedef DeclRefBase NativeType; @@ -651,7 +671,7 @@ struct ASTSerialTypeInfo auto& dst = *(NativeType*)native; dst.type = reader->getPointer(src.type).dynamicCast(); - dst.exp = reader->getPointer(src.type).dynamicCast(); + dst.exp = reader->getPointer(src.expr).dynamicCast(); } }; @@ -779,7 +799,15 @@ struct ASTSerialTypeInfo ASTSerialTypeInfo::toSerial(writer, &src.type, &dst.type); ASTSerialTypeInfo::toSerial(writer, &src.loc, &dst.loc); - dst.name = writer->addName(src.getName()); + + if (src.flags & TokenFlag::Name) + { + dst.name = writer->addName(src.getName()); + } + else + { + dst.name = writer->addString(src.getContent()); + } } static void toNative(ASTSerialReader* reader, const void* serial, void* native) { @@ -792,6 +820,7 @@ struct ASTSerialTypeInfo ASTSerialTypeInfo::toNative(reader, &src.type, &dst.type); ASTSerialTypeInfo::toNative(reader, &src.loc, &dst.loc); + // At the other end all token content will appear as Names. if (src.name != ASTSerialIndex(0)) { dst.charsNameUnion.name = reader->getName(src.name); @@ -1682,7 +1711,7 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerializeUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!! -/* static */SlangResult ASTSerializeUtil::selfTest() +/* static */SlangResult ASTSerialTestUtil::selfTest() { RefPtr classes = new ASTSerialClasses; @@ -1720,5 +1749,92 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil return SLANG_OK; } +/* static */SlangResult ASTSerialTestUtil::testSerialize(NodeBase* node, RootNamePool* rootNamePool, SharedASTBuilder* sharedASTBuilder, SourceManager* sourceManager) +{ + RefPtr classes = new ASTSerialClasses; + + List contents; + + { + OwnedMemoryStream stream(FileAccess::ReadWrite); + + ASTSerialWriter writer(classes); + + // Lets serialize it all + writer.addPointer(node); + // Let's stick it all in a stream + writer.write(&stream); + + stream.swapContents(contents); + + NamePool namePool; + namePool.setRootNamePool(rootNamePool); + + ASTSerialReader reader(classes); + + ASTBuilder builder(sharedASTBuilder, "Serialize Check"); + + // We could now check that the loaded data matches + + { + const List& writtenEntries = writer.getEntries(); + List readEntries; + + SlangResult res = reader.loadEntries(contents.getBuffer(), contents.getCount(), readEntries); + SLANG_UNUSED(res); + + SLANG_ASSERT(writtenEntries.getCount() == readEntries.getCount()); + + // They should be identical up to the + for (Index i = 1; i < readEntries.getCount(); ++i) + { + auto writtenEntry = writtenEntries[i]; + auto readEntry = readEntries[i]; + + const size_t writtenSize = writtenEntry->calcSize(classes); + const size_t readSize = readEntry->calcSize(classes); + + SLANG_ASSERT(readSize == writtenSize); + // Check the payload is the same + SLANG_ASSERT(memcmp(readEntry, writtenEntry, readSize) == 0); + } + + } + + { + SlangResult res = reader.load(contents.getBuffer(), contents.getCount(), &builder, &namePool); + SLANG_UNUSED(res); + } + + // Lets see what we have + const ASTDumpUtil::Flags dumpFlags = ASTDumpUtil::Flag::HideSourceLoc | ASTDumpUtil::Flag::HideScope; + + String readDump; + { + SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None); + ASTDumpUtil::dump(reader.getPointer(ASTSerialIndex(1)).dynamicCast(), ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter); + readDump = sourceWriter.getContentAndClear(); + + } + String origDump; + { + SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None); + ASTDumpUtil::dump(node, ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter); + origDump = sourceWriter.getContentAndClear(); + } + + // Write out + File::writeAllText("ast-read.ast-dump", readDump); + File::writeAllText("ast-orig.ast-dump", origDump); + + if (readDump != origDump) + { + return SLANG_FAIL; + } + } + + return SLANG_OK; +} + } // namespace Slang -- cgit v1.2.3