/tests/language-feature/

• ../
• 1-vector.slang
• anonymous-struct.slang
• bit-cast
• bitfield
• capability
• constants
• dynamic-uniform
• enums
• extensions
• general-inline.slang
• general-inline.slang.expected.txt
• generics
• higher-order-functions
• if-let
• ifunc
• inheritance
• initializer-lists
• interfaces
• modules
• multi-level-break-switch.slang
• multi-level-break-switch.slang.expected.txt
• multi-level-break.slang
• multi-level-break.slang.expected.txt
• namespaces
• nested-empty-initializer-list.slang
• no-hlsl-binding.slang
• no-hlsl-pack-constant-buffer-elements.slang
• no-mangle.slang
• non-copyable-return.slang
• operators
• overload-resolution.slang
• parameters
• pointer
• properties
• reinterpret
• resource-specialization-nested-specialization.slang
• resource-specialization-struct-out.slang
• resource-specialization-struct-return.slang
• saturated-cooperation
• shader-params
• shared-memory-initializer-error.slang
• simple-inline.slang
• simple-inline.slang.expected.txt
• spirv-asm
• static-members
• static_assert.slang
• struct-field-initializers
• struct-in-func.slang
• struct-in-func.slang.expected.txt
• swizzles
• tuple
• types
• unsized-array.slang
• zero-initialize

// slang-serialize-container.cpp
#include "slang-serialize-container.h"

#include "../core/slang-text-io.h"
#include "../core/slang-byte-encode-util.h"

#include "../core/slang-math.h"

#include "slang-compiler.h"
#include "slang-serialize-ast.h"
#include "slang-serialize-ir.h"
#include "slang-serialize-source-loc.h"
#include "slang-serialize-factory.h"
#include "../core/slang-stream.h"
#include "slang-parser.h"

#include "slang-mangled-lexer.h"

namespace Slang {

/* static */SlangResult SerialContainerUtil::write(Module* module, const WriteOptions& options, Stream* stream)
{
    RiffContainer container;
    {
        SerialContainerData data;
        SLANG_RETURN_ON_FAIL(SerialContainerUtil::addModuleToData(module, options, data));
        SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container));
    }
    // We now write the RiffContainer to the stream
    SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream));
    return SLANG_OK;
}

/* static */SlangResult SerialContainerUtil::write(FrontEndCompileRequest* frontEndReq, const WriteOptions& options, Stream* stream)
{
    RiffContainer container;
    {
        SerialContainerData data;
        SLANG_RETURN_ON_FAIL(SerialContainerUtil::addFrontEndRequestToData(frontEndReq, options, data));
        SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container));
    }
    // We now write the RiffContainer to the stream
    SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream));
    return SLANG_OK;
}

/* static */SlangResult SerialContainerUtil::write(EndToEndCompileRequest* request, const WriteOptions& options, Stream* stream)
{
    RiffContainer container;
    {
        SerialContainerData data;
        SLANG_RETURN_ON_FAIL(SerialContainerUtil::addEndToEndRequestToData(request, options, data));
        SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container));
    }
    // We now write the RiffContainer to the stream
    SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream));
    return SLANG_OK;
}

/* static */SlangResult SerialContainerUtil::addModuleToData(Module* module, const WriteOptions& options, SerialContainerData& outData)
{
    if (options.optionFlags & (SerialOptionFlag::ASTModule | SerialOptionFlag::IRModule))
    {
        SerialContainerData::Module dstModule;

        // NOTE: The astBuilder is not set here, as not needed to be scoped for serialization (it is assumed the
        // TranslationUnitRequest stays in scope)

        if (options.optionFlags & SerialOptionFlag::ASTModule)
        {
            // Root AST node
            auto moduleDecl = module->getModuleDecl();
            SLANG_ASSERT(moduleDecl);

            dstModule.astRootNode = moduleDecl;
        }
        if (options.optionFlags & SerialOptionFlag::IRModule)
        {
            // IR module
            dstModule.irModule = module->getIRModule();
            SLANG_ASSERT(dstModule.irModule);
        }
        DigestBuilder<SHA1> digestBuilder;
        auto version = String(getBuildTagString());
        digestBuilder.append(version);
        module->getOptionSet().buildHash(digestBuilder);
        auto fileDependencies = module->getFileDependencies();
        String canonicalModulePath, moduleDir;
        if (auto modulePath = module->getFilePath())
        {
            canonicalModulePath = modulePath;
            Path::getCanonical(canonicalModulePath, canonicalModulePath);
            moduleDir = Path::getParentDirectory(canonicalModulePath);
        }
        for (auto file : fileDependencies)
        {
            digestBuilder.append(file->getDigest());
            if (file->getPathInfo().hasFoundPath())
            {
                if (file->getPathInfo().foundPath == canonicalModulePath)
                {
                    dstModule.dependentFiles.add(Path::getFileName(canonicalModulePath));
                }
                else
                {
                    String canonicalFilePath = file->getPathInfo().foundPath;
                    Path::getCanonical(file->getPathInfo().foundPath, canonicalFilePath);
                    canonicalFilePath = Path::getRelativePath(moduleDir, canonicalFilePath);
                    dstModule.dependentFiles.add(canonicalFilePath);
                }
            }
            else
            {
                dstModule.dependentFiles.add(file->getPathInfo().getMostUniqueIdentity());
            }
        }
        dstModule.digest = digestBuilder.finalize();
        outData.modules.add(dstModule);
    }

    return SLANG_OK;
}

/* static */SlangResult SerialContainerUtil::addFrontEndRequestToData(FrontEndCompileRequest* frontEndReq, const WriteOptions& options, SerialContainerData& outData)
{
    // Go through translation units, adding modules
    for (TranslationUnitRequest* translationUnit : frontEndReq->translationUnits)
    {
        SLANG_RETURN_ON_FAIL(addModuleToData(translationUnit->module, options, outData));        
    }

    return SLANG_OK;
}

/* static */SlangResult SerialContainerUtil::addEndToEndRequestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& out)
{
    auto linkage = request->getLinkage();
    auto sink = request->getSink();

    // Output the parsed modules.
    addFrontEndRequestToData(request->getFrontEndReq(), options, out);

    //
    auto program = request->getSpecializedGlobalAndEntryPointsComponentType();

    // Add all the target modules
    {
        for (auto target : linkage->targets)
        {
            auto targetProgram = program->getTargetProgram(target);
            auto irModule = targetProgram->getOrCreateIRModuleForLayout(sink);

            SerialContainerData::TargetComponent targetComponent;

            targetComponent.irModule = irModule;

            auto& dstTarget = targetComponent.target;

            dstTarget.floatingPointMode = target->getOptionSet().getFloatingPointMode();
            dstTarget.profile = target->getOptionSet().getProfile();
            dstTarget.flags = target->getOptionSet().getTargetFlags();
            dstTarget.codeGenTarget = target->getTarget();

            out.targetComponents.add(targetComponent);
        }
    }

    // Entry points
    {
        auto entryPointCount = program->getEntryPointCount();
        for (Index ii = 0; ii < entryPointCount; ++ii)
        {
            auto entryPoint = program->getEntryPoint(ii);
            auto entryPointMangledName = program->getEntryPointMangledName(ii);

            SerialContainerData::EntryPoint dstEntryPoint;

            dstEntryPoint.name = entryPoint->getName();
            dstEntryPoint.mangledName = entryPointMangledName;
            dstEntryPoint.profile = entryPoint->getProfile();

            out.entryPoints.add(dstEntryPoint);
        }
    }

    return SLANG_OK;
}

/* static */SlangResult SerialContainerUtil::write(const SerialContainerData& data, const WriteOptions& options, RiffContainer* container)
{
    RefPtr<SerialSourceLocWriter> sourceLocWriter;

    // The string pool used across the whole of the container
    StringSlicePool containerStringPool(StringSlicePool::Style::Default);

    RiffContainer::ScopeChunk scopeModule(container, RiffContainer::Chunk::Kind::List, SerialBinary::kContainerFourCc);

    // Write the header
    {
        SerialBinary::ContainerHeader containerHeader;
        // Save the compression type if used - as can only serialize with a single compression type
        containerHeader.compressionType = uint32_t(options.compressionType);

        RiffContainer::ScopeChunk scopeHeader(container, RiffContainer::Chunk::Kind::Data, SerialBinary::kContainerHeaderFourCc);
        container->write(&containerHeader, sizeof(containerHeader));
    }

    if (data.modules.getCount() && (options.optionFlags & (SerialOptionFlag::IRModule | SerialOptionFlag::ASTModule)))
    {
        // Module list
        RiffContainer::ScopeChunk moduleListScope(container, RiffContainer::Chunk::Kind::List, SerialBinary::kModuleListFourCc);

        if (options.optionFlags & SerialOptionFlag::SourceLocation)
        {
            sourceLocWriter = new SerialSourceLocWriter(options.sourceManager);
        }

        RefPtr<SerialClasses> serialClasses;

        for (const auto& module : data.modules)
        {
            // Okay, we need to serialize this module to our container file.
            // We currently don't serialize it's name..., but support for that could be added.

            // First, we write a header that can be used to verify if the precompiled module is up-to-date.
            // The header has:
            // 1) a digest of all compile options and dependent source files.
            // 2) a list of source file paths.
            //
            {
                RiffContainer::ScopeChunk scopeHeader(container, RiffContainer::Chunk::Kind::Data, SerialBinary::kModuleHeaderFourCc);
                OwnedMemoryStream headerMemStream(FileAccess::Write);
                StringBuilder filePathsSB;
                for (auto fileDependency : module.dependentFiles)
                    filePathsSB << fileDependency << "\n";
                headerMemStream.write(module.digest.data, sizeof(module.digest.data));
                uint32_t fileListLength = (uint32_t)filePathsSB.getLength();
                headerMemStream.write(&fileListLength, sizeof(uint32_t));
                headerMemStream.write(filePathsSB.getBuffer(), fileListLength);
                container->write(headerMemStream.getContents().getBuffer(), headerMemStream.getContents().getCount());
            }

            // Write the IR information
            if ((options.optionFlags & SerialOptionFlag::IRModule) && module.irModule)
            {
                IRSerialData serialData;
                IRSerialWriter writer;
                SLANG_RETURN_ON_FAIL(writer.write(module.irModule, sourceLocWriter, options.optionFlags, &serialData));
                SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, options.compressionType, container));
            }

            // Write the AST information

            if (options.optionFlags & SerialOptionFlag::ASTModule)
            {
                if (ModuleDecl* moduleDecl = as<ModuleDecl>(module.astRootNode))
                {
                    // Put in AST module
                    RiffContainer::ScopeChunk scopeASTModule(container, RiffContainer::Chunk::Kind::List, ASTSerialBinary::kSlangASTModuleFourCC);

                    if (!serialClasses)
                    {
                        SLANG_RETURN_ON_FAIL(SerialClassesUtil::create(serialClasses));
                    }

                    ModuleSerialFilter filter(moduleDecl);
                    auto astWriterFlag = SerialWriter::Flag::ZeroInitialize;
                    if ((options.optionFlags & SerialOptionFlag::ASTFunctionBody) == 0)
                        astWriterFlag = (SerialWriter::Flag::Enum)(astWriterFlag | SerialWriter::Flag::SkipFunctionBody);

                    SerialWriter writer(serialClasses, &filter, astWriterFlag);

                    writer.getExtraObjects().set(sourceLocWriter);
                    
                    // Add the module and everything that isn't filtered out in the filter.
                    writer.addPointer(moduleDecl);


                    // We can now serialize it into the riff container.
                    SLANG_RETURN_ON_FAIL(writer.writeIntoContainer(ASTSerialBinary::kSlangASTModuleDataFourCC, container));
                }
            }
        }

        if (data.targetComponents.getCount() && (options.optionFlags & SerialOptionFlag::IRModule))
        {
            // TODO: in the case where we have specialization, we might need
            // to serialize IR related to `program`...

            for (const auto& targetComponent : data.targetComponents)
            {
                IRModule* irModule = targetComponent.irModule;

                // Okay, we need to serialize this target program and its IR too...
                IRSerialData serialData;
                IRSerialWriter writer;

                SLANG_RETURN_ON_FAIL(writer.write(irModule, sourceLocWriter, options.optionFlags, &serialData));
                SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, options.compressionType, container));
            }
        }
    }

    if (data.entryPoints.getCount())
    {
        for (const auto& entryPoint : data.entryPoints)
        {
            RiffContainer::ScopeChunk entryPointScope(container, RiffContainer::Chunk::Kind::Data, SerialBinary::kEntryPointFourCc);

            SerialContainerBinary::EntryPoint dst;

            dst.name = uint32_t(containerStringPool.add(entryPoint.name->text));
            dst.profile = entryPoint.profile.raw;
            dst.mangledName = uint32_t(containerStringPool.add(entryPoint.mangledName));

            container->write(&dst, sizeof(dst));
        }
    }

    // We can now output the debug information. This is for all IR and AST 
    if (sourceLocWriter)
    {
        // Write out the debug info
        SerialSourceLocData debugData;
        sourceLocWriter->write(&debugData);

        debugData.writeContainer(options.compressionType, container);
    }

    // Write the container string table
    if (containerStringPool.getAdded().getCount() > 0)
    {
        RiffContainer::ScopeChunk stringTableScope(container, RiffContainer::Chunk::Kind::Data, SerialBinary::kStringTableFourCc);

        List<char> encodedTable;
        SerialStringTableUtil::encodeStringTable(containerStringPool, encodedTable);
        
        container->write(encodedTable.getBuffer(), encodedTable.getCount());
    }

    return SLANG_OK;
}


static List<ExtensionDecl*>& _getCandidateExtensionList(
    AggTypeDecl* typeDecl,
    Dictionary<AggTypeDecl*, RefPtr<CandidateExtensionList>>& mapTypeToCandidateExtensions)
{
    RefPtr<CandidateExtensionList> entry;
    if (!mapTypeToCandidateExtensions.tryGetValue(typeDecl, entry))
    {
        entry = new CandidateExtensionList();
        mapTypeToCandidateExtensions.add(typeDecl, entry);
    }
    return entry->candidateExtensions;
}

/* static */Result SerialContainerUtil::read(RiffContainer* container, const ReadOptions& options, const LoadedModuleDictionary* additionalLoadedModules, SerialContainerData& out)
{
    out.clear();

    RiffContainer::ListChunk* containerChunk = container->getRoot()->findListRec(SerialBinary::kContainerFourCc);
    if (!containerChunk)
    {
        // Must be a container
        return SLANG_FAIL;
    }

    SerialBinary::ContainerHeader* containerHeader = containerChunk->findContainedData<SerialBinary::ContainerHeader>(SerialBinary::kContainerHeaderFourCc);
    if (!containerHeader)
    {
        // Didn't find the header
        return SLANG_FAIL;
    }

    const SerialCompressionType containerCompressionType = SerialCompressionType(containerHeader->compressionType);

    StringSlicePool containerStringPool(StringSlicePool::Style::Default);

    if (RiffContainer::Data* stringTableData = containerChunk->findContainedData(SerialBinary::kStringTableFourCc))
    {
        SerialStringTableUtil::decodeStringTable((const char*)stringTableData->getPayload(), stringTableData->getSize(), containerStringPool);
    }

    RefPtr<SerialSourceLocReader> sourceLocReader;
    RefPtr<SerialClasses> serialClasses;

    // Debug information
    if (auto debugChunk = containerChunk->findContainedList(SerialSourceLocData::kDebugFourCc))
    {
        // Read into data
        SerialSourceLocData sourceLocData;
        SLANG_RETURN_ON_FAIL(sourceLocData.readContainer(containerCompressionType, debugChunk));

        // Turn into DebugReader
        sourceLocReader = new SerialSourceLocReader;
        SLANG_RETURN_ON_FAIL(sourceLocReader->read(&sourceLocData, options.sourceManager));
    }

    // Create a source loc representing the binary module.
    SourceLoc binaryModuleLoc = SourceLoc();

    if (options.modulePath.getLength())
    {
        auto srcManager = options.linkage->getSourceManager();
        auto modulePathInfo = PathInfo::makePath(options.modulePath);
        auto srcFile = srcManager->createSourceFileWithString(modulePathInfo, String());
        srcManager->addSourceFile(options.modulePath, srcFile);
        auto srcView = srcManager->createSourceView(srcFile, &modulePathInfo, SourceLoc());
        binaryModuleLoc = srcView->getRange().begin;
    }

    // Add modules
    if (RiffContainer::ListChunk* moduleList = containerChunk->findContainedList(SerialBinary::kModuleListFourCc))
    {
        RiffContainer::Chunk* chunk = moduleList->getFirstContainedChunk();
        while (chunk)
        {
            auto startChunk = chunk;

            RefPtr<ASTBuilder> astBuilder = options.astBuilder;
            NodeBase* astRootNode = nullptr;
            RefPtr<IRModule> irModule;
            SerialContainerData::Module module;
            if (auto headerChunk = as<RiffContainer::DataChunk>(chunk, SerialBinary::kModuleHeaderFourCc))
            {
                MemoryStreamBase memStream(
                    FileAccess::Read,
                    headerChunk->getSingleData()->getPayload(),
                    headerChunk->getSingleData()->getSize());
                size_t readSize = 0;
                memStream.read(module.digest.data, sizeof(SHA1::Digest), readSize);
                if (readSize != sizeof(SHA1::Digest))
                    return SLANG_FAIL;
                uint32_t fileListLength = 0;
                memStream.read(&fileListLength, sizeof(uint32_t), readSize);
                if (readSize != sizeof(uint32_t))
                    return SLANG_FAIL;
                List<uint8_t> fileListContent;
                fileListContent.setCount(fileListLength);
                memStream.read(fileListContent.getBuffer(), fileListContent.getCount(), readSize);
                if (readSize != (size_t)fileListContent.getCount())
                    return SLANG_FAIL;
                UnownedStringSlice fileListString((const char*)fileListContent.getBuffer(), fileListContent.getCount());
                List<UnownedStringSlice> fileList;
                StringUtil::split(fileListString, '\n', fileList);
                for (auto file : fileList)
                {
                    if (file.getLength())
                    {
                        module.dependentFiles.add(file);
                    }
                }
                // Onto next chunk
                chunk = chunk->m_next;
            }

            if (auto irChunk = as<RiffContainer::ListChunk>(chunk, IRSerialBinary::kIRModuleFourCc))
            {
                if (!options.readHeaderOnly)
                {
                    IRSerialData serialData;
                    SLANG_RETURN_ON_FAIL(IRSerialReader::readContainer(irChunk, containerCompressionType, &serialData));

                    // Read IR back from serialData
                    IRSerialReader reader;
                    SLANG_RETURN_ON_FAIL(reader.read(serialData, options.session, sourceLocReader, irModule));
                }

                // Onto next chunk
                chunk = chunk->m_next;
            }

            if (auto astChunk = as<RiffContainer::ListChunk>(chunk, ASTSerialBinary::kSlangASTModuleFourCC))
            {
                if (!options.readHeaderOnly)
                {
                    RiffContainer::Data* astData = astChunk->findContainedData(ASTSerialBinary::kSlangASTModuleDataFourCC);

                    if (astData)
                    {
                        if (!serialClasses)
                        {
                            SLANG_RETURN_ON_FAIL(SerialClassesUtil::create(serialClasses));
                        }

                        // TODO(JS): We probably want to store off better information about each of the translation unit
                        // including some kind of 'name'.
                        // For now we just generate a name.

                        StringBuilder buf;
                        buf << "tu" << out.modules.getCount();
                        if (!astBuilder)
                        {
                            astBuilder = new ASTBuilder(options.sharedASTBuilder, buf.produceString());
                        }

                        /// We need to make the current ASTBuilder available for access via thread_local global.
                        SetASTBuilderContextRAII astBuilderRAII(astBuilder);

                        DefaultSerialObjectFactory objectFactory(astBuilder);

                        SerialReader reader(serialClasses, &objectFactory);

                        // Sets up the entry table - one entry for each 'object'.
                        // No native objects are constructed. No objects are deserialized.
                        SLANG_RETURN_ON_FAIL(reader.loadEntries((const uint8_t*)astData->getPayload(), astData->getSize()));

                        // Construct a native object for each table entry (where appropriate).
                        // Note that this *doesn't* set all object pointers - some are special cased and created on demand (strings)
                        // and imported symbols will have their object pointers unset (they are resolved in next step)
                        SLANG_RETURN_ON_FAIL(reader.constructObjects(options.namePool));

                        // Resolve external references if the linkage is specified
                        if (options.linkage)
                        {
                            const auto& entries = reader.getEntries();
                            auto& objects = reader.getObjects();
                            const Index entriesCount = entries.getCount();

                            String currentModuleName;
                            Module* currentModule = nullptr;

                            // Index from 1 (0 is null)
                            for (Index i = 1; i < entriesCount; ++i)
                            {
                                const SerialInfo::Entry* entry = entries[i];
                                if (entry->typeKind == SerialTypeKind::ImportSymbol)
                                {
                                    // Import symbols are always serialized with a mangled name in the form of
                                    // <module_name>!<symbol_mangled_name>.
                                    // As symbol_mangled_name may not contain the name of its parent module
                                    // in the case of an `extern` or `export` symbol.
                                    //
                                    UnownedStringSlice mangledName = reader.getStringSlice(SerialIndex(i));
                                    List<UnownedStringSlice> slicesOut;
                                    StringUtil::split(mangledName, '!', slicesOut);
                                    if (slicesOut.getCount() != 2)
                                        return SLANG_FAIL;
                                    auto moduleName = slicesOut[0];