From dfbe3cfcdb308cdb0f89cb2844fb3aba96cfcaec Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Thu, 18 Jun 2020 14:17:57 -0400 Subject: Fix and improvements around repro (#1397) * * Fix output in slang repro command line * Profile uses lowerCamel method names (had mix of upper and lower) * Rename slang-serialize-state/SerializeStateUtil to slang-repro and ReproUtil. --- source/slang/slang-check-shader.cpp | 2 +- source/slang/slang-compiler.cpp | 6 +- source/slang/slang-compiler.h | 4 +- source/slang/slang-dxc-support.cpp | 4 +- source/slang/slang-emit-cpp.cpp | 4 +- source/slang/slang-emit-cuda.cpp | 2 +- source/slang/slang-emit-glsl.cpp | 4 +- source/slang/slang-emit-hlsl.cpp | 4 +- source/slang/slang-emit-spirv.cpp | 6 +- source/slang/slang-emit.cpp | 2 +- source/slang/slang-ir-glsl-legalize.cpp | 2 +- source/slang/slang-options.cpp | 24 +- source/slang/slang-parameter-binding.cpp | 2 +- source/slang/slang-profile.h | 8 +- source/slang/slang-reflection.cpp | 4 +- source/slang/slang-repro.cpp | 1569 ++++++++++++++++++++++++++++++ source/slang/slang-repro.h | 200 ++++ source/slang/slang-state-serialize.cpp | 1564 ----------------------------- source/slang/slang-state-serialize.h | 194 ---- source/slang/slang-type-layout.cpp | 4 +- source/slang/slang.cpp | 42 +- source/slang/slang.vcxproj | 4 +- source/slang/slang.vcxproj.filters | 8 +- 23 files changed, 1837 insertions(+), 1826 deletions(-) create mode 100644 source/slang/slang-repro.cpp create mode 100644 source/slang/slang-repro.h delete mode 100644 source/slang/slang-state-serialize.cpp delete mode 100644 source/slang/slang-state-serialize.h (limited to 'source/slang') diff --git a/source/slang/slang-check-shader.cpp b/source/slang/slang-check-shader.cpp index ab1c7f580..17ca9fc78 100644 --- a/source/slang/slang-check-shader.cpp +++ b/source/slang/slang-check-shader.cpp @@ -544,7 +544,7 @@ namespace Slang auto entryPointProfile = entryPointReq->getProfile(); if( auto entryPointAttribute = entryPointFuncDecl->findModifier() ) { - auto entryPointStage = entryPointProfile.GetStage(); + auto entryPointStage = entryPointProfile.getStage(); if( entryPointStage == Stage::Unknown ) { entryPointProfile.setStage(entryPointAttribute->stage); diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp index 729c0077a..b64169db4 100644 --- a/source/slang/slang-compiler.cpp +++ b/source/slang/slang-compiler.cpp @@ -320,7 +320,7 @@ namespace Slang // - Profile Profile::LookUp(char const* name) + Profile Profile::lookUp(char const* name) { #define PROFILE(TAG, NAME, STAGE, VERSION) if(strcmp(name, #NAME) == 0) return Profile::TAG; #define PROFILE_ALIAS(TAG, DEF, NAME) if(strcmp(name, #NAME) == 0) return Profile::TAG; @@ -634,7 +634,7 @@ namespace Slang } char const* stagePrefix = nullptr; - switch( profile.GetStage() ) + switch( profile.getStage() ) { // Note: All of the raytracing-related stages require // compiling for a `lib_*` profile, even when only a @@ -669,7 +669,7 @@ namespace Slang } char const* versionSuffix = nullptr; - switch(profile.GetVersion()) + switch(profile.getVersion()) { #define CASE(TAG, SUFFIX) case ProfileVersion::TAG: versionSuffix = #SUFFIX; break CASE(DX_4_0, _4_0); diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index b1edbe602..ba0c94e99 100644 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -180,7 +180,7 @@ namespace Slang Name* getName() { return m_name; } /// Get the stage that the entry point is to be compiled for - Stage getStage() { return m_profile.GetStage(); } + Stage getStage() { return m_profile.getStage(); } /// Get the profile that the entry point is to be compiled for Profile getProfile() { return m_profile; } @@ -725,7 +725,7 @@ namespace Slang Profile getProfile() { return m_profile; } /// Get the stage that the entry point is for. - Stage getStage() { return m_profile.GetStage(); } + Stage getStage() { return m_profile.getStage(); } /// Get the module that contains the entry point. Module* getModule(); diff --git a/source/slang/slang-dxc-support.cpp b/source/slang/slang-dxc-support.cpp index 4fe8e64ca..ffcc405c8 100644 --- a/source/slang/slang-dxc-support.cpp +++ b/source/slang/slang-dxc-support.cpp @@ -185,7 +185,7 @@ namespace Slang // TODO: Ideally the dxc back-end should be passed some information // on the "capabilities" that were used and/or requested in the code. // - if( profile.GetVersion() >= ProfileVersion::DX_6_2 ) + if( profile.getVersion() >= ProfileVersion::DX_6_2 ) { args[argCount++] = L"-enable-16bit-types"; } @@ -195,7 +195,7 @@ namespace Slang ComPtr dxcResult; SLANG_RETURN_ON_FAIL(dxcCompiler->Compile(dxcSourceBlob, sourcePath.toWString().begin(), - profile.GetStage() == Stage::Unknown ? L"" : wideEntryPointName.begin(), + profile.getStage() == Stage::Unknown ? L"" : wideEntryPointName.begin(), wideProfileName.begin(), args, argCount, diff --git a/source/slang/slang-emit-cpp.cpp b/source/slang/slang-emit-cpp.cpp index 53a110632..032f04ff3 100644 --- a/source/slang/slang-emit-cpp.cpp +++ b/source/slang/slang-emit-cpp.cpp @@ -1742,7 +1742,7 @@ void CPPSourceEmitter::emitEntryPointAttributesImpl(IRFunc* irFunc, IREntryPoint SLANG_UNUSED(entryPointDecor); auto profile = m_effectiveProfile; - auto stage = profile.GetStage(); + auto stage = profile.getStage(); switch (stage) { @@ -2694,7 +2694,7 @@ void CPPSourceEmitter::emitModuleImpl(IRModule* module) IREntryPointDecoration* entryPointDecor = func->findDecoration(); - if (entryPointDecor && entryPointDecor->getProfile().GetStage() == Stage::Compute) + if (entryPointDecor && entryPointDecor->getProfile().getStage() == Stage::Compute) { // https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/sv-dispatchthreadid // SV_DispatchThreadID is the sum of SV_GroupID * numthreads and GroupThreadID. diff --git a/source/slang/slang-emit-cuda.cpp b/source/slang/slang-emit-cuda.cpp index 06bbbea96..6272ea02e 100644 --- a/source/slang/slang-emit-cuda.cpp +++ b/source/slang/slang-emit-cuda.cpp @@ -799,7 +799,7 @@ void CUDASourceEmitter::emitModuleImpl(IRModule* module) // must be prefixed to indicate to the OptiX runtime what // stage it is to be compiled for. // - auto stage = entryPointDecor->getProfile().GetStage(); + auto stage = entryPointDecor->getProfile().getStage(); switch( stage ) { default: diff --git a/source/slang/slang-emit-glsl.cpp b/source/slang/slang-emit-glsl.cpp index a73ea529a..0154f9741 100644 --- a/source/slang/slang-emit-glsl.cpp +++ b/source/slang/slang-emit-glsl.cpp @@ -806,7 +806,7 @@ void GLSLSourceEmitter::emitEntryPointAttributesImpl(IRFunc* irFunc, IREntryPoin SLANG_ASSERT(entryPointDecor); auto profile = entryPointDecor->getProfile(); - auto stage = profile.GetStage(); + auto stage = profile.getStage(); switch (stage) { @@ -1498,7 +1498,7 @@ void GLSLSourceEmitter::emitPreprocessorDirectivesImpl() auto effectiveProfile = m_effectiveProfile; if (effectiveProfile.getFamily() == ProfileFamily::GLSL) { - _requireGLSLVersion(effectiveProfile.GetVersion()); + _requireGLSLVersion(effectiveProfile.getVersion()); } // HACK: We aren't picking GLSL versions carefully right now, diff --git a/source/slang/slang-emit-hlsl.cpp b/source/slang/slang-emit-hlsl.cpp index 83bfb8f2a..f33697045 100644 --- a/source/slang/slang-emit-hlsl.cpp +++ b/source/slang/slang-emit-hlsl.cpp @@ -290,11 +290,11 @@ void HLSLSourceEmitter::emitParameterGroupImpl(IRGlobalParam* varDecl, IRUniform void HLSLSourceEmitter::emitEntryPointAttributesImpl(IRFunc* irFunc, IREntryPointDecoration* entryPointDecor) { auto profile = m_effectiveProfile; - auto stage = entryPointDecor->getProfile().GetStage(); + auto stage = entryPointDecor->getProfile().getStage(); if (profile.getFamily() == ProfileFamily::DX) { - if (profile.GetVersion() >= ProfileVersion::DX_6_1) + if (profile.getVersion() >= ProfileVersion::DX_6_1) { char const* stageName = getStageName(stage); if (stageName) diff --git a/source/slang/slang-emit-spirv.cpp b/source/slang/slang-emit-spirv.cpp index c6f2f7468..0ca523ecf 100644 --- a/source/slang/slang-emit-spirv.cpp +++ b/source/slang/slang-emit-spirv.cpp @@ -208,7 +208,7 @@ struct SpvInst : SpvInstParent // // > Word Count: The complete number of words taken by an instruction, // > including the word holding the word count and opcode, and any optional - // > operands. An instruction’s word count is the total space taken by the instruction. + // > operands. An instruction’s word count is the total space taken by the instruction. // SpvWord wordCount = 1 + SpvWord(operandWords.getCount()); @@ -531,7 +531,7 @@ struct SPIRVEmitContext cursor += 4; } // - // > The final word contains the string’s nul-termination character (0), and + // > The final word contains the string’s nul-termination character (0), and // > all contents past the end of the string in the final word are padded with 0. // // For the last word, the low-order bytes will @@ -1052,7 +1052,7 @@ struct SPIRVEmitContext // to the new globals, which would be used in the SPIR-V emit case. auto entryPointDecor = cast(decoration); - auto spvStage = mapStageToExecutionModel(entryPointDecor->getProfile().GetStage()); + auto spvStage = mapStageToExecutionModel(entryPointDecor->getProfile().getStage()); auto name = entryPointDecor->getName()->getStringSlice(); emitInst(section, decoration, SpvOpEntryPoint, spvStage, dstID, name); } diff --git a/source/slang/slang-emit.cpp b/source/slang/slang-emit.cpp index 4af7b5989..1afca6127 100644 --- a/source/slang/slang-emit.cpp +++ b/source/slang/slang-emit.cpp @@ -495,7 +495,7 @@ Result linkAndOptimizeIR( auto profile = targetRequest->targetProfile; if( profile.getFamily() == ProfileFamily::DX ) { - if(profile.GetVersion() <= ProfileVersion::DX_5_0) + if(profile.getVersion() <= ProfileVersion::DX_5_0) { // Fxc and earlier dxc versions do not support // a templates `.Load` operation on byte-address diff --git a/source/slang/slang-ir-glsl-legalize.cpp b/source/slang/slang-ir-glsl-legalize.cpp index afb67a081..794024d46 100644 --- a/source/slang/slang-ir-glsl-legalize.cpp +++ b/source/slang/slang-ir-glsl-legalize.cpp @@ -1602,7 +1602,7 @@ void legalizeEntryPointForGLSL( auto entryPointDecor = func->findDecoration(); SLANG_ASSERT(entryPointDecor); - auto stage = entryPointDecor->getProfile().GetStage(); + auto stage = entryPointDecor->getProfile().getStage(); auto layoutDecoration = func->findDecoration(); SLANG_ASSERT(layoutDecoration); diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp index 5bf0c6c9d..f3c702c26 100644 --- a/source/slang/slang-options.cpp +++ b/source/slang/slang-options.cpp @@ -12,7 +12,7 @@ #include "slang-file-system.h" -#include "slang-state-serialize.h" +#include "slang-repro.h" #include "slang-ir-serialize.h" #include "../core/slang-type-text-util.h" @@ -483,7 +483,7 @@ struct OptionsParser String reproName; SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, reproName)); - SLANG_RETURN_ON_FAIL(StateSerializeUtil::extractFilesToDirectory(reproName)); + SLANG_RETURN_ON_FAIL(ReproUtil::extractFilesToDirectory(reproName)); } else if (argStr == "-module-name") { @@ -498,16 +498,16 @@ struct OptionsParser SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, reproName)); List buffer; - SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState(reproName, buffer)); + SLANG_RETURN_ON_FAIL(ReproUtil::loadState(reproName, buffer)); - auto requestState = StateSerializeUtil::getRequest(buffer); + auto requestState = ReproUtil::getRequest(buffer); MemoryOffsetBase base; base.set(buffer.getBuffer(), buffer.getCount()); // If we can find a directory, that exists, we will set up a file system to load from that directory ComPtr fileSystem; String dirPath; - if (SLANG_SUCCEEDED(StateSerializeUtil::calcDirectoryPathFromFilename(reproName, dirPath))) + if (SLANG_SUCCEEDED(ReproUtil::calcDirectoryPathFromFilename(reproName, dirPath))) { SlangPathType pathType; if (SLANG_SUCCEEDED(Path::getPathType(dirPath, &pathType)) && pathType == SLANG_PATH_TYPE_DIRECTORY) @@ -516,7 +516,7 @@ struct OptionsParser } } - SLANG_RETURN_ON_FAIL(StateSerializeUtil::load(base, requestState, fileSystem, requestImpl)); + SLANG_RETURN_ON_FAIL(ReproUtil::load(base, requestState, fileSystem, requestImpl)); hasLoadedRepro = true; } @@ -526,16 +526,16 @@ struct OptionsParser SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, reproName)); List buffer; - SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState(reproName, buffer)); + SLANG_RETURN_ON_FAIL(ReproUtil::loadState(reproName, buffer)); - auto requestState = StateSerializeUtil::getRequest(buffer); + auto requestState = ReproUtil::getRequest(buffer); MemoryOffsetBase base; base.set(buffer.getBuffer(), buffer.getCount()); // If we can find a directory, that exists, we will set up a file system to load from that directory ComPtr dirFileSystem; String dirPath; - if (SLANG_SUCCEEDED(StateSerializeUtil::calcDirectoryPathFromFilename(reproName, dirPath))) + if (SLANG_SUCCEEDED(ReproUtil::calcDirectoryPathFromFilename(reproName, dirPath))) { SlangPathType pathType; if (SLANG_SUCCEEDED(Path::getPathType(dirPath, &pathType)) && pathType == SLANG_PATH_TYPE_DIRECTORY) @@ -545,7 +545,7 @@ struct OptionsParser } RefPtr cacheFileSystem; - SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadFileSystem(base, requestState, dirFileSystem, cacheFileSystem)); + SLANG_RETURN_ON_FAIL(ReproUtil::loadFileSystem(base, requestState, dirFileSystem, cacheFileSystem)); // I might want to make the dir file system the fallback file system... cacheFileSystem->setInnerFileSystem(dirFileSystem, cacheFileSystem->getUniqueIdentityMode(), cacheFileSystem->getPathStyle()); @@ -624,11 +624,11 @@ struct OptionsParser { auto profile = Profile(profileID); - setProfileVersion(getCurrentTarget(), profile.GetVersion()); + setProfileVersion(getCurrentTarget(), profile.getVersion()); // A `-profile` option that also specifies a stage (e.g., `-profile vs_5_0`) // should be treated like a composite (e.g., `-profile sm_5_0 -stage vertex`) - auto stage = profile.GetStage(); + auto stage = profile.getStage(); if(stage != Stage::Unknown) { setStage(getCurrentEntryPoint(), stage); diff --git a/source/slang/slang-parameter-binding.cpp b/source/slang/slang-parameter-binding.cpp index ea45612ec..ee3ef1234 100644 --- a/source/slang/slang-parameter-binding.cpp +++ b/source/slang/slang-parameter-binding.cpp @@ -1425,7 +1425,7 @@ static RefPtr processSimpleEntryPointParameter( // if( isD3DTarget(context->getTargetRequest()) ) { - auto version = context->getTargetRequest()->targetProfile.GetVersion(); + auto version = context->getTargetRequest()->targetProfile.getVersion(); if( version <= ProfileVersion::DX_5_0 ) { // We will address the conflict here by claiming the corresponding diff --git a/source/slang/slang-profile.h b/source/slang/slang-profile.h index c90ca66d0..f5b15eda6 100644 --- a/source/slang/slang-profile.h +++ b/source/slang/slang-profile.h @@ -89,21 +89,21 @@ namespace Slang bool operator==(Profile const& other) const { return raw == other.raw; } bool operator!=(Profile const& other) const { return raw != other.raw; } - Stage GetStage() const { return Stage(uint32_t(raw) & 0xFFFF); } + Stage getStage() const { return Stage(uint32_t(raw) & 0xFFFF); } void setStage(Stage stage) { raw = (raw & ~0xFFFF) | uint32_t(stage); } - ProfileVersion GetVersion() const { return ProfileVersion((uint32_t(raw) >> 16) & 0xFFFF); } + ProfileVersion getVersion() const { return ProfileVersion((uint32_t(raw) >> 16) & 0xFFFF); } void setVersion(ProfileVersion version) { raw = (raw & 0x0000FFFF) | (uint32_t(version) << 16); } - ProfileFamily getFamily() const { return getProfileFamily(GetVersion()); } + ProfileFamily getFamily() const { return getProfileFamily(getVersion()); } - static Profile LookUp(char const* name); + static Profile lookUp(char const* name); char const* getName(); RawVal raw = Unknown; diff --git a/source/slang/slang-reflection.cpp b/source/slang/slang-reflection.cpp index c1e533140..4ed1c3e5d 100644 --- a/source/slang/slang-reflection.cpp +++ b/source/slang/slang-reflection.cpp @@ -1277,7 +1277,7 @@ SLANG_API SlangStage spReflectionEntryPoint_getStage(SlangReflectionEntryPoint* if(!entryPointLayout) return SLANG_STAGE_NONE; - return SlangStage(entryPointLayout->profile.GetStage()); + return SlangStage(entryPointLayout->profile.getStage()); } SLANG_API void spReflectionEntryPoint_getComputeThreadGroupSize( @@ -1323,7 +1323,7 @@ SLANG_API int spReflectionEntryPoint_usesAnySampleRateInput( if(!entryPointLayout) return 0; - if (entryPointLayout->profile.GetStage() != Stage::Fragment) + if (entryPointLayout->profile.getStage() != Stage::Fragment) return 0; return (entryPointLayout->flags & EntryPointLayout::Flag::usesAnySampleRateInput) != 0; diff --git a/source/slang/slang-repro.cpp b/source/slang/slang-repro.cpp new file mode 100644 index 000000000..80ef0322e --- /dev/null +++ b/source/slang/slang-repro.cpp @@ -0,0 +1,1569 @@ +// slang-repro.cpp +#include "slang-repro.h" + +#include "../core/slang-text-io.h" + +#include "../core/slang-stream.h" + +#include "../core/slang-math.h" +#include "../core/slang-type-text-util.h" + +#include "slang-options.h" + +#include "slang-source-loc.h" + +namespace Slang { + +/* static */const RiffSemanticVersion ReproUtil::g_semanticVersion = + RiffSemanticVersion::make(ReproUtil::kMajorVersion, ReproUtil::kMinorVersion, ReproUtil::kPatchVersion); + +// We can't just use sizeof for the sizes of these types, because the hash will be dependent on the ptr size, +// which isn't an issue for serialization (we turn all pointers into Offset32Ptr -> uint32_t). So we use an x macro +// to set up the thing to hash. +// +// Note that bool is in the list because size of bool can change between compilers. +#define SLANG_STATE_TYPES(x) \ + x(Util::FileState) \ + x(Util::PathInfoState) \ + x(Util::PathInfoState::CompressedResult) \ + x(SlangPathType) \ + x(Util::PathAndPathInfo) \ + x(Util::TargetRequestState) \ + x(Profile) \ + x(CodeGenTarget) \ + x(SlangTargetFlags) \ + x(FloatingPointMode) \ + x(Util::StringPair) \ + x(Util::SourceFileState) \ + x(PathInfo::Type) \ + x(Util::TranslationUnitRequestState) \ + x(SourceLanguage) \ + x(Util::EntryPointState) \ + x(Profile) \ + x(Util::RequestState) \ + x(SlangCompileFlags) \ + x(bool) \ + x(LineDirectiveMode) \ + x(DebugInfoLevel) \ + x(OptimizationLevel) \ + x(ContainerFormat) \ + x(PassThroughMode) \ + x(SlangMatrixLayoutMode) \ + +#define SLANG_STATE_TYPE_SIZE(x) uint32_t(sizeof(x)), + +// A function to calculate the hash related in list in part to how the types used are sized. Can catch crude breaking binary differences. +static HashCode32 _calcTypeHash() +{ + typedef ReproUtil Util; + const uint32_t sizes[] = + { + SLANG_STATE_TYPES(SLANG_STATE_TYPE_SIZE) + }; + return getStableHashCode32((const char*)&sizes, sizeof(sizes)); +} + +static HashCode32 _getTypeHash() +{ + static HashCode32 s_hash = _calcTypeHash(); + return s_hash; +} + + +namespace { // anonymous + +struct StoreContext +{ + typedef ReproUtil::FileState FileState; + typedef ReproUtil::SourceFileState SourceFileState; + typedef ReproUtil::PathInfoState PathInfoState; + + StoreContext(OffsetContainer* container) + { + m_container = container; + } + + Offset32Ptr findFile(const String& uniqueIdentity) + { + Offset32Ptr file; + m_uniqueToFileMap.TryGetValue(uniqueIdentity, file); + return file; + } + + Offset32Ptr addFile(const String& uniqueIdentity, const UnownedStringSlice* content) + { + OffsetBase& base = m_container->asBase(); + + Offset32Ptr file; + + // Get the file, if it has an identity + if (uniqueIdentity.getLength()) + { + if (!m_uniqueToFileMap.TryGetValue(uniqueIdentity, file)) + { + // If file was not found create it + // Create the file + file = m_container->newObject(); + // Add it + m_uniqueToFileMap.Add(uniqueIdentity, file); + + // Set the identity + auto offsetUniqueIdentity = m_container->newString(uniqueIdentity.getUnownedSlice()); + base[file]->uniqueIdentity = offsetUniqueIdentity; + + // Add the file + m_files.add(file); + } + } + else + { + // Create a file, but we know it can't have unique identity + file = m_container->newObject(); + // Add the file + m_files.add(file); + } + + // If the contents is not set add it + if (!base[file]->contents && content) + { + auto offsetContent = m_container->newString(*content); + base[file]->contents = offsetContent; + } + + return file; + } + + Offset32Ptr addSourceFile(SourceFile* sourceFile) + { + if (!sourceFile) + { + return Offset32Ptr(); + } + + auto& base = m_container->asBase(); + + Offset32Ptr sourceFileState; + if (m_sourceFileMap.TryGetValue(sourceFile, sourceFileState)) + { + return sourceFileState; + } + + const PathInfo& pathInfo = sourceFile->getPathInfo(); + + UnownedStringSlice content = sourceFile->getContent(); + Offset32Ptr file = addFile(pathInfo.uniqueIdentity, &content); + + Offset32Ptr foundPath; + + if (pathInfo.foundPath.getLength() && base[file]->foundPath.isNull()) + { + foundPath = fromString(pathInfo.foundPath.getUnownedSlice()); + } + // Set on the file + base[file]->foundPath = foundPath; + + // Create the source file + sourceFileState = m_container->newObject(); + + { + auto dst = base[sourceFileState]; + dst->file = file; + dst->foundPath = foundPath; + dst->type = pathInfo.type; + } + + m_sourceFileMap.Add(sourceFile, sourceFileState); + + return sourceFileState; + } + + Offset32Ptr fromString(const String& in) + { + Offset32Ptr value; + + if (m_stringMap.TryGetValue(in, value)) + { + return value; + } + value = m_container->newString(in.getUnownedSlice()); + m_stringMap.Add(in, value); + return value; + } + Offset32Ptr fromName(Name* name) + { + if (name) + { + return fromString(name->text); + } + return Offset32Ptr(); + } + + Offset32Ptr addPathInfo(const CacheFileSystem::PathInfo* srcPathInfo) + { + if (!srcPathInfo) + { + return Offset32Ptr(); + } + + OffsetBase& base = m_container->asBase(); + + Offset32Ptr pathInfo; + if (!m_pathInfoMap.TryGetValue(srcPathInfo, pathInfo)) + { + // Get the associated file + Offset32Ptr fileState; + + // Only store as file if we have the contents + if(ISlangBlob* fileBlob = srcPathInfo->m_fileBlob) + { + UnownedStringSlice content((const char*)fileBlob->getBufferPointer(), fileBlob->getBufferSize()); + + fileState = addFile(srcPathInfo->getUniqueIdentity(), &content); + } + + // Save the rest of the state + pathInfo = m_container->newObject(); + PathInfoState& dst = base[*pathInfo]; + + dst.file = fileState; + + // Save any other info + dst.getCanonicalPathResult = srcPathInfo->m_getCanonicalPathResult; + dst.getPathTypeResult = srcPathInfo->m_getPathTypeResult; + dst.loadFileResult = srcPathInfo->m_loadFileResult; + dst.pathType = srcPathInfo->m_pathType; + + m_pathInfoMap.Add(srcPathInfo, pathInfo); + } + + // Fill in info on the file + auto fileState(base[pathInfo]->file); + + // If have fileState add any missing element + if (fileState) + { + if (srcPathInfo->m_fileBlob && base[fileState]->contents.isNull()) + { + UnownedStringSlice contents((const char*)srcPathInfo->m_fileBlob->getBufferPointer(), srcPathInfo->m_fileBlob->getBufferSize()); + auto offsetContents = m_container->newString(contents); + base[fileState]->contents = offsetContents; + } + + if (srcPathInfo->m_canonicalPath && base[fileState]->canonicalPath.isNull()) + { + auto offsetCanonicalPath = fromString(srcPathInfo->m_canonicalPath->getString()); + base[fileState]->canonicalPath = offsetCanonicalPath; + } + + if (srcPathInfo->m_uniqueIdentity && base[fileState]->uniqueIdentity.isNull()) + { + auto offsetUniqueIdentity = fromString(srcPathInfo->m_uniqueIdentity->getString()); + base[fileState]->uniqueIdentity = offsetUniqueIdentity; + } + } + + return pathInfo; + } + + const Offset32Array calcDefines(const Dictionary& srcDefines) + { + typedef ReproUtil::StringPair StringPair; + + Offset32Array dstDefines = m_container->newArray(srcDefines.Count()); + + OffsetBase& base = m_container->asBase(); + + Index index = 0; + for (const auto& srcDefine : srcDefines) + { + // Do allocation before setting + auto key = fromString(srcDefine.Key); + auto value = fromString(srcDefine.Value); + + auto& dstDefine = base[dstDefines[index]]; + dstDefine.first = key; + dstDefine.second = value; + + index++; + } + + return dstDefines; + } + + const Offset32Array> fromList(const List& src) + { + Offset32Array> dst = m_container->newArray>(src.getCount()); + OffsetBase& base = m_container->asBase(); + + for (Index j = 0; j < src.getCount(); ++j) + { + base[dst[j]] = fromString(src[j]); + } + return dst; + } + + Dictionary > m_stringMap; + + Dictionary > m_sourceFileMap; + + Dictionary > m_uniqueToFileMap; + + Dictionary > m_pathInfoMap; + + List > m_files; + + OffsetContainer* m_container; +}; + +} // + +static bool _isStorable(const PathInfo::Type type) +{ + switch (type) + { + case PathInfo::Type::Unknown: + case PathInfo::Type::Normal: + case PathInfo::Type::FoundPath: + case PathInfo::Type::FromString: + { + return true; + } + default: return false; + } +} + +/* static */SlangResult ReproUtil::store(EndToEndCompileRequest* request, OffsetContainer& inOutContainer, Offset32Ptr& outRequest) +{ + StoreContext context(&inOutContainer); + + OffsetBase& base = inOutContainer.asBase(); + + auto linkage = request->getLinkage(); + + Offset32Ptr requestState = inOutContainer.newObject(); + + { + RequestState* dst = base[requestState]; + + dst->compileFlags = request->getFrontEndReq()->compileFlags; + dst->shouldDumpIntermediates = request->getBackEndReq()->shouldDumpIntermediates; + dst->lineDirectiveMode = request->getBackEndReq()->lineDirectiveMode; + + dst->debugInfoLevel = linkage->debugInfoLevel; + dst->optimizationLevel = linkage->optimizationLevel; + dst->containerFormat = request->m_containerFormat; + dst->passThroughMode = request->passThrough; + + + dst->useUnknownImageFormatAsDefault = request->getBackEndReq()->useUnknownImageFormatAsDefault; + dst->obfuscateCode = linkage->m_obfuscateCode; + + dst->defaultMatrixLayoutMode = linkage->defaultMatrixLayoutMode; + } + + // Entry points + { + const auto& srcEntryPoints = request->getFrontEndReq()->m_entryPointReqs; + const auto& srcEndToEndEntryPoints = request->entryPoints; + + SLANG_ASSERT(srcEntryPoints.getCount() == srcEndToEndEntryPoints.getCount()); + + Offset32Array dstEntryPoints = inOutContainer.newArray(srcEntryPoints.getCount()); + + for (Index i = 0; i < srcEntryPoints.getCount(); ++i) + { + FrontEndEntryPointRequest* srcEntryPoint = srcEntryPoints[i]; + const auto& srcEndToEndEntryPoint = srcEndToEndEntryPoints[i]; + + auto dstSpecializationArgStrings = context.fromList(srcEndToEndEntryPoint.specializationArgStrings); + Offset32Ptr dstName = context.fromName(srcEntryPoint->getName()); + + EntryPointState& dst = base[dstEntryPoints[i]]; + + dst.profile = srcEntryPoint->getProfile(); + dst.translationUnitIndex = uint32_t(srcEntryPoint->getTranslationUnitIndex()); + dst.specializationArgStrings = dstSpecializationArgStrings; + dst.name = dstName; + } + + base[requestState]->entryPoints = dstEntryPoints; + } + + + // Add all of the source files + { + SourceManager* sourceManager = request->getFrontEndReq()->getSourceManager(); + const List& sourceFiles = sourceManager->getSourceFiles(); + + for (SourceFile* sourceFile : sourceFiles) + { + const PathInfo& pathInfo = sourceFile->getPathInfo(); + if (_isStorable(pathInfo.type)) + { + context.addSourceFile(sourceFile); + } + } + } + + // Add all the target requests + { + Offset32Array dstTargets = inOutContainer.newArray(linkage->targets.getCount()); + + for (Index i = 0; i < linkage->targets.getCount(); ++i) + { + TargetRequest* srcTargetRequest = linkage->targets[i]; + + // Copy the simple stuff + { + auto& dst = base[dstTargets[i]]; + dst.target = srcTargetRequest->getTarget(); + dst.profile = srcTargetRequest->getTargetProfile(); + dst.targetFlags = srcTargetRequest->targetFlags; + dst.floatingPointMode = srcTargetRequest->floatingPointMode; + } + + // Copy the entry point/target output names + { + const auto& srcTargetInfos = request->targetInfos; + + if (RefPtr* infosPtr = srcTargetInfos.TryGetValue(srcTargetRequest)) + { + EndToEndCompileRequest::TargetInfo* infos = *infosPtr; + + const auto& entryPointOutputPaths = infos->entryPointOutputPaths; + + Offset32Array dstOutputStates = inOutContainer.newArray(entryPointOutputPaths.Count()); + + Index index = 0; + for (const auto& pair : entryPointOutputPaths) + { + Offset32Ptr outputPath = inOutContainer.newString(pair.Value.getUnownedSlice()); + + auto& dstOutputState = base[dstOutputStates[index]]; + + dstOutputState.entryPointIndex = int32_t(pair.Key); + dstOutputState.outputPath = outputPath; + + index++; + } + + base[dstTargets[i]].outputStates = dstOutputStates; + } + } + } + + // Save the result + base[requestState]->targetRequests = dstTargets; + } + + // Add the search paths + { + const auto& srcPaths = linkage->searchDirectories.searchDirectories; + Offset32Array > dstPaths = inOutContainer.newArray >(srcPaths.getCount()); + + // We don't handle parents here + SLANG_ASSERT(linkage->searchDirectories.parent == nullptr); + for (Index i = 0; i < srcPaths.getCount(); ++i) + { + base[dstPaths[i]] = context.fromString(srcPaths[i].path); + } + base[requestState]->searchPaths = dstPaths; + } + + // Add preprocessor definitions + base[requestState]->preprocessorDefinitions = context.calcDefines(linkage->preprocessorDefinitions); + + { + const auto& srcTranslationUnits = request->getFrontEndReq()->translationUnits; + Offset32Array dstTranslationUnits = inOutContainer.newArray(srcTranslationUnits.getCount()); + + for (Index i = 0; i < srcTranslationUnits.getCount(); ++i) + { + TranslationUnitRequest* srcTranslationUnit = srcTranslationUnits[i]; + + // Do before setting, because this can allocate, and therefore break, the following section + auto defines = context.calcDefines(srcTranslationUnit->preprocessorDefinitions); + auto moduleName = context.fromName(srcTranslationUnit->moduleName); + + Offset32Array> dstSourceFiles; + { + const auto& srcFiles = srcTranslationUnit->getSourceFiles(); + dstSourceFiles = inOutContainer.newArray >(srcFiles.getCount()); + + for (Index j = 0; j < srcFiles.getCount(); ++j) + { + base[dstSourceFiles[j]] = context.addSourceFile(srcFiles[j]); + } + } + + TranslationUnitRequestState& dstTranslationUnit = base[dstTranslationUnits[i]]; + + dstTranslationUnit.language = srcTranslationUnit->sourceLanguage; + dstTranslationUnit.moduleName = moduleName; + dstTranslationUnit.sourceFiles = dstSourceFiles; + dstTranslationUnit.preprocessorDefinitions = defines; + } + + base[requestState]->translationUnits = dstTranslationUnits; + } + + // Find files from the file system, and mapping paths to files + { + CacheFileSystem* cacheFileSystem = linkage->getCacheFileSystem(); + if (!cacheFileSystem) + { + return SLANG_FAIL; + } + + // Traverse the references (in process we will construct the map from PathInfo) + { + const auto& srcFiles = cacheFileSystem->getPathMap(); + + Offset32Array pathMap = inOutContainer.newArray(srcFiles.Count()); + + Index index = 0; + for (const auto& pair : srcFiles) + { + Offset32Ptr path = context.fromString(pair.Key); + Offset32Ptr pathInfo = context.addPathInfo(pair.Value); + + PathAndPathInfo& dstInfo = base[pathMap[index]]; + dstInfo.path = path; + dstInfo.pathInfo = pathInfo; + + index++; + } + + base[requestState]->pathInfoMap = pathMap; + } + } + + // Save all of the files + { + Dictionary uniqueNameMap; + + auto files = inOutContainer.newArray>(context.m_files.getCount()); + for (Index i = 0; i < context.m_files.getCount(); ++i) + { + Offset32Ptr file = context.m_files[i]; + + // Need to come up with unique names + String path; + + if (auto canonicalPath = base[file]->canonicalPath) + { + path = base[canonicalPath]->getSlice(); + } + else if (auto foundPath = base[file]->foundPath) + { + path = base[foundPath]->getSlice(); + } + else if (auto uniqueIdentity = base[file]->uniqueIdentity) + { + path = base[uniqueIdentity]->getSlice(); + } + + if (path.getLength() == 0) + { + StringBuilder builder; + builder << "unnamed" << i; + path = builder; + } + + String filename = Path::getFileNameWithoutExt(path); + String ext = Path::getPathExt(path); + + StringBuilder uniqueName; + for (Index j = 0; j < 0x10000; j++) + { + uniqueName.Clear(); + uniqueName << filename; + + if (j > 0) + { + uniqueName << "-" << j; + } + + if (ext.getLength()) + { + uniqueName << "." << ext; + } + + int dummy = 0; + if (!uniqueNameMap.TryGetValueOrAdd(uniqueName, dummy)) + { + // It was added so we are done + break; + } + } + + // Save the unique generated name + base[file]->uniqueName = inOutContainer.newString(uniqueName.getUnownedSlice()); + + base[files[i]] = file; + } + + base[requestState]->files = files; + } + + // Save all the SourceFile state + { + const auto& srcSourceFiles = context.m_sourceFileMap; + auto dstSourceFiles = inOutContainer.newArray>(srcSourceFiles.Count()); + + Index index = 0; + for (const auto& pair : srcSourceFiles) + { + base[dstSourceFiles[index]] = pair.Value; + index++; + } + base[requestState]->sourceFiles = dstSourceFiles; + } + + outRequest = requestState; + return SLANG_OK; +} + +namespace { // anonymous + +struct LoadContext +{ + typedef ReproUtil::SourceFileState SourceFileState; + typedef ReproUtil::FileState FileState; + typedef ReproUtil::PathInfoState PathInfoState; + + CacheFileSystem::PathInfo* getPathInfoFromFile(FileState* file) + { + if (!file) + { + return nullptr; + } + + CacheFileSystem::PathInfo* dstInfo = nullptr; + if (!m_fileToPathInfoMap.TryGetValue(file, dstInfo)) + { + ComPtr blob; + + if (m_fileSystem && file->uniqueName) + { + // Try loading from the file system + m_fileSystem->loadFile(m_base->asRaw(file->uniqueName)->getCstr(), blob.writeRef()); + } + + // If wasn't loaded, and has contents, use that + if (!blob && file->contents) + { + blob = new StringBlob(m_base->asRaw(file->contents)->getSlice()); + } + + dstInfo = new CacheFileSystem::PathInfo(String()); + + if (file->uniqueIdentity) + { + String uniqueIdentity = m_base->asRaw(file->uniqueIdentity)->getSlice(); + dstInfo->m_uniqueIdentity = new StringBlob(uniqueIdentity); + } + + if (file->canonicalPath) + { + dstInfo->m_canonicalPath = new StringBlob(m_base->asRaw(file->canonicalPath)->getSlice()); + } + + if (blob) + { + dstInfo->m_loadFileResult = CacheFileSystem::CompressedResult::Ok; + dstInfo->m_getPathTypeResult = CacheFileSystem::CompressedResult::Ok; + dstInfo->m_pathType = SLANG_PATH_TYPE_FILE; + } + + dstInfo->m_fileBlob = blob; + + // Add to map, even if the blob is nullptr (say from a failed read) + m_fileToPathInfoMap.Add(file, dstInfo); + } + + return dstInfo; + } + + ISlangBlob* getFileBlobFromFile(FileState* file) + { + CacheFileSystem::PathInfo* pathInfo = getPathInfoFromFile(file); + return pathInfo ? pathInfo->m_fileBlob.get() : nullptr; + } + + SourceFile* getSourceFile(SourceFileState* sourceFile) + { + if (sourceFile == nullptr) + { + return nullptr; + } + + SourceFile* dstFile; + if (!m_sourceFileMap.TryGetValue(sourceFile, dstFile)) + { + FileState* file = m_base->asRaw(sourceFile->file); + ISlangBlob* blob = getFileBlobFromFile(file); + + PathInfo pathInfo; + + pathInfo.type = sourceFile->type; + + if (sourceFile->foundPath) + { + pathInfo.foundPath = m_base->asRaw(sourceFile->foundPath)->getSlice(); + } + else if (file->foundPath) + { + pathInfo.foundPath = m_base->asRaw(file->foundPath)->getSlice(); + } + + if (file->uniqueIdentity) + { + pathInfo.uniqueIdentity = m_base->asRaw(file->uniqueIdentity)->getSlice(); + } + + dstFile = new SourceFile(m_sourceManager, pathInfo, blob->getBufferSize()); + dstFile->setContents(blob); + + // Add to map + m_sourceFileMap.Add(sourceFile, dstFile); + + // Add to manager + m_sourceManager->addSourceFile(pathInfo.uniqueIdentity, dstFile); + } + return dstFile; + } + + CacheFileSystem::PathInfo* addPathInfo(const PathInfoState* srcInfo) + { + CacheFileSystem::PathInfo* pathInfo; + if (m_pathInfoMap.TryGetValue(srcInfo, pathInfo)) + { + return pathInfo; + } + + FileState* file = m_base->asRaw(srcInfo->file); + CacheFileSystem::PathInfo* dstInfo; + + if (file) + { + dstInfo = getPathInfoFromFile(file); + } + else + { + // TODO(JS): Hmmm... this could end up not being cleared up + // Because it is not added to the unique set (as unique set is for files and this isn't a file) + dstInfo = new CacheFileSystem::PathInfo(String()); + } + + dstInfo->m_getCanonicalPathResult = srcInfo->getCanonicalPathResult; + dstInfo->m_getPathTypeResult = srcInfo->getPathTypeResult; + dstInfo->m_loadFileResult = srcInfo->loadFileResult; + dstInfo->m_pathType = srcInfo->pathType; + + m_pathInfoMap.Add(srcInfo, dstInfo); + return dstInfo; + } + + List toList(const Offset32Array>& src) + { + List dst; + dst.setCount(src.getCount()); + for (Index i = 0; i < src.getCount(); ++i) + { + OffsetString* srcString = m_base->asRaw(m_base->asRaw(src[i])); + dst[i] = srcString ? srcString->getCstr() : nullptr; + } + return dst; + } + + + void loadDefines(const Offset32Array& in, Dictionary& out) + { + out.Clear(); + + for (const auto& define : in) + { + out.Add(m_base->asRaw(m_base->asRaw(define).first)->getSlice(), m_base->asRaw(m_base->asRaw(define).second)->getSlice()); + } + } + + LoadContext(SourceManager* sourceManger, ISlangFileSystem* fileSystem, OffsetBase* base): + m_sourceManager(sourceManger), + m_fileSystem(fileSystem), + m_base(base) + { + } + + ISlangFileSystem* m_fileSystem; + + OffsetBase* m_base; + + SourceManager* m_sourceManager; + + Dictionary m_sourceFileMap; + Dictionary m_fileToPathInfoMap; + Dictionary m_pathInfoMap; +}; + +} // anonymous + + +/* static */SlangResult ReproUtil::loadFileSystem(OffsetBase& base, RequestState* requestState, ISlangFileSystem* fileSystem, RefPtr& outFileSystem) +{ + LoadContext context(nullptr, fileSystem, &base); + + RefPtr cacheFileSystem = new CacheFileSystem(nullptr); + auto& dstUniqueMap = cacheFileSystem->getUniqueMap(); + auto& dstPathMap = cacheFileSystem->getPathMap(); + + for (auto fileOffset : requestState->files) + { + // add the file + FileState* fileState = base.asRaw(base.asRaw(fileOffset)); + CacheFileSystem::PathInfo* pathInfo = context.getPathInfoFromFile(fileState); + + if (fileState->foundPath) + { + String foundPath = base.asRaw(fileState->foundPath)->getSlice(); + dstPathMap.AddIfNotExists(foundPath, pathInfo); + } + } + + // Put all the paths to path info + { + for (const auto& pairOffset : requestState->pathInfoMap) + { + const auto& pair = base.asRaw(pairOffset); + CacheFileSystem::PathInfo* pathInfo = context.addPathInfo(base.asRaw(pair.pathInfo)); + dstPathMap.AddIfNotExists(base.asRaw(pair.path)->getSlice(), pathInfo); + } + } + + // Put all the path infos in the cache system + { + for (const auto& pair : context.m_fileToPathInfoMap) + { + CacheFileSystem::PathInfo* pathInfo = pair.Value; + SLANG_ASSERT(pathInfo->m_uniqueIdentity); + dstUniqueMap.Add(pathInfo->m_uniqueIdentity->getString(), pathInfo); + + // Add canonical paths too.. + if (pathInfo->m_canonicalPath) + { + String canonicalPath = pathInfo->m_canonicalPath->getString(); + + dstPathMap.AddIfNotExists(canonicalPath, pathInfo); + } + } + } + + outFileSystem = cacheFileSystem; + return SLANG_OK; +} + +/* static */SlangResult ReproUtil::load(OffsetBase& base, RequestState* requestState, ISlangFileSystem* fileSystem, EndToEndCompileRequest* request) +{ + auto externalRequest = asExternal(request); + + auto linkage = request->getLinkage(); + + // TODO(JS): Really should be more exhaustive here, and set up to initial state ideally + // Reset state + { + request->targetInfos.Clear(); + // Remove any requests + linkage->targets.clear(); + } + + LoadContext context(linkage->getSourceManager(), fileSystem, &base); + + // Try to set state through API - as doing so means if state stored in multiple places it will be ok + + { + spSetCompileFlags(externalRequest, (SlangCompileFlags)requestState->compileFlags); + spSetDumpIntermediates(externalRequest, int(requestState->shouldDumpIntermediates)); + spSetLineDirectiveMode(externalRequest, SlangLineDirectiveMode(requestState->lineDirectiveMode)); + spSetDebugInfoLevel(externalRequest, SlangDebugInfoLevel(requestState->debugInfoLevel)); + spSetOptimizationLevel(externalRequest, SlangOptimizationLevel(requestState->optimizationLevel)); + spSetOutputContainerFormat(externalRequest, SlangContainerFormat(requestState->containerFormat)); + spSetPassThrough(externalRequest, SlangPassThrough(request->passThrough)); + + request->getBackEndReq()->useUnknownImageFormatAsDefault = requestState->useUnknownImageFormatAsDefault; + linkage->m_obfuscateCode = requestState->obfuscateCode; + + linkage->setMatrixLayoutMode(requestState->defaultMatrixLayoutMode); + } + + // Add the target requests + { + for (Index i = 0; i < requestState->targetRequests.getCount(); ++i) + { + TargetRequestState& src = base.asRaw(requestState->targetRequests[i]); + int index = spAddCodeGenTarget(externalRequest, SlangCompileTarget(src.target)); + SLANG_ASSERT(index == i); + + auto dstTarget = linkage->targets[index]; + + SLANG_ASSERT(dstTarget->getTarget() == src.target); + dstTarget->targetProfile = src.profile; + dstTarget->targetFlags = src.targetFlags; + dstTarget->floatingPointMode = src.floatingPointMode; + + // If there is output state (like output filenames) add here + if (src.outputStates.getCount()) + { + RefPtr dstTargetInfo(new EndToEndCompileRequest::TargetInfo); + request->targetInfos[dstTarget] = dstTargetInfo; + + for (const auto& srcOutputStateOffset : src.outputStates) + { + const auto& srcOutputState = base.asRaw(srcOutputStateOffset); + + SLANG_ASSERT(srcOutputState.entryPointIndex < requestState->entryPoints.getCount()); + + String entryPointPath; + if (srcOutputState.outputPath) + { + entryPointPath = base.asRaw(srcOutputState.outputPath)->getSlice(); + } + + dstTargetInfo->entryPointOutputPaths.Add(srcOutputState.entryPointIndex, entryPointPath); + } + } + } + } + + { + const auto& srcPaths = requestState->searchPaths; + auto& dstPaths = linkage->searchDirectories.searchDirectories; + dstPaths.setCount(srcPaths.getCount()); + for (Index i = 0; i < srcPaths.getCount(); ++i) + { + dstPaths[i].path = base.asRaw(base.asRaw(srcPaths[i]))->getSlice(); + } + } + + context.loadDefines(requestState->preprocessorDefinitions, linkage->preprocessorDefinitions); + + { + auto frontEndReq = request->getFrontEndReq(); + + const auto& srcTranslationUnits = requestState->translationUnits; + auto& dstTranslationUnits = frontEndReq->translationUnits; + + dstTranslationUnits.clear(); + + for (Index i = 0; i < srcTranslationUnits.getCount(); ++i) + { + const auto& srcTranslationUnit = base.asRaw(srcTranslationUnits[i]); + + // TODO(JS): We should probably serialize off the module name + // Passing in nullptr will just generate the module name + int index = frontEndReq->addTranslationUnit(srcTranslationUnit.language, nullptr); + SLANG_UNUSED(index); + SLANG_ASSERT(index == i); + + TranslationUnitRequest* dstTranslationUnit = dstTranslationUnits[i]; + + context.loadDefines(srcTranslationUnit.preprocessorDefinitions, dstTranslationUnit->preprocessorDefinitions); + + Name* moduleName = nullptr; + if (srcTranslationUnit.moduleName) + { + moduleName = request->getNamePool()->getName(base.asRaw(srcTranslationUnit.moduleName)->getSlice()); + } + + dstTranslationUnit->moduleName = moduleName; + + const auto& srcSourceFiles = srcTranslationUnit.sourceFiles; + auto& dstSourceFiles = dstTranslationUnit->m_sourceFiles; + + dstSourceFiles.clear(); + + for (Index j = 0; j < srcSourceFiles.getCount(); ++j) + { + SourceFile* sourceFile = context.getSourceFile(base.asRaw(base.asRaw(srcSourceFiles[i]))); + // Add to translation unit + dstTranslationUnit->addSourceFile(sourceFile); + } + } + } + + // Entry points + { + // Check there aren't any set entry point + SLANG_ASSERT(request->getFrontEndReq()->m_entryPointReqs.getCount() == 0); + + for (const auto& srcEntryPointOffset : requestState->entryPoints) + { + const auto srcEntryPoint = base.asRaw(srcEntryPointOffset); + + const char* name = srcEntryPoint.name ? base.asRaw(srcEntryPoint.name)->getCstr() : nullptr; + + Stage stage = srcEntryPoint.profile.getStage(); + + List args = context.toList(srcEntryPoint.specializationArgStrings); + + spAddEntryPointEx(externalRequest, int(srcEntryPoint.translationUnitIndex), name, SlangStage(stage), int(args.getCount()), args.getBuffer()); + } + } + + { + RefPtr cacheFileSystem = new CacheFileSystem(nullptr); + auto& dstUniqueMap = cacheFileSystem->getUniqueMap(); + auto& dstPathMap = cacheFileSystem->getPathMap(); + + // Put all the paths to path info + { + for (const auto& pairOffset : requestState->pathInfoMap) + { + const auto& pair = base.asRaw(pairOffset); + CacheFileSystem::PathInfo* pathInfo = context.addPathInfo(base.asRaw(pair.pathInfo)); + dstPathMap.Add(base.asRaw(pair.path)->getSlice(), pathInfo); + } + } + // Put all the path infos in the cache system + { + for (const auto& pair : context.m_fileToPathInfoMap) + { + CacheFileSystem::PathInfo* pathInfo = pair.Value; + SLANG_ASSERT(pathInfo->m_uniqueIdentity); + dstUniqueMap.Add(pathInfo->m_uniqueIdentity->getString(), pathInfo); + } + } + + // This is a bit of a hack, we are going to replace the file system, with our one which is filled in + // with what was read from the file. + + linkage->m_fileSystemExt = cacheFileSystem; + linkage->m_cacheFileSystem = cacheFileSystem; + } + + return SLANG_OK; +} + + +/* static */SlangResult ReproUtil::saveState(EndToEndCompileRequest* request, Stream* stream) +{ + OffsetContainer container; + Offset32Ptr requestState; + SLANG_RETURN_ON_FAIL(store(request, container, requestState)); + + Header header; + header.m_chunk.type = kSlangStateFourCC; + header.m_semanticVersion = g_semanticVersion; + header.m_typeHash = uint32_t(_getTypeHash()); + + return RiffUtil::writeData(&header.m_chunk, sizeof(header),container.getData(), container.getDataCount(), stream); +} + +/* static */SlangResult ReproUtil::saveState(EndToEndCompileRequest* request, const String& filename) +{ + RefPtr stream(new FileStream(filename, FileMode::Create, FileAccess::Write, FileShare::ReadWrite)); + return saveState(request, stream); +} + +/* static */ SlangResult ReproUtil::loadState(const String& filename, List& outBuffer) +{ + RefPtr stream; + try + { + stream = new FileStream(filename, FileMode::Open, FileAccess::Read, FileShare::ReadWrite); + } + catch (const IOException&) + { + return SLANG_FAIL; + } + + return loadState(stream, outBuffer); +} + +/* static */ SlangResult ReproUtil::loadState(Stream* stream, List& buffer) +{ + Header header; + + SLANG_RETURN_ON_FAIL(RiffUtil::readData(stream, &header.m_chunk, sizeof(header), buffer)); + if (header.m_chunk.type != kSlangStateFourCC) + { + return SLANG_FAIL; + } + + if (!RiffSemanticVersion::areCompatible(g_semanticVersion, header.m_semanticVersion)) + { + return SLANG_FAIL; + } + + if (header.m_typeHash != uint32_t(_getTypeHash())) + { + return SLANG_FAIL; + } + + return SLANG_OK; +} + +/* static */SlangResult ReproUtil::loadState(const uint8_t* data, size_t size, List& outBuffer) +{ + MemoryStreamBase stream(FileAccess::Read, data, size); + return loadState(&stream, outBuffer); +} + +/* static */ ReproUtil::RequestState* ReproUtil::getRequest(const List& buffer) +{ + return (ReproUtil::RequestState*)(buffer.getBuffer() + kStartOffset); +} + +/* static */SlangResult ReproUtil::calcDirectoryPathFromFilename(const String& filename, String& outPath) +{ + String absPath; + SLANG_RETURN_ON_FAIL(Path::getCanonical(filename, absPath)); + + String parentDir = Path::getParentDirectory(absPath); + + String baseName = Path::getFileNameWithoutExt(filename); + String ext = Path::getPathExt(filename); + + if (ext.getLength() == 0) + { + StringBuilder builder; + builder << baseName << "-files"; + baseName = builder; + } + + outPath = Path::combine(parentDir, baseName); + return SLANG_OK; +} + +/* static */SlangResult ReproUtil::extractFilesToDirectory(const String& filename) +{ + List buffer; + SLANG_RETURN_ON_FAIL(ReproUtil::loadState(filename, buffer)); + + MemoryOffsetBase base; + base.set(buffer.getBuffer(), buffer.getCount()); + + RequestState* requestState = ReproUtil::getRequest(buffer); + + String dirPath; + SLANG_RETURN_ON_FAIL(ReproUtil::calcDirectoryPathFromFilename(filename, dirPath)); + + Path::createDirectory(dirPath); + // Set up a file system to write into this directory + RelativeFileSystem relFileSystem(OSFileSystemExt::getSingleton(), dirPath); + + return extractFiles(base, requestState, &relFileSystem); +} + +static void _calcPreprocessorDefines(OffsetBase& base, const Offset32Array& srcDefines, CommandLine& cmd) +{ + for (const auto& define : srcDefines) + { + StringBuilder builder; + builder << "-D" << base.asRaw(base.asRaw(define).first)->getSlice(); + if (base.asRaw(define).second) + { + builder << "=" << base.asRaw(base.asRaw(define).second)->getSlice(); + } + + cmd.addArg(builder); + } +} + +static SlangResult _calcCommandLine(OffsetBase& base, ReproUtil::RequestState* requestState, CommandLine& cmd) +{ + typedef ReproUtil::TargetRequestState TargetRequestState; + typedef ReproUtil::SourceFileState SourceFileState; + + { + SlangCompileFlags flags = (SlangCompileFlags)requestState->compileFlags; + while (flags) + { + // Extract a bit + const SlangCompileFlags isolatedBit = flags & SlangCompileFlags(-int(flags)); + + switch (isolatedBit) + { + case SLANG_COMPILE_FLAG_NO_MANGLING: cmd.addArg("-no-mangle"); break; + case SLANG_COMPILE_FLAG_NO_CODEGEN: cmd.addArg("-no-codegen"); break; + default: break; + } + + // Remove the bit + flags &= ~isolatedBit; + } + //spSetDumpIntermediates(externalRequest, int(requestState->shouldDumpIntermediates)); + + switch (SlangLineDirectiveMode(requestState->lineDirectiveMode)) + { + case SLANG_LINE_DIRECTIVE_MODE_DEFAULT: break; + case SLANG_LINE_DIRECTIVE_MODE_NONE: + { + cmd.addArg("-line-directive-mode"); + cmd.addArg("none"); + break; + } + default: break; + } + + switch (SlangDebugInfoLevel(requestState->debugInfoLevel)) + { + case SLANG_DEBUG_INFO_LEVEL_STANDARD: cmd.addArg("-g"); break; + case SLANG_DEBUG_INFO_LEVEL_NONE: cmd.addArg("-g0"); break; + case SLANG_DEBUG_INFO_LEVEL_MINIMAL: cmd.addArg("-g1"); break; + case SLANG_DEBUG_INFO_LEVEL_MAXIMAL: cmd.addArg("-g3"); break; + default: break; + } + + switch (SlangOptimizationLevel(requestState->optimizationLevel)) + { + case SLANG_OPTIMIZATION_LEVEL_NONE: cmd.addArg("-O0"); break; + case SLANG_OPTIMIZATION_LEVEL_DEFAULT: cmd.addArg("-O"); break; + case SLANG_OPTIMIZATION_LEVEL_HIGH: cmd.addArg("-O2"); break; + case SLANG_OPTIMIZATION_LEVEL_MAXIMAL: cmd.addArg("-O3"); break; + default: break; + } + + //spSetOutputContainerFormat(externalRequest, SlangContainerFormat(requestState->containerFormat)); + + switch (SlangPassThrough(requestState->passThroughMode)) + { + case SLANG_PASS_THROUGH_NONE: break; + default: + { + cmd.addArg("-pass-through"); + cmd.addArg(TypeTextUtil::getPassThroughName(SlangPassThrough(requestState->passThroughMode))); + break; + } + } + + //request->getBackEndReq()->useUnknownImageFormatAsDefault = requestState->useUnknownImageFormatAsDefault; + //request->getBackEndReq()->obfuscateCode = requestState->obfuscateCode; + //request->getFrontEndReq()->obfuscateCode = requestState->obfuscateCode; + + switch (requestState->defaultMatrixLayoutMode) + { + case SLANG_MATRIX_LAYOUT_ROW_MAJOR: cmd.addArg("-matrix-layout-row-major"); break; + case SLANG_MATRIX_LAYOUT_COLUMN_MAJOR: cmd.addArg("-matrix-layout-column-major"); break; + default: break; + } + } + + // Add the target requests + { + for (Index i = 0; i < requestState->targetRequests.getCount(); ++i) + { + TargetRequestState& src = base.asRaw(requestState->targetRequests[i]); + + cmd.addArg("-target"); + cmd.addArg(TypeTextUtil::getCompileTargetName(SlangCompileTarget(src.target))); + + if (src.profile != Profile::Unknown) + { + cmd.addArg("-profile"); + cmd.addArg(Profile(src.profile).getName()); + } + + if (src.targetFlags & SLANG_TARGET_FLAG_PARAMETER_BLOCKS_USE_REGISTER_SPACES) + { + cmd.addArg("-parameter-blocks-use-register-spaces"); + } + + switch (src.floatingPointMode) + { + case FloatingPointMode::Fast: + { + cmd.addArg("-fp-mode"); + cmd.addArg("fast"); + break; + } + case FloatingPointMode::Precise: + { + cmd.addArg("-fp-mode"); + cmd.addArg("precise"); + break; + } + default: break; + } + +#if 0 + // If there is output state (like output filenames) add here + if (src.outputStates.getCount()) + { + RefPtr dstTargetInfo(new EndToEndCompileRequest::TargetInfo); + request->targetInfos[dstTarget] = dstTargetInfo; + + for (const auto& srcOutputStateOffset : src.outputStates) + { + const auto& srcOutputState = base.asRaw(srcOutputStateOffset); + + SLANG_ASSERT(srcOutputState.entryPointIndex < requestState->entryPoints.getCount()); + + String entryPointPath; + if (srcOutputState.outputPath) + { + entryPointPath = base.asRaw(srcOutputState.outputPath)->getSlice(); + } + + dstTargetInfo->entryPointOutputPaths.Add(srcOutputState.entryPointIndex, entryPointPath); + } + } +#endif + } + } + + { + const auto& srcPaths = requestState->searchPaths; + for (Index i = 0; i < srcPaths.getCount(); ++i) + { + cmd.addArg("-I"); + cmd.addArg(base.asRaw(base.asRaw(srcPaths[i]))->getSlice()); + } + } + + _calcPreprocessorDefines(base, requestState->preprocessorDefinitions, cmd); + + { + const auto& srcTranslationUnits = requestState->translationUnits; + + for (Index i = 0; i < srcTranslationUnits.getCount(); ++i) + { + const auto& srcTranslationUnit = base.asRaw(srcTranslationUnits[i]); + + _calcPreprocessorDefines(base, srcTranslationUnit.preprocessorDefinitions, cmd); + + +#if 0 + if (srcTranslationUnit.moduleName) + { + moduleName = base[srcTranslationUnit].moduleName->getSlice()); + } +#endif + + const auto& srcSourceFiles = srcTranslationUnit.sourceFiles; + + for (Index j = 0; j < srcSourceFiles.getCount(); ++j) + { + SourceFileState* sourceFile = base.asRaw(base.asRaw(srcSourceFiles[i])); + OffsetString* path = base[sourceFile->foundPath]; + + if (path) + { + cmd.addArg(path->getSlice()); + } + } + } + } + + // Entry points + { + for (const auto& srcEntryPointOffset : requestState->entryPoints) + { + const auto srcEntryPoint = base.asRaw(srcEntryPointOffset); + + const char* name = srcEntryPoint.name ? base.asRaw(srcEntryPoint.name)->getCstr() : nullptr; + + cmd.addArg("-entry"); + cmd.addArg(name); + + cmd.addArg("-stage"); + UnownedStringSlice stageText = getStageText(srcEntryPoint.profile.getStage()); + cmd.addArg(stageText); + + //cmd.addArg("-profile"); + //cmd.addArg(Profile(srcEntryPoint.profile).getName()); + + + //List args = context.toList(srcEntryPoint.specializationArgStrings); + + //spAddEntryPointEx(externalRequest, int(srcEntryPoint.translationUnitIndex), name, SlangStage(stage), int(args.getCount()), args.getBuffer()); + } + } + + return SLANG_OK; +} + +/* static */SlangResult ReproUtil::extractFiles(OffsetBase& base, RequestState* requestState, ISlangFileSystemExt* fileSystem) +{ + StringBuilder builder; + + builder << "[command-line]\n"; + + { + CommandLine cmdLine; + _calcCommandLine(base, requestState, cmdLine); + String text = ProcessUtil::getCommandLineString(cmdLine); + builder << text << "\n"; + } + + builder << "[files]\n"; + + for (auto fileOffset : requestState->files) + { + auto file = base.asRaw(base.asRaw(fileOffset)); + + if (file->contents) + { + UnownedStringSlice contents = base.asRaw(file->contents)->getSlice(); + + SLANG_RETURN_ON_FAIL(fileSystem->saveFile(base.asRaw(file->uniqueName)->getCstr(), contents.begin(), contents.getLength())); + + OffsetString* originalName = nullptr; + if (file->canonicalPath) + { + originalName = base.asRaw(file->canonicalPath); + } + else if (file->foundPath) + { + originalName = base.asRaw(file->foundPath); + } + else if (file->uniqueIdentity) + { + originalName = base.asRaw(file->uniqueIdentity); + } + + builder << base.asRaw(file->uniqueName)->getSlice() << " -> "; + if (originalName) + { + builder << originalName->getSlice(); + } + + if (builder.getLength() == 0) + { + builder << "?"; + } + + builder << "\n"; + } + } + + builder << "[paths]\n"; + for (const auto pathOffset : requestState->pathInfoMap) + { + const auto& path = base.asRaw(pathOffset); + + builder << base.asRaw(path.path)->getSlice() << " -> "; + + const auto pathInfo = base.asRaw(path.pathInfo); + + if (pathInfo->file) + { + builder << base.asRaw(base.asRaw(pathInfo->file)->uniqueName)->getSlice(); + } + else + { + typedef CacheFileSystem::CompressedResult CompressedResult; + if (pathInfo->getPathTypeResult == CompressedResult::Ok) + { + switch (pathInfo->pathType) + { + case SLANG_PATH_TYPE_FILE: builder << "file "; break; + case SLANG_PATH_TYPE_DIRECTORY: builder << "directory "; break; + default: builder << "?"; break; + } + } + + CompressedResult curRes = pathInfo->getCanonicalPathResult; + CompressedResult results[] = + { + pathInfo->getPathTypeResult, + pathInfo->loadFileResult, + }; + + for (auto compRes : results) + { + if (int(compRes) > int(curRes)) + { + curRes = compRes; + } + } + + switch (curRes) + { + default: + case CompressedResult::Uninitialized: break; + case CompressedResult::Ok: break; + + case CompressedResult::NotFound: builder << " [not found]"; break; + case CompressedResult::CannotOpen: builder << "[cannot open]"; break; + case CompressedResult::Fail: builder << "[fail]"; break; + } + } + + builder << "\n"; + } + + SLANG_RETURN_ON_FAIL(fileSystem->saveFile("manifest.txt", builder.getBuffer(), builder.getLength())); + return SLANG_OK; +} + +static SlangResult _findFirstSourcePath(EndToEndCompileRequest* request, String& outFilename) +{ + // We are going to look through all of the srcTranlationUnits, looking for the first filename + + auto frontEndReq = request->getFrontEndReq(); + const auto& srcTranslationUnits = frontEndReq->translationUnits; + + for (Index i = 0; i < srcTranslationUnits.getCount(); ++i) + { + TranslationUnitRequest* srcTranslationUnit = srcTranslationUnits[i]; + const auto& srcSourceFiles = srcTranslationUnit->getSourceFiles(); + + for (Index j = 0; j < srcSourceFiles.getCount(); ++j) + { + SourceFile* sourceFile = srcSourceFiles[j]; + + const PathInfo& pathInfo = sourceFile->getPathInfo(); + + if (pathInfo.foundPath.getLength()) + { + outFilename = pathInfo.foundPath; + return SLANG_OK; + } + } + } + return SLANG_FAIL; +} + +/* static */SlangResult ReproUtil::findUniqueReproDumpStream(EndToEndCompileRequest* request, String& outFileName, RefPtr& outStream) +{ + String sourcePath; + + if (SLANG_FAILED(_findFirstSourcePath(request, sourcePath))) + { + sourcePath = "unknown.slang"; + } + + String sourceFileName = Path::getFileName(sourcePath); + String sourceBaseName = Path::getFileNameWithoutExt(sourceFileName); + + // Okay we need a unique number to make sure the name is unique + const int maxTries = 100; + for (int triesCount = 0; triesCount < maxTries; ++triesCount) + { + // We could include the count in some way perhaps, but for now let's just go with ticks + auto tick = ProcessUtil::getClockTick(); + + StringBuilder builder; + builder << sourceBaseName << "-" << tick << ".slang-repro"; + + // We write out the file name tried even if it fails, as might be useful in reporting + outFileName = builder; + + // We could have clashes, as we use ticks, we should get to a point where the clashes stop + try + { + outStream = new FileStream(builder, FileMode::CreateNew, FileAccess::Write, FileShare::WriteOnly); + return SLANG_OK; + } + catch (const IOException&) + { + } + + // TODO(JS): + // Might make sense to sleep here - but don't seem to have cross platform func for that yet. + } + + return SLANG_FAIL; +} + +} // namespace Slang diff --git a/source/slang/slang-repro.h b/source/slang/slang-repro.h new file mode 100644 index 000000000..bd35d3ee9 --- /dev/null +++ b/source/slang/slang-repro.h @@ -0,0 +1,200 @@ +// slang-repro.h +#ifndef SLANG_REPRO_H_INCLUDED +#define SLANG_REPRO_H_INCLUDED + +#include "../core/slang-riff.h" +#include "../core/slang-string.h" + +// For TranslationUnitRequest +#include "slang-compiler.h" + +#include "../core/slang-offset-container.h" + +#include "slang-file-system.h" + +namespace Slang { + +/* Facilities to be able to save and load the full state of a compilation, including source files, +and all compilation options into 'slang-repro' files. Repro is short for 'reproducible' and it's +main purposes is to make compilations easily reproducible as everything that is needed from +a compilation environment is packaged up into a single file. The single file can be used to +repeat the compilation, or extracted such that everything that was specified in the compilation +can be inspected and modified. */ +struct ReproUtil +{ + enum + { + kMajorVersion = 0, + kMinorVersion = 0, + kPatchVersion = 0, + }; + + static const uint32_t kSlangStateFourCC = SLANG_FOUR_CC('S', 'L', 'S', 'T'); ///< Holds all the slang specific chunks + static const RiffSemanticVersion g_semanticVersion; + + struct Header + { + RiffHeader m_chunk; ///< The chunk + RiffSemanticVersion m_semanticVersion; ///< The semantic version + uint32_t m_typeHash; ///< A hash based on the binary representation. If doesn't match then not binary compatible (extra check over semantic versioning) + }; + + struct FileState + { + Offset32Ptr uniqueIdentity; ///< The unique identity for the file (from ISlangFileSystem), or nullptr + Offset32Ptr contents; ///< The contents of this file + Offset32Ptr canonicalPath; ///< The canonical name of this file (or nullptr) + Offset32Ptr foundPath; ///< The 'found' path + + Offset32Ptr uniqueName; ///< A generated unique name (not used by slang, but used as mechanism to replace files) + }; + + struct PathInfoState + { + typedef CacheFileSystem::CompressedResult CompressedResult; + + SlangPathType pathType = SLANG_PATH_TYPE_FILE; + CompressedResult loadFileResult = CompressedResult::Uninitialized; + CompressedResult getPathTypeResult = CompressedResult::Uninitialized; + CompressedResult getCanonicalPathResult = CompressedResult::Uninitialized; + + Offset32Ptr file; ///< File contents + }; + + struct PathAndPathInfo + { + Offset32Ptr path; + Offset32Ptr pathInfo; + }; + + struct OutputState + { + int32_t entryPointIndex; + Offset32Ptr outputPath; + }; + + // spSetCodeGenTarget/spAddCodeGenTarget + // spSetTargetProfile + // spSetTargetFlags + // spSetTargetFloatingPointMode + // spSetTargetMatrixLayoutMode + struct TargetRequestState + { + Profile profile; + CodeGenTarget target; + SlangTargetFlags targetFlags; + FloatingPointMode floatingPointMode; + + Offset32Array outputStates; + }; + + struct StringPair + { + Offset32Ptr first; + Offset32Ptr second; + }; + + struct SourceFileState + { + PathInfo::Type type; ///< The type of this file + Offset32Ptr foundPath; ///< The Path this was found along + Offset32Ptr file; ///< The file contents + }; + + // spAddTranslationUnit + struct TranslationUnitRequestState + { + SourceLanguage language; + + Offset32Ptr moduleName; + + // spTranslationUnit_addPreprocessorDefine + Offset32Array preprocessorDefinitions; + + Offset32Array > sourceFiles; + }; + + struct EntryPointState + { + Offset32Ptr name; + Profile profile; + uint32_t translationUnitIndex; + Offset32Array> specializationArgStrings; + }; + + struct RequestState + { + Offset32Array> files; ///< All of the files + Offset32Array> sourceFiles; ///< All of the source files (from source manager) + + // spSetCompileFlags + SlangCompileFlags compileFlags; + // spSetDumpIntermediates + bool shouldDumpIntermediates; + // spSetLineDirectiveMode + LineDirectiveMode lineDirectiveMode; + + Offset32Array targetRequests; + + // spSetDebugInfoLevel + DebugInfoLevel debugInfoLevel; + // spSetOptimizationLevel + OptimizationLevel optimizationLevel; + // spSetOutputContainerFormat + ContainerFormat containerFormat; + // spSetPassThrough + PassThroughMode passThroughMode; + + // spAddSearchPath + Offset32Array > searchPaths; + + // spAddPreprocessorDefine + Offset32Array preprocessorDefinitions; + + bool useUnknownImageFormatAsDefault = false; + bool obfuscateCode = false; + + Offset32Array pathInfoMap; ///< Stores all the accesses to the file system + + Offset32Array translationUnits; + + Offset32Array entryPoints; + + SlangMatrixLayoutMode defaultMatrixLayoutMode; + }; + + static SlangResult store(EndToEndCompileRequest* request, OffsetContainer& inOutContainer, Offset32Ptr& outRequest); + + static SlangResult saveState(EndToEndCompileRequest* request, const String& filename); + + static SlangResult saveState(EndToEndCompileRequest* request, Stream* stream); + + /// Create a cache file system that uses contents of the request state. + /// The passed in fileSystem is used for accessing any file accesses not found in the cache + static SlangResult loadFileSystem(OffsetBase& base, RequestState* requestState, ISlangFileSystem* fileSystem, RefPtr& outFileSystem); + + /// Load the requestState into request + /// The fileSystem is optional and can be passed as nullptr. If set, as each file is loaded + /// it will attempt to load from fileSystem the *uniqueName* + static SlangResult load(OffsetBase& base, RequestState* requestState, ISlangFileSystem* fileSystem, EndToEndCompileRequest* request); + + static SlangResult loadState(const String& filename, List& outBuffer); + static SlangResult loadState(Stream* stream, List& outBuffer); + static SlangResult loadState(const uint8_t* data, size_t size, List& outBuffer); + + static RequestState* getRequest(const List& inBuffer); + + static SlangResult extractFilesToDirectory(const String& file); + + static SlangResult extractFiles(OffsetBase& base, RequestState* requestState, ISlangFileSystemExt* fileSystem); + + /// Given the repo file work out a suitable path + static SlangResult calcDirectoryPathFromFilename(const String& filename, String& outPath); + + /// Given a request trys to determine a suitable dump file name, that is unique. + static SlangResult findUniqueReproDumpStream(EndToEndCompileRequest* request, String& outFileName, RefPtr& outStream); +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-state-serialize.cpp b/source/slang/slang-state-serialize.cpp deleted file mode 100644 index 3d98f182d..000000000 --- a/source/slang/slang-state-serialize.cpp +++ /dev/null @@ -1,1564 +0,0 @@ -// slang-state-serialize.cpp -#include "slang-state-serialize.h" - -#include "../core/slang-text-io.h" - -#include "../core/slang-stream.h" - -#include "../core/slang-math.h" -#include "../core/slang-type-text-util.h" - -#include "slang-options.h" - -#include "slang-source-loc.h" - -namespace Slang { - -/* static */const RiffSemanticVersion StateSerializeUtil::g_semanticVersion = - RiffSemanticVersion::make(StateSerializeUtil::kMajorVersion, StateSerializeUtil::kMinorVersion, StateSerializeUtil::kPatchVersion); - -// We can't just use sizeof for the sizes of these types, because the hash will be dependent on the ptr size, -// which isn't an issue for serialization (we turn all pointers into Offset32Ptr -> uint32_t). So we use an x macro -// to set up the thing to hash. -// -// Note that bool is in the list because size of bool can change between compilers. -#define SLANG_STATE_TYPES(x) \ - x(Util::FileState) \ - x(Util::PathInfoState) \ - x(Util::PathInfoState::CompressedResult) \ - x(SlangPathType) \ - x(Util::PathAndPathInfo) \ - x(Util::TargetRequestState) \ - x(Profile) \ - x(CodeGenTarget) \ - x(SlangTargetFlags) \ - x(FloatingPointMode) \ - x(Util::StringPair) \ - x(Util::SourceFileState) \ - x(PathInfo::Type) \ - x(Util::TranslationUnitRequestState) \ - x(SourceLanguage) \ - x(Util::EntryPointState) \ - x(Profile) \ - x(Util::RequestState) \ - x(SlangCompileFlags) \ - x(bool) \ - x(LineDirectiveMode) \ - x(DebugInfoLevel) \ - x(OptimizationLevel) \ - x(ContainerFormat) \ - x(PassThroughMode) \ - x(SlangMatrixLayoutMode) \ - -#define SLANG_STATE_TYPE_SIZE(x) uint32_t(sizeof(x)), - -// A function to calculate the hash related in list in part to how the types used are sized. Can catch crude breaking binary differences. -static HashCode32 _calcTypeHash() -{ - typedef StateSerializeUtil Util; - const uint32_t sizes[] = - { - SLANG_STATE_TYPES(SLANG_STATE_TYPE_SIZE) - }; - return getStableHashCode32((const char*)&sizes, sizeof(sizes)); -} - -static HashCode32 _getTypeHash() -{ - static HashCode32 s_hash = _calcTypeHash(); - return s_hash; -} - - -namespace { // anonymous - -struct StoreContext -{ - typedef StateSerializeUtil::FileState FileState; - typedef StateSerializeUtil::SourceFileState SourceFileState; - typedef StateSerializeUtil::PathInfoState PathInfoState; - - StoreContext(OffsetContainer* container) - { - m_container = container; - } - - Offset32Ptr findFile(const String& uniqueIdentity) - { - Offset32Ptr file; - m_uniqueToFileMap.TryGetValue(uniqueIdentity, file); - return file; - } - - Offset32Ptr addFile(const String& uniqueIdentity, const UnownedStringSlice* content) - { - OffsetBase& base = m_container->asBase(); - - Offset32Ptr file; - - // Get the file, if it has an identity - if (uniqueIdentity.getLength()) - { - if (!m_uniqueToFileMap.TryGetValue(uniqueIdentity, file)) - { - // If file was not found create it - // Create the file - file = m_container->newObject(); - // Add it - m_uniqueToFileMap.Add(uniqueIdentity, file); - - // Set the identity - auto offsetUniqueIdentity = m_container->newString(uniqueIdentity.getUnownedSlice()); - base[file]->uniqueIdentity = offsetUniqueIdentity; - - // Add the file - m_files.add(file); - } - } - else - { - // Create a file, but we know it can't have unique identity - file = m_container->newObject(); - // Add the file - m_files.add(file); - } - - // If the contents is not set add it - if (!base[file]->contents && content) - { - auto offsetContent = m_container->newString(*content); - base[file]->contents = offsetContent; - } - - return file; - } - - Offset32Ptr addSourceFile(SourceFile* sourceFile) - { - if (!sourceFile) - { - return Offset32Ptr(); - } - - auto& base = m_container->asBase(); - - Offset32Ptr sourceFileState; - if (m_sourceFileMap.TryGetValue(sourceFile, sourceFileState)) - { - return sourceFileState; - } - - const PathInfo& pathInfo = sourceFile->getPathInfo(); - - UnownedStringSlice content = sourceFile->getContent(); - Offset32Ptr file = addFile(pathInfo.uniqueIdentity, &content); - - Offset32Ptr foundPath; - - if (pathInfo.foundPath.getLength() && base[file]->foundPath.isNull()) - { - foundPath = fromString(pathInfo.foundPath.getUnownedSlice()); - } - // Set on the file - base[file]->foundPath = foundPath; - - // Create the source file - sourceFileState = m_container->newObject(); - - { - auto dst = base[sourceFileState]; - dst->file = file; - dst->foundPath = foundPath; - dst->type = pathInfo.type; - } - - m_sourceFileMap.Add(sourceFile, sourceFileState); - - return sourceFileState; - } - - Offset32Ptr fromString(const String& in) - { - Offset32Ptr value; - - if (m_stringMap.TryGetValue(in, value)) - { - return value; - } - value = m_container->newString(in.getUnownedSlice()); - m_stringMap.Add(in, value); - return value; - } - Offset32Ptr fromName(Name* name) - { - if (name) - { - return fromString(name->text); - } - return Offset32Ptr(); - } - - Offset32Ptr addPathInfo(const CacheFileSystem::PathInfo* srcPathInfo) - { - if (!srcPathInfo) - { - return Offset32Ptr(); - } - - OffsetBase& base = m_container->asBase(); - - Offset32Ptr pathInfo; - if (!m_pathInfoMap.TryGetValue(srcPathInfo, pathInfo)) - { - // Get the associated file - Offset32Ptr fileState; - - // Only store as file if we have the contents - if(ISlangBlob* fileBlob = srcPathInfo->m_fileBlob) - { - UnownedStringSlice content((const char*)fileBlob->getBufferPointer(), fileBlob->getBufferSize()); - - fileState = addFile(srcPathInfo->getUniqueIdentity(), &content); - } - - // Save the rest of the state - pathInfo = m_container->newObject(); - PathInfoState& dst = base[*pathInfo]; - - dst.file = fileState; - - // Save any other info - dst.getCanonicalPathResult = srcPathInfo->m_getCanonicalPathResult; - dst.getPathTypeResult = srcPathInfo->m_getPathTypeResult; - dst.loadFileResult = srcPathInfo->m_loadFileResult; - dst.pathType = srcPathInfo->m_pathType; - - m_pathInfoMap.Add(srcPathInfo, pathInfo); - } - - // Fill in info on the file - auto fileState(base[pathInfo]->file); - - // If have fileState add any missing element - if (fileState) - { - if (srcPathInfo->m_fileBlob && base[fileState]->contents.isNull()) - { - UnownedStringSlice contents((const char*)srcPathInfo->m_fileBlob->getBufferPointer(), srcPathInfo->m_fileBlob->getBufferSize()); - auto offsetContents = m_container->newString(contents); - base[fileState]->contents = offsetContents; - } - - if (srcPathInfo->m_canonicalPath && base[fileState]->canonicalPath.isNull()) - { - auto offsetCanonicalPath = fromString(srcPathInfo->m_canonicalPath->getString()); - base[fileState]->canonicalPath = offsetCanonicalPath; - } - - if (srcPathInfo->m_uniqueIdentity && base[fileState]->uniqueIdentity.isNull()) - { - auto offsetUniqueIdentity = fromString(srcPathInfo->m_uniqueIdentity->getString()); - base[fileState]->uniqueIdentity = offsetUniqueIdentity; - } - } - - return pathInfo; - } - - const Offset32Array calcDefines(const Dictionary& srcDefines) - { - typedef StateSerializeUtil::StringPair StringPair; - - Offset32Array dstDefines = m_container->newArray(srcDefines.Count()); - - OffsetBase& base = m_container->asBase(); - - Index index = 0; - for (const auto& srcDefine : srcDefines) - { - // Do allocation before setting - auto key = fromString(srcDefine.Key); - auto value = fromString(srcDefine.Value); - - auto& dstDefine = base[dstDefines[index]]; - dstDefine.first = key; - dstDefine.second = value; - - index++; - } - - return dstDefines; - } - - const Offset32Array> fromList(const List& src) - { - Offset32Array> dst = m_container->newArray>(src.getCount()); - OffsetBase& base = m_container->asBase(); - - for (Index j = 0; j < src.getCount(); ++j) - { - base[dst[j]] = fromString(src[j]); - } - return dst; - } - - Dictionary > m_stringMap; - - Dictionary > m_sourceFileMap; - - Dictionary > m_uniqueToFileMap; - - Dictionary > m_pathInfoMap; - - List > m_files; - - OffsetContainer* m_container; -}; - -} // - -static bool _isStorable(const PathInfo::Type type) -{ - switch (type) - { - case PathInfo::Type::Unknown: - case PathInfo::Type::Normal: - case PathInfo::Type::FoundPath: - case PathInfo::Type::FromString: - { - return true; - } - default: return false; - } -} - -/* static */SlangResult StateSerializeUtil::store(EndToEndCompileRequest* request, OffsetContainer& inOutContainer, Offset32Ptr& outRequest) -{ - StoreContext context(&inOutContainer); - - OffsetBase& base = inOutContainer.asBase(); - - auto linkage = request->getLinkage(); - - Offset32Ptr requestState = inOutContainer.newObject(); - - { - RequestState* dst = base[requestState]; - - dst->compileFlags = request->getFrontEndReq()->compileFlags; - dst->shouldDumpIntermediates = request->getBackEndReq()->shouldDumpIntermediates; - dst->lineDirectiveMode = request->getBackEndReq()->lineDirectiveMode; - - dst->debugInfoLevel = linkage->debugInfoLevel; - dst->optimizationLevel = linkage->optimizationLevel; - dst->containerFormat = request->m_containerFormat; - dst->passThroughMode = request->passThrough; - - - dst->useUnknownImageFormatAsDefault = request->getBackEndReq()->useUnknownImageFormatAsDefault; - dst->obfuscateCode = linkage->m_obfuscateCode; - - dst->defaultMatrixLayoutMode = linkage->defaultMatrixLayoutMode; - } - - // Entry points - { - const auto& srcEntryPoints = request->getFrontEndReq()->m_entryPointReqs; - const auto& srcEndToEndEntryPoints = request->entryPoints; - - SLANG_ASSERT(srcEntryPoints.getCount() == srcEndToEndEntryPoints.getCount()); - - Offset32Array dstEntryPoints = inOutContainer.newArray(srcEntryPoints.getCount()); - - for (Index i = 0; i < srcEntryPoints.getCount(); ++i) - { - FrontEndEntryPointRequest* srcEntryPoint = srcEntryPoints[i]; - const auto& srcEndToEndEntryPoint = srcEndToEndEntryPoints[i]; - - auto dstSpecializationArgStrings = context.fromList(srcEndToEndEntryPoint.specializationArgStrings); - Offset32Ptr dstName = context.fromName(srcEntryPoint->getName()); - - EntryPointState& dst = base[dstEntryPoints[i]]; - - dst.profile = srcEntryPoint->getProfile(); - dst.translationUnitIndex = uint32_t(srcEntryPoint->getTranslationUnitIndex()); - dst.specializationArgStrings = dstSpecializationArgStrings; - dst.name = dstName; - } - - base[requestState]->entryPoints = dstEntryPoints; - } - - - // Add all of the source files - { - SourceManager* sourceManager = request->getFrontEndReq()->getSourceManager(); - const List& sourceFiles = sourceManager->getSourceFiles(); - - for (SourceFile* sourceFile : sourceFiles) - { - const PathInfo& pathInfo = sourceFile->getPathInfo(); - if (_isStorable(pathInfo.type)) - { - context.addSourceFile(sourceFile); - } - } - } - - // Add all the target requests - { - Offset32Array dstTargets = inOutContainer.newArray(linkage->targets.getCount()); - - for (Index i = 0; i < linkage->targets.getCount(); ++i) - { - TargetRequest* srcTargetRequest = linkage->targets[i]; - - // Copy the simple stuff - { - auto& dst = base[dstTargets[i]]; - dst.target = srcTargetRequest->getTarget(); - dst.profile = srcTargetRequest->getTargetProfile(); - dst.targetFlags = srcTargetRequest->targetFlags; - dst.floatingPointMode = srcTargetRequest->floatingPointMode; - } - - // Copy the entry point/target output names - { - const auto& srcTargetInfos = request->targetInfos; - - if (RefPtr* infosPtr = srcTargetInfos.TryGetValue(srcTargetRequest)) - { - EndToEndCompileRequest::TargetInfo* infos = *infosPtr; - - const auto& entryPointOutputPaths = infos->entryPointOutputPaths; - - Offset32Array dstOutputStates = inOutContainer.newArray(entryPointOutputPaths.Count()); - - Index index = 0; - for (const auto& pair : entryPointOutputPaths) - { - Offset32Ptr outputPath = inOutContainer.newString(pair.Value.getUnownedSlice()); - - auto& dstOutputState = base[dstOutputStates[index]]; - - dstOutputState.entryPointIndex = int32_t(pair.Key); - dstOutputState.outputPath = outputPath; - - index++; - } - - base[dstTargets[i]].outputStates = dstOutputStates; - } - } - } - - // Save the result - base[requestState]->targetRequests = dstTargets; - } - - // Add the search paths - { - const auto& srcPaths = linkage->searchDirectories.searchDirectories; - Offset32Array > dstPaths = inOutContainer.newArray >(srcPaths.getCount()); - - // We don't handle parents here - SLANG_ASSERT(linkage->searchDirectories.parent == nullptr); - for (Index i = 0; i < srcPaths.getCount(); ++i) - { - base[dstPaths[i]] = context.fromString(srcPaths[i].path); - } - base[requestState]->searchPaths = dstPaths; - } - - // Add preprocessor definitions - base[requestState]->preprocessorDefinitions = context.calcDefines(linkage->preprocessorDefinitions); - - { - const auto& srcTranslationUnits = request->getFrontEndReq()->translationUnits; - Offset32Array dstTranslationUnits = inOutContainer.newArray(srcTranslationUnits.getCount()); - - for (Index i = 0; i < srcTranslationUnits.getCount(); ++i) - { - TranslationUnitRequest* srcTranslationUnit = srcTranslationUnits[i]; - - // Do before setting, because this can allocate, and therefore break, the following section - auto defines = context.calcDefines(srcTranslationUnit->preprocessorDefinitions); - auto moduleName = context.fromName(srcTranslationUnit->moduleName); - - Offset32Array> dstSourceFiles; - { - const auto& srcFiles = srcTranslationUnit->getSourceFiles(); - dstSourceFiles = inOutContainer.newArray >(srcFiles.getCount()); - - for (Index j = 0; j < srcFiles.getCount(); ++j) - { - base[dstSourceFiles[j]] = context.addSourceFile(srcFiles[j]); - } - } - - TranslationUnitRequestState& dstTranslationUnit = base[dstTranslationUnits[i]]; - - dstTranslationUnit.language = srcTranslationUnit->sourceLanguage; - dstTranslationUnit.moduleName = moduleName; - dstTranslationUnit.sourceFiles = dstSourceFiles; - dstTranslationUnit.preprocessorDefinitions = defines; - } - - base[requestState]->translationUnits = dstTranslationUnits; - } - - // Find files from the file system, and mapping paths to files - { - CacheFileSystem* cacheFileSystem = linkage->getCacheFileSystem(); - if (!cacheFileSystem) - { - return SLANG_FAIL; - } - - // Traverse the references (in process we will construct the map from PathInfo) - { - const auto& srcFiles = cacheFileSystem->getPathMap(); - - Offset32Array pathMap = inOutContainer.newArray(srcFiles.Count()); - - Index index = 0; - for (const auto& pair : srcFiles) - { - Offset32Ptr path = context.fromString(pair.Key); - Offset32Ptr pathInfo = context.addPathInfo(pair.Value); - - PathAndPathInfo& dstInfo = base[pathMap[index]]; - dstInfo.path = path; - dstInfo.pathInfo = pathInfo; - - index++; - } - - base[requestState]->pathInfoMap = pathMap; - } - } - - // Save all of the files - { - Dictionary uniqueNameMap; - - auto files = inOutContainer.newArray>(context.m_files.getCount()); - for (Index i = 0; i < context.m_files.getCount(); ++i) - { - Offset32Ptr file = context.m_files[i]; - - // Need to come up with unique names - String path; - - if (auto canonicalPath = base[file]->canonicalPath) - { - path = base[canonicalPath]->getSlice(); - } - else if (auto foundPath = base[file]->foundPath) - { - path = base[foundPath]->getSlice(); - } - else if (auto uniqueIdentity = base[file]->uniqueIdentity) - { - path = base[uniqueIdentity]->getSlice(); - } - - if (path.getLength() == 0) - { - StringBuilder builder; - builder << "unnamed" << i; - path = builder; - } - - String filename = Path::getFileNameWithoutExt(path); - String ext = Path::getPathExt(path); - - StringBuilder uniqueName; - for (Index j = 0; j < 0x10000; j++) - { - uniqueName.Clear(); - uniqueName << filename; - - if (j > 0) - { - uniqueName << "-" << j; - } - - if (ext.getLength()) - { - uniqueName << "." << ext; - } - - int dummy = 0; - if (!uniqueNameMap.TryGetValueOrAdd(uniqueName, dummy)) - { - // It was added so we are done - break; - } - } - - // Save the unique generated name - base[file]->uniqueName = inOutContainer.newString(uniqueName.getUnownedSlice()); - - base[files[i]] = file; - } - - base[requestState]->files = files; - } - - // Save all the SourceFile state - { - const auto& srcSourceFiles = context.m_sourceFileMap; - auto dstSourceFiles = inOutContainer.newArray>(srcSourceFiles.Count()); - - Index index = 0; - for (const auto& pair : srcSourceFiles) - { - base[dstSourceFiles[index]] = pair.Value; - index++; - } - base[requestState]->sourceFiles = dstSourceFiles; - } - - outRequest = requestState; - return SLANG_OK; -} - -namespace { // anonymous - -struct LoadContext -{ - typedef StateSerializeUtil::SourceFileState SourceFileState; - typedef StateSerializeUtil::FileState FileState; - typedef StateSerializeUtil::PathInfoState PathInfoState; - - CacheFileSystem::PathInfo* getPathInfoFromFile(FileState* file) - { - if (!file) - { - return nullptr; - } - - CacheFileSystem::PathInfo* dstInfo = nullptr; - if (!m_fileToPathInfoMap.TryGetValue(file, dstInfo)) - { - ComPtr blob; - - if (m_fileSystem && file->uniqueName) - { - // Try loading from the file system - m_fileSystem->loadFile(m_base->asRaw(file->uniqueName)->getCstr(), blob.writeRef()); - } - - // If wasn't loaded, and has contents, use that - if (!blob && file->contents) - { - blob = new StringBlob(m_base->asRaw(file->contents)->getSlice()); - } - - dstInfo = new CacheFileSystem::PathInfo(String()); - - if (file->uniqueIdentity) - { - String uniqueIdentity = m_base->asRaw(file->uniqueIdentity)->getSlice(); - dstInfo->m_uniqueIdentity = new StringBlob(uniqueIdentity); - } - - if (file->canonicalPath) - { - dstInfo->m_canonicalPath = new StringBlob(m_base->asRaw(file->canonicalPath)->getSlice()); - } - - if (blob) - { - dstInfo->m_loadFileResult = CacheFileSystem::CompressedResult::Ok; - dstInfo->m_getPathTypeResult = CacheFileSystem::CompressedResult::Ok; - dstInfo->m_pathType = SLANG_PATH_TYPE_FILE; - } - - dstInfo->m_fileBlob = blob; - - // Add to map, even if the blob is nullptr (say from a failed read) - m_fileToPathInfoMap.Add(file, dstInfo); - } - - return dstInfo; - } - - ISlangBlob* getFileBlobFromFile(FileState* file) - { - CacheFileSystem::PathInfo* pathInfo = getPathInfoFromFile(file); - return pathInfo ? pathInfo->m_fileBlob.get() : nullptr; - } - - SourceFile* getSourceFile(SourceFileState* sourceFile) - { - if (sourceFile == nullptr) - { - return nullptr; - } - - SourceFile* dstFile; - if (!m_sourceFileMap.TryGetValue(sourceFile, dstFile)) - { - FileState* file = m_base->asRaw(sourceFile->file); - ISlangBlob* blob = getFileBlobFromFile(file); - - PathInfo pathInfo; - - pathInfo.type = sourceFile->type; - - if (sourceFile->foundPath) - { - pathInfo.foundPath = m_base->asRaw(sourceFile->foundPath)->getSlice(); - } - else if (file->foundPath) - { - pathInfo.foundPath = m_base->asRaw(file->foundPath)->getSlice(); - } - - if (file->uniqueIdentity) - { - pathInfo.uniqueIdentity = m_base->asRaw(file->uniqueIdentity)->getSlice(); - } - - dstFile = new SourceFile(m_sourceManager, pathInfo, blob->getBufferSize()); - dstFile->setContents(blob); - - // Add to map - m_sourceFileMap.Add(sourceFile, dstFile); - - // Add to manager - m_sourceManager->addSourceFile(pathInfo.uniqueIdentity, dstFile); - } - return dstFile; - } - - CacheFileSystem::PathInfo* addPathInfo(const PathInfoState* srcInfo) - { - CacheFileSystem::PathInfo* pathInfo; - if (m_pathInfoMap.TryGetValue(srcInfo, pathInfo)) - { - return pathInfo; - } - - FileState* file = m_base->asRaw(srcInfo->file); - CacheFileSystem::PathInfo* dstInfo; - - if (file) - { - dstInfo = getPathInfoFromFile(file); - } - else - { - // TODO(JS): Hmmm... this could end up not being cleared up - // Because it is not added to the unique set (as unique set is for files and this isn't a file) - dstInfo = new CacheFileSystem::PathInfo(String()); - } - - dstInfo->m_getCanonicalPathResult = srcInfo->getCanonicalPathResult; - dstInfo->m_getPathTypeResult = srcInfo->getPathTypeResult; - dstInfo->m_loadFileResult = srcInfo->loadFileResult; - dstInfo->m_pathType = srcInfo->pathType; - - m_pathInfoMap.Add(srcInfo, dstInfo); - return dstInfo; - } - - List toList(const Offset32Array>& src) - { - List dst; - dst.setCount(src.getCount()); - for (Index i = 0; i < src.getCount(); ++i) - { - OffsetString* srcString = m_base->asRaw(m_base->asRaw(src[i])); - dst[i] = srcString ? srcString->getCstr() : nullptr; - } - return dst; - } - - - void loadDefines(const Offset32Array& in, Dictionary& out) - { - out.Clear(); - - for (const auto& define : in) - { - out.Add(m_base->asRaw(m_base->asRaw(define).first)->getSlice(), m_base->asRaw(m_base->asRaw(define).second)->getSlice()); - } - } - - LoadContext(SourceManager* sourceManger, ISlangFileSystem* fileSystem, OffsetBase* base): - m_sourceManager(sourceManger), - m_fileSystem(fileSystem), - m_base(base) - { - } - - ISlangFileSystem* m_fileSystem; - - OffsetBase* m_base; - - SourceManager* m_sourceManager; - - Dictionary m_sourceFileMap; - Dictionary m_fileToPathInfoMap; - Dictionary m_pathInfoMap; -}; - -} // anonymous - - -/* static */SlangResult StateSerializeUtil::loadFileSystem(OffsetBase& base, RequestState* requestState, ISlangFileSystem* fileSystem, RefPtr& outFileSystem) -{ - LoadContext context(nullptr, fileSystem, &base); - - RefPtr cacheFileSystem = new CacheFileSystem(nullptr); - auto& dstUniqueMap = cacheFileSystem->getUniqueMap(); - auto& dstPathMap = cacheFileSystem->getPathMap(); - - for (auto fileOffset : requestState->files) - { - // add the file - FileState* fileState = base.asRaw(base.asRaw(fileOffset)); - CacheFileSystem::PathInfo* pathInfo = context.getPathInfoFromFile(fileState); - - if (fileState->foundPath) - { - String foundPath = base.asRaw(fileState->foundPath)->getSlice(); - dstPathMap.AddIfNotExists(foundPath, pathInfo); - } - } - - // Put all the paths to path info - { - for (const auto& pairOffset : requestState->pathInfoMap) - { - const auto& pair = base.asRaw(pairOffset); - CacheFileSystem::PathInfo* pathInfo = context.addPathInfo(base.asRaw(pair.pathInfo)); - dstPathMap.AddIfNotExists(base.asRaw(pair.path)->getSlice(), pathInfo); - } - } - - // Put all the path infos in the cache system - { - for (const auto& pair : context.m_fileToPathInfoMap) - { - CacheFileSystem::PathInfo* pathInfo = pair.Value; - SLANG_ASSERT(pathInfo->m_uniqueIdentity); - dstUniqueMap.Add(pathInfo->m_uniqueIdentity->getString(), pathInfo); - - // Add canonical paths too.. - if (pathInfo->m_canonicalPath) - { - String canonicalPath = pathInfo->m_canonicalPath->getString(); - - dstPathMap.AddIfNotExists(canonicalPath, pathInfo); - } - } - } - - outFileSystem = cacheFileSystem; - return SLANG_OK; -} - -/* static */SlangResult StateSerializeUtil::load(OffsetBase& base, RequestState* requestState, ISlangFileSystem* fileSystem, EndToEndCompileRequest* request) -{ - auto externalRequest = asExternal(request); - - auto linkage = request->getLinkage(); - - // TODO(JS): Really should be more exhaustive here, and set up to initial state ideally - // Reset state - { - request->targetInfos.Clear(); - // Remove any requests - linkage->targets.clear(); - } - - LoadContext context(linkage->getSourceManager(), fileSystem, &base); - - // Try to set state through API - as doing so means if state stored in multiple places it will be ok - - { - spSetCompileFlags(externalRequest, (SlangCompileFlags)requestState->compileFlags); - spSetDumpIntermediates(externalRequest, int(requestState->shouldDumpIntermediates)); - spSetLineDirectiveMode(externalRequest, SlangLineDirectiveMode(requestState->lineDirectiveMode)); - spSetDebugInfoLevel(externalRequest, SlangDebugInfoLevel(requestState->debugInfoLevel)); - spSetOptimizationLevel(externalRequest, SlangOptimizationLevel(requestState->optimizationLevel)); - spSetOutputContainerFormat(externalRequest, SlangContainerFormat(requestState->containerFormat)); - spSetPassThrough(externalRequest, SlangPassThrough(request->passThrough)); - - request->getBackEndReq()->useUnknownImageFormatAsDefault = requestState->useUnknownImageFormatAsDefault; - linkage->m_obfuscateCode = requestState->obfuscateCode; - - linkage->setMatrixLayoutMode(requestState->defaultMatrixLayoutMode); - } - - // Add the target requests - { - for (Index i = 0; i < requestState->targetRequests.getCount(); ++i) - { - TargetRequestState& src = base.asRaw(requestState->targetRequests[i]); - int index = spAddCodeGenTarget(externalRequest, SlangCompileTarget(src.target)); - SLANG_ASSERT(index == i); - - auto dstTarget = linkage->targets[index]; - - SLANG_ASSERT(dstTarget->getTarget() == src.target); - dstTarget->targetProfile = src.profile; - dstTarget->targetFlags = src.targetFlags; - dstTarget->floatingPointMode = src.floatingPointMode; - - // If there is output state (like output filenames) add here - if (src.outputStates.getCount()) - { - RefPtr dstTargetInfo(new EndToEndCompileRequest::TargetInfo); - request->targetInfos[dstTarget] = dstTargetInfo; - - for (const auto& srcOutputStateOffset : src.outputStates) - { - const auto& srcOutputState = base.asRaw(srcOutputStateOffset); - - SLANG_ASSERT(srcOutputState.entryPointIndex < requestState->entryPoints.getCount()); - - String entryPointPath; - if (srcOutputState.outputPath) - { - entryPointPath = base.asRaw(srcOutputState.outputPath)->getSlice(); - } - - dstTargetInfo->entryPointOutputPaths.Add(srcOutputState.entryPointIndex, entryPointPath); - } - } - } - } - - { - const auto& srcPaths = requestState->searchPaths; - auto& dstPaths = linkage->searchDirectories.searchDirectories; - dstPaths.setCount(srcPaths.getCount()); - for (Index i = 0; i < srcPaths.getCount(); ++i) - { - dstPaths[i].path = base.asRaw(base.asRaw(srcPaths[i]))->getSlice(); - } - } - - context.loadDefines(requestState->preprocessorDefinitions, linkage->preprocessorDefinitions); - - { - auto frontEndReq = request->getFrontEndReq(); - - const auto& srcTranslationUnits = requestState->translationUnits; - auto& dstTranslationUnits = frontEndReq->translationUnits; - - dstTranslationUnits.clear(); - - for (Index i = 0; i < srcTranslationUnits.getCount(); ++i) - { - const auto& srcTranslationUnit = base.asRaw(srcTranslationUnits[i]); - - // TODO(JS): We should probably serialize off the module name - // Passing in nullptr will just generate the module name - int index = frontEndReq->addTranslationUnit(srcTranslationUnit.language, nullptr); - SLANG_UNUSED(index); - SLANG_ASSERT(index == i); - - TranslationUnitRequest* dstTranslationUnit = dstTranslationUnits[i]; - - context.loadDefines(srcTranslationUnit.preprocessorDefinitions, dstTranslationUnit->preprocessorDefinitions); - - Name* moduleName = nullptr; - if (srcTranslationUnit.moduleName) - { - moduleName = request->getNamePool()->getName(base.asRaw(srcTranslationUnit.moduleName)->getSlice()); - } - - dstTranslationUnit->moduleName = moduleName; - - const auto& srcSourceFiles = srcTranslationUnit.sourceFiles; - auto& dstSourceFiles = dstTranslationUnit->m_sourceFiles; - - dstSourceFiles.clear(); - - for (Index j = 0; j < srcSourceFiles.getCount(); ++j) - { - SourceFile* sourceFile = context.getSourceFile(base.asRaw(base.asRaw(srcSourceFiles[i]))); - // Add to translation unit - dstTranslationUnit->addSourceFile(sourceFile); - } - } - } - - // Entry points - { - // Check there aren't any set entry point - SLANG_ASSERT(request->getFrontEndReq()->m_entryPointReqs.getCount() == 0); - - for (const auto& srcEntryPointOffset : requestState->entryPoints) - { - const auto srcEntryPoint = base.asRaw(srcEntryPointOffset); - - const char* name = srcEntryPoint.name ? base.asRaw(srcEntryPoint.name)->getCstr() : nullptr; - - Stage stage = srcEntryPoint.profile.GetStage(); - - List args = context.toList(srcEntryPoint.specializationArgStrings); - - spAddEntryPointEx(externalRequest, int(srcEntryPoint.translationUnitIndex), name, SlangStage(stage), int(args.getCount()), args.getBuffer()); - } - } - - { - RefPtr cacheFileSystem = new CacheFileSystem(nullptr); - auto& dstUniqueMap = cacheFileSystem->getUniqueMap(); - auto& dstPathMap = cacheFileSystem->getPathMap(); - - // Put all the paths to path info - { - for (const auto& pairOffset : requestState->pathInfoMap) - { - const auto& pair = base.asRaw(pairOffset); - CacheFileSystem::PathInfo* pathInfo = context.addPathInfo(base.asRaw(pair.pathInfo)); - dstPathMap.Add(base.asRaw(pair.path)->getSlice(), pathInfo); - } - } - // Put all the path infos in the cache system - { - for (const auto& pair : context.m_fileToPathInfoMap) - { - CacheFileSystem::PathInfo* pathInfo = pair.Value; - SLANG_ASSERT(pathInfo->m_uniqueIdentity); - dstUniqueMap.Add(pathInfo->m_uniqueIdentity->getString(), pathInfo); - } - } - - // This is a bit of a hack, we are going to replace the file system, with our one which is filled in - // with what was read from the file. - - linkage->m_fileSystemExt = cacheFileSystem; - linkage->m_cacheFileSystem = cacheFileSystem; - } - - return SLANG_OK; -} - - -/* static */SlangResult StateSerializeUtil::saveState(EndToEndCompileRequest* request, Stream* stream) -{ - OffsetContainer container; - Offset32Ptr requestState; - SLANG_RETURN_ON_FAIL(store(request, container, requestState)); - - Header header; - header.m_chunk.type = kSlangStateFourCC; - header.m_semanticVersion = g_semanticVersion; - header.m_typeHash = uint32_t(_getTypeHash()); - - return RiffUtil::writeData(&header.m_chunk, sizeof(header),container.getData(), container.getDataCount(), stream); -} - -/* static */SlangResult StateSerializeUtil::saveState(EndToEndCompileRequest* request, const String& filename) -{ - RefPtr stream(new FileStream(filename, FileMode::Create, FileAccess::Write, FileShare::ReadWrite)); - return saveState(request, stream); -} - -/* static */ SlangResult StateSerializeUtil::loadState(const String& filename, List& outBuffer) -{ - RefPtr stream; - try - { - stream = new FileStream(filename, FileMode::Open, FileAccess::Read, FileShare::ReadWrite); - } - catch (const IOException&) - { - return SLANG_FAIL; - } - - return loadState(stream, outBuffer); -} - -/* static */ SlangResult StateSerializeUtil::loadState(Stream* stream, List& buffer) -{ - Header header; - - SLANG_RETURN_ON_FAIL(RiffUtil::readData(stream, &header.m_chunk, sizeof(header), buffer)); - if (header.m_chunk.type != kSlangStateFourCC) - { - return SLANG_FAIL; - } - - if (!RiffSemanticVersion::areCompatible(g_semanticVersion, header.m_semanticVersion)) - { - return SLANG_FAIL; - } - - if (header.m_typeHash != uint32_t(_getTypeHash())) - { - return SLANG_FAIL; - } - - return SLANG_OK; -} - -/* static */SlangResult StateSerializeUtil::loadState(const uint8_t* data, size_t size, List& outBuffer) -{ - MemoryStreamBase stream(FileAccess::Read, data, size); - return loadState(&stream, outBuffer); -} - -/* static */ StateSerializeUtil::RequestState* StateSerializeUtil::getRequest(const List& buffer) -{ - return (StateSerializeUtil::RequestState*)(buffer.getBuffer() + kStartOffset); -} - -/* static */SlangResult StateSerializeUtil::calcDirectoryPathFromFilename(const String& filename, String& outPath) -{ - String absPath; - SLANG_RETURN_ON_FAIL(Path::getCanonical(filename, absPath)); - - String parentDir = Path::getParentDirectory(absPath); - - String baseName = Path::getFileNameWithoutExt(filename); - String ext = Path::getPathExt(filename); - - if (ext.getLength() == 0) - { - StringBuilder builder; - builder << baseName << "-files"; - baseName = builder; - } - - outPath = Path::combine(parentDir, baseName); - return SLANG_OK; -} - -/* static */SlangResult StateSerializeUtil::extractFilesToDirectory(const String& filename) -{ - List buffer; - SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState(filename, buffer)); - - MemoryOffsetBase base; - base.set(buffer.getBuffer(), buffer.getCount()); - - RequestState* requestState = StateSerializeUtil::getRequest(buffer); - - String dirPath; - SLANG_RETURN_ON_FAIL(StateSerializeUtil::calcDirectoryPathFromFilename(filename, dirPath)); - - Path::createDirectory(dirPath); - // Set up a file system to write into this directory - RelativeFileSystem relFileSystem(OSFileSystemExt::getSingleton(), dirPath); - - return extractFiles(base, requestState, &relFileSystem); -} - -static void _calcPreprocessorDefines(OffsetBase& base, const Offset32Array& srcDefines, CommandLine& cmd) -{ - for (const auto& define : srcDefines) - { - StringBuilder builder; - builder << "-D" << base.asRaw(base.asRaw(define).first)->getSlice(); - if (base.asRaw(define).second) - { - builder << "=" << base.asRaw(base.asRaw(define).second)->getSlice(); - } - - cmd.addArg(builder); - } -} - -static SlangResult _calcCommandLine(OffsetBase& base, StateSerializeUtil::RequestState* requestState, CommandLine& cmd) -{ - typedef StateSerializeUtil::TargetRequestState TargetRequestState; - typedef StateSerializeUtil::SourceFileState SourceFileState; - - { - SlangCompileFlags flags = (SlangCompileFlags)requestState->compileFlags; - while (flags) - { - // Extract a bit - const SlangCompileFlags isolatedBit = flags & SlangCompileFlags(-int(flags)); - - switch (isolatedBit) - { - case SLANG_COMPILE_FLAG_NO_MANGLING: cmd.addArg("-no-mangle"); break; - case SLANG_COMPILE_FLAG_NO_CODEGEN: cmd.addArg("-no-codegen"); break; - default: break; - } - - // Remove the bit - flags &= ~isolatedBit; - } - //spSetDumpIntermediates(externalRequest, int(requestState->shouldDumpIntermediates)); - - switch (SlangLineDirectiveMode(requestState->lineDirectiveMode)) - { - case SLANG_LINE_DIRECTIVE_MODE_DEFAULT: break; - case SLANG_LINE_DIRECTIVE_MODE_NONE: - { - cmd.addArg("-line-directive-mode none"); break; - } - default: break; - } - - switch (SlangDebugInfoLevel(requestState->debugInfoLevel)) - { - case SLANG_DEBUG_INFO_LEVEL_STANDARD: cmd.addArg("-g"); break; - case SLANG_DEBUG_INFO_LEVEL_NONE: cmd.addArg("-g0"); break; - case SLANG_DEBUG_INFO_LEVEL_MINIMAL: cmd.addArg("-g1"); break; - case SLANG_DEBUG_INFO_LEVEL_MAXIMAL: cmd.addArg("-g3"); break; - default: break; - } - - switch (SlangOptimizationLevel(requestState->optimizationLevel)) - { - case SLANG_OPTIMIZATION_LEVEL_NONE: cmd.addArg("-O0"); break; - case SLANG_OPTIMIZATION_LEVEL_DEFAULT: cmd.addArg("-O"); break; - case SLANG_OPTIMIZATION_LEVEL_HIGH: cmd.addArg("-O2"); break; - case SLANG_OPTIMIZATION_LEVEL_MAXIMAL: cmd.addArg("-O3"); break; - default: break; - } - - //spSetOutputContainerFormat(externalRequest, SlangContainerFormat(requestState->containerFormat)); - - switch (SlangPassThrough(requestState->passThroughMode)) - { - case SLANG_PASS_THROUGH_NONE: break; - default: - { - cmd.addArg("-pass-through"); - cmd.addArg(TypeTextUtil::getPassThroughName(SlangPassThrough(requestState->passThroughMode))); - break; - } - } - - //request->getBackEndReq()->useUnknownImageFormatAsDefault = requestState->useUnknownImageFormatAsDefault; - //request->getBackEndReq()->obfuscateCode = requestState->obfuscateCode; - //request->getFrontEndReq()->obfuscateCode = requestState->obfuscateCode; - - switch (requestState->defaultMatrixLayoutMode) - { - case SLANG_MATRIX_LAYOUT_ROW_MAJOR: cmd.addArg("-matrix-layout-row-major"); break; - case SLANG_MATRIX_LAYOUT_COLUMN_MAJOR: cmd.addArg("-matrix-layout-column-major"); break; - default: break; - } - } - - // Add the target requests - { - for (Index i = 0; i < requestState->targetRequests.getCount(); ++i) - { - TargetRequestState& src = base.asRaw(requestState->targetRequests[i]); - - cmd.addArg("-target"); - cmd.addArg(TypeTextUtil::getCompileTargetName(SlangCompileTarget(src.target))); - - cmd.addArg("-profile"); - cmd.addArg(Profile(src.profile).getName()); - - if (src.targetFlags & SLANG_TARGET_FLAG_PARAMETER_BLOCKS_USE_REGISTER_SPACES) - { - cmd.addArg("-parameter-blocks-use-register-spaces"); - } - - switch (src.floatingPointMode) - { - case FloatingPointMode::Fast: - { - cmd.addArg("-fp-mode"); - cmd.addArg("fast"); - break; - } - case FloatingPointMode::Precise: - { - cmd.addArg("-fp-mode"); - cmd.addArg("precise"); - break; - } - default: break; - } - -#if 0 - // If there is output state (like output filenames) add here - if (src.outputStates.getCount()) - { - RefPtr dstTargetInfo(new EndToEndCompileRequest::TargetInfo); - request->targetInfos[dstTarget] = dstTargetInfo; - - for (const auto& srcOutputStateOffset : src.outputStates) - { - const auto& srcOutputState = base.asRaw(srcOutputStateOffset); - - SLANG_ASSERT(srcOutputState.entryPointIndex < requestState->entryPoints.getCount()); - - String entryPointPath; - if (srcOutputState.outputPath) - { - entryPointPath = base.asRaw(srcOutputState.outputPath)->getSlice(); - } - - dstTargetInfo->entryPointOutputPaths.Add(srcOutputState.entryPointIndex, entryPointPath); - } - } -#endif - } - } - - { - const auto& srcPaths = requestState->searchPaths; - for (Index i = 0; i < srcPaths.getCount(); ++i) - { - cmd.addArg("-I"); - cmd.addArg(base.asRaw(base.asRaw(srcPaths[i]))->getSlice()); - } - } - - _calcPreprocessorDefines(base, requestState->preprocessorDefinitions, cmd); - - { - const auto& srcTranslationUnits = requestState->translationUnits; - - for (Index i = 0; i < srcTranslationUnits.getCount(); ++i) - { - const auto& srcTranslationUnit = base.asRaw(srcTranslationUnits[i]); - - _calcPreprocessorDefines(base, srcTranslationUnit.preprocessorDefinitions, cmd); - - -#if 0 - if (srcTranslationUnit.moduleName) - { - moduleName = base[srcTranslationUnit].moduleName->getSlice()); - } -#endif - - const auto& srcSourceFiles = srcTranslationUnit.sourceFiles; - - for (Index j = 0; j < srcSourceFiles.getCount(); ++j) - { - SourceFileState* sourceFile = base.asRaw(base.asRaw(srcSourceFiles[i])); - OffsetString* path = base[sourceFile->foundPath]; - - if (path) - { - cmd.addArg(path->getSlice()); - } - } - } - } - - // Entry points - { - for (const auto& srcEntryPointOffset : requestState->entryPoints) - { - const auto srcEntryPoint = base.asRaw(srcEntryPointOffset); - - const char* name = srcEntryPoint.name ? base.asRaw(srcEntryPoint.name)->getCstr() : nullptr; - - cmd.addArg("-entry"); - cmd.addArg(name); - - cmd.addArg("-stage"); - UnownedStringSlice stageText = getStageText(srcEntryPoint.profile.GetStage()); - cmd.addArg(stageText); - - //cmd.addArg("-profile"); - //cmd.addArg(Profile(srcEntryPoint.profile).getName()); - - - //List args = context.toList(srcEntryPoint.specializationArgStrings); - - //spAddEntryPointEx(externalRequest, int(srcEntryPoint.translationUnitIndex), name, SlangStage(stage), int(args.getCount()), args.getBuffer()); - } - } - - return SLANG_OK; -} - -/* static */SlangResult StateSerializeUtil::extractFiles(OffsetBase& base, RequestState* requestState, ISlangFileSystemExt* fileSystem) -{ - StringBuilder builder; - - builder << "[command-line]\n"; - - { - CommandLine cmdLine; - _calcCommandLine(base, requestState, cmdLine); - String text = ProcessUtil::getCommandLineString(cmdLine); - builder << text << "\n"; - } - - builder << "[files]\n"; - - for (auto fileOffset : requestState->files) - { - auto file = base.asRaw(base.asRaw(fileOffset)); - - if (file->contents) - { - UnownedStringSlice contents = base.asRaw(file->contents)->getSlice(); - - SLANG_RETURN_ON_FAIL(fileSystem->saveFile(base.asRaw(file->uniqueName)->getCstr(), contents.begin(), contents.getLength())); - - OffsetString* originalName = nullptr; - if (file->canonicalPath) - { - originalName = base.asRaw(file->canonicalPath); - } - else if (file->foundPath) - { - originalName = base.asRaw(file->foundPath); - } - else if (file->uniqueIdentity) - { - originalName = base.asRaw(file->uniqueIdentity); - } - - builder << base.asRaw(file->uniqueName)->getSlice() << " -> "; - if (originalName) - { - builder << originalName->getSlice(); - } - - if (builder.getLength() == 0) - { - builder << "?"; - } - - builder << "\n"; - } - } - - builder << "[paths]\n"; - for (const auto pathOffset : requestState->pathInfoMap) - { - const auto& path = base.asRaw(pathOffset); - - builder << base.asRaw(path.path)->getSlice() << " -> "; - - const auto pathInfo = base.asRaw(path.pathInfo); - - if (pathInfo->file) - { - builder << base.asRaw(base.asRaw(pathInfo->file)->uniqueName)->getSlice(); - } - else - { - typedef CacheFileSystem::CompressedResult CompressedResult; - if (pathInfo->getPathTypeResult == CompressedResult::Ok) - { - switch (pathInfo->pathType) - { - case SLANG_PATH_TYPE_FILE: builder << "file "; break; - case SLANG_PATH_TYPE_DIRECTORY: builder << "directory "; break; - default: builder << "?"; break; - } - } - - CompressedResult curRes = pathInfo->getCanonicalPathResult; - CompressedResult results[] = - { - pathInfo->getPathTypeResult, - pathInfo->loadFileResult, - }; - - for (auto compRes : results) - { - if (int(compRes) > int(curRes)) - { - curRes = compRes; - } - } - - switch (curRes) - { - default: - case CompressedResult::Uninitialized: break; - case CompressedResult::Ok: break; - - case CompressedResult::NotFound: builder << " [not found]"; break; - case CompressedResult::CannotOpen: builder << "[cannot open]"; break; - case CompressedResult::Fail: builder << "[fail]"; break; - } - } - - builder << "\n"; - } - - SLANG_RETURN_ON_FAIL(fileSystem->saveFile("manifest.txt", builder.getBuffer(), builder.getLength())); - return SLANG_OK; -} - -static SlangResult _findFirstSourcePath(EndToEndCompileRequest* request, String& outFilename) -{ - // We are going to look through all of the srcTranlationUnits, looking for the first filename - - auto frontEndReq = request->getFrontEndReq(); - const auto& srcTranslationUnits = frontEndReq->translationUnits; - - for (Index i = 0; i < srcTranslationUnits.getCount(); ++i) - { - TranslationUnitRequest* srcTranslationUnit = srcTranslationUnits[i]; - const auto& srcSourceFiles = srcTranslationUnit->getSourceFiles(); - - for (Index j = 0; j < srcSourceFiles.getCount(); ++j) - { - SourceFile* sourceFile = srcSourceFiles[j]; - - const PathInfo& pathInfo = sourceFile->getPathInfo(); - - if (pathInfo.foundPath.getLength()) - { - outFilename = pathInfo.foundPath; - return SLANG_OK; - } - } - } - return SLANG_FAIL; -} - -/* static */SlangResult StateSerializeUtil::findUniqueReproDumpStream(EndToEndCompileRequest* request, String& outFileName, RefPtr& outStream) -{ - String sourcePath; - - if (SLANG_FAILED(_findFirstSourcePath(request, sourcePath))) - { - sourcePath = "unknown.slang"; - } - - String sourceFileName = Path::getFileName(sourcePath); - String sourceBaseName = Path::getFileNameWithoutExt(sourceFileName); - - // Okay we need a unique number to make sure the name is unique - const int maxTries = 100; - for (int triesCount = 0; triesCount < maxTries; ++triesCount) - { - // We could include the count in some way perhaps, but for now let's just go with ticks - auto tick = ProcessUtil::getClockTick(); - - StringBuilder builder; - builder << sourceBaseName << "-" << tick << ".slang-repro"; - - // We write out the file name tried even if it fails, as might be useful in reporting - outFileName = builder; - - // We could have clashes, as we use ticks, we should get to a point where the clashes stop - try - { - outStream = new FileStream(builder, FileMode::CreateNew, FileAccess::Write, FileShare::WriteOnly); - return SLANG_OK; - } - catch (const IOException&) - { - } - - // TODO(JS): - // Might make sense to sleep here - but don't seem to have cross platform func for that yet. - } - - return SLANG_FAIL; -} - -} // namespace Slang diff --git a/source/slang/slang-state-serialize.h b/source/slang/slang-state-serialize.h deleted file mode 100644 index 9a404138e..000000000 --- a/source/slang/slang-state-serialize.h +++ /dev/null @@ -1,194 +0,0 @@ -// slang-state-serialize.h -#ifndef SLANG_STATE_SERIALIZE_H_INCLUDED -#define SLANG_STATE_SERIALIZE_H_INCLUDED - -#include "../core/slang-riff.h" -#include "../core/slang-string.h" - -// For TranslationUnitRequest -#include "slang-compiler.h" - -#include "../core/slang-offset-container.h" - -#include "slang-file-system.h" - -namespace Slang { - -struct StateSerializeUtil -{ - enum - { - kMajorVersion = 0, - kMinorVersion = 0, - kPatchVersion = 0, - }; - - static const uint32_t kSlangStateFourCC = SLANG_FOUR_CC('S', 'L', 'S', 'T'); ///< Holds all the slang specific chunks - static const RiffSemanticVersion g_semanticVersion; - - struct Header - { - RiffHeader m_chunk; ///< The chunk - RiffSemanticVersion m_semanticVersion; ///< The semantic version - uint32_t m_typeHash; ///< A hash based on the binary representation. If doesn't match then not binary compatible (extra check over semantic versioning) - }; - - struct FileState - { - Offset32Ptr uniqueIdentity; ///< The unique identity for the file (from ISlangFileSystem), or nullptr - Offset32Ptr contents; ///< The contents of this file - Offset32Ptr canonicalPath; ///< The canonical name of this file (or nullptr) - Offset32Ptr foundPath; ///< The 'found' path - - Offset32Ptr uniqueName; ///< A generated unique name (not used by slang, but used as mechanism to replace files) - }; - - struct PathInfoState - { - typedef CacheFileSystem::CompressedResult CompressedResult; - - SlangPathType pathType = SLANG_PATH_TYPE_FILE; - CompressedResult loadFileResult = CompressedResult::Uninitialized; - CompressedResult getPathTypeResult = CompressedResult::Uninitialized; - CompressedResult getCanonicalPathResult = CompressedResult::Uninitialized; - - Offset32Ptr file; ///< File contents - }; - - struct PathAndPathInfo - { - Offset32Ptr path; - Offset32Ptr pathInfo; - }; - - struct OutputState - { - int32_t entryPointIndex; - Offset32Ptr outputPath; - }; - - // spSetCodeGenTarget/spAddCodeGenTarget - // spSetTargetProfile - // spSetTargetFlags - // spSetTargetFloatingPointMode - // spSetTargetMatrixLayoutMode - struct TargetRequestState - { - Profile profile; - CodeGenTarget target; - SlangTargetFlags targetFlags; - FloatingPointMode floatingPointMode; - - Offset32Array outputStates; - }; - - struct StringPair - { - Offset32Ptr first; - Offset32Ptr second; - }; - - struct SourceFileState - { - PathInfo::Type type; ///< The type of this file - Offset32Ptr foundPath; ///< The Path this was found along - Offset32Ptr file; ///< The file contents - }; - - // spAddTranslationUnit - struct TranslationUnitRequestState - { - SourceLanguage language; - - Offset32Ptr moduleName; - - // spTranslationUnit_addPreprocessorDefine - Offset32Array preprocessorDefinitions; - - Offset32Array > sourceFiles; - }; - - struct EntryPointState - { - Offset32Ptr name; - Profile profile; - uint32_t translationUnitIndex; - Offset32Array> specializationArgStrings; - }; - - struct RequestState - { - Offset32Array> files; ///< All of the files - Offset32Array> sourceFiles; ///< All of the source files (from source manager) - - // spSetCompileFlags - SlangCompileFlags compileFlags; - // spSetDumpIntermediates - bool shouldDumpIntermediates; - // spSetLineDirectiveMode - LineDirectiveMode lineDirectiveMode; - - Offset32Array targetRequests; - - // spSetDebugInfoLevel - DebugInfoLevel debugInfoLevel; - // spSetOptimizationLevel - OptimizationLevel optimizationLevel; - // spSetOutputContainerFormat - ContainerFormat containerFormat; - // spSetPassThrough - PassThroughMode passThroughMode; - - // spAddSearchPath - Offset32Array > searchPaths; - - // spAddPreprocessorDefine - Offset32Array preprocessorDefinitions; - - bool useUnknownImageFormatAsDefault = false; - bool obfuscateCode = false; - - Offset32Array pathInfoMap; ///< Stores all the accesses to the file system - - Offset32Array translationUnits; - - Offset32Array entryPoints; - - SlangMatrixLayoutMode defaultMatrixLayoutMode; - }; - - static SlangResult store(EndToEndCompileRequest* request, OffsetContainer& inOutContainer, Offset32Ptr& outRequest); - - static SlangResult saveState(EndToEndCompileRequest* request, const String& filename); - - static SlangResult saveState(EndToEndCompileRequest* request, Stream* stream); - - /// Create a cache file system that uses contents of the request state. - /// The passed in fileSystem is used for accessing any file accesses not found in the cache - static SlangResult loadFileSystem(OffsetBase& base, RequestState* requestState, ISlangFileSystem* fileSystem, RefPtr& outFileSystem); - - /// Load the requestState into request - /// The fileSystem is optional and can be passed as nullptr. If set, as each file is loaded - /// it will attempt to load from fileSystem the *uniqueName* - static SlangResult load(OffsetBase& base, RequestState* requestState, ISlangFileSystem* fileSystem, EndToEndCompileRequest* request); - - static SlangResult loadState(const String& filename, List& outBuffer); - static SlangResult loadState(Stream* stream, List& outBuffer); - static SlangResult loadState(const uint8_t* data, size_t size, List& outBuffer); - - static RequestState* getRequest(const List& inBuffer); - - static SlangResult extractFilesToDirectory(const String& file); - - static SlangResult extractFiles(OffsetBase& base, RequestState* requestState, ISlangFileSystemExt* fileSystem); - - /// Given the repo file work out a suitable path - static SlangResult calcDirectoryPathFromFilename(const String& filename, String& outPath); - - /// Given a request trys to determine a suitable dump file name, that is unique. - static SlangResult findUniqueReproDumpStream(EndToEndCompileRequest* request, String& outFileName, RefPtr& outStream); -}; - -} // namespace Slang - -#endif diff --git a/source/slang/slang-type-layout.cpp b/source/slang/slang-type-layout.cpp index 3b30e6ee0..d6138f2af 100644 --- a/source/slang/slang-type-layout.cpp +++ b/source/slang/slang-type-layout.cpp @@ -1487,7 +1487,7 @@ static bool isSM5OrEarlier(TargetRequest* targetReq) if(profile.getFamily() == ProfileFamily::DX) { - if(profile.GetVersion() <= ProfileVersion::DX_5_0) + if(profile.getVersion() <= ProfileVersion::DX_5_0) return true; } @@ -1503,7 +1503,7 @@ static bool isSM5_1OrLater(TargetRequest* targetReq) if(profile.getFamily() == ProfileFamily::DX) { - if(profile.GetVersion() >= ProfileVersion::DX_5_1) + if(profile.getVersion() >= ProfileVersion::DX_5_1) return true; } diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 1710359bf..02c9e25e3 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -16,7 +16,7 @@ #include "slang-ast-dump.h" -#include "slang-state-serialize.h" +#include "slang-repro.h" #include "slang-file-system.h" @@ -231,7 +231,7 @@ SLANG_NO_THROW SlangResult SLANG_MCALL Session::createSession( SLANG_NO_THROW SlangProfileID SLANG_MCALL Session::findProfile( char const* name) { - return Slang::Profile::LookUp(name).raw; + return Slang::Profile::lookUp(name).raw; } SLANG_NO_THROW void SLANG_MCALL Session::setDownstreamCompilerPath( @@ -426,8 +426,8 @@ Profile getEffectiveProfile(EntryPoint* entryPoint, TargetRequest* target) break; } - auto entryPointProfileVersion = entryPointProfile.GetVersion(); - auto targetProfileVersion = targetProfile.GetVersion(); + auto entryPointProfileVersion = entryPointProfile.getVersion(); + auto targetProfileVersion = targetProfile.getVersion(); // Default to the entry point profile, since we know that has the right stage. Profile effectiveProfile = entryPointProfile; @@ -450,7 +450,7 @@ Profile getEffectiveProfile(EntryPoint* entryPoint, TargetRequest* target) switch( effectiveProfile.getFamily() ) { case ProfileFamily::DX: - switch(effectiveProfile.GetStage()) + switch(effectiveProfile.getStage()) { default: break; @@ -472,7 +472,7 @@ Profile getEffectiveProfile(EntryPoint* entryPoint, TargetRequest* target) break; case ProfileFamily::GLSL: - switch(effectiveProfile.GetStage()) + switch(effectiveProfile.getStage()) { default: break; @@ -494,7 +494,7 @@ Profile getEffectiveProfile(EntryPoint* entryPoint, TargetRequest* target) break; } - if( stageMinVersion > effectiveProfile.GetVersion() ) + if( stageMinVersion > effectiveProfile.getVersion() ) { effectiveProfile.setVersion(stageMinVersion); } @@ -3166,7 +3166,7 @@ SLANG_API SlangProfileID spFindProfile( SlangSession*, char const* name) { - return Slang::Profile::LookUp(name).raw; + return Slang::Profile::lookUp(name).raw; } SLANG_API int spAddEntryPoint( @@ -3327,7 +3327,7 @@ SLANG_API SlangResult spCompile( { if (req->dumpRepro.getLength()) { - SlangResult saveRes = StateSerializeUtil::saveState(req, req->dumpRepro); + SlangResult saveRes = ReproUtil::saveState(req, req->dumpRepro); if (SLANG_FAILED(saveRes)) { req->getSink()->diagnose(SourceLoc(), Diagnostics::unableToWriteReproFile, req->dumpRepro); @@ -3340,9 +3340,9 @@ SLANG_API SlangResult spCompile( SlangResult saveRes = SLANG_FAIL; RefPtr stream; - if (SLANG_SUCCEEDED(StateSerializeUtil::findUniqueReproDumpStream(req, reproFileName, stream))) + if (SLANG_SUCCEEDED(ReproUtil::findUniqueReproDumpStream(req, reproFileName, stream))) { - saveRes = StateSerializeUtil::saveState(req, stream); + saveRes = ReproUtil::saveState(req, stream); } if (SLANG_FAILED(saveRes)) @@ -3566,14 +3566,14 @@ SLANG_API SlangResult spLoadRepro( auto request = asInternal(inRequest); List buffer; - SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState((const uint8_t*)data, size, buffer)); + SLANG_RETURN_ON_FAIL(ReproUtil::loadState((const uint8_t*)data, size, buffer)); MemoryOffsetBase base; base.set(buffer.getBuffer(), buffer.getCount()); - StateSerializeUtil::RequestState* requestState = StateSerializeUtil::getRequest(buffer); + ReproUtil::RequestState* requestState = ReproUtil::getRequest(buffer); - SLANG_RETURN_ON_FAIL(StateSerializeUtil::load(base, requestState, fileSystem, request)); + SLANG_RETURN_ON_FAIL(ReproUtil::load(base, requestState, fileSystem, request)); return SLANG_OK; } @@ -3586,7 +3586,7 @@ SLANG_API SlangResult spSaveRepro( OwnedMemoryStream stream(FileAccess::Write); - SLANG_RETURN_ON_FAIL(StateSerializeUtil::saveState(request, &stream)); + SLANG_RETURN_ON_FAIL(ReproUtil::saveState(request, &stream)); RefPtr listBlob(new ListBlob); @@ -3615,14 +3615,14 @@ SLANG_API SlangResult spExtractRepro(SlangSession* session, const void* reproDat List buffer; { MemoryStreamBase memoryStream(FileAccess::Read, reproData, reproDataSize); - SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState(&memoryStream, buffer)); + SLANG_RETURN_ON_FAIL(ReproUtil::loadState(&memoryStream, buffer)); } MemoryOffsetBase base; base.set(buffer.getBuffer(), buffer.getCount()); - StateSerializeUtil::RequestState* requestState = StateSerializeUtil::getRequest(buffer); - return StateSerializeUtil::extractFiles(base, requestState, fileSystem); + ReproUtil::RequestState* requestState = ReproUtil::getRequest(buffer); + return ReproUtil::extractFiles(base, requestState, fileSystem); } SLANG_API SlangResult spLoadReproAsFileSystem( @@ -3639,14 +3639,14 @@ SLANG_API SlangResult spLoadReproAsFileSystem( MemoryStreamBase stream(FileAccess::Read, reproData, reproDataSize); List buffer; - SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadState(&stream, buffer)); + SLANG_RETURN_ON_FAIL(ReproUtil::loadState(&stream, buffer)); - auto requestState = StateSerializeUtil::getRequest(buffer); + auto requestState = ReproUtil::getRequest(buffer); MemoryOffsetBase base; base.set(buffer.getBuffer(), buffer.getCount()); RefPtr cacheFileSystem; - SLANG_RETURN_ON_FAIL(StateSerializeUtil::loadFileSystem(base, requestState, replaceFileSystem, cacheFileSystem)); + SLANG_RETURN_ON_FAIL(ReproUtil::loadFileSystem(base, requestState, replaceFileSystem, cacheFileSystem)); *outFileSystem = cacheFileSystem.detach(); return SLANG_OK; diff --git a/source/slang/slang.vcxproj b/source/slang/slang.vcxproj index f6b321b7d..3be09765d 100644 --- a/source/slang/slang.vcxproj +++ b/source/slang/slang.vcxproj @@ -265,8 +265,8 @@ + - @@ -352,8 +352,8 @@ + - diff --git a/source/slang/slang.vcxproj.filters b/source/slang/slang.vcxproj.filters index a636e338a..64f912110 100644 --- a/source/slang/slang.vcxproj.filters +++ b/source/slang/slang.vcxproj.filters @@ -246,10 +246,10 @@ Header Files - + Header Files - + Header Files @@ -503,10 +503,10 @@ Source Files - + Source Files - + Source Files -- cgit v1.2.3