diff options
| author | jsmall-nvidia <jsmall@nvidia.com> | 2019-01-21 16:41:54 -0500 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-01-21 16:41:54 -0500 |
| commit | bd815f02d846a50e16dab67e6512db2a6215c41f (patch) | |
| tree | 01e391757bdb8f2d15bdc010227d522bddac3936 | |
| parent | 0a3ef7b4ae02983ea3f986ba8211e7c6af9d254b (diff) | |
Feature/file unique identity (#789)
* * Fix memory bug around expanding va_args - needed buffer to have space for terminating 0
* Fix problem with FileWriter defaults being globals, as memory they allocate, will only be freed after return from main - work around by making StdWriters RefObject derived, and kept in scope such the writers are destroyed before checks for leaks is found
* Added SimplifyPathAndHash mode for CacheFileSystem - will simplify the path and see if simplified path is in cache before reading file (limiting amout of underlying file requests)
* * Added calcReplaceChar
* Renamed DefaultFileSystem to OSFileSystem
* Made OSFileSystem convert windows \ to / on linux
* Simplified logic for caching in CacheFileSystem.
* Added pragma-once-c to add extra test, but also so there is an 'include' directory in preprocessor tests.
* Small fixes in pragma once test.
* Simplified cache handling path, so that paths/simplified paths area always added.
* Improve naming of methods for different caches.
* Removed references to 'canonicalPath' and made 'uniqueIdentity'
* * Re-add support for canonicalPath to ISlangFileSystem -> not for uniqueIdentifier but as a way to display 'canonicalPath'
* Added peliminary support for being able to display verbose paths in a diagnostic
* Added 'clearCache' support
* Added verbose path support to SourceManager (now needs a ISlangFileSystemExt to do this)
* Added support for '-verbose-path' option to slangc and slang-test.
| -rw-r--r-- | docs/command-line-slangc.md | 1 | ||||
| -rw-r--r-- | slang.h | 42 | ||||
| -rw-r--r-- | source/slang/compiler.cpp | 21 | ||||
| -rw-r--r-- | source/slang/compiler.h | 4 | ||||
| -rw-r--r-- | source/slang/diagnostic-defs.h | 2 | ||||
| -rw-r--r-- | source/slang/diagnostics.cpp | 60 | ||||
| -rw-r--r-- | source/slang/diagnostics.h | 15 | ||||
| -rw-r--r-- | source/slang/ir-serialize.cpp | 2 | ||||
| -rw-r--r-- | source/slang/options.cpp | 4 | ||||
| -rw-r--r-- | source/slang/preprocessor.cpp | 37 | ||||
| -rw-r--r-- | source/slang/slang-file-system.cpp | 185 | ||||
| -rw-r--r-- | source/slang/slang-file-system.h | 79 | ||||
| -rw-r--r-- | source/slang/slang.cpp | 89 | ||||
| -rw-r--r-- | source/slang/source-loc.cpp | 59 | ||||
| -rw-r--r-- | source/slang/source-loc.h | 55 | ||||
| -rw-r--r-- | tools/slang-test/options.cpp | 4 | ||||
| -rw-r--r-- | tools/slang-test/options.h | 3 | ||||
| -rw-r--r-- | tools/slang-test/slang-test-main.cpp | 28 |
18 files changed, 445 insertions, 245 deletions
diff --git a/docs/command-line-slangc.md b/docs/command-line-slangc.md index 8d59526d4..5e706f695 100644 --- a/docs/command-line-slangc.md +++ b/docs/command-line-slangc.md @@ -133,6 +133,7 @@ For completeness, here are the options that `slangc` currently accepts: * 'dxc': Use DirectXShaderCompiler (https://github.com/Microsoft/DirectXShaderCompiler) * These are intended for debugging/testing purposes, when you want to be able to see what these existing compilers do with the "same" input and options +* `-verbose-paths`: When displaying diagnostic output aim to display more detailed path information. In practice this is typically the complete 'canonical' path to the source file used. * `--`: Stop parsing options, and treat the rest of the command line as input paths @@ -778,25 +778,25 @@ extern "C" struct ISlangFileSystemExt : public ISlangFileSystem { public: - /** Get a canonical path which uniquely identifies an object of the file system. + /** Get a uniqueIdentity which uniquely identifies an object of the file system. - Given a path, returns a 'canonical' path which will return the same path for the same file/directory. - The canonical path is used to compare if two includes are the same file. - The string for the canonical path is held zero terminated in the ISlangBlob of canonicalPathOut. + Given a path, returns a 'uniqueIdentity' path will return the same value for the same file/directory on the file system. + The uniqueIdentity is used to compare if two includes are the same file. + The string for the uniqueIdentity is held zero terminated in the ISlangBlob of outUniqueIdentity. - Note that a canonical path doesn't *have* to be a 'canonical' path, or a path at all - - it can just be a string that uniquely identifies a file. For example another possible mechanism + Note that there are many ways a uniqueIdentity may be generated for a file. For example it could be the + 'canonical path' - assuming it is available and unambitious for a file system. Another possible mechanism could be to store the filename combined with the file date time to uniquely identify it. The client must ensure the blob be released when no longer used, otherwise memory will leak. @param path - @param canonicalPathOut - @returns A `SlangResult` to indicate success or failure getting the canonical path. + @param outUniqueIdentity + @returns A `SlangResult` to indicate success or failure getting the uniqueIdentity. */ - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanoncialPath( + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getFileUniqueIdentity( const char* path, - ISlangBlob** canonicalPathOut) = 0; + ISlangBlob** outUniqueIdentity) = 0; /** Calculate a path combining the 'fromPath' with 'path' @@ -821,7 +821,27 @@ extern "C" */ virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType( const char* path, - SlangPathType* pathTypeOut) = 0; + SlangPathType* pathTypeOut) = 0; + + /** Get a canonical path identifies an object of the file system. + + Given a path, returns a 'canonicalPath' the file. This may be a file system 'canonical path' to + show where a file was read from. Also though if the filesystem is say a zip file - it might include the path to the zip + container as well as the absolute path to the specific file. The main purpose of the method is to be able + to display to users unambiguously where a file was read from in diagnostics. + + This method is optional and if not implemented will display 'found paths'. If not implemented return SLANG_E_NOT_IMPLEMENTED. + + @param path + @param outCanonicalPath + @returns A `SlangResult` + */ + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanonicalPath( + const char* path, + ISlangBlob** outCanonicalPath) = 0; + + /** Clears any cached information */ + virtual SLANG_NO_THROW void SLANG_MCALL clearCache() = 0; }; #define SLANG_UUID_ISlangFileSystemExt { 0x5fb632d2, 0x979d, 0x4481, { 0x9f, 0xee, 0x66, 0x3c, 0x3f, 0x14, 0x49, 0xe1 } } diff --git a/source/slang/compiler.cpp b/source/slang/compiler.cpp index a003cf33d..29f7a95d9 100644 --- a/source/slang/compiler.cpp +++ b/source/slang/compiler.cpp @@ -473,6 +473,18 @@ namespace Slang sink->diagnoseRaw(SLANG_FAILED(res) ? Severity::Error : Severity::Warning, builder.getUnownedSlice()); } + static String _getDisplayPath(const DiagnosticSink& sink, SourceFile* sourceFile) + { + if (sink.flags & DiagnosticSink::Flag::VerbosePath) + { + return sourceFile->calcVerbosePath(); + } + else + { + return sourceFile->getPathInfo().foundPath; + } + } + String calcTranslationUnitSourcePath(TranslationUnitRequest* translationUnitRequest) { CompileRequest* compileRequest = translationUnitRequest->compileRequest; @@ -481,6 +493,8 @@ namespace Slang return "slang-generated"; } + auto& sink = translationUnitRequest->compileRequest->mSink; + const auto& sourceFiles = translationUnitRequest->sourceFiles; const int numSourceFiles = int(sourceFiles.Count()); @@ -488,16 +502,15 @@ namespace Slang switch (numSourceFiles) { case 0: return "unknown"; - case 1: return sourceFiles[0]->getPathInfo().foundPath; + case 1: return _getDisplayPath(sink, sourceFiles[0]); default: { StringBuilder builder; - builder << sourceFiles[0]->getPathInfo().foundPath; + builder << _getDisplayPath(sink, sourceFiles[0]); for (int i = 1; i < numSourceFiles; ++i) { - builder << ";" << sourceFiles[i]->getPathInfo().foundPath; + builder << ";" << _getDisplayPath(sink, sourceFiles[i]); } - return builder; } } diff --git a/source/slang/compiler.h b/source/slang/compiler.h index 41ba027c6..ca82950be 100644 --- a/source/slang/compiler.h +++ b/source/slang/compiler.h @@ -399,7 +399,7 @@ namespace Slang // This is a list of unique modules loaded, in the order they were encountered. List<RefPtr<LoadedModule> > loadedModulesList; - // Map from the path of a module file to its definition + // Map from the path (or uniqueIdentity if available) of a module file to its definition Dictionary<String, RefPtr<LoadedModule>> mapPathToLoadedModule; // Map from the logical name of a module to its definition @@ -509,6 +509,8 @@ namespace Slang mSink.sourceManager = sm; } + void setFileSystem(ISlangFileSystem* fileSystem); + /// During propagation of an exception for an internal /// error, note that this source location was involved void noteInternalErrorLoc(SourceLoc const& loc); diff --git a/source/slang/diagnostic-defs.h b/source/slang/diagnostic-defs.h index a7c89321d..ceb4706fd 100644 --- a/source/slang/diagnostic-defs.h +++ b/source/slang/diagnostic-defs.h @@ -149,7 +149,7 @@ DIAGNOSTIC(-1, Note, seeOpeningToken, "see opening '$0'") DIAGNOSTIC(15300, Error, includeFailed, "failed to find include file '$0'") DIAGNOSTIC(15301, Error, importFailed, "failed to find imported file '$0'") DIAGNOSTIC(-1, Error, noIncludeHandlerSpecified, "no `#include` handler was specified") -DIAGNOSTIC(15302, Error, noCanonicalPath, "`#include` handler didn't generate a canonical path for '$0'") +DIAGNOSTIC(15302, Error, noUniqueIdentity, "`#include` handler didn't generate a unique identity for file '$0'") // 154xx - macro definition diff --git a/source/slang/diagnostics.cpp b/source/slang/diagnostics.cpp index 9a88c041d..00205fed3 100644 --- a/source/slang/diagnostics.cpp +++ b/source/slang/diagnostics.cpp @@ -199,30 +199,54 @@ static void formatDiagnosticMessage(StringBuilder& sb, char const* format, int a } } +static void formatDiagnostic(const HumaneSourceLoc& humaneLoc, Diagnostic const& diagnostic, StringBuilder& outBuilder) +{ + outBuilder << humaneLoc.pathInfo.foundPath; + outBuilder << "("; + outBuilder << Int32(humaneLoc.line); + outBuilder << "): "; + + outBuilder << getSeverityName(diagnostic.severity); + + if (diagnostic.ErrorID >= 0) + { + outBuilder << " "; + outBuilder << diagnostic.ErrorID; + } + + outBuilder << ": "; + outBuilder << diagnostic.Message; + outBuilder << "\n"; +} + static void formatDiagnostic( DiagnosticSink* sink, - StringBuilder& sb, - Diagnostic const& diagnostic) + Diagnostic const& diagnostic, + StringBuilder& sb) { auto sourceManager = sink->sourceManager; - auto humaneLoc = sourceManager->getHumaneLoc(diagnostic.loc); - - sb << humaneLoc.pathInfo.foundPath; - sb << "("; - sb << Int32(humaneLoc.line); - sb << "): "; - sb << getSeverityName(diagnostic.severity); - - if( diagnostic.ErrorID >= 0 ) + SourceView* sourceView = nullptr; + HumaneSourceLoc humaneLoc; + const auto sourceLoc = diagnostic.loc; { - sb << " "; - sb << diagnostic.ErrorID; + sourceView = sourceManager->findSourceViewRecursively(sourceLoc); + if (sourceView) + { + humaneLoc = sourceView->getHumaneLoc(sourceLoc); + } + formatDiagnostic(humaneLoc, diagnostic, sb); } + + if (sourceView && (sink->flags & DiagnosticSink::Flag::VerbosePath)) + { + auto actualLoc = sourceView->getHumaneLoc(diagnostic.loc, SourceLocType::Actual); + // Look up the full path + SourceFile* sourceFile = sourceView->getSourceFile(); + actualLoc.pathInfo.foundPath = sourceFile->calcVerbosePath(); - sb << ": "; - sb << diagnostic.Message; - sb << "\n"; + formatDiagnostic(actualLoc, diagnostic, sb); + } } void DiagnosticSink::diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args) @@ -246,7 +270,7 @@ void DiagnosticSink::diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& in { // If so, pass the error string along to them StringBuilder messageBuilder; - formatDiagnostic(this, messageBuilder, diagnostic); + formatDiagnostic(this, diagnostic, messageBuilder); writer->write(messageBuilder.Buffer(), messageBuilder.Length()); } @@ -254,7 +278,7 @@ void DiagnosticSink::diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& in { // If the user doesn't have a callback, then just // collect our diagnostic messages into a buffer - formatDiagnostic(this, outputBuffer, diagnostic); + formatDiagnostic(this, diagnostic, outputBuffer); } if (diagnostic.severity >= Severity::Fatal) diff --git a/source/slang/diagnostics.h b/source/slang/diagnostics.h index 8e5bbcb64..fed492bc7 100644 --- a/source/slang/diagnostics.h +++ b/source/slang/diagnostics.h @@ -141,12 +141,23 @@ namespace Slang // The source manager to use when mapping source locations to file+line info SourceManager* sourceManager; + struct Flag + { + enum Enum: uint32_t + { + VerbosePath = 0x1, ///< Will try and display a + }; + }; + typedef uint32_t Flags; + StringBuilder outputBuffer; // List<Diagnostic> diagnostics; int errorCount = 0; - ISlangWriter* writer = nullptr; + ISlangWriter* writer = nullptr; + Flags flags = 0; + /* void Error(int id, const String & msg, const SourceLoc & pos) { @@ -163,7 +174,7 @@ namespace Slang void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info) { - diagnoseImpl(pos, info, 0, NULL); + diagnoseImpl(pos, info, 0, nullptr); } void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0) diff --git a/source/slang/ir-serialize.cpp b/source/slang/ir-serialize.cpp index 61b21f1ef..4ff7cd733 100644 --- a/source/slang/ir-serialize.cpp +++ b/source/slang/ir-serialize.cpp @@ -2005,7 +2005,7 @@ static int _calcFixSourceLoc(const IRSerialData::DebugSourceInfo& info, SourceVi RefPtr<IRModule> irReadModule; SourceManager workSourceManager; - workSourceManager.initialize(sourceManager); + workSourceManager.initialize(sourceManager, nullptr); { IRSerialReader reader; diff --git a/source/slang/options.cpp b/source/slang/options.cpp index b87e19e46..8a4cf35b7 100644 --- a/source/slang/options.cpp +++ b/source/slang/options.cpp @@ -456,6 +456,10 @@ struct OptionsParser { requestImpl->useSerialIRBottleneck = true; } + else if (argStr == "-verbose-paths") + { + requestImpl->mSink.flags |= DiagnosticSink::Flag::VerbosePath; + } else if (argStr == "-verify-debug-serial-ir") { requestImpl->verifyDebugSerialization = true; diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp index c038737b7..4ccb5d50e 100644 --- a/source/slang/preprocessor.cpp +++ b/source/slang/preprocessor.cpp @@ -197,10 +197,9 @@ struct Preprocessor // The translation unit that is being parsed TranslationUnitRequest* translationUnit; - // Any paths that have issued `#pragma once` directives to + // The unique identities of any paths that have issued `#pragma once` directives to // stop them from being included again. - HashSet<String> pragmaOncePaths; - + HashSet<String> pragmaOnceUniqueIdentities; TranslationUnitRequest* getTranslationUnit() { @@ -1610,10 +1609,10 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context) return; } - // We must have a canonical path to be compare - if (!filePathInfo.hasCanonicalPath()) + // We must have a uniqueIdentity to be compare + if (!filePathInfo.hasUniqueIdentity()) { - GetSink(context)->diagnose(pathToken.loc, Diagnostics::noCanonicalPath, path); + GetSink(context)->diagnose(pathToken.loc, Diagnostics::noUniqueIdentity, path); return; } @@ -1623,7 +1622,7 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context) expectEndOfDirective(context); // Check whether we've previously included this file and seen a `#pragma once` directive - if(context->preprocessor->pragmaOncePaths.Contains(filePathInfo.canonicalPath)) + if(context->preprocessor->pragmaOnceUniqueIdentities.Contains(filePathInfo.uniqueIdentity)) { return; } @@ -1633,7 +1632,7 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context) auto sourceManager = context->preprocessor->getCompileRequest()->getSourceManager(); // See if this an already loaded source file - SourceFile* sourceFile = sourceManager->findSourceFileRecursively(filePathInfo.canonicalPath); + SourceFile* sourceFile = sourceManager->findSourceFileRecursively(filePathInfo.uniqueIdentity); // If not create a new one, and add to the list of known source files if (!sourceFile) { @@ -1645,7 +1644,7 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context) } sourceFile = sourceManager->createSourceFileWithBlob(filePathInfo, foundSourceBlob); - sourceManager->addSourceFile(filePathInfo.canonicalPath, sourceFile); + sourceManager->addSourceFile(filePathInfo.uniqueIdentity, sourceFile); } // This is a new parse (even if it's a pre-existing source file), so create a new SourceUnit @@ -1873,29 +1872,19 @@ SLANG_PRAGMA_DIRECTIVE_CALLBACK(handlePragmaOnceDirective) // We need to identify the path of the file we are preprocessing, // so that we can avoid including it again. // - // Note: for now we are doing a very simplistic check where - // we use the raw file path as the key for our duplicate checking. - // - // TODO: a more refined implementation should probably apply Unicode - // normalization and case-folding to the path, and then use that - // plus a hash of the file contents to determine whether things - // represent the "same" file. - // - // TODO: even for our simplistic implementation, we need to add - // logic to deal with `../` segments in path names to detect - // trivial cases of the "same" path. - // + // We are using the 'uniqueIdentity' as determined by the ISlangFileSystemEx interface to determine file identities. + auto directiveLoc = GetDirectiveLoc(context); auto issuedFromPathInfo = context->preprocessor->translationUnit->compileRequest->getSourceManager()->getPathInfo(directiveLoc, SourceLocType::Actual); - // Must have a canonical path for a #pragma once to work - if (!issuedFromPathInfo.hasCanonicalPath()) + // Must have uniqueIdentity for a #pragma once to work + if (!issuedFromPathInfo.hasUniqueIdentity()) { GetSink(context)->diagnose(subDirectiveToken, Diagnostics::pragmaOnceIgnored); return; } - context->preprocessor->pragmaOncePaths.Add(issuedFromPathInfo.canonicalPath); + context->preprocessor->pragmaOnceUniqueIdentities.Add(issuedFromPathInfo.uniqueIdentity); } // Information about a specific `#pragma` directive diff --git a/source/slang/slang-file-system.cpp b/source/slang/slang-file-system.cpp index 96dcc7558..b3ba692bd 100644 --- a/source/slang/slang-file-system.cpp +++ b/source/slang/slang-file-system.cpp @@ -62,12 +62,17 @@ static String _fixPathDelimiters(const char* pathIn) #endif } -SlangResult OSFileSystem::getCanoncialPath(const char* pathIn, ISlangBlob** canonicalPathOut) +SlangResult OSFileSystem::getFileUniqueIdentity(const char* pathIn, ISlangBlob** outUniqueIdentity) +{ + // By default we use the canonical path to uniquely identify a file + return getCanonicalPath(pathIn, outUniqueIdentity); +} + +SlangResult OSFileSystem::getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) { String canonicalPath; - SLANG_RETURN_ON_FAIL(Path::GetCanonical(_fixPathDelimiters(pathIn), canonicalPath)); - - *canonicalPathOut = StringUtil::createStringBlob(canonicalPath).detach(); + SLANG_RETURN_ON_FAIL(Path::GetCanonical(_fixPathDelimiters(path), canonicalPath)); + *outCanonicalPath = StringUtil::createStringBlob(canonicalPath).detach(); return SLANG_OK; } @@ -138,44 +143,61 @@ ISlangUnknown* CacheFileSystem::getInterface(const Guid& guid) return _getInterface(this, guid); } -CacheFileSystem::CacheFileSystem(ISlangFileSystem* fileSystem, CanonicalMode canonicalMode) : +CacheFileSystem::CacheFileSystem(ISlangFileSystem* fileSystem, UniqueIdentityMode uniqueIdentityMode) : m_fileSystem(fileSystem), - m_canonicalMode(canonicalMode) + m_uniqueIdentityMode(uniqueIdentityMode) { // Try to get the more sophisticated interface fileSystem->queryInterface(IID_ISlangFileSystemExt, (void**)m_fileSystemExt.writeRef()); - switch (canonicalMode) + switch (uniqueIdentityMode) { - case CanonicalMode::Default: - case CanonicalMode::FileSystemExt: + case UniqueIdentityMode::Default: + case UniqueIdentityMode::FileSystemExt: { - m_canonicalMode = m_fileSystemExt ? CanonicalMode::FileSystemExt : CanonicalMode::SimplifyPathAndHash; + // If it's not a complete file system, we will default to SimplifyAndHash style by default + m_uniqueIdentityMode = m_fileSystemExt ? UniqueIdentityMode::FileSystemExt : UniqueIdentityMode::SimplifyPathAndHash; break; } default: break; } // It can't be default - SLANG_ASSERT(m_canonicalMode != CanonicalMode::Default); + SLANG_ASSERT(m_uniqueIdentityMode != UniqueIdentityMode::Default); } CacheFileSystem::~CacheFileSystem() { - for (const auto& pair : m_canonicalMap) + for (const auto& pair : m_uniqueIdentityMap) { delete pair.Value; } } +void CacheFileSystem::clearCache() +{ + for (const auto& pair : m_uniqueIdentityMap) + { + delete pair.Value; + } + + m_uniqueIdentityMap.Clear(); + m_pathMap.Clear(); + + if (m_fileSystemExt) + { + m_fileSystemExt->clearCache(); + } +} + // Determines if we can simplify a path for a given mode -static bool _canSimplifyPath(CacheFileSystem::CanonicalMode mode) +static bool _canSimplifyPath(CacheFileSystem::UniqueIdentityMode mode) { - typedef CacheFileSystem::CanonicalMode CanonicalMode; + typedef CacheFileSystem::UniqueIdentityMode UniqueIdentityMode; switch (mode) { - case CanonicalMode::SimplifyPath: - case CanonicalMode::SimplifyPathAndHash: + case UniqueIdentityMode::SimplifyPath: + case UniqueIdentityMode::SimplifyPathAndHash: { return true; } @@ -186,33 +208,32 @@ static bool _canSimplifyPath(CacheFileSystem::CanonicalMode mode) } } -SlangResult CacheFileSystem::_calcCanonicalPath(const String& path, String& outCanonicalPath, ComPtr<ISlangBlob>& outFileContents) +SlangResult CacheFileSystem::_calcUniqueIdentity(const String& path, String& outUniqueIdentity, ComPtr<ISlangBlob>& outFileContents) { - switch (m_canonicalMode) + switch (m_uniqueIdentityMode) { - case CanonicalMode::FileSystemExt: + case UniqueIdentityMode::FileSystemExt: { - // Try getting the canonical path - // Okay request from the underlying file system the canonical path - ComPtr<ISlangBlob> canonicalBlob; - SLANG_RETURN_ON_FAIL(m_fileSystemExt->getCanoncialPath(path.Buffer(), canonicalBlob.writeRef())); + // Try getting the uniqueIdentity by asking underlying file system + ComPtr<ISlangBlob> uniqueIdentity; + SLANG_RETURN_ON_FAIL(m_fileSystemExt->getFileUniqueIdentity(path.Buffer(), uniqueIdentity.writeRef())); // Get the path as a string - outCanonicalPath = StringUtil::getString(canonicalBlob); + outUniqueIdentity = StringUtil::getString(uniqueIdentity); return SLANG_OK; } - case CanonicalMode::Path: + case UniqueIdentityMode::Path: { - outCanonicalPath = path; + outUniqueIdentity = path; return SLANG_OK; } - case CanonicalMode::SimplifyPath: + case UniqueIdentityMode::SimplifyPath: { - outCanonicalPath = Path::Simplify(path); + outUniqueIdentity = Path::Simplify(path); // If it still has relative elements can't uniquely identify, so give up - return Path::IsRelative(outCanonicalPath) ? SLANG_FAIL : SLANG_OK; + return Path::IsRelative(outUniqueIdentity) ? SLANG_FAIL : SLANG_OK; } - case CanonicalMode::SimplifyPathAndHash: - case CanonicalMode::Hash: + case UniqueIdentityMode::SimplifyPathAndHash: + case UniqueIdentityMode::Hash: { // I can only see if this is the same file as already loaded by loading the file and doing a hash Result res = m_fileSystem->loadFile(path.Buffer(), outFileContents.writeRef()); @@ -229,10 +250,10 @@ SlangResult CacheFileSystem::_calcCanonicalPath(const String& path, String& outC hashString.append(':'); - // The canonical name is.. combination of name and hash + // The uniqueIdentity is a combination of name and hash hashString.append(hash, 16); - outCanonicalPath = hashString; + outUniqueIdentity = hashString; return SLANG_OK; } } @@ -240,32 +261,32 @@ SlangResult CacheFileSystem::_calcCanonicalPath(const String& path, String& outC return SLANG_FAIL; } -CacheFileSystem::PathInfo* CacheFileSystem::_getOrCreateCanonicalCacheInfo(const String& path) +CacheFileSystem::PathInfo* CacheFileSystem::_resolveUniqueIdentityCacheInfo(const String& path) { - // Use the path to produce canonicalPath information + // Use the path to produce uniqueIdentity information ComPtr<ISlangBlob> fileContents; - String canonicalPath; + String uniqueIdentity; - SlangResult res = _calcCanonicalPath(path, canonicalPath, fileContents); + SlangResult res = _calcUniqueIdentity(path, uniqueIdentity, fileContents); if (SLANG_FAILED(res)) { - // Was not able to create a canonical path.. so mark in path map the problem + // Was not able to create a uniqueIdentity - return failure as nullptr return nullptr; } - // Now try looking up by canonical path. If not found, add a new result + // Now try looking up by uniqueIdentity path. If not found, add a new result PathInfo* pathInfo = nullptr; - if (!m_canonicalMap.TryGetValue(canonicalPath, pathInfo)) + if (!m_uniqueIdentityMap.TryGetValue(uniqueIdentity, pathInfo)) { - // Create with found canonicalPath - pathInfo = new PathInfo(canonicalPath); - m_canonicalMap.Add(canonicalPath, pathInfo); + // Create with found uniqueIdentity + pathInfo = new PathInfo(uniqueIdentity); + m_uniqueIdentityMap.Add(uniqueIdentity, pathInfo); } - // At this point they must have same canonicalPath - SLANG_ASSERT(pathInfo->getCanonicalPath() == canonicalPath); + // At this point they must have same uniqueIdentity + SLANG_ASSERT(pathInfo->getUniqueIdentity() == uniqueIdentity); - // If we have the file contents (because of calcing canonical), and there isn't a read fileblob already + // If we have the file contents (because of calcing uniqueIdentity), and there isn't a read fileblob already // store the data as if read, so doesn't get read again if (fileContents && !pathInfo->m_fileBlob) { @@ -276,24 +297,24 @@ CacheFileSystem::PathInfo* CacheFileSystem::_getOrCreateCanonicalCacheInfo(const return pathInfo; } -CacheFileSystem::PathInfo* CacheFileSystem::_getOrCreateSimplifiedPathCacheInfo(const String& path) +CacheFileSystem::PathInfo* CacheFileSystem::_resolveSimplifiedPathCacheInfo(const String& path) { // If we can simplify the path, try looking up in path cache with simplified path (as long as it's different!) - if (_canSimplifyPath(m_canonicalMode)) + if (_canSimplifyPath(m_uniqueIdentityMode)) { const String simplifiedPath = Path::Simplify(path); // Only lookup if the path is different - because otherwise will recurse forever... if (simplifiedPath != path) { // This is a recursive call - and will ensure the simplified path is added to the cache - return _getOrCreatePathCacheInfo(simplifiedPath); + return _resolvePathCacheInfo(simplifiedPath); } } - return _getOrCreateCanonicalCacheInfo(path); + return _resolveUniqueIdentityCacheInfo(path); } -CacheFileSystem::PathInfo* CacheFileSystem::_getOrCreatePathCacheInfo(const String& path) +CacheFileSystem::PathInfo* CacheFileSystem::_resolvePathCacheInfo(const String& path) { // Lookup in path cache PathInfo* pathInfo; @@ -304,7 +325,7 @@ CacheFileSystem::PathInfo* CacheFileSystem::_getOrCreatePathCacheInfo(const Stri } // Try getting or creating taking into account possible path simplification - pathInfo = _getOrCreateSimplifiedPathCacheInfo(path); + pathInfo = _resolveSimplifiedPathCacheInfo(path); // Always add the result to the path cache (even if null) m_pathMap.Add(path, pathInfo); return pathInfo; @@ -314,7 +335,7 @@ SlangResult CacheFileSystem::loadFile(char const* pathIn, ISlangBlob** blobOut) { *blobOut = nullptr; String path(pathIn); - PathInfo* info = _getOrCreatePathCacheInfo(path); + PathInfo* info = _resolvePathCacheInfo(path); if (!info) { return SLANG_FAIL; @@ -333,15 +354,15 @@ SlangResult CacheFileSystem::loadFile(char const* pathIn, ISlangBlob** blobOut) return toResult(info->m_loadFileResult); } -SlangResult CacheFileSystem::getCanoncialPath(const char* path, ISlangBlob** canonicalPathOut) +SlangResult CacheFileSystem::getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity) { - PathInfo* info = _getOrCreatePathCacheInfo(path); + PathInfo* info = _resolvePathCacheInfo(path); if (!info) { return SLANG_E_NOT_FOUND; } - info->m_canonicalPath->addRef(); - *canonicalPathOut = info->m_canonicalPath; + info->m_uniqueIdentity->addRef(); + *outUniqueIdentity = info->m_uniqueIdentity; return SLANG_OK; } @@ -362,7 +383,7 @@ SlangResult CacheFileSystem::calcCombinedPath(SlangPathType fromPathType, const SlangResult CacheFileSystem::getPathType(const char* pathIn, SlangPathType* pathTypeOut) { String path(pathIn); - PathInfo* info = _getOrCreatePathCacheInfo(path); + PathInfo* info = _resolvePathCacheInfo(path); if (!info) { return SLANG_E_NOT_FOUND; @@ -393,4 +414,52 @@ SlangResult CacheFileSystem::getPathType(const char* pathIn, SlangPathType* path return toResult(info->m_getPathTypeResult); } +SlangResult CacheFileSystem::getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) +{ + // If we don't have a backing full file system, we can't produce a canonical path with just ISlangFileSystem::loadFile + if (!m_fileSystemExt) + { + return SLANG_E_NOT_IMPLEMENTED; + } + + // A file must exist to get a canonical path... + PathInfo* info = _resolvePathCacheInfo(path); + if (!info) + { + return SLANG_E_NOT_FOUND; + } + + // We don't have this -> so read it ... + if (info->m_getCanonicalPathResult == CompressedResult::Uninitialized) + { + // Try getting the canonicalPath by asking underlying file system + ComPtr<ISlangBlob> canonicalPathBlob; + SlangResult res = m_fileSystemExt->getCanonicalPath(path, canonicalPathBlob.writeRef()); + + if (SLANG_SUCCEEDED(res)) + { + // Get the path as a string + String canonicalPath = StringUtil::getString(canonicalPathBlob); + if (canonicalPath.Length() > 0) + { + info->m_canonicalPath = new StringBlob(canonicalPath); + } + else + { + res = SLANG_FAIL; + } + } + + // Save the result + info->m_getCanonicalPathResult = toCompressedResult(res); + } + + if (info->m_canonicalPath) + { + info->m_canonicalPath->addRef(); + } + *outCanonicalPath = info->m_canonicalPath; + return SLANG_OK; +} + } diff --git a/source/slang/slang-file-system.h b/source/slang/slang-file-system.h index 94d89b58b..e341bf649 100644 --- a/source/slang/slang-file-system.h +++ b/source/slang/slang-file-system.h @@ -26,9 +26,9 @@ public: ISlangBlob** outBlob) SLANG_OVERRIDE; // ISlangFileSystemExt - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanoncialPath( + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getFileUniqueIdentity( const char* path, - ISlangBlob** canonicalPathOut) SLANG_OVERRIDE; + ISlangBlob** uniqueIdentityOut) SLANG_OVERRIDE; virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcCombinedPath( SlangPathType fromPathType, @@ -40,6 +40,12 @@ public: const char* path, SlangPathType* pathTypeOut) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanonicalPath( + const char* path, + ISlangBlob** outCanonicalPath); + + virtual SLANG_NO_THROW void SLANG_MCALL clearCache() {} + /// Get a default instance static ISlangFileSystemExt* getSingleton() { return &s_singleton; } @@ -59,16 +65,13 @@ of the interface on the constructor. NOTE! That this behavior is the same as previously in that.... 1) calcRelativePath, just returns the path as processed by the Path:: methods -2) getCanonicalPath, just returns the input path as the 'canonical' path. This will be wrong with a file multiply referenced through paths with .. and or . but -doing it this way means it works as before and requires no new functions. - -You can use a more sophisticated canonical style if you pass true to useSimplifyForCanonicalPath. This will simplify relative path to create a canonical path. +2) getUniqueIdentity behavior depends on the UniqueIdentityMode. */ class CacheFileSystem: public ISlangFileSystemExt, public RefObject { public: - enum CanonicalMode + enum UniqueIdentityMode { Default, ///< If passed, will default to the others depending on what kind of ISlangFileSystem is passed in Path, ///< Just use the path as is (old style slang behavior) @@ -98,9 +101,9 @@ class CacheFileSystem: public ISlangFileSystemExt, public RefObject ISlangBlob** outBlob) SLANG_OVERRIDE; // ISlangFileSystemExt - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanoncialPath( + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getFileUniqueIdentity( const char* path, - ISlangBlob** canonicalPathOut) SLANG_OVERRIDE; + ISlangBlob** outUniqueIdentity) SLANG_OVERRIDE; virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcCombinedPath( SlangPathType fromPathType, @@ -110,10 +113,16 @@ class CacheFileSystem: public ISlangFileSystemExt, public RefObject virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType( const char* path, - SlangPathType* pathTypeOut) SLANG_OVERRIDE; + SlangPathType* outPathType) SLANG_OVERRIDE; + + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanonicalPath( + const char* path, + ISlangBlob** outCanonicalPath); + + virtual SLANG_NO_THROW void SLANG_MCALL clearCache(); /// Ctor - CacheFileSystem(ISlangFileSystem* fileSystem, CanonicalMode canonicalMode = CanonicalMode::Default); + CacheFileSystem(ISlangFileSystem* fileSystem, UniqueIdentityMode uniqueIdentityMode = UniqueIdentityMode::Default); /// Dtor virtual ~CacheFileSystem(); @@ -126,52 +135,52 @@ protected: struct PathInfo { - PathInfo(const String& canonicalPath) + PathInfo(const String& uniqueIdentity) { - m_canonicalPath = new StringBlob(canonicalPath); - m_canonicalPath->addRef(); + m_uniqueIdentity = new StringBlob(uniqueIdentity); + m_uniqueIdentity->addRef(); m_loadFileResult = CompressedResult::Uninitialized; m_getPathTypeResult = CompressedResult::Uninitialized; + m_getCanonicalPathResult = CompressedResult::Uninitialized; m_pathType = SLANG_PATH_TYPE_FILE; } - ~PathInfo() - { - m_canonicalPath->release(); - } - /// Get the canonical path as a string - const String& getCanonicalPath() const { SLANG_ASSERT(m_canonicalPath); return m_canonicalPath->getString(); } + + /// Get the unique identity path as a string + const String& getUniqueIdentity() const { SLANG_ASSERT(m_uniqueIdentity); return m_uniqueIdentity->getString(); } - StringBlob* m_canonicalPath; ///< The canonical path + RefPtr<StringBlob> m_uniqueIdentity; CompressedResult m_loadFileResult; - CompressedResult m_getPathTypeResult; + CompressedResult m_getPathTypeResult; + CompressedResult m_getCanonicalPathResult; + SlangPathType m_pathType; ComPtr<ISlangBlob> m_fileBlob; + RefPtr<StringBlob> m_canonicalPath; }; - /// Given a path works out a canonical path, based on the canonicalMode. outFileContents will be set if file had to be read to produce the canonicalPath (ie with Hash) - SlangResult _calcCanonicalPath(const String& path, String& outCanonicalPath, ComPtr<ISlangBlob>& outFileContents); - + /// Given a path, works out a uniqueIdentity, based on the uniqueIdentityMode. outFileContents will be set if file had to be read to produce the uniqueIdentity (ie with Hash) + SlangResult _calcUniqueIdentity(const String& path, String& outUniqueIdentity, ComPtr<ISlangBlob>& outFileContents); /// For a given path gets a PathInfo. Can return nullptr, if it is not possible to create the PathInfo for some reason - PathInfo* _getOrCreatePathCacheInfo(const String& path); - /// Turns the path into a canonical path, and then tries to look up in the canonicalPathMap. - PathInfo* _getOrCreateCanonicalCacheInfo(const String& path); - /// Will simplify the path (if possible) to lookup on the pathCache else will create on canonicalCache - PathInfo* _getOrCreateSimplifiedPathCacheInfo(const String& path); + PathInfo* _resolvePathCacheInfo(const String& path); + /// Turns the path into a uniqueIdentity, and then tries to look up in the uniqueIdentityMap. + PathInfo* _resolveUniqueIdentityCacheInfo(const String& path); + /// Will simplify the path (if possible) to lookup on the pathCache else will create on uniqueIdentityMap + PathInfo* _resolveSimplifiedPathCacheInfo(const String& path); /* TODO: This may be improved by mapping to a ISlangBlob. This makes output fast and easy, and if constructed as a StringBlob, we can just static_cast to get as a string to use internally, instead of constantly converting. It is probably the case we cannot do dynamic_cast on ISlangBlob if we don't know where constructed -> if outside of slang codebase doing such a cast can cause an exception. So we *never* want to do dynamic cast from blobs which could be created by external code. */ - Dictionary<String, PathInfo*> m_pathMap; ///< Maps a path to a canonical path - Dictionary<String, PathInfo*> m_canonicalMap; ///< Maps a canonical path to a files contents. This OWNs the PathInfo. + Dictionary<String, PathInfo*> m_pathMap; ///< Maps a path to a PathInfo (and unique identity) + Dictionary<String, PathInfo*> m_uniqueIdentityMap; ///< Maps a unique identity for a file to its contents. This OWNs the PathInfo. - CanonicalMode m_canonicalMode; ///< Determines how 'canonicalPath' is produced. Cannot be Default in usage. + UniqueIdentityMode m_uniqueIdentityMode; ///< Determines how the 'uniqueIdentity' is produced. Cannot be Default in usage. - ComPtr<ISlangFileSystem> m_fileSystem; ///< Must always be set - ComPtr<ISlangFileSystemExt> m_fileSystemExt; ///< Optionally set -> if not will fall back on the m_fileSystem + ComPtr<ISlangFileSystem> m_fileSystem; ///< Must always be set + ComPtr<ISlangFileSystemExt> m_fileSystemExt; ///< Optionally set -> if nullptr will fall back on the m_fileSystem and emulate all the other methods of ISlangFileSystemExt }; } diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 247098434..359848d65 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -58,7 +58,7 @@ Session::Session() #include "object-meta-end.h" // Make sure our source manager is initialized - builtinSourceManager.initialize(nullptr); + builtinSourceManager.initialize(nullptr, nullptr); // Initialize representations of some very basic types: initializeTypes(); @@ -117,21 +117,21 @@ struct IncludeHandlerImpl : IncludeHandler return SLANG_E_NOT_FOUND; } - // Get the canonical path - ComPtr<ISlangBlob> canonicalPathBlob; - SLANG_RETURN_ON_FAIL(fileSystemExt->getCanoncialPath(combinedPath.begin(), canonicalPathBlob.writeRef())); + // Get the uniqueIdentity + ComPtr<ISlangBlob> uniqueIdentityBlob; + SLANG_RETURN_ON_FAIL(fileSystemExt->getFileUniqueIdentity(combinedPath.begin(), uniqueIdentityBlob.writeRef())); - // If the rel path exists -> the canonical path MUST exists too - String canonicalPath(StringUtil::getString(canonicalPathBlob)); - if (canonicalPath.Length() <= 0) + // If the rel path exists -> a uniqueIdentity MUST exists too + String uniqueIdentity(StringUtil::getString(uniqueIdentityBlob)); + if (uniqueIdentity.Length() <= 0) { - // Canonical path can't be empty + // Unique identity can't be empty return SLANG_FAIL; } pathInfoOut.type = PathInfo::Type::Normal; pathInfoOut.foundPath = combinedPath; - pathInfoOut.canonicalPath = canonicalPath; + pathInfoOut.uniqueIdentity = uniqueIdentity; return SLANG_OK; } @@ -306,7 +306,7 @@ CompileRequest::CompileRequest(Session* session) setSourceManager(&sourceManagerStorage); - sourceManager->initialize(session->getBuiltinSourceManager()); + sourceManager->initialize(session->getBuiltinSourceManager(), nullptr); // Set all the default writers for (int i = 0; i < int(WriterChannel::CountOf); ++i) @@ -314,9 +314,7 @@ CompileRequest::CompileRequest(Session* session) setWriter(WriterChannel(i), nullptr); } - // Set up the default file system - SLANG_ASSERT(fileSystem == nullptr); - fileSystemExt = new CacheFileSystem(OSFileSystem::getSingleton()); + setFileSystem(nullptr); } // Allocate static const storage for the various interface IDs that the Slang API needs to expose @@ -419,7 +417,7 @@ RefPtr<Expr> CompileRequest::parseTypeString(TranslationUnitRequest * translatio { // Create a SourceManager on the stack, so any allocations for 'SourceFile'/'SourceView' etc will be cleaned up SourceManager localSourceManager; - localSourceManager.initialize(sourceManager); + localSourceManager.initialize(sourceManager, nullptr); Slang::SourceFile* srcFile = localSourceManager.createSourceFileWithString(PathInfo::makeTypeParse(), typeStr); @@ -859,10 +857,10 @@ void CompileRequest::loadParsedModule( RefPtr<LoadedModule> loadedModule = new LoadedModule(); // Get a path - String mostUniquePath = pathInfo.getMostUniquePath(); - SLANG_ASSERT(mostUniquePath.Length() > 0); + String mostUniqueIdentity = pathInfo.getMostUniqueIdentity(); + SLANG_ASSERT(mostUniqueIdentity.Length() > 0); - mapPathToLoadedModule.Add(mostUniquePath, loadedModule); + mapPathToLoadedModule.Add(mostUniqueIdentity, loadedModule); mapNameToLoadedModules.Add(name, loadedModule); int errorCountBefore = mSink.GetErrorCount(); @@ -985,7 +983,7 @@ RefPtr<ModuleDecl> CompileRequest::findOrImportModule( PathInfo pathIncludedFromInfo = getSourceManager()->getPathInfo(loc, SourceLocType::Actual); PathInfo filePathInfo; - // We are going to allow canonicalPath to be able to hold strings other than paths (like hashes), therefore we have to load via the found path + // We have to load via the found path - as that is how file was originally loaded if (SLANG_FAILED(includeHandler.findFile(fileName, pathIncludedFromInfo.foundPath, filePathInfo))) { this->mSink.diagnose(loc, Diagnostics::cannotFindFile, fileName); @@ -994,7 +992,7 @@ RefPtr<ModuleDecl> CompileRequest::findOrImportModule( } // Maybe this was loaded previously at a different relative name? - if (mapPathToLoadedModule.TryGetValue(filePathInfo.getMostUniquePath(), loadedModule)) + if (mapPathToLoadedModule.TryGetValue(filePathInfo.getMostUniqueIdentity(), loadedModule)) return loadedModule->moduleDecl; // Try to load it @@ -1047,6 +1045,34 @@ void CompileRequest::noteInternalErrorLoc(SourceLoc const& loc) internalErrorLocsNoted++; } +static const Slang::Guid IID_ISlangFileSystemExt = SLANG_UUID_ISlangFileSystemExt; + +void CompileRequest::setFileSystem(ISlangFileSystem* inFileSystem) +{ + // Set the fileSystem + fileSystem = inFileSystem; + + // Set up fileSystemExt appropriately + if (inFileSystem == nullptr) + { + fileSystemExt = new Slang::CacheFileSystem(Slang::OSFileSystem::getSingleton()); + } + else + { + // See if we have the interface + inFileSystem->queryInterface(IID_ISlangFileSystemExt, (void**)fileSystemExt.writeRef()); + + // If not wrap with WrapFileSytem that keeps the old behavior + if (!fileSystemExt) + { + // Construct a wrapper to emulate the extended interface behavior + fileSystemExt = new Slang::CacheFileSystem(fileSystem); + } + } + + // Set the file system used on the source manager + sourceManager->setFileSystemExt(fileSystemExt); +} RefPtr<ModuleDecl> findOrImportModule( CompileRequest* request, @@ -1236,35 +1262,12 @@ SLANG_API void spDestroyCompileRequest( delete req; } -static const Slang::Guid IID_ISlangFileSystemExt = SLANG_UUID_ISlangFileSystemExt; - SLANG_API void spSetFileSystem( SlangCompileRequest* request, ISlangFileSystem* fileSystem) { if(!request) return; - auto req = REQ(request); - - // Set the fileSystem - req->fileSystem = fileSystem; - - // Set up fileSystemExt appropriately - if (fileSystem == nullptr) - { - req->fileSystemExt = new Slang::CacheFileSystem(Slang::OSFileSystem::getSingleton()); - } - else - { - // See if we have the interface - fileSystem->queryInterface(IID_ISlangFileSystemExt, (void**)req->fileSystemExt.writeRef()); - - // If not wrap with WrapFileSytem that keeps the old behavior - if (!req->fileSystemExt) - { - // Construct a wrapper to emulate the extended interface behavior - req->fileSystemExt = new Slang::CacheFileSystem(fileSystem); - } - } + REQ(request)->setFileSystem(fileSystem); } SLANG_API void spSetCompileFlags( diff --git a/source/slang/source-loc.cpp b/source/slang/source-loc.cpp index 78477bb78..6507ea4b7 100644 --- a/source/slang/source-loc.cpp +++ b/source/slang/source-loc.cpp @@ -9,11 +9,11 @@ namespace Slang { /* !!!!!!!!!!!!!!!!!!!!!!!!! SourceView !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ -const String PathInfo::getMostUniquePath() const +const String PathInfo::getMostUniqueIdentity() const { switch (type) { - case Type::Normal: return canonicalPath; + case Type::Normal: return uniqueIdentity; case Type::FoundPath: return foundPath; default: return ""; } @@ -89,7 +89,7 @@ void SourceView::addLineDirective(SourceLoc directiveLoc, StringSlicePool::Handl void SourceView::addLineDirective(SourceLoc directiveLoc, const String& path, int line) { - StringSlicePool::Handle pathHandle = m_sourceManager->getStringSlicePool().add(path.getUnownedSlice()); + StringSlicePool::Handle pathHandle = getSourceManager()->getStringSlicePool().add(path.getUnownedSlice()); return addLineDirective(directiveLoc, pathHandle, line); } @@ -163,8 +163,7 @@ PathInfo SourceView::_getPathInfo(StringSlicePool::Handle pathHandle) const } else { - // We don't have a full normal path (including 'canonical') so just go with FoundPath - return PathInfo::makePath(m_sourceManager->getStringSlicePool().getSlice(pathHandle)); + return PathInfo::makePath(getSourceManager()->getStringSlicePool().getSlice(pathHandle)); } } @@ -295,7 +294,8 @@ void SourceFile::setContents(const String& content) setContents(contentBlob); } -SourceFile::SourceFile(const PathInfo& pathInfo, size_t contentSize) : +SourceFile::SourceFile(SourceManager* sourceManager, const PathInfo& pathInfo, size_t contentSize) : + m_sourceManager(sourceManager), m_pathInfo(pathInfo), m_contentSize(contentSize) { @@ -303,14 +303,37 @@ SourceFile::SourceFile(const PathInfo& pathInfo, size_t contentSize) : SourceFile::~SourceFile() { +} + +String SourceFile::calcVerbosePath() const +{ + ISlangFileSystemExt* fileSystemExt = getSourceManager()->getFileSystemExt(); + if (fileSystemExt) + { + String canonicalPath; + ComPtr<ISlangBlob> canonicalPathBlob; + if (SLANG_SUCCEEDED(fileSystemExt->getCanonicalPath(m_pathInfo.foundPath.Buffer(), canonicalPathBlob.writeRef()))) + { + canonicalPath = StringUtil::getString(canonicalPathBlob); + } + if (canonicalPath.Length() > 0) + { + return canonicalPath; + } + } + + return m_pathInfo.foundPath; } /* !!!!!!!!!!!!!!!!!!!!!!!!! SourceManager !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ void SourceManager::initialize( - SourceManager* p) + SourceManager* p, + ISlangFileSystemExt* fileSystemExt) { + m_fileSystemExt = fileSystemExt; + m_parent = p; if( p ) @@ -374,14 +397,14 @@ SourceRange SourceManager::allocateSourceRange(UInt size) SourceFile* SourceManager::createSourceFileWithSize(const PathInfo& pathInfo, size_t contentSize) { - SourceFile* sourceFile = new SourceFile(pathInfo, contentSize); + SourceFile* sourceFile = new SourceFile(this, pathInfo, contentSize); m_sourceFiles.Add(sourceFile); return sourceFile; } SourceFile* SourceManager::createSourceFileWithString(const PathInfo& pathInfo, const String& contents) { - SourceFile* sourceFile = new SourceFile(pathInfo, contents.Length()); + SourceFile* sourceFile = new SourceFile(this, pathInfo, contents.Length()); m_sourceFiles.Add(sourceFile); sourceFile->setContents(contents); return sourceFile; @@ -389,7 +412,7 @@ SourceFile* SourceManager::createSourceFileWithString(const PathInfo& pathInfo, SourceFile* SourceManager::createSourceFileWithBlob(const PathInfo& pathInfo, ISlangBlob* blob) { - SourceFile* sourceFile = new SourceFile(pathInfo, blob->getBufferSize()); + SourceFile* sourceFile = new SourceFile(this, pathInfo, blob->getBufferSize()); m_sourceFiles.Add(sourceFile); sourceFile->setContents(blob); return sourceFile; @@ -398,7 +421,7 @@ SourceFile* SourceManager::createSourceFileWithBlob(const PathInfo& pathInfo, IS SourceView* SourceManager::createSourceView(SourceFile* sourceFile) { SourceRange range = allocateSourceRange(sourceFile->getContentSize()); - SourceView* sourceView = new SourceView(this, sourceFile, range); + SourceView* sourceView = new SourceView(sourceFile, range); m_sourceViews.Add(sourceView); return sourceView; @@ -479,18 +502,18 @@ SourceView* SourceManager::findSourceViewRecursively(SourceLoc loc) const return nullptr; } -SourceFile* SourceManager::findSourceFile(const String& canonicalPath) const +SourceFile* SourceManager::findSourceFile(const String& uniqueIdentity) const { - SourceFile*const* filePtr = m_sourceFileMap.TryGetValue(canonicalPath); + SourceFile*const* filePtr = m_sourceFileMap.TryGetValue(uniqueIdentity); return (filePtr) ? *filePtr : nullptr; } -SourceFile* SourceManager::findSourceFileRecursively(const String& canonicalPath) const +SourceFile* SourceManager::findSourceFileRecursively(const String& uniqueIdentity) const { const SourceManager* manager = this; do { - SourceFile* sourceFile = manager->findSourceFile(canonicalPath); + SourceFile* sourceFile = manager->findSourceFile(uniqueIdentity); if (sourceFile) { return sourceFile; @@ -500,10 +523,10 @@ SourceFile* SourceManager::findSourceFileRecursively(const String& canonicalPath return nullptr; } -void SourceManager::addSourceFile(const String& canonicalPath, SourceFile* sourceFile) +void SourceManager::addSourceFile(const String& uniqueIdentity, SourceFile* sourceFile) { - SLANG_ASSERT(!findSourceFileRecursively(canonicalPath)); - m_sourceFileMap.Add(canonicalPath, sourceFile); + SLANG_ASSERT(!findSourceFileRecursively(uniqueIdentity)); + m_sourceFileMap.Add(uniqueIdentity, sourceFile); } HumaneSourceLoc SourceManager::getHumaneLoc(SourceLoc loc, SourceLocType type) diff --git a/source/slang/source-loc.h b/source/slang/source-loc.h index ee4049473..aa0623e3d 100644 --- a/source/slang/source-loc.h +++ b/source/slang/source-loc.h @@ -44,32 +44,32 @@ struct PathInfo enum class Type { Unknown, ///< The path is not known - Normal, ///< Normal has both path and canonical path - FoundPath, ///< Just has a found path (canonical path is unknown, or even 'unknowable') + Normal, ///< Normal has both path and uniqueIdentity + FoundPath, ///< Just has a found path (uniqueIdentity is unknown, or even 'unknowable') TokenPaste, ///< No paths, just created to do a macro expansion TypeParse, ///< No path, just created to do a type parse CommandLine, ///< A macro constructed from the command line }; /// True if has a canonical path - SLANG_FORCE_INLINE bool hasCanonicalPath() const { return type == Type::Normal && canonicalPath.Length() > 0; } + SLANG_FORCE_INLINE bool hasUniqueIdentity() const { return type == Type::Normal && uniqueIdentity.Length() > 0; } /// True if has a regular found path SLANG_FORCE_INLINE bool hasFoundPath() const { return type == Type::Normal || type == Type::FoundPath; } - /// The canonical path is unique, so return that if we have it. If not the regular path, otherwise the empty string. - const String getMostUniquePath() const; + /// Returns the 'most unique' identity for the path. If has a 'uniqueIdentity' returns that, else the foundPath, else "". + const String getMostUniqueIdentity() const; // So simplify construction. In normal usage it's safer to use make methods over constructing directly. static PathInfo makeUnknown() { return PathInfo { Type::Unknown, "unknown", String() }; } static PathInfo makeTokenPaste() { return PathInfo{ Type::TokenPaste, "token paste", String()}; } - static PathInfo makeNormal(const String& foundPathIn, const String& canonicalPathIn) { SLANG_ASSERT(canonicalPathIn.Length() > 0 && foundPathIn.Length() > 0); return PathInfo { Type::Normal, foundPathIn, canonicalPathIn }; } + static PathInfo makeNormal(const String& foundPathIn, const String& uniqueIdentity) { SLANG_ASSERT(uniqueIdentity.Length() > 0 && foundPathIn.Length() > 0); return PathInfo { Type::Normal, foundPathIn, uniqueIdentity }; } static PathInfo makePath(const String& pathIn) { SLANG_ASSERT(pathIn.Length() > 0); return PathInfo { Type::FoundPath, pathIn, String()}; } static PathInfo makeTypeParse() { return PathInfo { Type::TypeParse, "type string", String() }; } static PathInfo makeCommandLine() { return PathInfo { Type::CommandLine, "command line", String() }; } Type type; ///< The type of path String foundPath; ///< The path where the file was found (might contain relative elements) - String canonicalPath; ///< Canonical version of the found path + String uniqueIdentity; ///< The unique identity of the file on the path found }; class SourceLoc @@ -139,6 +139,9 @@ struct SourceRange SourceLoc end; }; +// Pre-declare +struct SourceManager; + // A logical or physical storage object for a range of input code // that has logically contiguous source locations. class SourceFile @@ -178,13 +181,20 @@ public: /// Set the content as a string void setContents(const String& content); + /// Calculate a display path -> can canonicalize if necessary + String calcVerbosePath() const; + + /// Get the source manager this was created on + SourceManager* getSourceManager() const { return m_sourceManager; } + /// Ctor - SourceFile(const PathInfo& pathInfo, size_t contentSize); + SourceFile(SourceManager* sourceManager, const PathInfo& pathInfo, size_t contentSize); /// Dtor ~SourceFile(); protected: + SourceManager * m_sourceManager; ///< The source manager this belongs to PathInfo m_pathInfo; ///< The path The logical file path to report for locations inside this span. ComPtr<ISlangBlob> m_contentBlob; ///< A blob that owns the storage for the file contents. If nullptr, there is no contents UnownedStringSlice m_content; ///< The actual contents of the file. @@ -210,8 +220,6 @@ struct HumaneSourceLoc Int column = 0; }; -// Pre-declare -struct SourceManager; /* A SourceView maps to a single span of SourceLoc range and is equivalent to a single include or more precisely use of a source file. It is distinct from a SourceFile - because a SourceFile may be included multiple times, with different interpretations (depending @@ -257,7 +265,7 @@ class SourceView /// Get the source file holds the contents this view SourceFile* getSourceFile() const { return m_sourceFile; } /// Get the source manager - SourceManager* getSourceManager() const { return m_sourceManager; } + SourceManager* getSourceManager() const { return m_sourceFile->getSourceManager(); } /// Get the associated 'content' (the source text) const UnownedStringSlice& getContent() const { return m_sourceFile->getContent(); } @@ -273,8 +281,7 @@ class SourceView PathInfo getPathInfo(SourceLoc loc, SourceLocType type = SourceLocType::Nominal); /// Ctor - SourceView(SourceManager* sourceManager, SourceFile* sourceFile, SourceRange range): - m_sourceManager(sourceManager), + SourceView(SourceFile* sourceFile, SourceRange range): m_range(range), m_sourceFile(sourceFile) { @@ -283,7 +290,6 @@ class SourceView protected: PathInfo _getPathInfo(StringSlicePool::Handle pathHandle) const; - SourceManager* m_sourceManager; /// Get the manager this belongs to SourceRange m_range; ///< The range that this SourceView applies to SourceFile* m_sourceFile; ///< The source file. Can hold the line breaks List<Entry> m_entries; ///< An array entries describing how we should interpret a range, starting from the start location. @@ -292,7 +298,7 @@ class SourceView struct SourceManager { // Initialize a source manager, with an optional parent - void initialize(SourceManager* parent); + void initialize(SourceManager* parent, ISlangFileSystemExt* fileSystemExt); /// Allocate a range of SourceLoc locations, these can be used to identify a specific location in the source SourceRange allocateSourceRange(UInt size); @@ -322,12 +328,17 @@ struct SourceManager /// Searches this manager, and then the parent to see if can find a match for path. /// If not found returns nullptr. - SourceFile* findSourceFileRecursively(const String& canonicalPath) const; + SourceFile* findSourceFileRecursively(const String& uniqueIdentity) const; /// Find if the source file is defined on this manager. - SourceFile* findSourceFile(const String& canonicalPath) const; + SourceFile* findSourceFile(const String& uniqueIdentity) const; - /// Add a source file, path must be unique for this manager AND any parents - void addSourceFile(const String& canonicalPath, SourceFile* sourceFile); + /// Get the file system associated with this source manager + ISlangFileSystemExt* getFileSystemExt() const { return m_fileSystemExt; } + /// Get the file system associated with this source manager + void setFileSystemExt(ISlangFileSystemExt* fileSystemExt) { m_fileSystemExt = fileSystemExt; } + + /// Add a source file, uniqueIdentity must be unique for this manager AND any parents + void addSourceFile(const String& uniqueIdentity, SourceFile* sourceFile); /// Get the slice pool StringSlicePool& getStringSlicePool() { return m_slicePool; } @@ -374,8 +385,10 @@ struct SourceManager // Can be used for storing the decoded contents of Token. Content for example. MemoryArena m_memoryArena; - // Maps canonical paths to source files - Dictionary<String, SourceFile*> m_sourceFileMap; + // Maps uniqueIdentities to source files + Dictionary<String, SourceFile*> m_sourceFileMap; + + ComPtr<ISlangFileSystemExt> m_fileSystemExt; }; } // namespace Slang diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp index 0a6023733..70b7a4533 100644 --- a/tools/slang-test/options.cpp +++ b/tools/slang-test/options.cpp @@ -134,6 +134,10 @@ static bool _isSubCommand(const char* arg) { optionsOut->shouldBeVerbose = true; } + else if (strcmp(arg, "-verbose-paths") == 0) + { + optionsOut->verbosePaths = true; + } else if (strcmp(arg, "-generate-hlsl-baselines") == 0) { optionsOut->generateHLSLBaselines = true; diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h index 4858ffd03..77460e190 100644 --- a/tools/slang-test/options.h +++ b/tools/slang-test/options.h @@ -49,6 +49,9 @@ struct Options // generate extra output (notably: command lines we run) bool shouldBeVerbose = false; + // Use verbose paths + bool verbosePaths = false; + // force generation of baselines for HLSL tests bool generateHLSLBaselines = false; diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp index d0da4fc61..cbd2974da 100644 --- a/tools/slang-test/slang-test-main.cpp +++ b/tools/slang-test/slang-test-main.cpp @@ -519,6 +519,15 @@ String findExpectedPath(const TestInput& input, const char* postFix) return ""; } +static void _initSlangCompiler(TestContext* context, OSProcessSpawner& spawnerOut) +{ + spawnerOut.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix()); + + if (context->options.verbosePaths) + { + spawnerOut.pushArgument("-verbose-paths"); + } +} TestResult runSimpleTest(TestContext* context, TestInput& input) { @@ -528,8 +537,8 @@ TestResult runSimpleTest(TestContext* context, TestInput& input) auto outputStem = input.outputStem; OSProcessSpawner spawner; + _initSlangCompiler(context, spawner); - spawner.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix()); spawner.pushArgument(filePath999); for( auto arg : input.testOptions->args ) @@ -699,6 +708,8 @@ static SlangCompileTarget _getCompileTarget(const UnownedStringSlice& name) return SLANG_TARGET_UNKNOWN; } + + TestResult runCrossCompilerTest(TestContext* context, TestInput& input) { // need to execute the stand-alone Slang compiler on the file @@ -710,11 +721,11 @@ TestResult runCrossCompilerTest(TestContext* context, TestInput& input) OSProcessSpawner actualSpawner; OSProcessSpawner expectedSpawner; - actualSpawner.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix()); - expectedSpawner.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix()); - + _initSlangCompiler(context, actualSpawner); + _initSlangCompiler(context, expectedSpawner); + actualSpawner.pushArgument(filePath); - + const auto& args = input.testOptions->args; const UInt targetIndex = args.IndexOf("-target"); @@ -821,7 +832,8 @@ TestResult generateHLSLBaseline(TestContext* context, TestInput& input) auto outputStem = input.outputStem; OSProcessSpawner spawner; - spawner.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix()); + _initSlangCompiler(context, spawner); + spawner.pushArgument(filePath999); for( auto arg : input.testOptions->args ) @@ -866,8 +878,8 @@ TestResult runHLSLComparisonTest(TestContext* context, TestInput& input) // need to execute the stand-alone Slang compiler on the file, and compare its output to what we expect OSProcessSpawner spawner; + _initSlangCompiler(context, spawner); - spawner.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix()); spawner.pushArgument(filePath999); for( auto arg : input.testOptions->args ) @@ -967,8 +979,8 @@ TestResult doGLSLComparisonTestRun(TestContext* context, auto outputStem = input.outputStem; OSProcessSpawner spawner; + _initSlangCompiler(context, spawner); - spawner.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix()); spawner.pushArgument(filePath999); if( langDefine ) |
