From cb9d679a3a93c65c44904bf77811b9d74e431e23 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Fri, 26 Oct 2018 08:16:54 -0400 Subject: Feature/file system cache (#692) * First pass at caching file system. * default-file-system -> slang-file-system fix problem with location("build.linux") confusing windows build for now. * Added CompressedResult Fix problem in Result construction with it being unsigned * Add support for Path simplification. * Testing for Path::Simplify. * Refactored CacheFileSystem - automatically handles ISlangFileSystem or ISlangFileSystemExt appropriately. Removed WrapFileSystem - because wasn't possible to emulate some of the behavior if just loadFile is implemented. Split out StringBlob - so that no need to convert between ISlangBlob and String repeatidly. * Remove unwanted code in ~CompileRequest --- source/core/core.vcxproj | 2 +- source/core/core.vcxproj.filters | 4 +- source/core/slang-io.cpp | 99 +++++++++++++ source/core/slang-io.h | 11 ++ source/core/slang-string-util.cpp | 37 +++++ source/core/slang-string-util.h | 41 ++++++ source/slang/compiler.cpp | 4 +- source/slang/compiler.h | 5 - source/slang/default-file-system.cpp | 127 ---------------- source/slang/default-file-system.h | 104 ------------- source/slang/preprocessor.cpp | 2 +- source/slang/slang-file-system.cpp | 275 +++++++++++++++++++++++++++++++++++ source/slang/slang-file-system.h | 161 ++++++++++++++++++++ source/slang/slang.cpp | 71 ++------- source/slang/slang.vcxproj | 14 +- source/slang/slang.vcxproj.filters | 22 +-- source/slang/source-loc.cpp | 4 +- 17 files changed, 664 insertions(+), 319 deletions(-) delete mode 100644 source/slang/default-file-system.cpp delete mode 100644 source/slang/default-file-system.h create mode 100644 source/slang/slang-file-system.cpp create mode 100644 source/slang/slang-file-system.h (limited to 'source') diff --git a/source/core/core.vcxproj b/source/core/core.vcxproj index d1ed4715f..2d0c67342 100644 --- a/source/core/core.vcxproj +++ b/source/core/core.vcxproj @@ -213,7 +213,7 @@ - + diff --git a/source/core/core.vcxproj.filters b/source/core/core.vcxproj.filters index 7936ca11b..1548217b4 100644 --- a/source/core/core.vcxproj.filters +++ b/source/core/core.vcxproj.filters @@ -130,8 +130,8 @@ - + Source Files - + \ No newline at end of file diff --git a/source/core/slang-io.cpp b/source/core/slang-io.cpp index 2b443a62b..f1948200c 100644 --- a/source/core/slang-io.cpp +++ b/source/core/slang-io.cpp @@ -123,6 +123,105 @@ namespace Slang return sb.ProduceString(); } + /* static */ bool Path::IsDriveSpecification(const UnownedStringSlice& element) + { + switch (element.size()) + { + case 0: + { + // We'll just assume it is + return true; + } + case 2: + { + // Look for a windows like drive spec + const char firstChar = element[0]; + return element[1] == ':' && ((firstChar >= 'a' && firstChar <= 'z') || (firstChar >= 'A' && firstChar <= 'Z')); + } + default: return false; + } + + } + + /* static */void Path::Split(const UnownedStringSlice& path, List& splitOut) + { + splitOut.Clear(); + + const char* start = path.begin(); + const char* end = path.end(); + + while (start < end) + { + const char* cur = start; + // Find the split + while (cur < end && !IsDelimiter(*cur)) cur++; + + splitOut.Add(UnownedStringSlice(start, cur)); + + // Next + start = cur + 1; + } + + // Okay if the end is empty. And we aren't with a spec like // or c:/ , then drop the final slash + if (splitOut.Count() > 1 && splitOut.Last().size() == 0) + { + if (splitOut.Count() == 2 && IsDriveSpecification(splitOut[0])) + { + return; + } + // Remove the last + splitOut.RemoveLast(); + } + } + + /* static */String Path::Simplify(const UnownedStringSlice& path) + { + List split; + Split(path, split); + + // Strictly speaking we could do something about case on platforms like window, but here we won't worry about that + for (int i = 0; i < int(split.Count()); i++) + { + const UnownedStringSlice& cur = split[i]; + if (cur == "." && split.Count() > 1) + { + // Just remove it + split.RemoveAt(i); + i--; + } + else if (cur == ".." && i > 0) + { + // Can we remove this and the one before ? + UnownedStringSlice& before = split[i - 1]; + if (before == ".." || (i == 1 && IsDriveSpecification(before))) + { + // Can't do it + continue; + } + split.RemoveRange(i - 1, 2); + i -= 2; + } + } + + // If its empty it must be . + if (split.Count() == 0) + { + split.Add(UnownedStringSlice::fromLiteral(".")); + } + + // Reconstruct the string + StringBuilder builder; + for (int i = 0; i < int(split.Count()); i++) + { + if (i > 0) + { + builder.Append(PathDelimiter); + } + builder.Append(split[i]); + } + return builder; + } + bool Path::CreateDir(const String & path) { #if defined(_WIN32) diff --git a/source/core/slang-io.h b/source/core/slang-io.h index ff287254c..3703aee08 100644 --- a/source/core/slang-io.h +++ b/source/core/slang-io.h @@ -32,6 +32,17 @@ namespace Slang static String Combine(const String & path1, const String & path2, const String & path3); static bool CreateDir(const String & path); + /// Accept either style of delimiter + SLANG_FORCE_INLINE static bool IsDelimiter(char c) { return c == '/' || c == '\\'; } + + static bool IsDriveSpecification(const UnownedStringSlice& element); + + /// Splits the path into it's individual bits + static void Split(const UnownedStringSlice& path, List& splitOut); + /// Strips .. and . as much as it can + static String Simplify(const UnownedStringSlice& path); + static String Simplify(const String& path) { return Simplify(path.getUnownedSlice()); } + static SlangResult GetPathType(const String & path, SlangPathType* pathTypeOut); static SlangResult GetCanonical(const String & path, String& canonicalPathOut); diff --git a/source/core/slang-string-util.cpp b/source/core/slang-string-util.cpp index 29f8dc0ca..6d0d896a1 100644 --- a/source/core/slang-string-util.cpp +++ b/source/core/slang-string-util.cpp @@ -2,6 +2,19 @@ namespace Slang { +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! StringBlob !!!!!!!!!!!!!!!!!!!!!!!!!!! + +// Allocate static const storage for the various interface IDs that the Slang API needs to expose +static const Guid IID_ISlangUnknown = SLANG_UUID_ISlangUnknown; +static const Guid IID_ISlangBlob = SLANG_UUID_ISlangBlob; + +/* static */ISlangUnknown* StringBlob::getInterface(const Guid& guid) +{ + return (guid == IID_ISlangUnknown || guid == IID_ISlangBlob) ? static_cast(this) : nullptr; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! StringUtil !!!!!!!!!!!!!!!!!!!!!!!!!!! + /* static */void StringUtil::split(const UnownedStringSlice& in, char splitChar, List& slicesOut) { slicesOut.Clear(); @@ -74,4 +87,28 @@ namespace Slang { return builder; } +/* static */String StringUtil::getString(ISlangBlob* blob) +{ + if (blob) + { + size_t size = blob->getBufferSize(); + if (size > 0) + { + const char* contents = (const char*)blob->getBufferPointer(); + // Check it has terminating 0, if not we must construct as if it does + if (contents[size - 1] == 0) + { + size--; + } + return String(contents, contents + size); + } + } + return String(); +} + +ComPtr StringUtil::createStringBlob(const String& string) +{ + return ComPtr(new StringBlob(string)); +} + } // namespace Slang diff --git a/source/core/slang-string-util.h b/source/core/slang-string-util.h index fc3258490..47d92f2fe 100644 --- a/source/core/slang-string-util.h +++ b/source/core/slang-string-util.h @@ -6,8 +6,41 @@ #include +#include "../../slang-com-helper.h" +#include "../../slang-com-ptr.h" + namespace Slang { +/** A blob that uses a `String` for its storage. +*/ +class StringBlob : public ISlangBlob +{ +public: + // ISlangUnknown + SLANG_IUNKNOWN_ALL + + // ISlangBlob + SLANG_NO_THROW void const* SLANG_MCALL getBufferPointer() SLANG_OVERRIDE { return m_string.Buffer(); } + SLANG_NO_THROW size_t SLANG_MCALL getBufferSize() SLANG_OVERRIDE { return m_string.Length(); } + + /// Get the contained string + SLANG_FORCE_INLINE const String& getString() const { return m_string; } + + explicit StringBlob(String const& string) + : m_string(string) + {} + + /// Need virtual dtor, because BlobBase is derived from and release impl used is the one in the base class (that doesn't know the derived type) + /// Alternatively could be implemented by always using SLANG_IUNKNOWN_RELEASE in derived types - this would make derived types slightly smaller/faster + virtual ~StringBlob() {} + +protected: + ISlangUnknown* getInterface(const Guid& guid); + + String m_string; + uint32_t m_refCount = 0; +}; + struct StringUtil { /// Split in, by specified splitChar into slices out @@ -22,6 +55,14 @@ struct StringUtil /// Create a string from the format string applying args (like sprintf) static String makeStringWithFormat(const char* format, ...); + + /// Given a string held in a blob, returns as a String + /// Returns an empty string if blob is nullptr + static String getString(ISlangBlob* blob); + + /// Create a blob from a string + static ComPtr createStringBlob(const String& string); + }; } // namespace Slang diff --git a/source/slang/compiler.cpp b/source/slang/compiler.cpp index 4b7fcb554..f16ad06b7 100644 --- a/source/slang/compiler.cpp +++ b/source/slang/compiler.cpp @@ -3,6 +3,8 @@ #include "../core/basic.h" #include "../core/platform.h" #include "../core/slang-io.h" +#include "../core/slang-string-util.h" + #include "bytecode.h" #include "compiler.h" #include "lexer.h" @@ -108,7 +110,7 @@ namespace Slang break; case ResultFormat::Text: - blob = createStringBlob(outputString); + blob = StringUtil::createStringBlob(outputString); break; case ResultFormat::Binary: diff --git a/source/slang/compiler.h b/source/slang/compiler.h index 04508837f..de099c40b 100644 --- a/source/slang/compiler.h +++ b/source/slang/compiler.h @@ -287,9 +287,6 @@ namespace Slang class Session; - /// Create a blob that will retain a string for its lifetime. - /// - ComPtr createStringBlob(String const& string); /// Create a blob that will retain (a copy of) raw data. /// @@ -411,8 +408,6 @@ namespace Slang CompileRequest(Session* session); - ~CompileRequest(); - RefPtr parseTypeString(TranslationUnitRequest * translationUnit, String typeStr, RefPtr scope); Type* getTypeFromString(String typeStr); diff --git a/source/slang/default-file-system.cpp b/source/slang/default-file-system.cpp deleted file mode 100644 index f9c66b7d7..000000000 --- a/source/slang/default-file-system.cpp +++ /dev/null @@ -1,127 +0,0 @@ -#include "default-file-system.h" - -#include "../../slang-com-ptr.h" -#include "../core/slang-io.h" -#include "compiler.h" - -namespace Slang -{ - -// Allocate static const storage for the various interface IDs that the Slang API needs to expose -static const Guid IID_ISlangUnknown = SLANG_UUID_ISlangUnknown; -static const Guid IID_ISlangFileSystem = SLANG_UUID_ISlangFileSystem; -static const Guid IID_ISlangFileSystemExt = SLANG_UUID_ISlangFileSystemExt; - -/* !!!!!!!!!!!!!!!!!!!!!!!!!! IncludeFileSystem !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/ - -/* static */DefaultFileSystem DefaultFileSystem::s_singleton; - -ISlangUnknown* DefaultFileSystem::getInterface(const Guid& guid) -{ - return (guid == IID_ISlangUnknown || guid == IID_ISlangFileSystem || guid == IID_ISlangFileSystemExt) ? static_cast(this) : nullptr; -} - -SlangResult DefaultFileSystem::getCanoncialPath(const char* path, ISlangBlob** canonicalPathOut) -{ - String canonicalPath; - SLANG_RETURN_ON_FAIL(Path::GetCanonical(path, canonicalPath)); - - *canonicalPathOut = createStringBlob(canonicalPath).detach(); - return SLANG_OK; -} - -SlangResult DefaultFileSystem::calcRelativePath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) -{ - String relPath; - switch (fromPathType) - { - case SLANG_PATH_TYPE_FILE: - { - relPath = Path::Combine(Path::GetDirectoryName(fromPath), path); - break; - } - case SLANG_PATH_TYPE_DIRECTORY: - { - relPath = Path::Combine(fromPath, path); - break; - } - } - - *pathOut = createStringBlob(relPath).detach(); - return SLANG_OK; -} - -SlangResult SLANG_MCALL DefaultFileSystem::getPathType( - const char* path, - SlangPathType* pathTypeOut) -{ - return Path::GetPathType(path, pathTypeOut); -} - -SlangResult DefaultFileSystem::loadFile(char const* path, ISlangBlob** outBlob) -{ - // Default implementation that uses the `core` libraries facilities for talking to the OS filesystem. - // - // TODO: we might want to conditionally compile these in, so that - // a user could create a build of Slang that doesn't include any OS - // filesystem calls. - - if (!File::Exists(path)) - { - return SLANG_E_NOT_FOUND; - } - - try - { - String sourceString = File::ReadAllText(path); - ComPtr sourceBlob = createStringBlob(sourceString); - *outBlob = sourceBlob.detach(); - return SLANG_OK; - } - catch (...) - { - } - return SLANG_E_CANNOT_OPEN; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!! WrapFileSystem !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/ - -ISlangUnknown* WrapFileSystem::getInterface(const Guid& guid) -{ - return (guid == IID_ISlangUnknown || guid == IID_ISlangFileSystem || guid == IID_ISlangFileSystemExt) ? static_cast(this) : nullptr; -} - -SlangResult WrapFileSystem::loadFile(char const* path, ISlangBlob** outBlob) -{ - // Used the wrapped file system to do the loading - return m_fileSystem->loadFile(path, outBlob); -} - -SlangResult WrapFileSystem::getCanoncialPath(const char* path, ISlangBlob** canonicalPathOut) -{ - // This isn't a very good 'canonical path' because the same file might be referenced - // multiple ways - for example by using relative paths. - // But it's simple and matches slangs previous behavior. - String canonicalPath(path); - *canonicalPathOut = createStringBlob(canonicalPath).detach(); - return SLANG_OK; -} - -SlangResult WrapFileSystem::calcRelativePath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) -{ - // Just defer to the default implementation - return DefaultFileSystem::getSingleton()->calcRelativePath(fromPathType, fromPath, path, pathOut); -} - -SlangResult WrapFileSystem::getPathType(const char* path, SlangPathType* pathTypeOut) -{ - // TODO: - // This might be undesirable in the longer term because it means that ISlangFileSystem will not be used - // to test file existence - but the file system will be. - // - // It would probably be better to use some kind of cache that uses 'loadFile' to load files, but also - // to test for existence. - return DefaultFileSystem::getSingleton()->getPathType(path, pathTypeOut); -} - -} \ No newline at end of file diff --git a/source/slang/default-file-system.h b/source/slang/default-file-system.h deleted file mode 100644 index 7805dd81a..000000000 --- a/source/slang/default-file-system.h +++ /dev/null @@ -1,104 +0,0 @@ -#ifndef SLANG_DEFAULT_FILE_SYSTEM_H_INCLUDED -#define SLANG_DEFAULT_FILE_SYSTEM_H_INCLUDED - -#include "../../slang.h" -#include "../../slang-com-helper.h" -#include "../../slang-com-ptr.h" - -namespace Slang -{ - -class DefaultFileSystem : public ISlangFileSystemExt -{ -public: - // ISlangUnknown - // override ref counting, as DefaultFileSystem is singleton - SLANG_IUNKNOWN_QUERY_INTERFACE - SLANG_NO_THROW uint32_t SLANG_MCALL addRef() SLANG_OVERRIDE { return 1; } - SLANG_NO_THROW uint32_t SLANG_MCALL release() SLANG_OVERRIDE { return 1; } - - // ISlangFileSystem - virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile( - char const* path, - ISlangBlob** outBlob) SLANG_OVERRIDE; - - // ISlangFileSystemExt - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanoncialPath( - const char* path, - ISlangBlob** canonicalPathOut) SLANG_OVERRIDE; - - virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcRelativePath( - SlangPathType fromPathType, - const char* fromPath, - const char* path, - ISlangBlob** pathOut) SLANG_OVERRIDE; - - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType( - const char* path, - SlangPathType* pathTypeOut) SLANG_OVERRIDE; - - /// Get a default instance - static ISlangFileSystemExt* getSingleton() { return &s_singleton; } - -private: - /// Make so not constructible - DefaultFileSystem() {} - virtual ~DefaultFileSystem() {} - - ISlangUnknown* getInterface(const Guid& guid); - - static DefaultFileSystem s_singleton; -}; - -/* Wraps an ISlangFileSystem, and provides the extra methods required to make a ISlangFileSystemExt -interface, deferring to the contained file system to do reading. - -NOTE! That this behavior is the same as previously in that.... -1) calcRelativePath, just returns the path as processed by the Path:: methods -2) getCanonicalPath, just returns the input path as the 'canonical' path. This will be wrong with a file multiply referenced through paths with .. and or . but -doing it this way means it works as before and requires no new functions. -*/ -class WrapFileSystem : public ISlangFileSystemExt -{ -public: - // ISlangUnknown - SLANG_IUNKNOWN_ALL - - // ISlangFileSystem - virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile( - char const* path, - ISlangBlob** outBlob) SLANG_OVERRIDE; - - // ISlangFileSystemExt - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanoncialPath( - const char* path, - ISlangBlob** canonicalPathOut) SLANG_OVERRIDE; - - virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcRelativePath( - SlangPathType fromPathType, - const char* fromPath, - const char* path, - ISlangBlob** pathOut) SLANG_OVERRIDE; - - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType( - const char* path, - SlangPathType* pathTypeOut) SLANG_OVERRIDE; - - /// Ctor - WrapFileSystem(ISlangFileSystem* fileSystem): - m_fileSystem(fileSystem) - { - } - virtual ~WrapFileSystem() {} - - -protected: - ISlangUnknown* getInterface(const Guid& guid); - - ComPtr m_fileSystem; ///< The wrapped file system - uint32_t m_refCount = 0; -}; - -} - -#endif \ No newline at end of file diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp index 47aa0cfd4..0bda70891 100644 --- a/source/slang/preprocessor.cpp +++ b/source/slang/preprocessor.cpp @@ -1637,7 +1637,7 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context) if (!sourceFile) { ComPtr foundSourceBlob; - if (SLANG_FAILED(includeHandler->readFile(filePathInfo.canonicalPath, foundSourceBlob.writeRef()))) + if (SLANG_FAILED(includeHandler->readFile(filePathInfo.foundPath, foundSourceBlob.writeRef()))) { GetSink(context)->diagnose(pathToken.loc, Diagnostics::includeFailed, path); return; diff --git a/source/slang/slang-file-system.cpp b/source/slang/slang-file-system.cpp new file mode 100644 index 000000000..ef77dbf33 --- /dev/null +++ b/source/slang/slang-file-system.cpp @@ -0,0 +1,275 @@ +#include "slang-file-system.h" + +#include "../../slang-com-ptr.h" +#include "../core/slang-io.h" +#include "../core/slang-string-util.h" + +#include "compiler.h" + +namespace Slang +{ + +// Allocate static const storage for the various interface IDs that the Slang API needs to expose +static const Guid IID_ISlangUnknown = SLANG_UUID_ISlangUnknown; +static const Guid IID_ISlangFileSystem = SLANG_UUID_ISlangFileSystem; +static const Guid IID_ISlangFileSystemExt = SLANG_UUID_ISlangFileSystemExt; + +// Cacluate a relative path, just using Path:: string processing +static SlangResult _calcRelativePath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) +{ + String relPath; + switch (fromPathType) + { + case SLANG_PATH_TYPE_FILE: + { + relPath = Path::Combine(Path::GetDirectoryName(fromPath), path); + break; + } + case SLANG_PATH_TYPE_DIRECTORY: + { + relPath = Path::Combine(fromPath, path); + break; + } + } + + *pathOut = StringUtil::createStringBlob(relPath).detach(); + return SLANG_OK; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!! IncludeFileSystem !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/ + +/* static */DefaultFileSystem DefaultFileSystem::s_singleton; + +template +static ISlangFileSystemExt* _getInterface(T* ptr, const Guid& guid) +{ + return (guid == IID_ISlangUnknown || guid == IID_ISlangFileSystem || guid == IID_ISlangFileSystemExt) ? static_cast(ptr) : nullptr; +} + +ISlangUnknown* DefaultFileSystem::getInterface(const Guid& guid) +{ + return _getInterface(this, guid); +} + +SlangResult DefaultFileSystem::getCanoncialPath(const char* path, ISlangBlob** canonicalPathOut) +{ + String canonicalPath; + SLANG_RETURN_ON_FAIL(Path::GetCanonical(path, canonicalPath)); + + *canonicalPathOut = StringUtil::createStringBlob(canonicalPath).detach(); + return SLANG_OK; +} + +SlangResult DefaultFileSystem::calcRelativePath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) +{ + return _calcRelativePath(fromPathType, fromPath, path, pathOut); +} + +SlangResult SLANG_MCALL DefaultFileSystem::getPathType( + const char* path, + SlangPathType* pathTypeOut) +{ + return Path::GetPathType(path, pathTypeOut); +} + +SlangResult DefaultFileSystem::loadFile(char const* path, ISlangBlob** outBlob) +{ + // Default implementation that uses the `core` libraries facilities for talking to the OS filesystem. + // + // TODO: we might want to conditionally compile these in, so that + // a user could create a build of Slang that doesn't include any OS + // filesystem calls. + + if (!File::Exists(path)) + { + return SLANG_E_NOT_FOUND; + } + + try + { + String sourceString = File::ReadAllText(path); + *outBlob = StringUtil::createStringBlob(sourceString).detach(); + return SLANG_OK; + } + catch (...) + { + } + return SLANG_E_CANNOT_OPEN; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CacheFileSystem !!!!!!!!!!!!!!!!!!!!!!!!!!! + +/* static */ const Result CacheFileSystem::s_compressedResultToResult[] = +{ + SLANG_E_UNINITIALIZED, + SLANG_OK, ///< Ok + SLANG_E_NOT_FOUND, ///< File not found + SLANG_E_CANNOT_OPEN, ///< CannotOpen, + SLANG_FAIL, ///< Fail +}; + +/* static */CacheFileSystem::CompressedResult CacheFileSystem::toCompressedResult(Result res) +{ + if (SLANG_SUCCEEDED(res)) + { + return CompressedResult::Ok; + } + switch (res) + { + case SLANG_E_CANNOT_OPEN: return CompressedResult::CannotOpen; + case SLANG_E_NOT_FOUND: return CompressedResult::NotFound; + default: return CompressedResult::Fail; + } +} + +ISlangUnknown* CacheFileSystem::getInterface(const Guid& guid) +{ + return _getInterface(this, guid); +} + +CacheFileSystem::CacheFileSystem(ISlangFileSystem* fileSystem, bool useSimplifyForCanonicalPath) : + m_fileSystem(fileSystem), + m_useSimplifyForCanonicalPath(useSimplifyForCanonicalPath) +{ + // Try to get the more sophisticated interface + fileSystem->queryInterface(IID_ISlangFileSystemExt, (void**)m_fileSystemExt.writeRef()); +} + +CacheFileSystem::~CacheFileSystem() +{ + for (const auto& pair : m_canonicalMap) + { + delete pair.Value; + } +} + +CacheFileSystem::PathInfo* CacheFileSystem::_getPathInfo(const String& relPath) +{ + PathInfo** infoPtr = m_pathMap.TryGetValue(relPath); + if (infoPtr) + { + return *infoPtr; + } + + String canonicalPath; + if (m_fileSystemExt) + { + // Try getting the canonical path + // Okay request from the underlying file system the canonical path + ComPtr canonicalBlob; + if (SLANG_FAILED(m_fileSystemExt->getCanoncialPath(relPath.Buffer(), canonicalBlob.writeRef()))) + { + // Write in result as being null ptr so not tried again + m_pathMap.Add(relPath, nullptr); + return nullptr; + } + // Get the path as a string + canonicalPath = StringUtil::getString(canonicalBlob); + } + else + { + canonicalPath = m_useSimplifyForCanonicalPath ? Path::Simplify(relPath.getUnownedSlice()) : relPath; + } + + PathInfo* pathInfo; + infoPtr = m_canonicalMap.TryGetValue(canonicalPath); + if (infoPtr) + { + pathInfo = *infoPtr; + } + else + { + // Create and add to canonical path + pathInfo = new PathInfo(canonicalPath); + m_canonicalMap.Add(canonicalPath, pathInfo); + } + + // Add the relPath + m_pathMap.Add(relPath, pathInfo); + return pathInfo; +} + +SlangResult CacheFileSystem::loadFile(char const* pathIn, ISlangBlob** blobOut) +{ + *blobOut = nullptr; + String path(pathIn); + PathInfo* info = _getPathInfo(path); + if (!info) + { + return SLANG_FAIL; + } + + if (info->m_loadFileResult == CompressedResult::Uninitialized) + { + info->m_loadFileResult = toCompressedResult(m_fileSystem->loadFile(path.Buffer(), info->m_fileBlob.writeRef())); + } + + *blobOut = info->m_fileBlob; + if (*blobOut) + { + (*blobOut)->addRef(); + } + return toResult(info->m_loadFileResult); +} + +SlangResult CacheFileSystem::getCanoncialPath(const char* path, ISlangBlob** canonicalPathOut) +{ + PathInfo* info = _getPathInfo(path); + if (!info) + { + return SLANG_E_NOT_FOUND; + } + info->m_canonicalPath->addRef(); + *canonicalPathOut = info->m_canonicalPath; + return SLANG_OK; +} + +SlangResult CacheFileSystem::calcRelativePath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) +{ + // Just defer to contained implementation + if (m_fileSystemExt) + { + return m_fileSystemExt->calcRelativePath(fromPathType, fromPath, path, pathOut); + } + else + { + // Just use the default implementation + return _calcRelativePath(fromPathType, fromPath, path, pathOut); + } +} + +SlangResult CacheFileSystem::getPathType(const char* pathIn, SlangPathType* pathTypeOut) +{ + String path(pathIn); + PathInfo* info = _getPathInfo(path); + if (!info) + { + return SLANG_E_NOT_FOUND; + } + + if (info->m_getPathTypeResult == CompressedResult::Uninitialized) + { + if (m_fileSystemExt) + { + info->m_getPathTypeResult = toCompressedResult(m_fileSystemExt->getPathType(pathIn, &info->m_pathType)); + } + else + { + // Okay try to load the file + if (info->m_loadFileResult == CompressedResult::Uninitialized) + { + info->m_loadFileResult = toCompressedResult(m_fileSystem->loadFile(path.Buffer(), info->m_fileBlob.writeRef())); + } + + // Make the getPathResult the same as the load result + info->m_getPathTypeResult = info->m_loadFileResult; + // Just set to file... the result is what matters in this case + info->m_pathType = SLANG_PATH_TYPE_FILE; + } + } + + *pathTypeOut = info->m_pathType; + return toResult(info->m_getPathTypeResult); +} + +} \ No newline at end of file diff --git a/source/slang/slang-file-system.h b/source/slang/slang-file-system.h new file mode 100644 index 000000000..c1358654a --- /dev/null +++ b/source/slang/slang-file-system.h @@ -0,0 +1,161 @@ +#ifndef SLANG_FILE_SYSTEM_H_INCLUDED +#define SLANG_FILE_SYSTEM_H_INCLUDED + +#include "../../slang.h" +#include "../../slang-com-helper.h" +#include "../../slang-com-ptr.h" + +#include "../core/slang-string-util.h" +#include "../core/dictionary.h" + +namespace Slang +{ + +class DefaultFileSystem : public ISlangFileSystemExt +{ +public: + // ISlangUnknown + // override ref counting, as DefaultFileSystem is singleton + SLANG_IUNKNOWN_QUERY_INTERFACE + SLANG_NO_THROW uint32_t SLANG_MCALL addRef() SLANG_OVERRIDE { return 1; } + SLANG_NO_THROW uint32_t SLANG_MCALL release() SLANG_OVERRIDE { return 1; } + + // ISlangFileSystem + virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile( + char const* path, + ISlangBlob** outBlob) SLANG_OVERRIDE; + + // ISlangFileSystemExt + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanoncialPath( + const char* path, + ISlangBlob** canonicalPathOut) SLANG_OVERRIDE; + + virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcRelativePath( + SlangPathType fromPathType, + const char* fromPath, + const char* path, + ISlangBlob** pathOut) SLANG_OVERRIDE; + + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType( + const char* path, + SlangPathType* pathTypeOut) SLANG_OVERRIDE; + + /// Get a default instance + static ISlangFileSystemExt* getSingleton() { return &s_singleton; } + +private: + /// Make so not constructible + DefaultFileSystem() {} + virtual ~DefaultFileSystem() {} + + ISlangUnknown* getInterface(const Guid& guid); + + static DefaultFileSystem s_singleton; +}; + +/* Wraps an underlying ISlangFileSystem or ISlangFileSystemExt and provides caching, +as well as emulation of methods if only has ISlangFileSystem interface. Will query capabilities +of the interface on the constructor. + +NOTE! That this behavior is the same as previously in that.... +1) calcRelativePath, just returns the path as processed by the Path:: methods +2) getCanonicalPath, just returns the input path as the 'canonical' path. This will be wrong with a file multiply referenced through paths with .. and or . but +doing it this way means it works as before and requires no new functions. + +You can use a more sophisticated canonical style if you pass true to useSimplifyForCanonicalPath. This will simplify relative path to create a canonical path. +*/ +class CacheFileSystem: public ISlangFileSystemExt +{ + public: + + /* Cannot change order/add members without changing s_compressedResultToResult */ + enum class CompressedResult: uint8_t + { + Uninitialized, ///< Holds no value + Ok, ///< Ok + NotFound, ///< File not found + CannotOpen, ///< Cannot open + Fail, ///< Generic failure + CountOf, + }; + + // ISlangUnknown + SLANG_IUNKNOWN_ALL + + // ISlangFileSystem + virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile( + char const* path, + ISlangBlob** outBlob) SLANG_OVERRIDE; + + // ISlangFileSystemExt + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanoncialPath( + const char* path, + ISlangBlob** canonicalPathOut) SLANG_OVERRIDE; + + virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcRelativePath( + SlangPathType fromPathType, + const char* fromPath, + const char* path, + ISlangBlob** pathOut) SLANG_OVERRIDE; + + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType( + const char* path, + SlangPathType* pathTypeOut) SLANG_OVERRIDE; + + /// Ctor + CacheFileSystem(ISlangFileSystem* fileSystem, bool useSimplifyForCanonicalPath = false); + /// Dtor + ~CacheFileSystem(); + + static CompressedResult toCompressedResult(Result res); + static Result toResult(CompressedResult compRes) { return s_compressedResultToResult[int(compRes)]; } + static const Result s_compressedResultToResult[int(CompressedResult::CountOf)]; + +protected: + ISlangUnknown* getInterface(const Guid& guid); + + struct PathInfo + { + PathInfo(const String& canonicalPath) + { + m_canonicalPath = new StringBlob(canonicalPath); + m_canonicalPath->addRef(); + + m_loadFileResult = CompressedResult::Uninitialized; + m_getPathTypeResult = CompressedResult::Uninitialized; + } + ~PathInfo() + { + m_canonicalPath->release(); + } + /// Get the canonical path as a string + const String& getCanonicalPath() const { SLANG_ASSERT(m_canonicalPath); return m_canonicalPath->getString(); } + + StringBlob* m_canonicalPath; ///< The canonical path + CompressedResult m_loadFileResult; + CompressedResult m_getPathTypeResult; + SlangPathType m_pathType; + ComPtr m_fileBlob; + }; + + /// For a given relPath gets a PathInfo + PathInfo* _getPathInfo(const String& relPath); + + /* TODO: This may be improved by mapping to a ISlangBlob. This makes output fast and easy, and if constructed + as a StringBlob, we can just static_cast to get as a string to use internally, instead of constantly converting. + It is probably the case we cannot do dynamic_cast on ISlangBlob if we don't know where constructed -> if outside of slang codebase + doing such a cast can cause an exception. So we *never* want to do dynamic cast from blobs which could be created by external code. */ + + Dictionary m_pathMap; ///< Maps a path to a canonical path + Dictionary m_canonicalMap; ///< Maps a canonical path to a files contents. This OWNs the PathInfo. + + bool m_useSimplifyForCanonicalPath; ///< If set will use Path::Simplify to create 'canonical' paths + + ComPtr m_fileSystem; ///< Must always be set + ComPtr m_fileSystemExt; ///< Optionally set -> if not will fall back on the m_fileSystem + uint32_t m_refCount = 0; +}; + +} + +#endif // SLANG_FILE_SYSTEM_H_INCLUDED \ No newline at end of file diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 996f8e32f..38fe88ef3 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -1,6 +1,8 @@ #include "../../slang.h" #include "../core/slang-io.h" +#include "../core/slang-string-util.h" + #include "parameter-binding.h" #include "lower-to-ir.h" #include "../slang/parser.h" @@ -9,7 +11,7 @@ #include "syntax-visitors.h" #include "../slang/type-layout.h" -#include "default-file-system.h" +#include "slang-file-system.h" #include "ir-serialize.h" @@ -77,25 +79,6 @@ Session::Session() addBuiltinSource(hlslLanguageScope, "hlsl", getHLSLLibraryCode()); } -static String getString(ISlangBlob* blob) -{ - if (blob) - { - size_t size = blob->getBufferSize(); - if (size > 0) - { - const char* contents = (const char*)blob->getBufferPointer(); - // Check it has terminating 0, if not we must construct as if it does - if (contents[size - 1] == 0) - { - size --; - } - return String(contents, contents + size); - } - } - return String(); -} - struct IncludeHandlerImpl : IncludeHandler { CompileRequest* request; @@ -112,7 +95,7 @@ struct IncludeHandlerImpl : IncludeHandler // Get relative path ComPtr relPathBlob; SLANG_RETURN_ON_FAIL(fileSystemExt->calcRelativePath(fromPathType, fromPath.begin(), path.begin(), relPathBlob.writeRef())); - String relPath(getString(relPathBlob)); + String relPath(StringUtil::getString(relPathBlob)); if (relPath.Length() <= 0) { return SLANG_FAIL; @@ -130,7 +113,7 @@ struct IncludeHandlerImpl : IncludeHandler SLANG_RETURN_ON_FAIL(fileSystemExt->getCanoncialPath(relPath.begin(), canonicalPathBlob.writeRef())); // If the rel path exists -> the canonical path MUST exists too - String canonicalPath(getString(canonicalPathBlob)); + String canonicalPath(StringUtil::getString(canonicalPathBlob)); if (canonicalPath.Length() <= 0) { // Canonical path can't be empty @@ -323,16 +306,7 @@ CompileRequest::CompileRequest(Session* session) // Set up the default file system SLANG_ASSERT(fileSystem == nullptr); - fileSystemExt = DefaultFileSystem::getSingleton(); -} - -CompileRequest::~CompileRequest() -{ - // delete things that may reference IR objects first - targets = decltype(targets)(); - translationUnits = decltype(translationUnits)(); - entryPoints = decltype(entryPoints)(); - types = decltype(types)(); + fileSystemExt = new CacheFileSystem(DefaultFileSystem::getSingleton()); } // Allocate static const storage for the various interface IDs that the Slang API needs to expose @@ -360,27 +334,6 @@ protected: } }; -/** A blob that uses a `String` for its storage. -*/ -class StringBlob : public BlobBase -{ -public: - // ISlangBlob - SLANG_NO_THROW void const* SLANG_MCALL getBufferPointer() SLANG_OVERRIDE { return m_string.Buffer(); } - SLANG_NO_THROW size_t SLANG_MCALL getBufferSize() SLANG_OVERRIDE { return m_string.Length(); } - - explicit StringBlob(String const& string) - : m_string(string) - {} -protected: - String m_string; -}; - -ComPtr createStringBlob(String const& string) -{ - return ComPtr(new StringBlob(string)); -} - /** A blob that manages some raw data that it owns. */ class RawBlob : public BlobBase @@ -972,7 +925,7 @@ RefPtr CompileRequest::findOrImportModule( PathInfo pathIncludedFromInfo = getSourceManager()->getPathInfo(loc, SourceLocType::Actual); PathInfo filePathInfo; - // There is an argument to passing in the 'canonicalPath' instead of the foundPath, but either should work here + // We are going to allow canonicalPath to be able to hold strings other than paths (like hashes), therefore we have to load via the found path if (SLANG_FAILED(includeHandler.findFile(fileName, pathIncludedFromInfo.foundPath, filePathInfo))) { this->mSink.diagnose(loc, Diagnostics::cannotFindFile, fileName); @@ -981,12 +934,12 @@ RefPtr CompileRequest::findOrImportModule( } // Maybe this was loaded previously at a different relative name? - if (mapPathToLoadedModule.TryGetValue(filePathInfo.canonicalPath, loadedModule)) + if (mapPathToLoadedModule.TryGetValue(filePathInfo.getMostUniquePath(), loadedModule)) return loadedModule->moduleDecl; // Try to load it ComPtr fileContents; - if (SLANG_FAILED(includeHandler.readFile(filePathInfo.canonicalPath, fileContents.writeRef()))) + if (SLANG_FAILED(includeHandler.readFile(filePathInfo.foundPath, fileContents.writeRef()))) { this->mSink.diagnose(loc, Diagnostics::cannotOpenFile, fileName); mapNameToLoadedModules[name] = nullptr; @@ -1188,7 +1141,7 @@ SLANG_API void spSetFileSystem( // Set up fileSystemExt appropriately if (fileSystem == nullptr) { - req->fileSystemExt = Slang::DefaultFileSystem::getSingleton(); + req->fileSystemExt = new Slang::CacheFileSystem(Slang::DefaultFileSystem::getSingleton()); } else { @@ -1199,7 +1152,7 @@ SLANG_API void spSetFileSystem( if (!req->fileSystemExt) { // Construct a wrapper to emulate the extended interface behavior - req->fileSystemExt = new Slang::WrapFileSystem(fileSystem); + req->fileSystemExt = new Slang::CacheFileSystem(fileSystem); } } } @@ -1342,7 +1295,7 @@ SLANG_API SlangResult spGetDiagnosticOutputBlob( if(!req->diagnosticOutputBlob) { - req->diagnosticOutputBlob = Slang::createStringBlob(req->mDiagnosticOutput); + req->diagnosticOutputBlob = Slang::StringUtil::createStringBlob(req->mDiagnosticOutput); } Slang::ComPtr resultBlob = req->diagnosticOutputBlob; diff --git a/source/slang/slang.vcxproj b/source/slang/slang.vcxproj index b899506b4..6bcaac3da 100644 --- a/source/slang/slang.vcxproj +++ b/source/slang/slang.vcxproj @@ -175,7 +175,6 @@ - @@ -209,6 +208,7 @@ + @@ -228,7 +228,6 @@ - @@ -255,6 +254,7 @@ + @@ -264,6 +264,9 @@ + + + Document @@ -271,7 +274,7 @@ "../../bin/windows-x64/debug/slang-generate" %(Identity) "../../bin/windows-x86/release/slang-generate" %(Identity) "../../bin/windows-x64/release/slang-generate" %(Identity) - ../../core.meta.slang.h + %(Identity).h slang-generate %(Identity) ../../bin/windows-x86/debug/slang-generate.exe ../../bin/windows-x64/debug/slang-generate.exe @@ -284,7 +287,7 @@ "../../bin/windows-x64/debug/slang-generate" %(Identity) "../../bin/windows-x86/release/slang-generate" %(Identity) "../../bin/windows-x64/release/slang-generate" %(Identity) - ../../hlsl.meta.slang.h + %(Identity).h slang-generate %(Identity) ../../bin/windows-x86/debug/slang-generate.exe ../../bin/windows-x64/debug/slang-generate.exe @@ -292,9 +295,6 @@ ../../bin/windows-x64/release/slang-generate.exe - - - {F9BE7957-8399-899E-0C49-E714FDDD4B65} diff --git a/source/slang/slang.vcxproj.filters b/source/slang/slang.vcxproj.filters index 6b51e16c8..ce76ec77e 100644 --- a/source/slang/slang.vcxproj.filters +++ b/source/slang/slang.vcxproj.filters @@ -24,9 +24,6 @@ Header Files - - Header Files - Header Files @@ -126,6 +123,9 @@ Header Files + + Header Files + Header Files @@ -179,9 +179,6 @@ Source Files - - Source Files - Source Files @@ -260,6 +257,9 @@ Source Files + + Source Files + Source Files @@ -285,6 +285,11 @@ Source Files + + + Source Files + + Source Files @@ -293,9 +298,4 @@ Source Files - - - Source Files - - \ No newline at end of file diff --git a/source/slang/source-loc.cpp b/source/slang/source-loc.cpp index 1c3b8c70a..62926aa8c 100644 --- a/source/slang/source-loc.cpp +++ b/source/slang/source-loc.cpp @@ -3,6 +3,8 @@ #include "compiler.h" +#include "../core/slang-string-util.h" + namespace Slang { /* !!!!!!!!!!!!!!!!!!!!!!!!! SourceView !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ @@ -323,7 +325,7 @@ SourceFile* SourceManager::createSourceFile(const PathInfo& pathInfo, ISlangBlob SourceFile* SourceManager::createSourceFile(const PathInfo& pathInfo, const String& content) { - ComPtr contentBlob = createStringBlob(content); + ComPtr contentBlob = StringUtil::createStringBlob(content); return createSourceFile(pathInfo, contentBlob); } -- cgit v1.2.3