diff options
| author | jsmall-nvidia <jsmall@nvidia.com> | 2021-01-11 15:24:11 -0500 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-01-11 15:24:11 -0500 |
| commit | 723796a0a0fed8e5b8c3222b1c90443189113098 (patch) | |
| tree | 41fa039f2f2fcec4c24746203ad119a8054f021a | |
| parent | 5554777188225266e2295db3588f6cb17cae0c4d (diff) | |
LZ4 compression support (#1654)
* #include an absolute path didn't work - because paths were taken to always be relative.
* Testing out use of lz4.
* Added ICompressionSystem, and LZ4 implementation.
* Add support for deflate compression.
Simplify compression interface - to make more easily work across apis.
* WIP on CompressedFileSystem.
* ImplicitDirectoryCollector
* SubStringIndexMap - > StringSliceIndexMap.
* WIP save stdlib in different containers.
* Support for different archive types for stdlib.
* Fix project.
* CompressedFileSystem -> ArchiveFileSystem.
Added CompressionSystemType::None
* Added ArchiveFileSystem
* Fix problem RiffFileSystem load withoug compression system.
* Test archive types.
Improve diagnostic message.
* Fix typo in testing file system archives.
* Split out archive detection.
* Fix gcc warning issue.
* Fix warning.
* RiffArchiveFileSystem -> RiffFileSystem
Co-authored-by: Tim Foley <tfoleyNV@users.noreply.github.com>
32 files changed, 1651 insertions, 334 deletions
diff --git a/.gitmodules b/.gitmodules index 77ed3e03e..d325a2609 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,3 +22,6 @@ [submodule "external/miniz"] path = external/miniz url = https://github.com/richgel999/miniz +[submodule "external/lz4"] + path = external/lz4 + url = https://github.com/lz4/lz4 diff --git a/build/visual-studio/core/core.vcxproj b/build/visual-studio/core/core.vcxproj index 7c97eed01..2bf02ecdd 100644 --- a/build/visual-studio/core/core.vcxproj +++ b/build/visual-studio/core/core.vcxproj @@ -171,6 +171,7 @@ </ItemDefinitionGroup> <ItemGroup> <ClInclude Include="..\..\..\source\core\slang-allocator.h" /> + <ClInclude Include="..\..\..\source\core\slang-archive-file-system.h" /> <ClInclude Include="..\..\..\source\core\slang-array-view.h" /> <ClInclude Include="..\..\..\source\core\slang-array.h" /> <ClInclude Include="..\..\..\source\core\slang-basic.h" /> @@ -178,6 +179,8 @@ <ClInclude Include="..\..\..\source\core\slang-byte-encode-util.h" /> <ClInclude Include="..\..\..\source\core\slang-char-util.h" /> <ClInclude Include="..\..\..\source\core\slang-common.h" /> + <ClInclude Include="..\..\..\source\core\slang-compression-system.h" /> + <ClInclude Include="..\..\..\source\core\slang-deflate-compression-system.h" /> <ClInclude Include="..\..\..\source\core\slang-dictionary.h" /> <ClInclude Include="..\..\..\source\core\slang-downstream-compiler.h" /> <ClInclude Include="..\..\..\source\core\slang-exception.h" /> @@ -188,6 +191,7 @@ <ClInclude Include="..\..\..\source\core\slang-io.h" /> <ClInclude Include="..\..\..\source\core\slang-linked-list.h" /> <ClInclude Include="..\..\..\source\core\slang-list.h" /> + <ClInclude Include="..\..\..\source\core\slang-lz4-compression-system.h" /> <ClInclude Include="..\..\..\source\core\slang-math.h" /> <ClInclude Include="..\..\..\source\core\slang-memory-arena.h" /> <ClInclude Include="..\..\..\source\core\slang-name-convention-util.h" /> @@ -197,6 +201,7 @@ <ClInclude Include="..\..\..\source\core\slang-process-util.h" /> <ClInclude Include="..\..\..\source\core\slang-random-generator.h" /> <ClInclude Include="..\..\..\source\core\slang-render-api-util.h" /> + <ClInclude Include="..\..\..\source\core\slang-riff-file-system.h" /> <ClInclude Include="..\..\..\source\core\slang-riff.h" /> <ClInclude Include="..\..\..\source\core\slang-secure-crt.h" /> <ClInclude Include="..\..\..\source\core\slang-semantic-version.h" /> @@ -220,14 +225,17 @@ <ClInclude Include="..\..\..\source\core\windows\slang-win-visual-studio-util.h" /> </ItemGroup> <ItemGroup> + <ClCompile Include="..\..\..\source\core\slang-archive-file-system.cpp" /> <ClCompile Include="..\..\..\source\core\slang-blob.cpp" /> <ClCompile Include="..\..\..\source\core\slang-byte-encode-util.cpp" /> <ClCompile Include="..\..\..\source\core\slang-char-util.cpp" /> + <ClCompile Include="..\..\..\source\core\slang-deflate-compression-system.cpp" /> <ClCompile Include="..\..\..\source\core\slang-downstream-compiler.cpp" /> <ClCompile Include="..\..\..\source\core\slang-free-list.cpp" /> <ClCompile Include="..\..\..\source\core\slang-gcc-compiler-util.cpp" /> <ClCompile Include="..\..\..\source\core\slang-hex-dump-util.cpp" /> <ClCompile Include="..\..\..\source\core\slang-io.cpp" /> + <ClCompile Include="..\..\..\source\core\slang-lz4-compression-system.cpp" /> <ClCompile Include="..\..\..\source\core\slang-memory-arena.cpp" /> <ClCompile Include="..\..\..\source\core\slang-name-convention-util.cpp" /> <ClCompile Include="..\..\..\source\core\slang-nvrtc-compiler.cpp" /> @@ -235,6 +243,7 @@ <ClCompile Include="..\..\..\source\core\slang-platform.cpp" /> <ClCompile Include="..\..\..\source\core\slang-random-generator.cpp" /> <ClCompile Include="..\..\..\source\core\slang-render-api-util.cpp" /> + <ClCompile Include="..\..\..\source\core\slang-riff-file-system.cpp" /> <ClCompile Include="..\..\..\source\core\slang-riff.cpp" /> <ClCompile Include="..\..\..\source\core\slang-semantic-version.cpp" /> <ClCompile Include="..\..\..\source\core\slang-shared-library.cpp" /> diff --git a/build/visual-studio/core/core.vcxproj.filters b/build/visual-studio/core/core.vcxproj.filters index 9a680cb39..99cc6ba6a 100644 --- a/build/visual-studio/core/core.vcxproj.filters +++ b/build/visual-studio/core/core.vcxproj.filters @@ -12,6 +12,9 @@ <ClInclude Include="..\..\..\source\core\slang-allocator.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\..\..\source\core\slang-archive-file-system.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="..\..\..\source\core\slang-array-view.h"> <Filter>Header Files</Filter> </ClInclude> @@ -33,6 +36,12 @@ <ClInclude Include="..\..\..\source\core\slang-common.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\..\..\source\core\slang-compression-system.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\..\..\source\core\slang-deflate-compression-system.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="..\..\..\source\core\slang-dictionary.h"> <Filter>Header Files</Filter> </ClInclude> @@ -63,6 +72,9 @@ <ClInclude Include="..\..\..\source\core\slang-list.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\..\..\source\core\slang-lz4-compression-system.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="..\..\..\source\core\slang-math.h"> <Filter>Header Files</Filter> </ClInclude> @@ -90,6 +102,9 @@ <ClInclude Include="..\..\..\source\core\slang-render-api-util.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\..\..\source\core\slang-riff-file-system.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="..\..\..\source\core\slang-riff.h"> <Filter>Header Files</Filter> </ClInclude> @@ -155,6 +170,9 @@ </ClInclude> </ItemGroup> <ItemGroup> + <ClCompile Include="..\..\..\source\core\slang-archive-file-system.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\..\source\core\slang-blob.cpp"> <Filter>Source Files</Filter> </ClCompile> @@ -164,6 +182,9 @@ <ClCompile Include="..\..\..\source\core\slang-char-util.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\..\source\core\slang-deflate-compression-system.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\..\source\core\slang-downstream-compiler.cpp"> <Filter>Source Files</Filter> </ClCompile> @@ -179,6 +200,9 @@ <ClCompile Include="..\..\..\source\core\slang-io.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\..\source\core\slang-lz4-compression-system.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\..\source\core\slang-memory-arena.cpp"> <Filter>Source Files</Filter> </ClCompile> @@ -200,6 +224,9 @@ <ClCompile Include="..\..\..\source\core\slang-render-api-util.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\..\source\core\slang-riff-file-system.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\..\source\core\slang-riff.cpp"> <Filter>Source Files</Filter> </ClCompile> diff --git a/build/visual-studio/slang-test/slang-test.vcxproj b/build/visual-studio/slang-test/slang-test.vcxproj index 4a56eb437..1626b6cca 100644 --- a/build/visual-studio/slang-test/slang-test.vcxproj +++ b/build/visual-studio/slang-test/slang-test.vcxproj @@ -196,6 +196,9 @@ <ProjectReference Include="..\miniz\miniz.vcxproj"> <Project>{E76ACB11-4A12-4F0A-BE1E-CE0B8836EB7F}</Project> </ProjectReference> + <ProjectReference Include="..\lz4\lz4.vcxproj"> + <Project>{E1EC8075-823E-46E5-BC38-C124CCCDF878}</Project> + </ProjectReference> </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> diff --git a/build/visual-studio/slang/slang.vcxproj b/build/visual-studio/slang/slang.vcxproj index 88c4b59e2..642ddab7a 100644 --- a/build/visual-studio/slang/slang.vcxproj +++ b/build/visual-studio/slang/slang.vcxproj @@ -443,6 +443,9 @@ <ProjectReference Include="..\miniz\miniz.vcxproj"> <Project>{E76ACB11-4A12-4F0A-BE1E-CE0B8836EB7F}</Project> </ProjectReference> + <ProjectReference Include="..\lz4\lz4.vcxproj"> + <Project>{E1EC8075-823E-46E5-BC38-C124CCCDF878}</Project> + </ProjectReference> </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> diff --git a/external/lz4 b/external/lz4 new file mode 160000 +Subproject d44371841a2f1728a3f36839fd4b7e872d0927d diff --git a/premake5.lua b/premake5.lua index 6fd2788a5..36a7ead0a 100644 --- a/premake5.lua +++ b/premake5.lua @@ -658,7 +658,7 @@ tool "slang-embed" tool "slang-test" uuid "0C768A18-1D25-4000-9F37-DA5FE99E3B64" includedirs { "." } - links { "core", "slang", "miniz" } + links { "core", "slang", "miniz", "lz4" } -- We want to set to the root of the project, but that doesn't seem to work with '.'. -- So set a path that resolves to the same place. @@ -982,7 +982,7 @@ if enableEmbedStdLib then standardProject("slangc-bootstrap", "source/slangc") uuid "6339BF31-AC99-4819-B719-679B63451EF0" kind "ConsoleApp" - links { "core", "miniz" } + links { "core", "miniz", "lz4" } -- We need to run all the generators to be able to build the main -- slang source in source/slang @@ -1057,7 +1057,7 @@ if enableEmbedStdLib then buildinputs { "%{cfg.targetdir}/slangc-bootstrap" .. executableSuffix } - local buildcmd = '"%{cfg.targetdir}/slangc-bootstrap" -save-stdlib-bin-source %{file.directory}/slang-stdlib-generated.h' + local buildcmd = '"%{cfg.targetdir}/slangc-bootstrap" -archive-type riff-lz4 -save-stdlib-bin-source %{file.directory}/slang-stdlib-generated.h' buildcommands { buildcmd } end @@ -1082,7 +1082,7 @@ end standardProject("slang", "source/slang") uuid "DB00DA62-0533-4AFD-B59F-A67D5B3A0808" kind "SharedLib" - links { "core", "miniz"} + links { "core", "miniz", "lz4"} warnings "Extra" flags { "FatalWarnings" } pic "On" @@ -1183,7 +1183,7 @@ if enableProfile then addSourceDir "source/slang" includedirs { "." } - links { "core", "miniz"} + links { "core", "miniz", "lz4"} filter { "system:linux" } linkoptions{ "-pg" } @@ -1208,6 +1208,20 @@ standardProject("miniz", nil) filter { "system:linux or macosx" } links { "dl"} +standardProject("lz4", nil) + uuid "E1EC8075-823E-46E5-BC38-C124CCCDF878" + kind "StaticLib" + pic "On" + + -- Add the files explicitly + files + { + "external/lz4/lib/lz4.c", + "external/lz4/lib/lz4.h", + } + + filter { "system:linux or macosx" } + links { "dl"} if buildGlslang then @@ -477,7 +477,8 @@ extern "C" #endif typedef bool SlangBool; - + + /*! @brief Severity of a diagnostic generated by the compiler. Values come from the enum below, with higher values representing more severe @@ -558,6 +559,18 @@ extern "C" SLANG_PASS_THROUGH_COUNT_OF, }; + /* Defines an archive type used to holds a 'file system' type structure. */ + typedef int SlangArchiveTypeIntegral; + enum SlangArchiveType : SlangArchiveTypeIntegral + { + SLANG_ARCHIVE_TYPE_UNDEFINED, + SLANG_ARCHIVE_TYPE_ZIP, + SLANG_ARCHIVE_TYPE_RIFF, ///< Riff container with no compression + SLANG_ARCHIVE_TYPE_RIFF_DEFLATE, + SLANG_ARCHIVE_TYPE_RIFF_LZ4, + SLANG_ARCHIVE_TYPE_COUNT_OF, + }; + /*! Flags to control compilation behavior. */ @@ -3125,10 +3138,11 @@ namespace slang virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadStdLib(const void* stdLib, size_t stdLibSizeInBytes) = 0; /** Save the StdLib modules to the file system + @param archiveType The type of archive used to hold the stdlib @param outBlob The serialized blob containing the standard library NOTE! API is experimental and not ready for production code */ - virtual SLANG_NO_THROW SlangResult SLANG_MCALL saveStdLib(ISlangBlob** outBlob) = 0; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL saveStdLib(SlangArchiveType archiveType, ISlangBlob** outBlob) = 0; /** Look up the internal ID of a capability by its `name`. @@ -26,6 +26,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "run-generators", "build\vis {7F773DD9-EB8F-2403-B43C-B49C2014B99C} = {7F773DD9-EB8F-2403-B43C-B49C2014B99C} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "build\visual-studio\lz4\lz4.vcxproj", "{E1EC8075-823E-46E5-BC38-C124CCCDF878}" +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "miniz", "build\visual-studio\miniz\miniz.vcxproj", "{E76ACB11-4A12-4F0A-BE1E-CE0B8836EB7F}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "slang", "build\visual-studio\slang\slang.vcxproj", "{DB00DA62-0533-4AFD-B59F-A67D5B3A0808}" @@ -127,6 +129,14 @@ Global {E145B2B8-CD13-A6BE-B6A7-16E5A2148223}.Release|Win32.Build.0 = Release|Win32 {E145B2B8-CD13-A6BE-B6A7-16E5A2148223}.Release|x64.ActiveCfg = Release|x64 {E145B2B8-CD13-A6BE-B6A7-16E5A2148223}.Release|x64.Build.0 = Release|x64 + {E1EC8075-823E-46E5-BC38-C124CCCDF878}.Debug|Win32.ActiveCfg = Debug|Win32 + {E1EC8075-823E-46E5-BC38-C124CCCDF878}.Debug|Win32.Build.0 = Debug|Win32 + {E1EC8075-823E-46E5-BC38-C124CCCDF878}.Debug|x64.ActiveCfg = Debug|x64 + {E1EC8075-823E-46E5-BC38-C124CCCDF878}.Debug|x64.Build.0 = Debug|x64 + {E1EC8075-823E-46E5-BC38-C124CCCDF878}.Release|Win32.ActiveCfg = Release|Win32 + {E1EC8075-823E-46E5-BC38-C124CCCDF878}.Release|Win32.Build.0 = Release|Win32 + {E1EC8075-823E-46E5-BC38-C124CCCDF878}.Release|x64.ActiveCfg = Release|x64 + {E1EC8075-823E-46E5-BC38-C124CCCDF878}.Release|x64.Build.0 = Release|x64 {E76ACB11-4A12-4F0A-BE1E-CE0B8836EB7F}.Debug|Win32.ActiveCfg = Debug|Win32 {E76ACB11-4A12-4F0A-BE1E-CE0B8836EB7F}.Debug|Win32.Build.0 = Debug|Win32 {E76ACB11-4A12-4F0A-BE1E-CE0B8836EB7F}.Debug|x64.ActiveCfg = Debug|x64 diff --git a/source/core/slang-archive-file-system.cpp b/source/core/slang-archive-file-system.cpp new file mode 100644 index 000000000..8c4d0ef2d --- /dev/null +++ b/source/core/slang-archive-file-system.cpp @@ -0,0 +1,183 @@ +#include "slang-archive-file-system.h" + +#include "../../slang-com-helper.h" +#include "../../slang-com-ptr.h" + +#include "slang-io.h" +#include "slang-string-util.h" +#include "slang-blob.h" +#include "slang-string-slice-pool.h" +#include "slang-uint-set.h" + +#include "slang-riff-file-system.h" + +// Compression systems +#include "slang-deflate-compression-system.h" +#include "slang-lz4-compression-system.h" + +// Zip file system +#include "slang-zip-file-system.h" + +#include "slang-riff.h" + +namespace Slang +{ + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! StringSliceIndexMap !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +StringSliceIndexMap::CountIndex StringSliceIndexMap::add(const UnownedStringSlice& key, Index valueIndex) +{ + StringSlicePool::Handle handle; + m_pool.findOrAdd(key, handle); + const CountIndex countIndex = StringSlicePool::asIndex(handle); + if (countIndex >= m_indexMap.getCount()) + { + SLANG_ASSERT(countIndex == m_indexMap.getCount()); + m_indexMap.add(valueIndex); + } + else + { + m_indexMap[countIndex] = valueIndex; + } + return countIndex; +} + +StringSliceIndexMap::CountIndex StringSliceIndexMap::findOrAdd(const UnownedStringSlice& key, Index defaultValueIndex) +{ + StringSlicePool::Handle handle; + m_pool.findOrAdd(key, handle); + const CountIndex countIndex = StringSlicePool::asIndex(handle); + if (countIndex >= m_indexMap.getCount()) + { + SLANG_ASSERT(countIndex == m_indexMap.getCount()); + m_indexMap.add(defaultValueIndex); + } + return countIndex; +} + +void StringSliceIndexMap::clear() +{ + m_pool.clear(); + m_indexMap.clear(); +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ImplicitDirectoryCollector !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +ImplicitDirectoryCollector::ImplicitDirectoryCollector(const String& canonicalPath, bool directoryExists) : + m_directoryExists(directoryExists) +{ + StringBuilder buffer; + if (canonicalPath != ".") + { + buffer << canonicalPath; + buffer.append('/'); + } + m_prefix = buffer.ProduceString(); +} + +void ImplicitDirectoryCollector::addRemainingPath(SlangPathType pathType, const UnownedStringSlice& inPathRemainder) +{ + // If it's zero length we probably don't want to add it + if (inPathRemainder.getLength() == 0) + { + // It's empty so don't add normal way - implies the directory exists + m_directoryExists = true; + return; + } + + UnownedStringSlice pathRemainder(inPathRemainder); + const Index slashIndex = pathRemainder.indexOf('/'); + + // If we have a following / that means it's an implicit directory. + if (slashIndex >= 0) + { + pathType = SLANG_PATH_TYPE_DIRECTORY; + pathRemainder = UnownedStringSlice(pathRemainder.begin(), pathRemainder.begin() + slashIndex); + } + + const Index countIndex = m_map.findOrAdd(pathRemainder, pathType); + // Make sure they are the same type + SLANG_ASSERT(SlangPathType(m_map.getValueAt(countIndex)) == pathType); +} + +void ImplicitDirectoryCollector::addPath(SlangPathType pathType, const UnownedStringSlice& canonicalPath) +{ + if (hasPrefix(canonicalPath)) + { + UnownedStringSlice remainder = getRemainder(canonicalPath); + addRemainingPath(pathType, remainder); + } +} + +SlangResult ImplicitDirectoryCollector::enumerate(FileSystemContentsCallBack callback, void* userData) +{ + const Int count = m_map.getCount(); + + for (Index i = 0; i < count; ++i) + { + const auto& pair = m_map.getAt(i); + + UnownedStringSlice path = pair.Key; + SlangPathType pathType = SlangPathType(pair.Value); + + // Note *is* 0 terminated in the pool + // Let's check tho + SLANG_ASSERT(path.begin()[path.getLength()] == 0); + callback(pathType, path.begin(), userData); + } + + return getDirectoryExists() ? SLANG_OK : SLANG_E_NOT_FOUND; +} + +SlangResult loadArchiveFileSystem(const void* data, size_t dataSizeInBytes, RefPtr<ArchiveFileSystem>& outFileSystem) +{ + RefPtr<ArchiveFileSystem> fileSystem; + if (ZipFileSystem::isArchive(data, dataSizeInBytes)) + { + // It's a zip + SLANG_RETURN_ON_FAIL(ZipFileSystem::create(fileSystem)); + } + else if (RiffFileSystem::isArchive(data, dataSizeInBytes)) + { + // It's riff contained (Slang specific) + fileSystem = new RiffFileSystem(nullptr); + } + else + { + return SLANG_FAIL; + } + SLANG_RETURN_ON_FAIL(fileSystem->loadArchive(data, dataSizeInBytes)); + + outFileSystem = fileSystem; + return SLANG_OK; +} + +SlangResult createArchiveFileSystem(SlangArchiveType type, RefPtr<ArchiveFileSystem>& outFileSystem) +{ + switch (type) + { + case SLANG_ARCHIVE_TYPE_ZIP: + { + return ZipFileSystem::create(outFileSystem); + } + case SLANG_ARCHIVE_TYPE_RIFF: + { + outFileSystem = new RiffFileSystem(nullptr); + return SLANG_OK; + } + case SLANG_ARCHIVE_TYPE_RIFF_DEFLATE: + { + outFileSystem = new RiffFileSystem(DeflateCompressionSystem::getSingleton()); + return SLANG_OK; + } + case SLANG_ARCHIVE_TYPE_RIFF_LZ4: + { + outFileSystem = new RiffFileSystem(LZ4CompressionSystem::getSingleton()); + return SLANG_OK; + } + } + + return SLANG_FAIL; +} + +} // namespace Slang diff --git a/source/core/slang-archive-file-system.h b/source/core/slang-archive-file-system.h new file mode 100644 index 000000000..6b4fe9e51 --- /dev/null +++ b/source/core/slang-archive-file-system.h @@ -0,0 +1,153 @@ +#ifndef SLANG_ARCHIVE_FILE_SYSTEM_H +#define SLANG_ARCHIVE_FILE_SYSTEM_H + +#include "slang-basic.h" + +#include "../../slang-com-ptr.h" + +#include "slang-compression-system.h" + +#include "slang-string-slice-pool.h" + +namespace Slang +{ + +class ArchiveFileSystem : public RefObject, public ISlangMutableFileSystem +{ +public: + /// Loads an archive. + virtual SlangResult loadArchive(const void* archive, size_t archiveSizeInBytes) = 0; + /// Get as an archive (that can be saved to disk) + /// NOTE! If the blob is not owned, it's contents can be invalidated by any call to a method of the file system or loss of scope + virtual SlangResult storeArchive(bool blobOwnsContent, ISlangBlob** outBlob) = 0; + /// Set the compression - used for any subsequent items added + virtual void setCompressionStyle(const CompressionStyle& style) = 0; +}; + +/* Maps an UnownedStringSlice to an index. All substrings are held internally in a StringSlicePool, and so +owned by the type. */ +class StringSliceIndexMap +{ +public: + /// An index that identifies a key value pair. + typedef Index CountIndex; + + /// Adds a key, value pair. Returns the CountIndex of the pair. + /// If there is already a value stored for the key it is replaced. + CountIndex add(const UnownedStringSlice& key, Index valueIndex); + + /// Finds or adds the slice. If the slice is added the defaultValueIndex is set. + /// If not the index associated with the slice remains the same. + /// Returns the CountIndex where the key,value pair are stored + CountIndex findOrAdd(const UnownedStringSlice& key, Index defaultValueIndex); + + /// Gets the index associated with the key. Returns -1 if there is no associated index. + SLANG_FORCE_INLINE Index getValue(const UnownedStringSlice& key); + + /// Get the amount of pairs in the map + Index getCount() const { return m_indexMap.getCount(); } + + /// Get the slice and the index at the specified index + SLANG_INLINE KeyValuePair<UnownedStringSlice, Index> getAt(CountIndex countIndex) const; + + /// Clear the contents of the map + void clear(); + + /// Get the key at the specified index + UnownedStringSlice getKeyAt(CountIndex index) const { return m_pool.getSlice(StringSlicePool::Handle(index)); } + /// Get the value at the specified index + Index& getValueAt(CountIndex index) { return m_indexMap[index]; } + + /// Get the amount of key,value pairs + Index getCount() { return m_indexMap.getCount(); } + + /// Ctor + StringSliceIndexMap() : + m_pool(StringSlicePool::Style::Empty) + { + } + +protected: + StringSlicePool m_pool; ///< Pool holds the substrings + List<Index> m_indexMap; ///< Maps a pool index to the output index +}; + +// --------------------------------------------------------------------------- +Index StringSliceIndexMap::getValue(const UnownedStringSlice& key) +{ + const Index poolIndex = m_pool.findIndex(key); + return (poolIndex >= 0) ? m_indexMap[poolIndex] : -1; +} + +// --------------------------------------------------------------------------- +KeyValuePair<UnownedStringSlice, Index> StringSliceIndexMap::getAt(CountIndex countIndex) const +{ + KeyValuePair<UnownedStringSlice, Index> pair; + pair.Key = m_pool.getSlice(StringSlicePool::Handle(countIndex)); + pair.Value = m_indexMap[countIndex]; + return pair; +} + + +/* This class helps to find the contents and/or existence of an implicit directory.This finds the contents of a directory. + +This is achieved by using a path prefix that any contained path must at least match. If the remainder of the path contains a folder + - detectable because it's not a leaf and so contains a delimiter - that directory is added. As a sub folder may contain many + files, and the directory itself may also be defined, it is necessary to dedup. The deduping is handled by the StringSliceIndexMap. */ +class ImplicitDirectoryCollector +{ +public: + + enum class State + { + None, ///< Neither the directory or content have been found + DirectoryExists, ///< The directory exists + HasContent, ///< If it has content, the directory must exist + }; + + /// Get the current state + State getState() const { return (m_map.getCount() > 0) ? State::HasContent : (m_directoryExists ? State::DirectoryExists : State::None); } + /// True if collector at least has the specified state + bool hasState(State state) { return Index(getState()) >= Index(state); } + + /// Set that it exists + void setDirectoryExists(bool directoryExists) { m_directoryExists = directoryExists; } + /// Get if it exists (implicitly or explicitly) + bool getDirectoryExists() const { return m_directoryExists || m_map.getCount() > 0; } + + /// True if the path matches the prefix + bool hasPrefix(const UnownedStringSlice& path) const { return path.startsWith(m_prefix.getUnownedSlice()); } + + /// True if the directory has content + bool hasContent() const { return m_map.getCount() > 0; } + + /// Gets the remainder or path after the prefix + UnownedStringSlice getRemainder(const UnownedStringSlice& path) const + { + SLANG_ASSERT(hasPrefix(path)); + return UnownedStringSlice(path.begin() + m_prefix.getLength(), path.end()); + } + + /// Add a remaining path + void addRemainingPath(SlangPathType pathType, const UnownedStringSlice& inPathRemainder); + /// Add a path + void addPath(SlangPathType pathType, const UnownedStringSlice& canonicalPath); + /// Enumerate the contents + SlangResult enumerate(FileSystemContentsCallBack callback, void* userData); + + /// Ctor + ImplicitDirectoryCollector(const String& canonicalPath, bool directoryExists = false); + + protected: + StringSliceIndexMap m_map; + String m_prefix; + bool m_directoryExists; +}; + + +SlangResult loadArchiveFileSystem(const void* data, size_t dataSizeInBytes, RefPtr<ArchiveFileSystem>& outFileSystem); +SlangResult createArchiveFileSystem(SlangArchiveType type, RefPtr<ArchiveFileSystem>& outFileSystem); + +} + +#endif diff --git a/source/core/slang-blob.h b/source/core/slang-blob.h index c98c41563..58984471f 100644 --- a/source/core/slang-blob.h +++ b/source/core/slang-blob.h @@ -93,6 +93,15 @@ public: } m_sizeInBytes = 0; } + // Reallocate so the buffer is the specified size. Contents of buffer up to size remain intact. + void reallocate(size_t size) + { + if (size != m_sizeInBytes) + { + m_data = ::realloc(m_data, size); + m_sizeInBytes = size; + } + } /// Makes this no longer own the allocation. Returns the allocated data (or nullptr if no allocation) void* detach() { @@ -189,7 +198,28 @@ protected: RawBlob() = default; ScopedAllocation m_data; +}; + +// A blob that does not own it's contained data. +class UnownedRawBlob : public BlobBase +{ +public: + // ISlangBlob + SLANG_NO_THROW void const* SLANG_MCALL getBufferPointer() SLANG_OVERRIDE { return m_data; } + SLANG_NO_THROW size_t SLANG_MCALL getBufferSize() SLANG_OVERRIDE { return m_dataSizeInBytes; } + // Ctor + UnownedRawBlob(const void* data, size_t size): + m_data(data), + m_dataSizeInBytes(size) + { + } + +protected: + UnownedRawBlob() = default; + + const void* m_data; + size_t m_dataSizeInBytes; }; /** A Blob that has no ref counting and exists typically for entire execution. diff --git a/source/core/slang-compression-system.h b/source/core/slang-compression-system.h new file mode 100644 index 000000000..f9fe9dfe1 --- /dev/null +++ b/source/core/slang-compression-system.h @@ -0,0 +1,58 @@ +#ifndef SLANG_COMPRESSION_SYSTEM_H +#define SLANG_COMPRESSION_SYSTEM_H + +#include "slang-basic.h" + +namespace Slang +{ + +struct CompressionStyle +{ + enum class Type + { + Level, ///< Use the value specified in 'level' to control compression + BestSpeed, ///< Best for speed (typically lower compression ration) + BestCompression, ///< Best compression (typically slower) + Default, ///< Default compression (a good balance between speed and size) + }; + Type m_type = Type::Default; ///< The type + float m_level = 1.0f; ///< 0 lowest compression, 1 highest compression (Ignored if m_type != Type::Level) +}; + +enum class CompressionSystemType +{ + None, + Deflate, + LZ4, + CountOf, +}; + +class ICompressionSystem : public ISlangUnknown +{ +public: + + /** Get the compression system type + @return The compression system type */ + virtual SLANG_NO_THROW CompressionSystemType SLANG_MCALL getSystemType() = 0; + + /** compress + @param src Points to the start of the data to compress + @param srcSizeInBytes The size of the source data to compress in bytes + @param outBlob The input data compressed + @return SLANG_OK if successful */ + virtual SLANG_NO_THROW SlangResult SLANG_MCALL compress(const CompressionStyle* style, const void* src, size_t srcSizeInBytes, ISlangBlob** outBlob) = 0; + + /* decompress + @param compressed The start of the compressed data + @param compressedSizeInBytes The compressed size in bytes + @param decompressedSizeInBytes The size of the decompressed buffer. MUST be exactly the same as the original source size. + @param outDecompressed Where decompressed data is written + @return SLANG_OK if successful */ + virtual SLANG_NO_THROW SlangResult SLANG_MCALL decompress(const void* compressed, size_t compressedSizeInBytes, size_t decompressedSizeInBytes, void* outDecompressed) = 0; +}; + +#define SLANG_UUID_ICompressionSystem { 0xcc935840, 0xe059, 0x4bb8, { 0xa2, 0x2d, 0x92, 0x7b, 0x3c, 0x73, 0x8f, 0x85 } }; + +} + +#endif diff --git a/source/core/slang-deflate-compression-system.cpp b/source/core/slang-deflate-compression-system.cpp new file mode 100644 index 000000000..a316be122 --- /dev/null +++ b/source/core/slang-deflate-compression-system.cpp @@ -0,0 +1,89 @@ +#include "slang-deflate-compression-system.h" + +#include "../../slang-com-helper.h" +#include "../../slang-com-ptr.h" + +// We don't want compress #define to clash +#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES 1 + +#include "../../external/miniz/miniz.h" +#include "../../external/miniz/miniz_common.h" +#include "../../external/miniz/miniz_tdef.h" +#include "../../external/miniz/miniz_tinfl.h" + +#include "slang-blob.h" + +namespace Slang +{ + +// Allocate static const storage for the various interface IDs that the Slang API needs to expose +static const Guid IID_ISlangUnknown = SLANG_UUID_ISlangUnknown; +static const Guid IID_ICompressionSystem = SLANG_UUID_ICompressionSystem; + +class DeflateCompressionSystemImpl : public RefObject, public ICompressionSystem +{ +public: + // ISlangUnknown + // override ref counting, as singleton + SLANG_IUNKNOWN_QUERY_INTERFACE + + SLANG_NO_THROW uint32_t SLANG_MCALL addRef() SLANG_OVERRIDE { return 1; } + SLANG_NO_THROW uint32_t SLANG_MCALL release() SLANG_OVERRIDE { return 1; } + + // ICompressionSystem + virtual SLANG_NO_THROW CompressionSystemType SLANG_MCALL getSystemType() SLANG_OVERRIDE { return CompressionSystemType::Deflate; } + virtual SLANG_NO_THROW SlangResult SLANG_MCALL compress(const CompressionStyle* style, const void* src, size_t srcSizeInBytes, ISlangBlob** outBlob) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL decompress(const void* compressed, size_t compressedSizeInBytes, size_t decompressedSizeInBytes, void* outDecompressed) SLANG_OVERRIDE; + +protected: + + ICompressionSystem* getInterface(const Guid& guid); +}; + +ICompressionSystem* DeflateCompressionSystemImpl::getInterface(const Guid& guid) +{ + return (guid == IID_ISlangUnknown || guid == IID_ICompressionSystem) ? static_cast<ICompressionSystem*>(this) : nullptr; +} + +SlangResult DeflateCompressionSystemImpl::compress(const CompressionStyle* style, const void* src, size_t srcSizeInBytes, ISlangBlob** outBlob) +{ + SLANG_UNUSED(style); + + size_t compressedSizeInBytes; + + const int flags = 0; + void* compressed = tdefl_compress_mem_to_heap(src, srcSizeInBytes, &compressedSizeInBytes, 0); + + if (!compressed) + { + return SLANG_FAIL; + } + + ScopedAllocation alloc; + alloc.attach(compressed, compressedSizeInBytes); + + auto blob = RawBlob::moveCreate(alloc); + *outBlob = blob.detach(); + return SLANG_OK; +} + +SlangResult DeflateCompressionSystemImpl::decompress(const void* compressed, size_t compressedSizeInBytes, size_t decompressedSizeInBytes, void* outDecompressed) +{ + const int flags = TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF; + + size_t size = tinfl_decompress_mem_to_mem(outDecompressed, decompressedSizeInBytes, compressed ,compressedSizeInBytes, flags); + if (size == TINFL_DECOMPRESS_MEM_TO_MEM_FAILED) + { + return SLANG_FAIL; + } + + return SLANG_OK; +} + +/* static */ICompressionSystem* DeflateCompressionSystem::getSingleton() +{ + static DeflateCompressionSystemImpl impl; + return &impl; +} + +} // namespace Slang diff --git a/source/core/slang-deflate-compression-system.h b/source/core/slang-deflate-compression-system.h new file mode 100644 index 000000000..6dc96af5d --- /dev/null +++ b/source/core/slang-deflate-compression-system.h @@ -0,0 +1,22 @@ +#ifndef SLANG_DEFLATE_COMPRESSION_SYSTEM_H +#define SLANG_DEFLATE_COMPRESSION_SYSTEM_H + +#include "slang-basic.h" + +#include "slang-compression-system.h" + +#include "../../slang-com-ptr.h" + +namespace Slang +{ + +class DeflateCompressionSystem +{ +public: + /* Get the Deflate compression system singleton. */ + static ICompressionSystem* getSingleton(); +}; + +} + +#endif diff --git a/source/core/slang-io.cpp b/source/core/slang-io.cpp index 2e2ec3d8b..d4f603109 100644 --- a/source/core/slang-io.cpp +++ b/source/core/slang-io.cpp @@ -3,6 +3,8 @@ #include "../../slang-com-helper.h" +#include "slang-string-util.h" + #ifndef __STDC__ # define __STDC__ 1 #endif @@ -400,52 +402,56 @@ namespace Slang return false; } - /* static */String Path::simplify(const UnownedStringSlice& path) + /* static */void Path::simplify(List<UnownedStringSlice>& ioSplit) { - List<UnownedStringSlice> splitPath; - split(path, splitPath); - // Strictly speaking we could do something about case on platforms like window, but here we won't worry about that - for (Index i = 0; i < splitPath.getCount(); i++) + for (Index i = 0; i < ioSplit.getCount(); i++) { - const UnownedStringSlice& cur = splitPath[i]; - if (cur == "." && splitPath.getCount() > 1) + const UnownedStringSlice& cur = ioSplit[i]; + if (cur == "." && ioSplit.getCount() > 1) { // Just remove it - splitPath.removeAt(i); + ioSplit.removeAt(i); i--; } else if (cur == ".." && i > 0) { // Can we remove this and the one before ? - UnownedStringSlice& before = splitPath[i - 1]; + UnownedStringSlice& before = ioSplit[i - 1]; if (before == ".." || (i == 1 && isDriveSpecification(before))) { - // Can't do it + // Can't do it, but we allow relative, so just leave for now continue; } - splitPath.removeRange(i - 1, 2); + ioSplit.removeRange(i - 1, 2); i -= 2; } } + } - // If its empty it must be . - if (splitPath.getCount() == 0) + /* static */void Path::join(const UnownedStringSlice* slices, Index count, StringBuilder& out) + { + out.Clear(); + + if (count == 0) { - splitPath.add(UnownedStringSlice::fromLiteral(".")); + out << "."; + return; } - + + StringUtil::join(slices, count, kPathDelimiter, out); + } + + + /* static */String Path::simplify(const UnownedStringSlice& path) + { + List<UnownedStringSlice> splitPath; + split(path, splitPath); + simplify(splitPath); + // Reconstruct the string StringBuilder builder; - for (Index i = 0; i < splitPath.getCount(); i++) - { - if (i > 0) - { - builder.Append(kPathDelimiter); - } - builder.Append(splitPath[i]); - } - + join(splitPath.getBuffer(), splitPath.getCount(), builder); return builder.ToString(); } diff --git a/source/core/slang-io.h b/source/core/slang-io.h index 5a611c445..ac3156b8a 100644 --- a/source/core/slang-io.h +++ b/source/core/slang-io.h @@ -104,6 +104,12 @@ namespace Slang static String simplify(const UnownedStringSlice& path); static String simplify(const String& path) { return simplify(path.getUnownedSlice()); } + /// Simplifies the path split up + static void simplify(List<UnownedStringSlice>& ioSplit); + + /// Join the parts of the path to produce an output path + static void join(const UnownedStringSlice* slices, Index count, StringBuilder& out); + /// Returns true if the path is absolute static bool isAbsolute(const UnownedStringSlice& path); static bool isAbsolute(const String& path) { return isAbsolute(path.getUnownedSlice()); } diff --git a/source/core/slang-lz4-compression-system.cpp b/source/core/slang-lz4-compression-system.cpp new file mode 100644 index 000000000..fa5c5f5ab --- /dev/null +++ b/source/core/slang-lz4-compression-system.cpp @@ -0,0 +1,71 @@ +#include "slang-lz4-compression-system.h" + +#include "../../slang-com-helper.h" +#include "../../slang-com-ptr.h" + +#include "slang-blob.h" + +#include "../../external/lz4/lib/lz4.h" + +namespace Slang +{ + +// Allocate static const storage for the various interface IDs that the Slang API needs to expose +static const Guid IID_ISlangUnknown = SLANG_UUID_ISlangUnknown; +static const Guid IID_ICompressionSystem = SLANG_UUID_ICompressionSystem; + +class LZ4CompressionSystemImpl : public RefObject, public ICompressionSystem +{ +public: + // ISlangUnknown + // override ref counting, as singleton + SLANG_IUNKNOWN_QUERY_INTERFACE + SLANG_NO_THROW uint32_t SLANG_MCALL addRef() SLANG_OVERRIDE { return 1; } + SLANG_NO_THROW uint32_t SLANG_MCALL release() SLANG_OVERRIDE { return 1; } + + // ICompressionSystem + virtual SLANG_NO_THROW CompressionSystemType SLANG_MCALL getSystemType() SLANG_OVERRIDE { return CompressionSystemType::LZ4; } + virtual SLANG_NO_THROW SlangResult SLANG_MCALL compress(const CompressionStyle* style, const void* src, size_t srcSizeInBytes, ISlangBlob** outBlob) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL decompress(const void* compressed, size_t compressedSizeInBytes, size_t decompressedSizeInBytes, void* outDecompressed) SLANG_OVERRIDE; + +protected: + + ICompressionSystem* getInterface(const Guid& guid); +}; + +ICompressionSystem* LZ4CompressionSystemImpl::getInterface(const Guid& guid) +{ + return (guid == IID_ISlangUnknown || guid == IID_ICompressionSystem) ? static_cast<ICompressionSystem*>(this) : nullptr; +} + +SlangResult LZ4CompressionSystemImpl::compress(const CompressionStyle* style, const void* src, size_t srcSizeInBytes, ISlangBlob** outBlob) +{ + SLANG_UNUSED(style); + const size_t compressedBound = LZ4_compressBound(int(srcSizeInBytes)); + + ScopedAllocation alloc; + void* compressedData = alloc.allocate(compressedBound); + + const int compressedSize = LZ4_compress_default((const char*)src, (char*)compressedData, int(srcSizeInBytes), int(compressedBound)); + alloc.reallocate(compressedSize); + + auto blob = RawBlob::moveCreate(alloc); + + *outBlob = blob.detach(); + return SLANG_OK; +} + +SlangResult LZ4CompressionSystemImpl::decompress(const void* compressed, size_t compressedSizeInBytes, size_t decompressedSizeInBytes, void* outDecompressed) +{ + const int decompressedSize = LZ4_decompress_safe((const char*)compressed, (char*)outDecompressed, int(compressedSizeInBytes), int(decompressedSizeInBytes)); + SLANG_ASSERT(size_t(decompressedSize) == decompressedSizeInBytes); + return SLANG_OK; +} + +/* static */ICompressionSystem* LZ4CompressionSystem::getSingleton() +{ + static LZ4CompressionSystemImpl impl; + return &impl; +} + +} // namespace Slang diff --git a/source/core/slang-lz4-compression-system.h b/source/core/slang-lz4-compression-system.h new file mode 100644 index 000000000..1bd7cefcf --- /dev/null +++ b/source/core/slang-lz4-compression-system.h @@ -0,0 +1,22 @@ +#ifndef SLANG_LZ4_COMPRESSION_SYSTEM_H +#define SLANG_LZ4_COMPRESSION_SYSTEM_H + +#include "slang-basic.h" + +#include "slang-compression-system.h" + +#include "../../slang-com-ptr.h" + +namespace Slang +{ + +class LZ4CompressionSystem +{ +public: + /* Get the LZ4 compression system singleton. */ + static ICompressionSystem* getSingleton(); +}; + +} + +#endif diff --git a/source/core/slang-riff-file-system.cpp b/source/core/slang-riff-file-system.cpp new file mode 100644 index 000000000..7084e1346 --- /dev/null +++ b/source/core/slang-riff-file-system.cpp @@ -0,0 +1,467 @@ +#include "slang-riff-file-system.h" + +#include "../../slang-com-helper.h" +#include "../../slang-com-ptr.h" + +#include "slang-io.h" +#include "slang-string-util.h" +#include "slang-blob.h" +#include "slang-string-slice-pool.h" +#include "slang-uint-set.h" + +// Compression systems +#include "slang-deflate-compression-system.h" +#include "slang-lz4-compression-system.h" + +namespace Slang +{ + +// Allocate static const storage for the various interface IDs that the Slang API needs to expose +static const Guid IID_ISlangUnknown = SLANG_UUID_ISlangUnknown; +static const Guid IID_ISlangFileSystem = SLANG_UUID_ISlangFileSystem; +static const Guid IID_ISlangFileSystemExt = SLANG_UUID_ISlangFileSystemExt; +static const Guid IID_ISlangMutableFileSystem = SLANG_UUID_ISlangMutableFileSystem; + +RiffFileSystem::RiffFileSystem(ICompressionSystem* compressionSystem): + m_compressionSystem(compressionSystem) +{ +} + +ISlangMutableFileSystem* RiffFileSystem::getInterface(const Guid& guid) +{ + return (guid == IID_ISlangUnknown || guid == IID_ISlangFileSystem || guid == IID_ISlangFileSystemExt || guid == IID_ISlangMutableFileSystem) ? static_cast<ISlangMutableFileSystem*>(this) : nullptr; +} + +SlangResult RiffFileSystem::_calcCanonicalPath(const char* path, StringBuilder& out) +{ + List<UnownedStringSlice> splitPath; + Path::split(UnownedStringSlice(path), splitPath); + + // If the first part of a path is "", it means path of form "/some/path". Turn into "some/path". + if (splitPath.getCount() > 1 && splitPath[0].getLength() == 0) + { + splitPath.removeAt(0); + } + + Path::simplify(splitPath); + + if (splitPath.indexOf(UnownedStringSlice::fromLiteral("..")) >= 0) + { + return SLANG_E_NOT_FOUND; + } + + if (splitPath.getCount() == 0) + { + // It's an empty path; + return SLANG_FAIL; + } + + Path::join(splitPath.getBuffer(), splitPath.getCount(), out); + return SLANG_OK; +} + +RiffFileSystem::Entry* RiffFileSystem::_getEntryFromCanonicalPath(const String& canonicalPath) +{ + RefPtr<Entry>* entryPtr = m_entries.TryGetValue(canonicalPath); + return entryPtr ? *entryPtr : nullptr; +} + +RiffFileSystem::Entry* RiffFileSystem::_getEntryFromPath(const char* path, String* outPath) +{ + StringBuilder buffer; + if (SLANG_FAILED(_calcCanonicalPath(path, buffer))) + { + return nullptr; + } + + if (outPath) + { + *outPath = buffer; + } + return _getEntryFromCanonicalPath(buffer); +} + +SlangResult RiffFileSystem::loadFile(char const* path, ISlangBlob** outBlob) +{ + Entry* entry = _getEntryFromPath(path); + if (entry == nullptr || entry->m_type != SLANG_PATH_TYPE_FILE) + { + return SLANG_E_NOT_FOUND; + } + + if (m_compressionSystem) + { + // Okay lets decompress into a blob + ScopedAllocation alloc; + void* dst = alloc.allocate(entry->m_uncompressedSizeInBytes); + + ISlangBlob* compressedData = entry->m_contents; + SLANG_RETURN_ON_FAIL(m_compressionSystem->decompress(compressedData->getBufferPointer(), compressedData->getBufferSize(), entry->m_uncompressedSizeInBytes, dst)); + + auto blob = RawBlob::moveCreate(alloc); + + *outBlob = blob.detach(); + } + else + { + // We don't have any compression, so can just return the blob + ISlangBlob* contents = entry->m_contents; + contents->addRef(); + *outBlob = contents; + } + + return SLANG_OK; +} + +SlangResult RiffFileSystem::getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity) +{ + return getCanonicalPath(path, outUniqueIdentity); +} + +SlangResult RiffFileSystem::calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) +{ + String combinedPath; + switch (fromPathType) + { + case SLANG_PATH_TYPE_FILE: + { + combinedPath = Path::combine(Path::getParentDirectory(fromPath), path); + break; + } + case SLANG_PATH_TYPE_DIRECTORY: + { + combinedPath = Path::combine(fromPath, path); + break; + } + } + + *pathOut = StringUtil::createStringBlob(combinedPath).detach(); + return SLANG_OK; +} + +SlangResult RiffFileSystem::getPathType(const char* path, SlangPathType* outPathType) +{ + String canonicalPath; + Entry* entry = _getEntryFromPath(path, &canonicalPath); + if (entry == nullptr) + { + // Could be an implicit path + ImplicitDirectoryCollector collector(canonicalPath); + for (const auto& pair : m_entries) + { + Entry* childEntry = pair.Value; + collector.addPath(childEntry->m_type, childEntry->m_canonicalPath.getUnownedSlice()); + // If on adding a path we determine a directory exists, then we are done + if (collector.getDirectoryExists()) + { + *outPathType = SLANG_PATH_TYPE_DIRECTORY; + return SLANG_OK; + } + } + + // If not implicit or explicit we are done. + return SLANG_E_NOT_FOUND; + } + + // Explicit type + *outPathType = entry->m_type; + return SLANG_OK; +} + +SlangResult RiffFileSystem::getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) +{ + String simplifiedPath = Path::simplify(path); + *outSimplifiedPath = StringUtil::createStringBlob(simplifiedPath).detach(); + return SLANG_OK; +} + +SlangResult RiffFileSystem::getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) +{ + StringBuilder buffer; + SLANG_RETURN_ON_FAIL(_calcCanonicalPath(path, buffer)); + *outCanonicalPath = StringUtil::createStringBlob(buffer).detach(); + return SLANG_OK; +} + + +SlangResult RiffFileSystem::enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) +{ + String canonicalPath; + Entry* entry = _getEntryFromPath(path, &canonicalPath); + if (entry && entry->m_type != SLANG_PATH_TYPE_DIRECTORY) + { + return SLANG_FAIL; + } + + // If we didn't find an explicit directory, lets handle an implicit one + ImplicitDirectoryCollector collector(canonicalPath); + + // If it is a directory, we need to see if there is anything in it + for (const auto& pair : m_entries) + { + Entry* childEntry = pair.Value; + collector.addPath(childEntry->m_type, childEntry->m_canonicalPath.getUnownedSlice()); + } + + return collector.enumerate(callback, userData); +} + +SlangResult RiffFileSystem::saveFile(const char* path, const void* data, size_t size) +{ + StringBuilder canonicalPath; + SLANG_RETURN_ON_FAIL(_calcCanonicalPath(path, canonicalPath)); + + ComPtr<ISlangBlob> contents; + + if (m_compressionSystem) + { + // Lets try compressing the input + SLANG_RETURN_ON_FAIL(m_compressionSystem->compress(&m_compressionStyle, data, size, contents.writeRef())); + } + else + { + // Just store the data directly. + contents = new RawBlob(data, size); + } + + Entry* entry = _getEntryFromCanonicalPath(canonicalPath); + if (!entry) + { + entry = new Entry; + entry->m_type = SLANG_PATH_TYPE_FILE; + entry->m_canonicalPath = canonicalPath; + entry->m_uncompressedSizeInBytes = size; + + m_entries.Add(canonicalPath, entry); + } + + entry->m_uncompressedSizeInBytes = size; + entry->m_contents = contents; + + return SLANG_OK; +} + +SlangResult RiffFileSystem::remove(const char* path) +{ + String canonicalPath; + Entry* entry = _getEntryFromPath(path, &canonicalPath); + + if (entry) + { + if (entry->m_type == SLANG_PATH_TYPE_FILE) + { + m_entries.Remove(canonicalPath); + return SLANG_OK; + } + + ImplicitDirectoryCollector collector(canonicalPath); + + // If it is a directory, we need to see if there is anything in it + for (const auto& pair : m_entries) + { + Entry* childEntry = pair.Value; + collector.addPath(childEntry->m_type, childEntry->m_canonicalPath.getUnownedSlice()); + if (collector.hasContent()) + { + // Directory is not empty + return SLANG_FAIL; + } + } + + m_entries.Remove(canonicalPath); + return SLANG_OK; + } + + return SLANG_E_NOT_FOUND; +} + +SlangResult RiffFileSystem::createDirectory(const char* path) +{ + String canonicalPath; + Entry* entry = _getEntryFromPath(path, &canonicalPath); + if (entry) + { + return SLANG_FAIL; + } + + entry = new Entry; + entry->m_type = SLANG_PATH_TYPE_DIRECTORY; + entry->m_canonicalPath = canonicalPath; + entry->m_uncompressedSizeInBytes = 0; + + m_entries.Add(canonicalPath, entry); + return SLANG_OK; +} + +SlangResult RiffFileSystem::loadArchive(const void* archive, size_t archiveSizeInBytes) +{ + // Load the riff + RiffContainer container; + + MemoryStreamBase stream(FileAccess::Read, archive, archiveSizeInBytes); + SLANG_RETURN_ON_FAIL(RiffUtil::read(&stream, container)); + + RiffContainer::ListChunk* rootList = container.getRoot(); + // Make sure it's the right type + if (rootList == nullptr || rootList->m_fourCC != RiffFileSystemBinary::kContainerFourCC) + { + return SLANG_FAIL; + } + + // Clear the contents + _clear(); + + // Find the header + const auto header = rootList->findContainedData<RiffFileSystemBinary::Header>(RiffFileSystemBinary::kHeaderFourCC); + + CompressionSystemType compressionType = CompressionSystemType(header->compressionSystemType); + switch (compressionType) + { + case CompressionSystemType::None: + { + // Null m_compressionSystem means no compression + m_compressionSystem.setNull(); + break; + } + case CompressionSystemType::Deflate: + { + m_compressionSystem = DeflateCompressionSystem::getSingleton(); + break; + } + case CompressionSystemType::LZ4: + { + m_compressionSystem = LZ4CompressionSystem::getSingleton(); + break; + } + default: return SLANG_FAIL; + } + + // Read all of the contained data + + { + List<RiffContainer::DataChunk*> srcEntries; + rootList->findContained(RiffFileSystemBinary::kEntryFourCC, srcEntries); + + for (auto chunk : srcEntries) + { + auto data = chunk->getSingleData(); + + const uint8_t* srcData = (const uint8_t*)data->getPayload(); + const size_t dataSize = data->getSize(); + + if (dataSize < sizeof(RiffFileSystemBinary::Entry)) + { + return SLANG_FAIL; + } + + auto srcEntry = (const RiffFileSystemBinary::Entry*)srcData; + srcData += sizeof(*srcEntry); + + // Check if seems plausible + if (sizeof(RiffFileSystemBinary::Entry) + srcEntry->compressedSize + srcEntry->pathSize != dataSize) + { + return SLANG_FAIL; + } + + RefPtr<Entry> dstEntry = new Entry; + + const char* path = (const char*)srcData; + srcData += srcEntry->pathSize; + + dstEntry->m_canonicalPath = UnownedStringSlice(path, srcEntry->pathSize - 1); + dstEntry->m_type = (SlangPathType)srcEntry->pathType; + dstEntry->m_uncompressedSizeInBytes = srcEntry->uncompressedSize; + + switch (dstEntry->m_type) + { + case SLANG_PATH_TYPE_FILE: + { + if (srcData + srcEntry->compressedSize != data->getPayloadEnd()) + { + return SLANG_FAIL; + } + + // Get the compressed data + dstEntry->m_contents = new RawBlob(srcData, srcEntry->compressedSize); + break; + } + case SLANG_PATH_TYPE_DIRECTORY: break; + default: return SLANG_FAIL; + } + + // Add to the list of entries + m_entries.Add(dstEntry->m_canonicalPath, dstEntry); + } + } + + return SLANG_OK; +} + +SlangResult RiffFileSystem::storeArchive(bool blobOwnsContent, ISlangBlob** outBlob) +{ + // All blobs are owned in this style + SLANG_UNUSED(blobOwnsContent) + + RiffContainer container; + RiffContainer::ScopeChunk scopeContainer(&container, RiffContainer::Chunk::Kind::List, RiffFileSystemBinary::kContainerFourCC); + + { + RiffFileSystemBinary::Header header; + CompressionSystemType compressionSystemType = m_compressionSystem ? m_compressionSystem->getSystemType() : CompressionSystemType::None; + header.compressionSystemType = uint32_t(compressionSystemType); + container.addDataChunk(RiffFileSystemBinary::kHeaderFourCC, &header, sizeof(header)); + } + + for (const auto& pair : m_entries) + { + RiffContainer::ScopeChunk scopeData(&container, RiffContainer::Chunk::Kind::Data, RiffFileSystemBinary::kEntryFourCC); + + const Entry* srcEntry = pair.Value; + + RiffFileSystemBinary::Entry dstEntry; + dstEntry.uncompressedSize = 0; + dstEntry.compressedSize = 0; + dstEntry.pathSize = uint32_t(srcEntry->m_canonicalPath.getLength() + 1); + dstEntry.pathType = srcEntry->m_type; + + ISlangBlob* blob = srcEntry->m_contents; + + if (srcEntry->m_type == SLANG_PATH_TYPE_FILE) + { + dstEntry.compressedSize = uint32_t(blob->getBufferSize()); + dstEntry.uncompressedSize = uint32_t(srcEntry->m_uncompressedSizeInBytes); + } + + // Entry header + container.write(&dstEntry, sizeof(dstEntry)); + + // Path + container.write(srcEntry->m_canonicalPath.getBuffer(), srcEntry->m_canonicalPath.getLength() + 1); + + // Add the contained data without copying + if (blob) + { + RiffContainer::Data* data = container.addData(); + container.setUnowned(data, const_cast<void*>(blob->getBufferPointer()), blob->getBufferSize()); + } + } + + OwnedMemoryStream stream(FileAccess::Write); + // We now write the RiffContainer to the stream + SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, &stream)); + + RefPtr<ListBlob> blob = new ListBlob; + stream.swapContents(blob->m_data); + + *outBlob = blob.detach(); + return SLANG_OK; +} + +/* static */bool RiffFileSystem::isArchive(const void* data, size_t sizeInBytes) +{ + MemoryStreamBase stream(FileAccess::Read, data, sizeInBytes); + RiffListHeader header; + return SLANG_SUCCEEDED(RiffUtil::readHeader(&stream, header)) && header.subType == RiffFileSystemBinary::kContainerFourCC; +} + +} // namespace Slang diff --git a/source/core/slang-riff-file-system.h b/source/core/slang-riff-file-system.h new file mode 100644 index 000000000..830fed71a --- /dev/null +++ b/source/core/slang-riff-file-system.h @@ -0,0 +1,98 @@ +#ifndef SLANG_RIFF_FILE_SYSTEM_H +#define SLANG_RIFF_FILE_SYSTEM_H + +#include "slang-archive-file-system.h" + +#include "slang-riff.h" +#include "slang-io.h" + +namespace Slang +{ + +// The riff information used for RiffArchiveFileSystem +struct RiffFileSystemBinary +{ + static const FourCC kContainerFourCC = SLANG_FOUR_CC('S', 'c', 'o', 'n'); + static const FourCC kEntryFourCC = SLANG_FOUR_CC('S', 'f', 'i', 'l'); + static const FourCC kHeaderFourCC = SLANG_FOUR_CC('S', 'h', 'e', 'a'); + + struct Header + { + uint32_t compressionSystemType; /// One of CompressionSystemType + }; + + struct Entry + { + uint32_t compressedSize; + uint32_t uncompressedSize; + uint32_t pathSize; ///< The size of the path in bytes, including terminating 0 + uint32_t pathType; ///< One of SlangPathType + + // Followed by the path (including terminating0) + // Followed by the compressed data + }; +}; + +class RiffFileSystem : public ArchiveFileSystem +{ +public: + + // ISlangUnknown + SLANG_REF_OBJECT_IUNKNOWN_ALL + + // ISlangFileSystem + virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile(char const* path, ISlangBlob** outBlob) SLANG_OVERRIDE; + + // ISlangFileSystemExt + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getFileUniqueIdentity(const char* path, ISlangBlob** uniqueIdentityOut) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType(const char* path, SlangPathType* pathTypeOut) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) SLANG_OVERRIDE; + virtual SLANG_NO_THROW void SLANG_MCALL clearCache() SLANG_OVERRIDE {} + virtual SLANG_NO_THROW SlangResult SLANG_MCALL enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) SLANG_OVERRIDE; + + // ISlangModifyableFileSystem + virtual SLANG_NO_THROW SlangResult SLANG_MCALL saveFile(const char* path, const void* data, size_t size) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL remove(const char* path) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL createDirectory(const char* path) SLANG_OVERRIDE; + + // ArchiveFileSystem + virtual SlangResult loadArchive(const void* archive, size_t archiveSizeInBytes) SLANG_OVERRIDE; + virtual SlangResult storeArchive(bool blobOwnsContent, ISlangBlob** outBlob) SLANG_OVERRIDE; + virtual void setCompressionStyle(const CompressionStyle& style) SLANG_OVERRIDE { m_compressionStyle = style; } + + RiffFileSystem(ICompressionSystem* compressionSystem); + + /// True if this appears to be Riff archive + static bool isArchive(const void* data, size_t sizeInBytes); + +protected: + + struct Entry : RefObject + { + SlangPathType m_type; + String m_canonicalPath; + size_t m_uncompressedSizeInBytes; ///< Needed if m_contents is compressed. + ComPtr<ISlangBlob> m_contents; ///< Can be compressed or not + }; + + ISlangMutableFileSystem* getInterface(const Guid& guid); + + SlangResult _calcCanonicalPath(const char* path, StringBuilder& out); + Entry* _getEntryFromPath(const char* path, String* outPath = nullptr); + Entry* _getEntryFromCanonicalPath(const String& canonicalPath); + + void _clear() { m_entries.Clear(); } + + // Maps a path to an entry + Dictionary<String, RefPtr<Entry>> m_entries; + + ComPtr<ICompressionSystem> m_compressionSystem; + + CompressionStyle m_compressionStyle; +}; + +} + +#endif diff --git a/source/core/slang-riff.cpp b/source/core/slang-riff.cpp index d64cd89c7..482c35e81 100644 --- a/source/core/slang-riff.cpp +++ b/source/core/slang-riff.cpp @@ -851,6 +851,13 @@ void RiffContainer::endChunk() SLANG_ASSERT(isChunkOk(chunk)); } +void RiffContainer::addDataChunk(FourCC dataFourCC, const void* data, size_t dataSizeInBytes) +{ + startChunk(Chunk::Kind::Data, dataFourCC); + write(data, dataSizeInBytes); + endChunk(); +} + void RiffContainer::setPayload(Data* data, const void* payload, size_t size) { // We must be in a data chunk diff --git a/source/core/slang-riff.h b/source/core/slang-riff.h index 5a753b6d6..83a522e81 100644 --- a/source/core/slang-riff.h +++ b/source/core/slang-riff.h @@ -195,6 +195,8 @@ public: { /// Get the payload void* getPayload() { return m_payload; } + /// Get the end pointer + void* getPayloadEnd() { return (void*)((uint8_t*)m_payload + m_size); } /// Get the size of the payload size_t getSize() const { return m_size; } /// Get the ownership of the data held in the payload @@ -361,6 +363,10 @@ public: virtual SlangResult leaveList(ListChunk* list) = 0; }; + + /// Add a complete data chunk + void addDataChunk(FourCC dataFourCC, const void* data, size_t dataSizeInBytes); + /// Start a chunk void startChunk(Chunk::Kind kind, FourCC type); diff --git a/source/core/slang-type-text-util.cpp b/source/core/slang-type-text-util.cpp index e9aa62048..80ef0027f 100644 --- a/source/core/slang-type-text-util.cpp +++ b/source/core/slang-type-text-util.cpp @@ -74,8 +74,34 @@ static const CompileTargetInfo s_compileTargetInfos[] = { SLANG_HOST_CALLABLE, "", "host-callable,callable" } }; +struct ArchiveTypeInfo +{ + SlangArchiveType type; + UnownedStringSlice text; +}; + +static const ArchiveTypeInfo s_archiveTypeInfos[] = +{ + { SLANG_ARCHIVE_TYPE_RIFF_DEFLATE, UnownedStringSlice::fromLiteral("riff-deflate")}, + { SLANG_ARCHIVE_TYPE_RIFF_LZ4, UnownedStringSlice::fromLiteral("riff-lz4")}, + { SLANG_ARCHIVE_TYPE_ZIP, UnownedStringSlice::fromLiteral("zip")}, + { SLANG_ARCHIVE_TYPE_RIFF, UnownedStringSlice::fromLiteral("riff")}, +}; + } // anonymous +/* static */SlangArchiveType TypeTextUtil::findArchiveType(const UnownedStringSlice& slice) +{ + for (const auto& entry : s_archiveTypeInfos) + { + if (slice == entry.text) + { + return entry.type; + } + } + return SLANG_ARCHIVE_TYPE_UNDEFINED; +} + /* static */UnownedStringSlice TypeTextUtil::getScalarTypeName(slang::TypeReflection::ScalarType scalarType) { typedef slang::TypeReflection::ScalarType ScalarType; diff --git a/source/core/slang-type-text-util.h b/source/core/slang-type-text-util.h index c4f9fb275..07426246e 100644 --- a/source/core/slang-type-text-util.h +++ b/source/core/slang-type-text-util.h @@ -41,6 +41,9 @@ struct TypeTextUtil /// Given a target returns the associated name. static UnownedStringSlice getCompileTargetName(SlangCompileTarget target); + + /// Returns SLANG_ARCHIVE_TYPE_UNKNOWN if a match is not found + static SlangArchiveType findArchiveType(const UnownedStringSlice& slice); }; } diff --git a/source/core/slang-zip-file-system.cpp b/source/core/slang-zip-file-system.cpp index 8547f7eac..a56ba09db 100644 --- a/source/core/slang-zip-file-system.cpp +++ b/source/core/slang-zip-file-system.cpp @@ -8,6 +8,7 @@ #include "slang-blob.h" #include "slang-string-slice-pool.h" #include "slang-uint-set.h" +#include "slang-riff.h" #include "../../external/miniz/miniz.h" #include "../../external/miniz/miniz_common.h" @@ -24,7 +25,7 @@ static const Guid IID_ISlangFileSystem = SLANG_UUID_ISlangFileSystem; static const Guid IID_ISlangFileSystemExt = SLANG_UUID_ISlangFileSystemExt; static const Guid IID_ISlangMutableFileSystem = SLANG_UUID_ISlangMutableFileSystem; -class ZipFileSystem : public CompressedFileSystem +class ZipFileSystemImpl : public ArchiveFileSystem { public: // ISlangUnknown @@ -48,67 +49,16 @@ public: virtual SLANG_NO_THROW SlangResult SLANG_MCALL remove(const char* path) SLANG_OVERRIDE; virtual SLANG_NO_THROW SlangResult SLANG_MCALL createDirectory(const char* path) SLANG_OVERRIDE; - // CompressedFileSystem - virtual ConstArrayView<uint8_t> getArchive() SLANG_OVERRIDE; - virtual void setCompressionType(CompressionType type) SLANG_OVERRIDE; + // ArchiveFileSystem + SlangResult loadArchive(const void* archive, size_t archiveSizeInBytes) SLANG_OVERRIDE; + virtual SlangResult storeArchive(bool blobOwnsContent, ISlangBlob** outBlob) SLANG_OVERRIDE; + virtual void setCompressionStyle(const CompressionStyle& style) SLANG_OVERRIDE; - ZipFileSystem(); - ~ZipFileSystem(); - - SlangResult init(const uint8_t* archive, size_t size); + ZipFileSystemImpl(); + ~ZipFileSystemImpl(); protected: - /// Maps a SubString (owned) to an index - struct SubStringIndexMap - { - void set(const UnownedStringSlice& slice, Index index) - { - StringSlicePool::Handle handle; - m_pool.findOrAdd(slice, handle); - const Index poolIndex = StringSlicePool::asIndex(handle); - - if (poolIndex >= m_indexMap.getCount()) - { - SLANG_ASSERT(poolIndex == m_indexMap.getCount()); - m_indexMap.add(index); - } - else - { - m_indexMap[poolIndex] = index; - } - } - Index get(const UnownedStringSlice& slice) - { - const Index poolIndex = m_pool.findIndex(slice); - return (poolIndex >= 0) ? m_indexMap[poolIndex] : -1; - } - - Index getCount() const { return m_indexMap.getCount(); } - - KeyValuePair<UnownedStringSlice, Index> getAt(Index index) const - { - KeyValuePair<UnownedStringSlice, Index> pair; - pair.Key = m_pool.getSlice(StringSlicePool::Handle(index)); - pair.Value = m_indexMap[index]; - return pair; - } - - void clear() - { - m_pool.clear(); - m_indexMap.clear(); - } - - SubStringIndexMap(): - m_pool(StringSlicePool::Style::Empty) - { - } - - StringSlicePool m_pool; ///< Pool holds the substrings - List<Index> m_indexMap; ///< Maps a pool index to the output index - }; - enum class Mode { None, // m_archive is not initialized @@ -128,8 +78,9 @@ protected: SlangResult _copyToAndInitWriter(mz_zip_archive& outWriter); /// Returns SLANG_E_NOT_FOUND if no directory or contents found + /// terminationState controls when search terminates. If State::Undefined, will enumerate everything. /// If outContents not set, will just determine if the directory exists - SlangResult _getPathContents(const String& fixedPath, SubStringIndexMap* outContents); + SlangResult _getPathContents(ImplicitDirectoryCollector::State terminationState, ImplicitDirectoryCollector* outCollector); void _rebuildMap(); @@ -141,7 +92,7 @@ protected: void _initReadWrite(mz_zip_archive& outWriter); // Maps from a path to an index in the m_archive - SubStringIndexMap m_pathMap; + StringSliceIndexMap m_pathMap; // If bit is set (at the archive index) this index has been deleted. UIntSet m_removedSet; @@ -155,7 +106,7 @@ protected: mz_zip_archive m_archive; }; -ISlangMutableFileSystem* ZipFileSystem::getInterface(const Guid& guid) +ISlangMutableFileSystem* ZipFileSystemImpl::getInterface(const Guid& guid) { return (guid == IID_ISlangUnknown || guid == IID_ISlangFileSystem || guid == IID_ISlangFileSystemExt || guid == IID_ISlangMutableFileSystem) ? static_cast<ISlangMutableFileSystem*>(this) : nullptr; } @@ -192,45 +143,18 @@ static mz_file_read_func _getReadFunc() return readFunc; } -ZipFileSystem::ZipFileSystem(): +ZipFileSystemImpl::ZipFileSystemImpl(): m_mode(Mode::None) { m_readFunc = _getReadFunc(); } - ZipFileSystem::~ZipFileSystem() + ZipFileSystemImpl::~ZipFileSystemImpl() { _requireMode(Mode::None); } -SlangResult ZipFileSystem::init(const uint8_t* archive, size_t size) -{ - SLANG_RETURN_ON_FAIL(_requireMode(Mode::None)); - - // Store a copy - if (!m_data.set(archive, size)) - { - return SLANG_E_OUT_OF_MEMORY; - } - - // Initialize archive - mz_zip_zero_struct(&m_archive); - - // Read the contents of the archive, and make m_archive own it - if (!mz_zip_reader_init_mem(&m_archive, m_data.getData(), size, 0)) - { - return SLANG_FAIL; - } - - m_mode = Mode::Read; - - // Set up the mapping from paths to indices - _rebuildMap(); - - return SLANG_OK; -} - -void ZipFileSystem::_rebuildMap() +void ZipFileSystemImpl::_rebuildMap() { m_pathMap.clear(); @@ -251,11 +175,11 @@ void ZipFileSystem::_rebuildMap() // Get rid of '/' currentName = currentName.trim('/'); - m_pathMap.set(currentName, Index(i)); + m_pathMap.add(currentName, Index(i)); } } -UnownedStringSlice ZipFileSystem::_getPathAtIndex(Index index) +UnownedStringSlice ZipFileSystemImpl::_getPathAtIndex(Index index) { SLANG_ASSERT(m_mode != Mode::None); @@ -269,14 +193,14 @@ UnownedStringSlice ZipFileSystem::_getPathAtIndex(Index index) return UnownedStringSlice(fileStat.m_filename).trim('/'); } -void ZipFileSystem::_initReadWrite(mz_zip_archive& outWriter) +void ZipFileSystemImpl::_initReadWrite(mz_zip_archive& outWriter) { mz_zip_zero_struct(&outWriter); mz_zip_writer_init_heap(&outWriter, 0, 0); outWriter.m_pRead = m_readFunc; } -SlangResult ZipFileSystem::_copyToAndInitWriter(mz_zip_archive& outWriter) +SlangResult ZipFileSystemImpl::_copyToAndInitWriter(mz_zip_archive& outWriter) { mz_zip_zero_struct(&outWriter); switch (m_mode) @@ -322,7 +246,7 @@ SlangResult ZipFileSystem::_copyToAndInitWriter(mz_zip_archive& outWriter) return SLANG_FAIL; } -SlangResult ZipFileSystem::_requireModeImpl(Mode newMode) +SlangResult ZipFileSystemImpl::_requireModeImpl(Mode newMode) { SLANG_ASSERT(newMode != m_mode); @@ -445,7 +369,7 @@ SlangResult ZipFileSystem::_requireModeImpl(Mode newMode) return SLANG_OK; } -SlangResult ZipFileSystem::_requireMode(Mode newMode) +SlangResult ZipFileSystemImpl::_requireMode(Mode newMode) { if (newMode == m_mode) { @@ -462,7 +386,7 @@ SlangResult ZipFileSystem::_requireMode(Mode newMode) return res; } -SlangResult ZipFileSystem::_getFixedPath(const char* path, String& outPath) +SlangResult ZipFileSystemImpl::_getFixedPath(const char* path, String& outPath) { String simplifiedPath = Path::simplify(UnownedStringSlice(path)); // Can simplify to just ., thats okay, if it otherwise has something relative it means it couldn't be simplified into the @@ -477,9 +401,9 @@ SlangResult ZipFileSystem::_getFixedPath(const char* path, String& outPath) return SLANG_OK; } -SlangResult ZipFileSystem::_findEntryIndexFromFixedPath(const String& fixedPath, mz_uint& outIndex) +SlangResult ZipFileSystemImpl::_findEntryIndexFromFixedPath(const String& fixedPath, mz_uint& outIndex) { - const Index index = m_pathMap.get(fixedPath.getUnownedSlice()); + const Index index = m_pathMap.getValue(fixedPath.getUnownedSlice()); // If not in list or deleted - it is removed if (index < 0 || m_removedSet.contains(index)) @@ -491,7 +415,7 @@ SlangResult ZipFileSystem::_findEntryIndexFromFixedPath(const String& fixedPath, return SLANG_OK; } -SlangResult ZipFileSystem::_findEntryIndex(const char* path, mz_uint& outIndex) +SlangResult ZipFileSystemImpl::_findEntryIndex(const char* path, mz_uint& outIndex) { String fixedPath; SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); @@ -499,7 +423,7 @@ SlangResult ZipFileSystem::_findEntryIndex(const char* path, mz_uint& outIndex) return SLANG_OK; } -SlangResult ZipFileSystem::loadFile(char const* path, ISlangBlob** outBlob) +SlangResult ZipFileSystemImpl::loadFile(char const* path, ISlangBlob** outBlob) { mz_uint index; SLANG_RETURN_ON_FAIL(_findEntryIndex(path, index)); @@ -529,7 +453,7 @@ SlangResult ZipFileSystem::loadFile(char const* path, ISlangBlob** outBlob) return SLANG_OK; } -SlangResult ZipFileSystem::getPathType(const char* path, SlangPathType* outPathType) +SlangResult ZipFileSystemImpl::getPathType(const char* path, SlangPathType* outPathType) { if (!_hasArchive()) { @@ -555,7 +479,9 @@ SlangResult ZipFileSystem::getPathType(const char* path, SlangPathType* outPathT else { // It could be an *implicit* directory (ie as part of a path). So lets look for that... - if (SLANG_SUCCEEDED(_getPathContents(fixedPath, nullptr))) + ImplicitDirectoryCollector collector(fixedPath); + SLANG_RETURN_ON_FAIL(_getPathContents(ImplicitDirectoryCollector::State::DirectoryExists, &collector)); + if (collector.getDirectoryExists()) { *outPathType = SLANG_PATH_TYPE_DIRECTORY; return SLANG_OK; @@ -565,7 +491,7 @@ SlangResult ZipFileSystem::getPathType(const char* path, SlangPathType* outPathT return SLANG_E_NOT_FOUND; } -SlangResult ZipFileSystem::getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) +SlangResult ZipFileSystemImpl::getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) { mz_uint index; SLANG_RETURN_ON_FAIL(_findEntryIndex(path, index)); @@ -581,12 +507,12 @@ SlangResult ZipFileSystem::getCanonicalPath(const char* path, ISlangBlob** outCa return SLANG_OK; } -SlangResult ZipFileSystem::getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity) +SlangResult ZipFileSystemImpl::getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity) { return getCanonicalPath(path, outUniqueIdentity); } -SlangResult ZipFileSystem::calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) +SlangResult ZipFileSystemImpl::calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) { String relPath; switch (fromPathType) @@ -607,35 +533,24 @@ SlangResult ZipFileSystem::calcCombinedPath(SlangPathType fromPathType, const ch return SLANG_OK; } -SlangResult ZipFileSystem::getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) +SlangResult ZipFileSystemImpl::getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) { *outSimplifiedPath = StringUtil::createStringBlob(Path::simplify(path)).detach(); return SLANG_OK; } -SlangResult ZipFileSystem::_getPathContents(const String& inFixedPath, SubStringIndexMap* outContents) +SlangResult ZipFileSystemImpl::_getPathContents(ImplicitDirectoryCollector::State terminationState, ImplicitDirectoryCollector* outCollector) { if (!_hasArchive()) { return SLANG_E_NOT_FOUND; } - String fixedPath(inFixedPath); - if (fixedPath == ".") - { - fixedPath = ""; - } - else - { - fixedPath.append('/'); - } - - bool foundDirectory = false; - // Okay - I want to iterate through all of the entries and look for the ones with this prefix const Index entryCount = Index(mz_zip_reader_get_num_files(&m_archive)); for (Index i = 0; i < entryCount; ++i) { + // Skip if it's been deleted. if (m_removedSet.contains(i)) { @@ -649,53 +564,20 @@ SlangResult ZipFileSystem::_getPathContents(const String& inFixedPath, SubString } UnownedStringSlice currentPath(fileStat.m_filename); - if (!currentPath.startsWith(fixedPath.getUnownedSlice())) - { - continue; - } - - UnownedStringSlice remaining(currentPath.begin() + fixedPath.getLength(), currentPath.end()); + SlangPathType pathType = fileStat.m_is_directory ? SLANG_PATH_TYPE_DIRECTORY : SLANG_PATH_TYPE_FILE; + outCollector->addPath(pathType, currentPath); - if (!outContents) + // If a termination state is defined, and we reach it, we are done + if (terminationState != ImplicitDirectoryCollector::State::None && outCollector->hasState(terminationState)) { - // We found the directory, as we found contents. And since we aren't adding to map, we are done return SLANG_OK; } - - // We found the directory (either implicitly or explicitly) - foundDirectory = true; - - if (remaining.getLength() == 0) - { - // It's the explicit directory to this path, we don't need to add - continue; - } - - // Work out if it's a file that implicitly implies the directory, by looking for it it contains a / - const Index delimiterIndex = remaining.indexOf('/'); - - SlangPathType pathType; - if (delimiterIndex >= 0) - { - // If we have the delimiter index, then it's an implicit *contained* directory, and we need to strip to just get the name. - remaining = UnownedStringSlice(remaining.begin(), delimiterIndex); - pathType = SLANG_PATH_TYPE_DIRECTORY; - } - else - { - // Just use what the zip archive says the type is - pathType = fileStat.m_is_directory ? SLANG_PATH_TYPE_DIRECTORY : SLANG_PATH_TYPE_FILE; - } - - // Set what type this path is - outContents->set(remaining, pathType); } - // Check we found the directory at all... - return foundDirectory ? SLANG_OK : SLANG_E_NOT_FOUND; + return outCollector->getDirectoryExists() ? SLANG_OK : SLANG_E_NOT_FOUND; } -SlangResult ZipFileSystem::enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) +SlangResult ZipFileSystemImpl::enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) { if (!_hasArchive()) { @@ -704,28 +586,12 @@ SlangResult ZipFileSystem::enumeratePathContents(const char* path, FileSystemCon String fixedPath; SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); - - // Maps the name to the SLANG_PATH_TYPE - SubStringIndexMap map; - SLANG_RETURN_ON_FAIL(_getPathContents(fixedPath, &map)); - - const Index entryCount = map.getCount(); - for (Index i = 0; i < entryCount; ++i) - { - auto pair = map.getAt(i); - SlangPathType pathType = SlangPathType(pair.Value); - UnownedStringSlice name = pair.Key; - - // Name is zero terminated (as in StringPool). Lets check that though.. - SLANG_ASSERT(name.begin()[name.getLength()] == 0); - - callback(pathType, name.begin(), userData); - } - - return SLANG_OK; + ImplicitDirectoryCollector collector(fixedPath); + SLANG_RETURN_ON_FAIL(_getPathContents(ImplicitDirectoryCollector::State::None, &collector)); + return collector.enumerate(callback, userData); } -SlangResult ZipFileSystem::saveFile(const char* path, const void* data, size_t size) +SlangResult ZipFileSystemImpl::saveFile(const char* path, const void* data, size_t size) { String fixedPath; SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); @@ -742,7 +608,7 @@ SlangResult ZipFileSystem::saveFile(const char* path, const void* data, size_t s // TODO(JS): // We may want to check the directory exists that holds the path exists - // Which is easy to do. Without this check it allows directories to come into exisitance + // Which is easy to do. Without this check it allows directories to come into existence // when the path to the file is used. // This behaviour *isn't* strictly the same as the file system, which requires the path // to a file to exist before it is written. @@ -764,11 +630,11 @@ SlangResult ZipFileSystem::saveFile(const char* path, const void* data, size_t s SLANG_ASSERT(_getPathAtIndex(entryCount) == fixedPath.getUnownedSlice()); // Set in the map - m_pathMap.set(fixedPath.getUnownedSlice(), entryCount); + m_pathMap.add(fixedPath.getUnownedSlice(), entryCount); return SLANG_OK; } -SlangResult ZipFileSystem::remove(const char* path) +SlangResult ZipFileSystemImpl::remove(const char* path) { String fixedPath; SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); @@ -785,10 +651,10 @@ SlangResult ZipFileSystem::remove(const char* path) if (fileStat.m_is_directory) { // Find the directory contents - SubStringIndexMap map; - SLANG_RETURN_ON_FAIL(_getPathContents(fixedPath, &map)); + ImplicitDirectoryCollector collector(fixedPath); + SLANG_RETURN_ON_FAIL(_getPathContents(ImplicitDirectoryCollector::State::HasContent, &collector)); - if (map.getCount() > 0) + if (collector.hasContent()) { // If it contains children we can't remove it return SLANG_FAIL; @@ -800,7 +666,7 @@ SlangResult ZipFileSystem::remove(const char* path) return SLANG_OK; } -SlangResult ZipFileSystem::createDirectory(const char* path) +SlangResult ZipFileSystemImpl::createDirectory(const char* path) { String fixedPath; SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); @@ -830,11 +696,11 @@ SlangResult ZipFileSystem::createDirectory(const char* path) SLANG_ASSERT(_getPathAtIndex(entryCount) == fixedPath.getUnownedSlice()); // Set the index, that we added at end - m_pathMap.set(fixedPath.getUnownedSlice(), entryCount); + m_pathMap.add(fixedPath.getUnownedSlice(), entryCount); return SLANG_OK; } -ConstArrayView<uint8_t> ZipFileSystem::getArchive() +SlangResult ZipFileSystemImpl::storeArchive(bool blobOwnsContent, ISlangBlob** outBlob) { // If we have anything deleted in 'Read', we need to convert to 'Write' and then back to read if (m_mode == Mode::Read && !m_removedSet.isEmpty()) @@ -843,31 +709,97 @@ ConstArrayView<uint8_t> ZipFileSystem::getArchive() } _requireMode(Mode::Read); - return ArrayView<uint8_t>((uint8_t*)m_data.getData(), Index(m_data.getSizeInBytes())); -} - void ZipFileSystem::setCompressionType(CompressionType type) - { - switch (type) - { - case CompressionType::BestSpeed: m_compressionLevel = MZ_BEST_SPEED; break; - case CompressionType::BestCompression: m_compressionLevel = MZ_BEST_COMPRESSION; break; - } - } + ComPtr<ISlangBlob> blob; + + if (blobOwnsContent) + { + // Takes a copy + blob = new RawBlob(m_data.getData(), Index(m_data.getSizeInBytes())); + } + else + { + // Doesn't take a copy... Must use with care(!) + blob = new UnownedRawBlob(m_data.getData(), Index(m_data.getSizeInBytes())); + } + *outBlob = blob.detach(); + return SLANG_OK; +} -/* static */SlangResult CompressedFileSystem::createZip(const void* data, size_t size, RefPtr<CompressedFileSystem>& out) +SlangResult ZipFileSystemImpl::loadArchive(const void* archive, size_t archiveSizeInBytes) { - RefPtr<ZipFileSystem> fileSystem(new ZipFileSystem); - SLANG_RETURN_ON_FAIL(fileSystem->init((const uint8_t*)data, size)); + // Making the mode None empties the archive + SLANG_RETURN_ON_FAIL(_requireMode(Mode::None)); + + // Store a copy of the archive contents + if (!m_data.set(archive, archiveSizeInBytes)) + { + return SLANG_E_OUT_OF_MEMORY; + } + + // Initialize archive + mz_zip_zero_struct(&m_archive); + + // Read the contents of the archive, and make m_archive own it + if (!mz_zip_reader_init_mem(&m_archive, m_data.getData(), archiveSizeInBytes, 0)) + { + return SLANG_FAIL; + } + + m_mode = Mode::Read; + + // Set up the mapping from paths to indices + _rebuildMap(); - out = fileSystem; return SLANG_OK; } -/* static */SlangResult CompressedFileSystem::createZip(RefPtr<CompressedFileSystem>& out) +void ZipFileSystemImpl::setCompressionStyle(const CompressionStyle& style) { - out = new ZipFileSystem; + switch (style.m_type) + { + case CompressionStyle::Type::BestSpeed: m_compressionLevel = MZ_BEST_SPEED; break; + case CompressionStyle::Type::BestCompression: m_compressionLevel = MZ_BEST_COMPRESSION; break; + case CompressionStyle::Type::Default: m_compressionLevel = MZ_DEFAULT_LEVEL; break; + case CompressionStyle::Type::Level: + { + int level = int(style.m_level * 10.0f + 0.5); + level = (level < 0) ? 0 : level; + level = (level > MZ_UBER_COMPRESSION) ? MZ_UBER_COMPRESSION : level; + m_compressionLevel = level; + break; + } + } +} + +/* static */SlangResult ZipFileSystem::create(RefPtr<ArchiveFileSystem>& out) +{ + out = new ZipFileSystemImpl; return SLANG_OK; } +/* static */bool ZipFileSystem::isArchive(const void* data, size_t dataSizeInBytes) +{ + if (dataSizeInBytes < sizeof(FourCC)) + { + return false; + } + + FourCC fourCC = 0; + ::memcpy(&fourCC, data, sizeof(FourCC)); + + // https://en.wikipedia.org/wiki/List_of_file_signatures + switch (fourCC) + { + case SLANG_FOUR_CC(0x50, 0x4B, 0x03, 0x04): + case SLANG_FOUR_CC(0x50, 0x4B, 0x05, 0x06): + case SLANG_FOUR_CC(0x50, 0x4B, 0x07, 0x08): + { + // It's a zip + return true; + } + } + return false; +} + } // namespace Slang diff --git a/source/core/slang-zip-file-system.h b/source/core/slang-zip-file-system.h index 4a95cd51c..189060822 100644 --- a/source/core/slang-zip-file-system.h +++ b/source/core/slang-zip-file-system.h @@ -3,31 +3,17 @@ #include "slang-basic.h" -#include "../../slang-com-ptr.h" +#include "slang-archive-file-system.h" namespace Slang { -class CompressedFileSystem : public RefObject, public ISlangMutableFileSystem +struct ZipFileSystem { -public: - - enum class CompressionType - { - BestSpeed, - BestCompression, - }; - - /// Get as an archive (that can be saved to disk) - virtual ConstArrayView<uint8_t> getArchive() = 0; - /// Set the compression - used for any subsequent items added - virtual void setCompressionType(CompressionType type) = 0; - - /// Create a zip with the contents of data/size (the contents of a zip file) - static SlangResult createZip(const void* data, size_t size, RefPtr<CompressedFileSystem>& out); - /// Create an empty zip - static SlangResult createZip(RefPtr<CompressedFileSystem>& out); + static SlangResult create(RefPtr<ArchiveFileSystem>& out); + /// True if this appears to be a zip archive + static bool isArchive(const void* data, size_t size); }; } diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index 008c5eb5a..62def6f0f 100755 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -5,6 +5,7 @@ #include "../core/slang-shared-library.h" #include "../core/slang-downstream-compiler.h" +#include "../core/slang-archive-file-system.h" #include "../../slang-com-ptr.h" @@ -2178,7 +2179,7 @@ namespace Slang SLANG_NO_THROW SlangResult SLANG_MCALL compileStdLib() override; SLANG_NO_THROW SlangResult SLANG_MCALL loadStdLib(const void* stdLib, size_t stdLibSizeInBytes) override; - SLANG_NO_THROW SlangResult SLANG_MCALL saveStdLib(ISlangBlob** outBlob) override; + SLANG_NO_THROW SlangResult SLANG_MCALL saveStdLib(SlangArchiveType archiveType, ISlangBlob** outBlob) override; SLANG_NO_THROW SlangCapabilityID SLANG_MCALL findCapability(char const* name) override; diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h index 9c65250cd..1739a1ee1 100644 --- a/source/slang/slang-diagnostic-defs.h +++ b/source/slang/slang-diagnostic-defs.h @@ -120,6 +120,7 @@ DIAGNOSTIC( 86, Error, unableToCreateModuleContainer, "unable to create modul DIAGNOSTIC( 87, Error, unableToSetDefaultDownstreamCompiler, "unable to set default downstream compiler for source language '%0' to '%1'") +DIAGNOSTIC( 88, Error, unknownArchiveType, "archive type '%0' is unknown") // // 001xx - Downstream Compilers diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp index a89818c4f..d28b50b88 100644 --- a/source/slang/slang-options.cpp +++ b/source/slang/slang-options.cpp @@ -434,6 +434,9 @@ struct OptionsParser SlangMatrixLayoutMode defaultMatrixLayoutMode = SLANG_MATRIX_LAYOUT_MODE_UNKNOWN; + // The default archive type is zip + SlangArchiveType archiveType = SLANG_ARCHIVE_TYPE_ZIP; + bool hasLoadedRepro = false; char const* const* argCursor = &argv[0]; @@ -463,6 +466,18 @@ struct OptionsParser { SLANG_RETURN_ON_FAIL(session->compileStdLib()); } + else if (argStr == "-archive-type") + { + String archiveTypeName; + SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, archiveTypeName)); + + archiveType = TypeTextUtil::findArchiveType(archiveTypeName.getUnownedSlice()); + if (archiveType == SLANG_ARCHIVE_TYPE_UNDEFINED) + { + sink->diagnose(SourceLoc(), Diagnostics::unknownArchiveType, archiveTypeName); + return SLANG_FAIL; + } + } else if (argStr == "-save-stdlib") { String fileName; @@ -470,7 +485,7 @@ struct OptionsParser ComPtr<ISlangBlob> blob; - SLANG_RETURN_ON_FAIL(session->saveStdLib(blob.writeRef())); + SLANG_RETURN_ON_FAIL(session->saveStdLib(archiveType, blob.writeRef())); SLANG_RETURN_ON_FAIL(File::writeAllBytes(fileName, blob->getBufferPointer(), blob->getBufferSize())); } else if (argStr == "-save-stdlib-bin-source") @@ -480,7 +495,7 @@ struct OptionsParser ComPtr<ISlangBlob> blob; - SLANG_RETURN_ON_FAIL(session->saveStdLib(blob.writeRef())); + SLANG_RETURN_ON_FAIL(session->saveStdLib(archiveType, blob.writeRef())); StringBuilder builder; StringWriter writer(&builder, 0); diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index fbcc97c51..c82d5cf65 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -3,8 +3,7 @@ #include "../core/slang-io.h" #include "../core/slang-string-util.h" #include "../core/slang-shared-library.h" - -#include "../core/slang-zip-file-system.h" +#include "../core/slang-archive-file-system.h" #include "slang-check.h" #include "slang-parameter-binding.h" @@ -266,8 +265,8 @@ SlangResult Session::loadStdLib(const void* stdLib, size_t stdLibSizeInBytes) } // Make a file system to read it from - RefPtr<CompressedFileSystem> fileSystem; - SLANG_RETURN_ON_FAIL(CompressedFileSystem::createZip(stdLib, stdLibSizeInBytes, fileSystem)); + RefPtr<ArchiveFileSystem> fileSystem; + SLANG_RETURN_ON_FAIL(loadArchiveFileSystem(stdLib, stdLibSizeInBytes, fileSystem)); // Let's try loading serialized modules and adding them SLANG_RETURN_ON_FAIL(_readBuiltinModule(fileSystem, coreLanguageScope, "core")); @@ -275,7 +274,7 @@ SlangResult Session::loadStdLib(const void* stdLib, size_t stdLibSizeInBytes) return SLANG_OK; } -SlangResult Session::saveStdLib(ISlangBlob** outBlob) +SlangResult Session::saveStdLib(SlangArchiveType archiveType, ISlangBlob** outBlob) { if (m_builtinLinkage->mapNameToLoadedModules.Count() == 0) { @@ -284,8 +283,8 @@ SlangResult Session::saveStdLib(ISlangBlob** outBlob) } // Make a file system to read it from - RefPtr<CompressedFileSystem> fileSystem; - SLANG_RETURN_ON_FAIL(CompressedFileSystem::createZip(fileSystem)); + RefPtr<ArchiveFileSystem> fileSystem; + SLANG_RETURN_ON_FAIL(createArchiveFileSystem(archiveType, fileSystem)); for (auto& pair : m_builtinLinkage->mapNameToLoadedModules) { @@ -315,11 +314,7 @@ SlangResult Session::saveStdLib(ISlangBlob** outBlob) } // Now need to convert into a blob - auto archiveContents = fileSystem->getArchive(); - - ComPtr<ISlangBlob> blob(new RawBlob(archiveContents.getBuffer(), archiveContents.getCount())); - *outBlob = blob.detach(); - + SLANG_RETURN_ON_FAIL(fileSystem->storeArchive(true, outBlob)); return SLANG_OK; } diff --git a/tools/slang-test/unit-test-compression.cpp b/tools/slang-test/unit-test-compression.cpp index d9900bbfa..b70bd8545 100644 --- a/tools/slang-test/unit-test-compression.cpp +++ b/tools/slang-test/unit-test-compression.cpp @@ -4,6 +4,9 @@ #include "../../source/core/slang-zip-file-system.h" +#include "../../source/core/slang-lz4-compression-system.h" +#include "../../source/core/slang-deflate-compression-system.h" + using namespace Slang; static bool _equals(const void* data, size_t size, ISlangBlob* blob) @@ -31,102 +34,156 @@ static List<String> _getContents(ISlangFileSystemExt* fileSystem, const char* pa static void compressionUnitTest() { - // Create a zip to add stuff to - RefPtr<CompressedFileSystem> buildFileSystem; - CompressedFileSystem::createZip(buildFileSystem); - - const char contents[] = "I'm compressed"; - const char contents2[] = "Some more stuff"; - const char contents3[] = "Replace it"; + const SlangArchiveType archiveTypes[] = + { + SLANG_ARCHIVE_TYPE_RIFF, + SLANG_ARCHIVE_TYPE_RIFF_DEFLATE, + SLANG_ARCHIVE_TYPE_RIFF_LZ4, + SLANG_ARCHIVE_TYPE_ZIP + }; + for (auto archiveType : archiveTypes) { - ISlangMutableFileSystem* fileSystem = buildFileSystem; + // Test out archive file systems + RefPtr<ArchiveFileSystem> archiveFileSystem; + SLANG_CHECK(SLANG_SUCCEEDED(createArchiveFileSystem(archiveType, archiveFileSystem))); + + const char contents[] = "I'm compressed"; + const char contents2[] = "Some more stuff"; + const char contents3[] = "Replace it"; - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->createDirectory("hello"))); - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->createDirectory("hello2"))); - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->remove("hello"))); - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->createDirectory("hello"))); + { + ISlangMutableFileSystem* fileSystem = archiveFileSystem; - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("file.txt", contents, SLANG_COUNT_OF(contents)))); + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->createDirectory("hello"))); + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->createDirectory("hello2"))); + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->remove("hello"))); + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->createDirectory("hello"))); - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("file2.txt", contents2, SLANG_COUNT_OF(contents2)))); + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("file.txt", contents, SLANG_COUNT_OF(contents)))); - ComPtr<ISlangBlob> blob; - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file.txt", blob.writeRef()))); - SLANG_CHECK(_equals(contents, blob)); + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("file2.txt", contents2, SLANG_COUNT_OF(contents2)))); - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file2.txt", blob.writeRef()))); - SLANG_CHECK(_equals(contents2, blob)); + ComPtr<ISlangBlob> blob; + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file.txt", blob.writeRef()))); + SLANG_CHECK(_equals(contents, blob)); - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("file2.txt", contents3, SLANG_COUNT_OF(contents3)))); + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file2.txt", blob.writeRef()))); + SLANG_CHECK(_equals(contents2, blob)); - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file2.txt", blob.writeRef()))); - SLANG_CHECK(_equals(contents3, blob)); + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("file2.txt", contents3, SLANG_COUNT_OF(contents3)))); - // Check the path type - { - SlangPathType pathType; - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType("file2.txt", &pathType))); - SLANG_CHECK(pathType == SLANG_PATH_TYPE_FILE); + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file2.txt", blob.writeRef()))); + SLANG_CHECK(_equals(contents3, blob)); - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType("hello", &pathType))); - SLANG_CHECK(pathType == SLANG_PATH_TYPE_DIRECTORY); - } - - // Enumerate - { - for (const auto& obj : _getContents(fileSystem, "")) + // Check the path type { - // All of these should exist SlangPathType pathType; - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType(obj.getBuffer(), &pathType))); + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType("file2.txt", &pathType))); + SLANG_CHECK(pathType == SLANG_PATH_TYPE_FILE); + + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType("hello", &pathType))); + SLANG_CHECK(pathType == SLANG_PATH_TYPE_DIRECTORY); } - } - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("implicit-path/file2.txt", contents3, SLANG_COUNT_OF(contents3)))); + // Enumerate + { + for (const auto& obj : _getContents(fileSystem, "")) + { + // All of these should exist + SlangPathType pathType; + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType(obj.getBuffer(), &pathType))); + } + } - { - SlangPathType pathType; - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType("implicit-path", &pathType))); + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("implicit-path/file2.txt", contents3, SLANG_COUNT_OF(contents3)))); - SLANG_CHECK(pathType == SLANG_PATH_TYPE_DIRECTORY); + { + SlangPathType pathType; + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType("implicit-path", &pathType))); - List<String> objs = _getContents(fileSystem, "implicit-path"); + SLANG_CHECK(pathType == SLANG_PATH_TYPE_DIRECTORY); - // It contains a file - SLANG_CHECK(objs.getCount() == 1); + List<String> objs = _getContents(fileSystem, "implicit-path"); - for (const auto& obj : objs) - { - String path = Path::combine("implicit-path", obj); + // It contains a file + SLANG_CHECK(objs.getCount() == 1); - // All of these should exist - SlangPathType pathType; - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType(path.getBuffer(), &pathType))); + for (const auto& obj : objs) + { + String path = Path::combine("implicit-path", obj); + + // All of these should exist + SlangPathType pathType; + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType(path.getBuffer(), &pathType))); + } + + // Make an explicit path, and see whe have the same results + fileSystem->createDirectory("implicit-path"); + + objs = _getContents(fileSystem, "implicit-path"); + SLANG_CHECK(objs.getCount() == 1); } + } + + + // Load and check its okay + + { + ComPtr<ISlangBlob> archiveBlob; + SLANG_CHECK(SLANG_SUCCEEDED(archiveFileSystem->storeArchive(false, archiveBlob.writeRef()))); + + + RefPtr<ArchiveFileSystem> fileSystem; +#if 0 + SLANG_CHECK(SLANG_SUCCEEDED(createArchiveFileSystem(archiveType, fileSystem))); + + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadArchive(archiveBlob->getBufferPointer(), archiveBlob->getBufferSize()))); +#else + SLANG_CHECK(SLANG_SUCCEEDED(loadArchiveFileSystem(archiveBlob->getBufferPointer(), archiveBlob->getBufferSize(), fileSystem))); +#endif - // Make an explicit path, and see whe have the same results - fileSystem->createDirectory("implicit-path"); + ComPtr<ISlangBlob> blob; - objs = _getContents(fileSystem, "implicit-path"); - SLANG_CHECK(objs.getCount() == 1); + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file.txt", blob.writeRef()))); + SLANG_CHECK(_equals(contents, blob)); + + SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file2.txt", blob.writeRef()))); + SLANG_CHECK(_equals(contents3, blob)); } } - // Load and check its okay + // Test out compression systems + for (Index i = 0; i < 2; ++i) { - const auto archive = buildFileSystem->getArchive(); + // Lets try lz4 + + ICompressionSystem* system = nullptr; + if (i == 0) + { + system = LZ4CompressionSystem::getSingleton(); + } + else + { + system = DeflateCompressionSystem::getSingleton(); + } + + const char src[] = "Some text to compress"; + size_t srcSize = sizeof(src); + + ComPtr<ISlangBlob> compressedBlob; - RefPtr<CompressedFileSystem> fileSystem; - CompressedFileSystem::createZip(archive.getBuffer(), archive.getCount(), fileSystem); + CompressionStyle style; - ComPtr<ISlangBlob> blob; + SLANG_CHECK(SLANG_SUCCEEDED(system->compress(&style, src, srcSize, compressedBlob.writeRef()))); + + // Now lets decompress - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file.txt", blob.writeRef()))); - SLANG_CHECK(_equals(contents, blob)); + List<char> decompressedData; + decompressedData.setCount(srcSize); - SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file2.txt", blob.writeRef()))); - SLANG_CHECK(_equals(contents3, blob)); + SLANG_CHECK(SLANG_SUCCEEDED(system->decompress(compressedBlob->getBufferPointer(), compressedBlob->getBufferSize(), srcSize, decompressedData.getBuffer()))); + SLANG_CHECK(memcmp(src, decompressedData.getBuffer(), srcSize) == 0); } } |