From 723796a0a0fed8e5b8c3222b1c90443189113098 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Mon, 11 Jan 2021 15:24:11 -0500 Subject: LZ4 compression support (#1654) * #include an absolute path didn't work - because paths were taken to always be relative. * Testing out use of lz4. * Added ICompressionSystem, and LZ4 implementation. * Add support for deflate compression. Simplify compression interface - to make more easily work across apis. * WIP on CompressedFileSystem. * ImplicitDirectoryCollector * SubStringIndexMap - > StringSliceIndexMap. * WIP save stdlib in different containers. * Support for different archive types for stdlib. * Fix project. * CompressedFileSystem -> ArchiveFileSystem. Added CompressionSystemType::None * Added ArchiveFileSystem * Fix problem RiffFileSystem load withoug compression system. * Test archive types. Improve diagnostic message. * Fix typo in testing file system archives. * Split out archive detection. * Fix gcc warning issue. * Fix warning. * RiffArchiveFileSystem -> RiffFileSystem Co-authored-by: Tim Foley --- source/core/slang-archive-file-system.cpp | 183 ++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 source/core/slang-archive-file-system.cpp (limited to 'source/core/slang-archive-file-system.cpp') diff --git a/source/core/slang-archive-file-system.cpp b/source/core/slang-archive-file-system.cpp new file mode 100644 index 000000000..8c4d0ef2d --- /dev/null +++ b/source/core/slang-archive-file-system.cpp @@ -0,0 +1,183 @@ +#include "slang-archive-file-system.h" + +#include "../../slang-com-helper.h" +#include "../../slang-com-ptr.h" + +#include "slang-io.h" +#include "slang-string-util.h" +#include "slang-blob.h" +#include "slang-string-slice-pool.h" +#include "slang-uint-set.h" + +#include "slang-riff-file-system.h" + +// Compression systems +#include "slang-deflate-compression-system.h" +#include "slang-lz4-compression-system.h" + +// Zip file system +#include "slang-zip-file-system.h" + +#include "slang-riff.h" + +namespace Slang +{ + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! StringSliceIndexMap !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +StringSliceIndexMap::CountIndex StringSliceIndexMap::add(const UnownedStringSlice& key, Index valueIndex) +{ + StringSlicePool::Handle handle; + m_pool.findOrAdd(key, handle); + const CountIndex countIndex = StringSlicePool::asIndex(handle); + if (countIndex >= m_indexMap.getCount()) + { + SLANG_ASSERT(countIndex == m_indexMap.getCount()); + m_indexMap.add(valueIndex); + } + else + { + m_indexMap[countIndex] = valueIndex; + } + return countIndex; +} + +StringSliceIndexMap::CountIndex StringSliceIndexMap::findOrAdd(const UnownedStringSlice& key, Index defaultValueIndex) +{ + StringSlicePool::Handle handle; + m_pool.findOrAdd(key, handle); + const CountIndex countIndex = StringSlicePool::asIndex(handle); + if (countIndex >= m_indexMap.getCount()) + { + SLANG_ASSERT(countIndex == m_indexMap.getCount()); + m_indexMap.add(defaultValueIndex); + } + return countIndex; +} + +void StringSliceIndexMap::clear() +{ + m_pool.clear(); + m_indexMap.clear(); +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ImplicitDirectoryCollector !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +ImplicitDirectoryCollector::ImplicitDirectoryCollector(const String& canonicalPath, bool directoryExists) : + m_directoryExists(directoryExists) +{ + StringBuilder buffer; + if (canonicalPath != ".") + { + buffer << canonicalPath; + buffer.append('/'); + } + m_prefix = buffer.ProduceString(); +} + +void ImplicitDirectoryCollector::addRemainingPath(SlangPathType pathType, const UnownedStringSlice& inPathRemainder) +{ + // If it's zero length we probably don't want to add it + if (inPathRemainder.getLength() == 0) + { + // It's empty so don't add normal way - implies the directory exists + m_directoryExists = true; + return; + } + + UnownedStringSlice pathRemainder(inPathRemainder); + const Index slashIndex = pathRemainder.indexOf('/'); + + // If we have a following / that means it's an implicit directory. + if (slashIndex >= 0) + { + pathType = SLANG_PATH_TYPE_DIRECTORY; + pathRemainder = UnownedStringSlice(pathRemainder.begin(), pathRemainder.begin() + slashIndex); + } + + const Index countIndex = m_map.findOrAdd(pathRemainder, pathType); + // Make sure they are the same type + SLANG_ASSERT(SlangPathType(m_map.getValueAt(countIndex)) == pathType); +} + +void ImplicitDirectoryCollector::addPath(SlangPathType pathType, const UnownedStringSlice& canonicalPath) +{ + if (hasPrefix(canonicalPath)) + { + UnownedStringSlice remainder = getRemainder(canonicalPath); + addRemainingPath(pathType, remainder); + } +} + +SlangResult ImplicitDirectoryCollector::enumerate(FileSystemContentsCallBack callback, void* userData) +{ + const Int count = m_map.getCount(); + + for (Index i = 0; i < count; ++i) + { + const auto& pair = m_map.getAt(i); + + UnownedStringSlice path = pair.Key; + SlangPathType pathType = SlangPathType(pair.Value); + + // Note *is* 0 terminated in the pool + // Let's check tho + SLANG_ASSERT(path.begin()[path.getLength()] == 0); + callback(pathType, path.begin(), userData); + } + + return getDirectoryExists() ? SLANG_OK : SLANG_E_NOT_FOUND; +} + +SlangResult loadArchiveFileSystem(const void* data, size_t dataSizeInBytes, RefPtr& outFileSystem) +{ + RefPtr fileSystem; + if (ZipFileSystem::isArchive(data, dataSizeInBytes)) + { + // It's a zip + SLANG_RETURN_ON_FAIL(ZipFileSystem::create(fileSystem)); + } + else if (RiffFileSystem::isArchive(data, dataSizeInBytes)) + { + // It's riff contained (Slang specific) + fileSystem = new RiffFileSystem(nullptr); + } + else + { + return SLANG_FAIL; + } + SLANG_RETURN_ON_FAIL(fileSystem->loadArchive(data, dataSizeInBytes)); + + outFileSystem = fileSystem; + return SLANG_OK; +} + +SlangResult createArchiveFileSystem(SlangArchiveType type, RefPtr& outFileSystem) +{ + switch (type) + { + case SLANG_ARCHIVE_TYPE_ZIP: + { + return ZipFileSystem::create(outFileSystem); + } + case SLANG_ARCHIVE_TYPE_RIFF: + { + outFileSystem = new RiffFileSystem(nullptr); + return SLANG_OK; + } + case SLANG_ARCHIVE_TYPE_RIFF_DEFLATE: + { + outFileSystem = new RiffFileSystem(DeflateCompressionSystem::getSingleton()); + return SLANG_OK; + } + case SLANG_ARCHIVE_TYPE_RIFF_LZ4: + { + outFileSystem = new RiffFileSystem(LZ4CompressionSystem::getSingleton()); + return SLANG_OK; + } + } + + return SLANG_FAIL; +} + +} // namespace Slang -- cgit v1.2.3