From 723796a0a0fed8e5b8c3222b1c90443189113098 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Mon, 11 Jan 2021 15:24:11 -0500 Subject: LZ4 compression support (#1654) * #include an absolute path didn't work - because paths were taken to always be relative. * Testing out use of lz4. * Added ICompressionSystem, and LZ4 implementation. * Add support for deflate compression. Simplify compression interface - to make more easily work across apis. * WIP on CompressedFileSystem. * ImplicitDirectoryCollector * SubStringIndexMap - > StringSliceIndexMap. * WIP save stdlib in different containers. * Support for different archive types for stdlib. * Fix project. * CompressedFileSystem -> ArchiveFileSystem. Added CompressionSystemType::None * Added ArchiveFileSystem * Fix problem RiffFileSystem load withoug compression system. * Test archive types. Improve diagnostic message. * Fix typo in testing file system archives. * Split out archive detection. * Fix gcc warning issue. * Fix warning. * RiffArchiveFileSystem -> RiffFileSystem Co-authored-by: Tim Foley --- source/slang/slang-compiler.h | 3 ++- source/slang/slang-diagnostic-defs.h | 1 + source/slang/slang-options.cpp | 19 +++++++++++++++++-- source/slang/slang.cpp | 19 +++++++------------ 4 files changed, 27 insertions(+), 15 deletions(-) (limited to 'source/slang') diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index 008c5eb5a..62def6f0f 100755 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -5,6 +5,7 @@ #include "../core/slang-shared-library.h" #include "../core/slang-downstream-compiler.h" +#include "../core/slang-archive-file-system.h" #include "../../slang-com-ptr.h" @@ -2178,7 +2179,7 @@ namespace Slang SLANG_NO_THROW SlangResult SLANG_MCALL compileStdLib() override; SLANG_NO_THROW SlangResult SLANG_MCALL loadStdLib(const void* stdLib, size_t stdLibSizeInBytes) override; - SLANG_NO_THROW SlangResult SLANG_MCALL saveStdLib(ISlangBlob** outBlob) override; + SLANG_NO_THROW SlangResult SLANG_MCALL saveStdLib(SlangArchiveType archiveType, ISlangBlob** outBlob) override; SLANG_NO_THROW SlangCapabilityID SLANG_MCALL findCapability(char const* name) override; diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h index 9c65250cd..1739a1ee1 100644 --- a/source/slang/slang-diagnostic-defs.h +++ b/source/slang/slang-diagnostic-defs.h @@ -120,6 +120,7 @@ DIAGNOSTIC( 86, Error, unableToCreateModuleContainer, "unable to create modul DIAGNOSTIC( 87, Error, unableToSetDefaultDownstreamCompiler, "unable to set default downstream compiler for source language '%0' to '%1'") +DIAGNOSTIC( 88, Error, unknownArchiveType, "archive type '%0' is unknown") // // 001xx - Downstream Compilers diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp index a89818c4f..d28b50b88 100644 --- a/source/slang/slang-options.cpp +++ b/source/slang/slang-options.cpp @@ -434,6 +434,9 @@ struct OptionsParser SlangMatrixLayoutMode defaultMatrixLayoutMode = SLANG_MATRIX_LAYOUT_MODE_UNKNOWN; + // The default archive type is zip + SlangArchiveType archiveType = SLANG_ARCHIVE_TYPE_ZIP; + bool hasLoadedRepro = false; char const* const* argCursor = &argv[0]; @@ -463,6 +466,18 @@ struct OptionsParser { SLANG_RETURN_ON_FAIL(session->compileStdLib()); } + else if (argStr == "-archive-type") + { + String archiveTypeName; + SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, archiveTypeName)); + + archiveType = TypeTextUtil::findArchiveType(archiveTypeName.getUnownedSlice()); + if (archiveType == SLANG_ARCHIVE_TYPE_UNDEFINED) + { + sink->diagnose(SourceLoc(), Diagnostics::unknownArchiveType, archiveTypeName); + return SLANG_FAIL; + } + } else if (argStr == "-save-stdlib") { String fileName; @@ -470,7 +485,7 @@ struct OptionsParser ComPtr blob; - SLANG_RETURN_ON_FAIL(session->saveStdLib(blob.writeRef())); + SLANG_RETURN_ON_FAIL(session->saveStdLib(archiveType, blob.writeRef())); SLANG_RETURN_ON_FAIL(File::writeAllBytes(fileName, blob->getBufferPointer(), blob->getBufferSize())); } else if (argStr == "-save-stdlib-bin-source") @@ -480,7 +495,7 @@ struct OptionsParser ComPtr blob; - SLANG_RETURN_ON_FAIL(session->saveStdLib(blob.writeRef())); + SLANG_RETURN_ON_FAIL(session->saveStdLib(archiveType, blob.writeRef())); StringBuilder builder; StringWriter writer(&builder, 0); diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index fbcc97c51..c82d5cf65 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -3,8 +3,7 @@ #include "../core/slang-io.h" #include "../core/slang-string-util.h" #include "../core/slang-shared-library.h" - -#include "../core/slang-zip-file-system.h" +#include "../core/slang-archive-file-system.h" #include "slang-check.h" #include "slang-parameter-binding.h" @@ -266,8 +265,8 @@ SlangResult Session::loadStdLib(const void* stdLib, size_t stdLibSizeInBytes) } // Make a file system to read it from - RefPtr fileSystem; - SLANG_RETURN_ON_FAIL(CompressedFileSystem::createZip(stdLib, stdLibSizeInBytes, fileSystem)); + RefPtr fileSystem; + SLANG_RETURN_ON_FAIL(loadArchiveFileSystem(stdLib, stdLibSizeInBytes, fileSystem)); // Let's try loading serialized modules and adding them SLANG_RETURN_ON_FAIL(_readBuiltinModule(fileSystem, coreLanguageScope, "core")); @@ -275,7 +274,7 @@ SlangResult Session::loadStdLib(const void* stdLib, size_t stdLibSizeInBytes) return SLANG_OK; } -SlangResult Session::saveStdLib(ISlangBlob** outBlob) +SlangResult Session::saveStdLib(SlangArchiveType archiveType, ISlangBlob** outBlob) { if (m_builtinLinkage->mapNameToLoadedModules.Count() == 0) { @@ -284,8 +283,8 @@ SlangResult Session::saveStdLib(ISlangBlob** outBlob) } // Make a file system to read it from - RefPtr fileSystem; - SLANG_RETURN_ON_FAIL(CompressedFileSystem::createZip(fileSystem)); + RefPtr fileSystem; + SLANG_RETURN_ON_FAIL(createArchiveFileSystem(archiveType, fileSystem)); for (auto& pair : m_builtinLinkage->mapNameToLoadedModules) { @@ -315,11 +314,7 @@ SlangResult Session::saveStdLib(ISlangBlob** outBlob) } // Now need to convert into a blob - auto archiveContents = fileSystem->getArchive(); - - ComPtr blob(new RawBlob(archiveContents.getBuffer(), archiveContents.getCount())); - *outBlob = blob.detach(); - + SLANG_RETURN_ON_FAIL(fileSystem->storeArchive(true, outBlob)); return SLANG_OK; } -- cgit v1.2.3