LZ4 compression support (#1654)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Testing out use of lz4.

* Added ICompressionSystem, and LZ4 implementation.

* Add support for deflate compression.
Simplify compression interface - to make more easily work across apis.

* WIP on CompressedFileSystem.

* ImplicitDirectoryCollector

* SubStringIndexMap - > StringSliceIndexMap.

* WIP save stdlib in different containers.

* Support for different archive types for stdlib.

* Fix project.

* CompressedFileSystem -> ArchiveFileSystem.
Added CompressionSystemType::None

* Added ArchiveFileSystem

* Fix problem RiffFileSystem load withoug compression system.

* Test archive types.
Improve diagnostic message.

* Fix typo in testing file system archives.

* Split out archive detection.

* Fix gcc warning issue.

* Fix warning.

* RiffArchiveFileSystem -> RiffFileSystem

Co-authored-by: Tim Foley <tfoleyNV@users.noreply.github.com>

4 files changed, 27 insertions, 15 deletions

diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index 008c5eb5a..62def6f0f 100755
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -5,6 +5,7 @@
 #include "../core/slang-shared-library.h"
 
 #include "../core/slang-downstream-compiler.h"
+#include "../core/slang-archive-file-system.h"
 
 #include "../../slang-com-ptr.h"
 
@@ -2178,7 +2179,7 @@ namespace Slang
 
         SLANG_NO_THROW SlangResult SLANG_MCALL compileStdLib() override;
         SLANG_NO_THROW SlangResult SLANG_MCALL loadStdLib(const void* stdLib, size_t stdLibSizeInBytes) override;
-        SLANG_NO_THROW SlangResult SLANG_MCALL saveStdLib(ISlangBlob** outBlob) override;
+        SLANG_NO_THROW SlangResult SLANG_MCALL saveStdLib(SlangArchiveType archiveType, ISlangBlob** outBlob) override;
 
         SLANG_NO_THROW SlangCapabilityID SLANG_MCALL findCapability(char const* name) override;
 
diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h
index 9c65250cd..1739a1ee1 100644
--- a/source/slang/slang-diagnostic-defs.h
+++ b/source/slang/slang-diagnostic-defs.h
@@ -120,6 +120,7 @@ DIAGNOSTIC(    86, Error, unableToCreateModuleContainer, "unable to create modul
 
 DIAGNOSTIC(    87, Error, unableToSetDefaultDownstreamCompiler, "unable to set default downstream compiler for source language '%0' to '%1'")
 
+DIAGNOSTIC(    88, Error, unknownArchiveType, "archive type '%0' is unknown")
 
 //
 // 001xx - Downstream Compilers
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index a89818c4f..d28b50b88 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -434,6 +434,9 @@ struct OptionsParser
 
         SlangMatrixLayoutMode defaultMatrixLayoutMode = SLANG_MATRIX_LAYOUT_MODE_UNKNOWN;
 
+        // The default archive type is zip
+        SlangArchiveType archiveType = SLANG_ARCHIVE_TYPE_ZIP;
+
         bool hasLoadedRepro = false;
 
         char const* const* argCursor = &argv[0];
@@ -463,6 +466,18 @@ struct OptionsParser
                 {
                     SLANG_RETURN_ON_FAIL(session->compileStdLib());
                 }
+                else if (argStr == "-archive-type")
+                {
+                    String archiveTypeName;
+                    SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, archiveTypeName));
+
+                    archiveType = TypeTextUtil::findArchiveType(archiveTypeName.getUnownedSlice());
+                    if (archiveType == SLANG_ARCHIVE_TYPE_UNDEFINED)
+                    {
+                        sink->diagnose(SourceLoc(), Diagnostics::unknownArchiveType, archiveTypeName);
+                        return SLANG_FAIL;
+                    }
+                }
                 else if (argStr == "-save-stdlib")
                 {
                     String fileName;
@@ -470,7 +485,7 @@ struct OptionsParser
 
                     ComPtr<ISlangBlob> blob;
 
-                    SLANG_RETURN_ON_FAIL(session->saveStdLib(blob.writeRef()));
+                    SLANG_RETURN_ON_FAIL(session->saveStdLib(archiveType, blob.writeRef()));
                     SLANG_RETURN_ON_FAIL(File::writeAllBytes(fileName, blob->getBufferPointer(), blob->getBufferSize()));
                 }
                 else if (argStr == "-save-stdlib-bin-source")
@@ -480,7 +495,7 @@ struct OptionsParser
 
                     ComPtr<ISlangBlob> blob;
 
-                    SLANG_RETURN_ON_FAIL(session->saveStdLib(blob.writeRef()));
+                    SLANG_RETURN_ON_FAIL(session->saveStdLib(archiveType, blob.writeRef()));
 
                     StringBuilder builder;
                     StringWriter writer(&builder, 0);
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index fbcc97c51..c82d5cf65 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -3,8 +3,7 @@
 #include "../core/slang-io.h"
 #include "../core/slang-string-util.h"
 #include "../core/slang-shared-library.h"
-
-#include "../core/slang-zip-file-system.h"
+#include "../core/slang-archive-file-system.h"
 
 #include "slang-check.h"
 #include "slang-parameter-binding.h"
@@ -266,8 +265,8 @@ SlangResult Session::loadStdLib(const void* stdLib, size_t stdLibSizeInBytes)
     }
 
     // Make a file system to read it from
-    RefPtr<CompressedFileSystem> fileSystem;
-    SLANG_RETURN_ON_FAIL(CompressedFileSystem::createZip(stdLib, stdLibSizeInBytes, fileSystem));
+    RefPtr<ArchiveFileSystem> fileSystem;
+    SLANG_RETURN_ON_FAIL(loadArchiveFileSystem(stdLib, stdLibSizeInBytes, fileSystem));
 
     // Let's try loading serialized modules and adding them
     SLANG_RETURN_ON_FAIL(_readBuiltinModule(fileSystem, coreLanguageScope, "core"));
@@ -275,7 +274,7 @@ SlangResult Session::loadStdLib(const void* stdLib, size_t stdLibSizeInBytes)
     return SLANG_OK;
 }
 
-SlangResult Session::saveStdLib(ISlangBlob** outBlob)
+SlangResult Session::saveStdLib(SlangArchiveType archiveType, ISlangBlob** outBlob)
 {
     if (m_builtinLinkage->mapNameToLoadedModules.Count() == 0)
     {
@@ -284,8 +283,8 @@ SlangResult Session::saveStdLib(ISlangBlob** outBlob)
     }
 
     // Make a file system to read it from
-    RefPtr<CompressedFileSystem> fileSystem;
-    SLANG_RETURN_ON_FAIL(CompressedFileSystem::createZip(fileSystem));
+    RefPtr<ArchiveFileSystem> fileSystem;
+    SLANG_RETURN_ON_FAIL(createArchiveFileSystem(archiveType, fileSystem));
 
     for (auto& pair : m_builtinLinkage->mapNameToLoadedModules)
     {
@@ -315,11 +314,7 @@ SlangResult Session::saveStdLib(ISlangBlob** outBlob)
     }
 
     // Now need to convert into a blob
-    auto archiveContents = fileSystem->getArchive();
-
-    ComPtr<ISlangBlob> blob(new RawBlob(archiveContents.getBuffer(), archiveContents.getCount()));
-    *outBlob = blob.detach();
-
+    SLANG_RETURN_ON_FAIL(fileSystem->storeArchive(true, outBlob));
     return SLANG_OK;
 }