LZ4 compression support (#1654)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Testing out use of lz4.

* Added ICompressionSystem, and LZ4 implementation.

* Add support for deflate compression.
Simplify compression interface - to make more easily work across apis.

* WIP on CompressedFileSystem.

* ImplicitDirectoryCollector

* SubStringIndexMap - > StringSliceIndexMap.

* WIP save stdlib in different containers.

* Support for different archive types for stdlib.

* Fix project.

* CompressedFileSystem -> ArchiveFileSystem.
Added CompressionSystemType::None

* Added ArchiveFileSystem

* Fix problem RiffFileSystem load withoug compression system.

* Test archive types.
Improve diagnostic message.

* Fix typo in testing file system archives.

* Split out archive detection.

* Fix gcc warning issue.

* Fix warning.

* RiffArchiveFileSystem -> RiffFileSystem

Co-authored-by: Tim Foley <tfoleyNV@users.noreply.github.com>

4 files changed, 42 insertions, 0 deletions

diff --git a/build/visual-studio/core/core.vcxproj b/build/visual-studio/core/core.vcxproj
index 7c97eed01..2bf02ecdd 100644
--- a/build/visual-studio/core/core.vcxproj
+++ b/build/visual-studio/core/core.vcxproj
@@ -171,6 +171,7 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\source\core\slang-allocator.h" />
+    <ClInclude Include="..\..\..\source\core\slang-archive-file-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-array-view.h" />
     <ClInclude Include="..\..\..\source\core\slang-array.h" />
     <ClInclude Include="..\..\..\source\core\slang-basic.h" />
@@ -178,6 +179,8 @@
     <ClInclude Include="..\..\..\source\core\slang-byte-encode-util.h" />
     <ClInclude Include="..\..\..\source\core\slang-char-util.h" />
     <ClInclude Include="..\..\..\source\core\slang-common.h" />
+    <ClInclude Include="..\..\..\source\core\slang-compression-system.h" />
+    <ClInclude Include="..\..\..\source\core\slang-deflate-compression-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-dictionary.h" />
     <ClInclude Include="..\..\..\source\core\slang-downstream-compiler.h" />
     <ClInclude Include="..\..\..\source\core\slang-exception.h" />
@@ -188,6 +191,7 @@
     <ClInclude Include="..\..\..\source\core\slang-io.h" />
     <ClInclude Include="..\..\..\source\core\slang-linked-list.h" />
     <ClInclude Include="..\..\..\source\core\slang-list.h" />
+    <ClInclude Include="..\..\..\source\core\slang-lz4-compression-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-math.h" />
     <ClInclude Include="..\..\..\source\core\slang-memory-arena.h" />
     <ClInclude Include="..\..\..\source\core\slang-name-convention-util.h" />
@@ -197,6 +201,7 @@
     <ClInclude Include="..\..\..\source\core\slang-process-util.h" />
     <ClInclude Include="..\..\..\source\core\slang-random-generator.h" />
     <ClInclude Include="..\..\..\source\core\slang-render-api-util.h" />
+    <ClInclude Include="..\..\..\source\core\slang-riff-file-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-riff.h" />
     <ClInclude Include="..\..\..\source\core\slang-secure-crt.h" />
     <ClInclude Include="..\..\..\source\core\slang-semantic-version.h" />
@@ -220,14 +225,17 @@
     <ClInclude Include="..\..\..\source\core\windows\slang-win-visual-studio-util.h" />
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\..\source\core\slang-archive-file-system.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-blob.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-byte-encode-util.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-char-util.cpp" />
+    <ClCompile Include="..\..\..\source\core\slang-deflate-compression-system.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-downstream-compiler.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-free-list.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-gcc-compiler-util.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-hex-dump-util.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-io.cpp" />
+    <ClCompile Include="..\..\..\source\core\slang-lz4-compression-system.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-memory-arena.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-name-convention-util.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-nvrtc-compiler.cpp" />
@@ -235,6 +243,7 @@
     <ClCompile Include="..\..\..\source\core\slang-platform.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-random-generator.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-render-api-util.cpp" />
+    <ClCompile Include="..\..\..\source\core\slang-riff-file-system.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-riff.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-semantic-version.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-shared-library.cpp" />
diff --git a/build/visual-studio/core/core.vcxproj.filters b/build/visual-studio/core/core.vcxproj.filters
index 9a680cb39..99cc6ba6a 100644
--- a/build/visual-studio/core/core.vcxproj.filters
+++ b/build/visual-studio/core/core.vcxproj.filters
@@ -12,6 +12,9 @@
     <ClInclude Include="..\..\..\source\core\slang-allocator.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\source\core\slang-archive-file-system.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\source\core\slang-array-view.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -33,6 +36,12 @@
     <ClInclude Include="..\..\..\source\core\slang-common.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\source\core\slang-compression-system.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\..\source\core\slang-deflate-compression-system.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\source\core\slang-dictionary.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -63,6 +72,9 @@
     <ClInclude Include="..\..\..\source\core\slang-list.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\source\core\slang-lz4-compression-system.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\source\core\slang-math.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -90,6 +102,9 @@
     <ClInclude Include="..\..\..\source\core\slang-render-api-util.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\source\core\slang-riff-file-system.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\source\core\slang-riff.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -155,6 +170,9 @@
     </ClInclude>
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\..\source\core\slang-archive-file-system.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\source\core\slang-blob.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -164,6 +182,9 @@
     <ClCompile Include="..\..\..\source\core\slang-char-util.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\source\core\slang-deflate-compression-system.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\source\core\slang-downstream-compiler.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -179,6 +200,9 @@
     <ClCompile Include="..\..\..\source\core\slang-io.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\source\core\slang-lz4-compression-system.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\source\core\slang-memory-arena.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -200,6 +224,9 @@
     <ClCompile Include="..\..\..\source\core\slang-render-api-util.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\source\core\slang-riff-file-system.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\source\core\slang-riff.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
diff --git a/build/visual-studio/slang-test/slang-test.vcxproj b/build/visual-studio/slang-test/slang-test.vcxproj
index 4a56eb437..1626b6cca 100644
--- a/build/visual-studio/slang-test/slang-test.vcxproj
+++ b/build/visual-studio/slang-test/slang-test.vcxproj
@@ -196,6 +196,9 @@
     <ProjectReference Include="..\miniz\miniz.vcxproj">
       <Project>{E76ACB11-4A12-4F0A-BE1E-CE0B8836EB7F}</Project>
     </ProjectReference>
+    <ProjectReference Include="..\lz4\lz4.vcxproj">
+      <Project>{E1EC8075-823E-46E5-BC38-C124CCCDF878}</Project>
+    </ProjectReference>
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
diff --git a/build/visual-studio/slang/slang.vcxproj b/build/visual-studio/slang/slang.vcxproj
index 88c4b59e2..642ddab7a 100644
--- a/build/visual-studio/slang/slang.vcxproj
+++ b/build/visual-studio/slang/slang.vcxproj
@@ -443,6 +443,9 @@
     <ProjectReference Include="..\miniz\miniz.vcxproj">
       <Project>{E76ACB11-4A12-4F0A-BE1E-CE0B8836EB7F}</Project>
     </ProjectReference>
+    <ProjectReference Include="..\lz4\lz4.vcxproj">
+      <Project>{E1EC8075-823E-46E5-BC38-C124CCCDF878}</Project>
+    </ProjectReference>
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">