Zip FileSystem support (#1617)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Add miniz

* Fix for separator in CacheFileSystem.
Add compression unit test for zip.

* Put zip compression into core.

* Remove delimiter stripping if simplifying a path - as stripping will fix delimiters.

* ZipFileSystem WIP.

* More ZipFileSystem working.

* Added isEmpty.
Fixed small bug is contains.

* First pass support for mutability on zip.

* Improvements to File::read/writeAllBytes

* Can access and save archive - but has memory leaks.

* Fix memory leak.

* Some ZIP compression tests.

* Fix memory leak on ScopedAllocation.
Fix off by one bug on UIntSet

* Bug fix in UIntSet

* Fix remaining ZipFileSystem issues.
Adde stand alone unit-test.

* Turn tabs to spaces in slang-io.h

* Renamed mode ReadWrite (instead of just Write)

* Make miniz it's own project.

* Fix windows warning on win32.

* Remove warnings needed when miniz was included as a header library.

* Set the C++ standard via 'flags' in premake.

* Add support for 'implicit' paths.

* Add testing for implicit directories.
Better handling of implicit directories.

* Improve comments in ZipFileSystem.

* Update comment around reader/writer transformation.

3 files changed, 140 insertions, 0 deletions

diff --git a/tools/slang-test/slang-test.vcxproj b/tools/slang-test/slang-test.vcxproj
index b161aa2c9..c667d294f 100644
--- a/tools/slang-test/slang-test.vcxproj
+++ b/tools/slang-test/slang-test.vcxproj
@@ -177,6 +177,7 @@
     <ClCompile Include="test-reporter.cpp" />
     <ClCompile Include="unit-offset-container.cpp" />
     <ClCompile Include="unit-test-byte-encode.cpp" />
+    <ClCompile Include="unit-test-compression.cpp" />
     <ClCompile Include="unit-test-find-type-by-name.cpp" />
     <ClCompile Include="unit-test-free-list.cpp" />
     <ClCompile Include="unit-test-memory-arena.cpp" />
@@ -192,6 +193,9 @@
     <ProjectReference Include="..\..\source\slang\slang.vcxproj">
       <Project>{DB00DA62-0533-4AFD-B59F-A67D5B3A0808}</Project>
     </ProjectReference>
+    <ProjectReference Include="..\..\source\miniz\miniz.vcxproj">
+      <Project>{E76ACB11-4A12-4F0A-BE1E-CE0B8836EB7F}</Project>
+    </ProjectReference>
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
diff --git a/tools/slang-test/slang-test.vcxproj.filters b/tools/slang-test/slang-test.vcxproj.filters
index bbcb30a1b..e31239bc6 100644
--- a/tools/slang-test/slang-test.vcxproj.filters
+++ b/tools/slang-test/slang-test.vcxproj.filters
@@ -50,6 +50,9 @@
     <ClCompile Include="unit-test-byte-encode.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="unit-test-compression.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="unit-test-find-type-by-name.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
diff --git a/tools/slang-test/unit-test-compression.cpp b/tools/slang-test/unit-test-compression.cpp
new file mode 100644
index 000000000..787c798b5
--- /dev/null
+++ b/tools/slang-test/unit-test-compression.cpp
@@ -0,0 +1,133 @@
+// unit-compression.cpp
+
+#include "test-context.h"
+
+#include "../../source/core/slang-zip-file-system.h"
+
+using namespace Slang;
+
+static bool _equals(const void* data, size_t size, ISlangBlob* blob)
+{
+    return blob && blob->getBufferSize() == size && memcmp(blob->getBufferPointer(), data, size) == 0;
+}
+
+template <size_t SIZE>
+static bool _equals(const char (&text)[SIZE], ISlangBlob* blob)
+{
+    return _equals(text, SIZE, blob);
+}
+
+static List<String> _getContents(ISlangFileSystemExt* fileSystem, char* path)
+{
+    List<String> objs;
+
+    fileSystem->enumeratePathContents(path, [](SlangPathType pathType, const char* name, void* userData) {
+        List<String>& out = *(List<String>*)userData;
+        out.add(name);
+    }, &objs);
+
+    return objs;
+}
+
+static void compressionUnitTest()
+{
+    // Create a zip to add stuff to
+    RefPtr<CompressedFileSystem> buildFileSystem;
+    CompressedFileSystem::createZip(buildFileSystem);
+
+    const char contents[] = "I'm compressed";
+    const char contents2[] = "Some more stuff";
+    const char contents3[] = "Replace it";
+
+    {
+        ISlangMutableFileSystem* fileSystem = buildFileSystem;
+
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->createDirectory("hello")));
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->createDirectory("hello2")));
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->remove("hello")));
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->createDirectory("hello")));
+
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("file.txt", contents, SLANG_COUNT_OF(contents))));
+
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("file2.txt", contents2, SLANG_COUNT_OF(contents2))));
+
+        ComPtr<ISlangBlob> blob;
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file.txt", blob.writeRef())));
+        SLANG_CHECK(_equals(contents, blob));
+
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file2.txt", blob.writeRef())));
+        SLANG_CHECK(_equals(contents2, blob));
+
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("file2.txt", contents3, SLANG_COUNT_OF(contents3))));
+
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file2.txt", blob.writeRef())));
+        SLANG_CHECK(_equals(contents3, blob));
+
+        // Check the path type
+        {
+            SlangPathType pathType;
+            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType("file2.txt", &pathType)));
+            SLANG_CHECK(pathType == SLANG_PATH_TYPE_FILE);
+
+            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType("hello", &pathType)));
+            SLANG_CHECK(pathType == SLANG_PATH_TYPE_DIRECTORY);
+        }
+
+        // Enumerate
+        {
+            for (const auto& obj : _getContents(fileSystem, ""))
+            {
+                // All of these should exist
+                SlangPathType pathType;
+                SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType(obj.getBuffer(), &pathType)));
+            }
+        }
+
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("implicit-path/file2.txt", contents3, SLANG_COUNT_OF(contents3))));
+
+        {
+            SlangPathType pathType;
+            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType("implicit-path", &pathType)));
+
+            SLANG_CHECK(pathType == SLANG_PATH_TYPE_DIRECTORY);
+
+            List<String> objs = _getContents(fileSystem, "implicit-path");
+
+            // It contains a file
+            SLANG_CHECK(objs.getCount() == 1);
+
+            for (const auto& obj : objs)
+            {
+                String path = Path::combine("implicit-path", obj);
+
+                // All of these should exist
+                SlangPathType pathType;
+                SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType(path.getBuffer(), &pathType)));
+            }
+
+            // Make an explicit path, and see whe have the same results
+            fileSystem->createDirectory("implicit-path");
+
+            objs = _getContents(fileSystem, "implicit-path");
+            SLANG_CHECK(objs.getCount() == 1);
+        }
+    }
+
+    // Load and check its okay
+    {
+        const auto archive = buildFileSystem->getArchive();
+
+        RefPtr<CompressedFileSystem> fileSystem;
+        CompressedFileSystem::createZip(archive.getBuffer(), archive.getCount(), fileSystem);
+
+        ComPtr<ISlangBlob> blob;
+
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file.txt", blob.writeRef())));
+        SLANG_CHECK(_equals(contents, blob));
+
+        SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file2.txt", blob.writeRef())));
+        SLANG_CHECK(_equals(contents3, blob));
+    }
+}
+
+SLANG_UNIT_TEST("Compression", compressionUnitTest);