IMutableFileSystem::saveFileBlob (#2427)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Remove ref count for Entry in RiffFileSystem.
Free up backing Entry types (to work around Dictionary not doing this).

* Some small improvements to RiffFileSystem.

* Add testing for file systems.

* Split out MemoryFileSystem.

* Add some documentation around different FileSystems.

* Small tiry up - removing unused headers, fixing some comments.
Use StringBlob::moveCreate where appropriate.

* Small improvement to MemoryFileSystem.
Improve documentation comments a little.

* Added PathKind

* * Make MemoryFileSystem not have implicit directories
* Make RelativeFileSystem only allow access to files in file system (kind of like chroot)
* Added Path::simplifyAbsolute
* Special handling for root of MemoryFileSystem
* Improvements around paths for different impls

* More improvements around RelativeFileSystem.
Special case root handling.

* Test archive serialization.
Move testinf from compression.
Remove the implicit directory test -> doesn't work on all file systems.

* Small optimization that removes need for check for a parent unless an item is being *created*.

* Add implicit path testing.

* Add support for saveFileBlob
Add testing for saveFileBlob

* Removed TemporaryFileSet
Added PlatformUtil::outputDebugMessage

* Some small improvements around RelativeFileSystem.

* Split out ImplicitDirectoryCollector so can use without requiring compression systems.

* Split out StringSliceIndexMap into own files.

2 files changed, 225 insertions, 193 deletions

diff --git a/tools/slang-unit-test/unit-test-compression.cpp b/tools/slang-unit-test/unit-test-compression.cpp
index e3bcbcf00..486a252b2 100644
--- a/tools/slang-unit-test/unit-test-compression.cpp
+++ b/tools/slang-unit-test/unit-test-compression.cpp
@@ -1,174 +1,32 @@
 // unit-compression.cpp
-
 #include "tools/unit-test/slang-unit-test.h"
 
-#include "source/core/slang-io.h"
-#include "../../source/core/slang-zip-file-system.h"
-
 #include "../../source/core/slang-lz4-compression-system.h"
 #include "../../source/core/slang-deflate-compression-system.h"
 
-#include "../../source/core/slang-destroyable.h"
-
 using namespace Slang;
 
-static bool _equals(const void* data, size_t size, ISlangBlob* blob)
-{
-    return blob && blob->getBufferSize() == size && memcmp(blob->getBufferPointer(), data, size) == 0;
-}
-
-template <size_t SIZE>
-static bool _equals(const char (&text)[SIZE], ISlangBlob* blob)
-{
-    return _equals(text, SIZE, blob);
-}
-
-static List<String> _getContents(ISlangFileSystemExt* fileSystem, const char* path)
+static ICompressionSystem* _getCompressionSystem(CompressionSystemType type)
 {
-    List<String> objs;
-
-    fileSystem->enumeratePathContents(path, [](SlangPathType pathType, const char* name, void* userData) {
-        List<String>& out = *(List<String>*)userData;
-        out.add(name);
-    }, &objs);
-
-    return objs;
+    switch (type)
+    {
+        case CompressionSystemType::Deflate: return DeflateCompressionSystem::getSingleton(); break;
+        case CompressionSystemType::LZ4:     return LZ4CompressionSystem::getSingleton(); break;
+        default: break;
+    }
+    return nullptr;
 }
 
 SLANG_UNIT_TEST(compression)
 {
-    const SlangArchiveType archiveTypes[] =
-    {
-        SLANG_ARCHIVE_TYPE_RIFF,
-        SLANG_ARCHIVE_TYPE_RIFF_DEFLATE,
-        SLANG_ARCHIVE_TYPE_RIFF_LZ4,
-        SLANG_ARCHIVE_TYPE_ZIP
-    };
-
-    for (auto archiveType : archiveTypes)
-    {
-        // Test out archive file systems
-        ComPtr<ISlangMutableFileSystem> fileSystem;
-        SLANG_CHECK(SLANG_SUCCEEDED(createArchiveFileSystem(archiveType, fileSystem)));
-        
-        const char contents[] = "I'm compressed";
-        const char contents2[] = "Some more stuff";
-        const char contents3[] = "Replace it";
-
-        {
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->createDirectory("hello")));
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->createDirectory("hello2")));
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->remove("hello")));
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->createDirectory("hello")));
-
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("file.txt", contents, SLANG_COUNT_OF(contents))));
-
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("file2.txt", contents2, SLANG_COUNT_OF(contents2))));
-
-            ComPtr<ISlangBlob> blob;
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file.txt", blob.writeRef())));
-            SLANG_CHECK(_equals(contents, blob));
-
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file2.txt", blob.writeRef())));
-            SLANG_CHECK(_equals(contents2, blob));
-
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("file2.txt", contents3, SLANG_COUNT_OF(contents3))));
-
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file2.txt", blob.writeRef())));
-            SLANG_CHECK(_equals(contents3, blob));
-
-            // Check the path type
-            {
-                SlangPathType pathType;
-                SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType("file2.txt", &pathType)));
-                SLANG_CHECK(pathType == SLANG_PATH_TYPE_FILE);
-
-                SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType("hello", &pathType)));
-                SLANG_CHECK(pathType == SLANG_PATH_TYPE_DIRECTORY);
-            }
-
-            // Enumerate
-            {
-                for (const auto& obj : _getContents(fileSystem, ""))
-                {
-                    // All of these should exist
-                    SlangPathType pathType;
-                    SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType(obj.getBuffer(), &pathType)));
-                }
-            }
-
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->saveFile("implicit-path/file2.txt", contents3, SLANG_COUNT_OF(contents3))));
-
-            {
-                SlangPathType pathType;
-                SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType("implicit-path", &pathType)));
-
-                SLANG_CHECK(pathType == SLANG_PATH_TYPE_DIRECTORY);
-
-                List<String> objs = _getContents(fileSystem, "implicit-path");
-
-                // It contains a file
-                SLANG_CHECK(objs.getCount() == 1);
-
-                for (const auto& obj : objs)
-                {
-                    String path = Path::combine("implicit-path", obj);
-
-                    // All of these should exist
-                    SlangPathType pathType;
-                    SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->getPathType(path.getBuffer(), &pathType)));
-                }
-
-                // Make an explicit path, and see whe have the same results
-                fileSystem->createDirectory("implicit-path");
-
-                objs = _getContents(fileSystem, "implicit-path");
-                SLANG_CHECK(objs.getCount() == 1);
-            }
-        }
-    
-
-        // Load and check its okay
- 
-        {
-            IArchiveFileSystem* archiveFileSystem = as<IArchiveFileSystem>(fileSystem);
-
-            ComPtr<ISlangBlob> archiveBlob;
-            SLANG_CHECK(SLANG_SUCCEEDED(archiveFileSystem->storeArchive(false, archiveBlob.writeRef())));
-
-
-            ComPtr<ISlangFileSystemExt> fileSystem;
-#if 0
-            SLANG_CHECK(SLANG_SUCCEEDED(createArchiveFileSystem(archiveType, fileSystem)));
-
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadArchive(archiveBlob->getBufferPointer(), archiveBlob->getBufferSize())));
-#else
-            SLANG_CHECK(SLANG_SUCCEEDED(loadArchiveFileSystem(archiveBlob->getBufferPointer(), archiveBlob->getBufferSize(), fileSystem)));
-#endif
-
-            ComPtr<ISlangBlob> blob;
-
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file.txt", blob.writeRef())));
-            SLANG_CHECK(_equals(contents, blob));
-
-            SLANG_CHECK(SLANG_SUCCEEDED(fileSystem->loadFile("file2.txt", blob.writeRef())));
-            SLANG_CHECK(_equals(contents3, blob));
-        }
-    }
-
     // Test out compression systems
-    for (Index i = 0; i < 2; ++i)
+    for (Index i = 0; i < Count(CompressionSystemType::CountOf); ++i)
     {
-        // Lets try lz4
-
-        ICompressionSystem* system = nullptr;
-        if (i == 0)
+        ICompressionSystem* system = _getCompressionSystem(CompressionSystemType(i));
+     
+        if (!system)
         {
-            system = LZ4CompressionSystem::getSingleton();
-        }
-        else
-        {
-            system = DeflateCompressionSystem::getSingleton();
+            continue;
         }
 
         const char src[] = "Some text to compress";
@@ -176,16 +34,16 @@ SLANG_UNIT_TEST(compression)
 
         ComPtr<ISlangBlob> compressedBlob;
 
+        // Use the default style
         CompressionStyle style;
 
         SLANG_CHECK(SLANG_SUCCEEDED(system->compress(&style, src, srcSize, compressedBlob.writeRef()))); 
         
         // Now lets decompress
-
         List<char> decompressedData;
         decompressedData.setCount(srcSize);
 
         SLANG_CHECK(SLANG_SUCCEEDED(system->decompress(compressedBlob->getBufferPointer(), compressedBlob->getBufferSize(), srcSize, decompressedData.getBuffer())));
-        SLANG_CHECK(memcmp(src, decompressedData.getBuffer(), srcSize) == 0);
+        SLANG_CHECK(::memcmp(src, decompressedData.getBuffer(), srcSize) == 0);
     }
 }
diff --git a/tools/slang-unit-test/unit-test-file-system.cpp b/tools/slang-unit-test/unit-test-file-system.cpp
index e3e2bb3cf..1575f4dfd 100644
--- a/tools/slang-unit-test/unit-test-file-system.cpp
+++ b/tools/slang-unit-test/unit-test-file-system.cpp
@@ -10,11 +10,15 @@
 #include "../../source/core/slang-deflate-compression-system.h"
 #include "../../source/core/slang-lz4-compression-system.h"
 
+#include "../../source/core/slang-destroyable.h"
+
+#include "../../source/core/slang-io.h"
+
 #include "tools/unit-test/slang-unit-test.h"
 
 using namespace Slang;
 
-namespace { 
+namespace { // anonymous
 
 enum class FileSystemType
 {
@@ -31,22 +35,18 @@ struct Entry
 {
 	typedef Entry ThisType;
 
-	bool operator<(const ThisType& rhs) const { return name < rhs.name; }
-	bool operator==(const ThisType& rhs) const { return name == rhs.name && type == rhs.type; }
+	bool operator<(const ThisType& rhs) const { return path < rhs.path; }
+	bool operator==(const ThisType& rhs) const { return path == rhs.path && type == rhs.type; }
 	bool operator!=(const ThisType& rhs) const { return !(*this == rhs); }
 
 	SlangPathType type;
-	String name;
+	String path;
 };
 
-} // 
+} // anonymous
 
-static SlangResult _createAndCheckFile(ISlangMutableFileSystem* fileSystem, const char* path, const char* contents)
+static SlangResult _checkFile(ISlangFileSystemExt* fileSystem, const char* path, const UnownedStringSlice& contentsSlice)
 {
-	UnownedStringSlice contentsSlice(contents);
-
-	SLANG_RETURN_ON_FAIL(fileSystem->saveFile(path, contentsSlice.begin(), contentsSlice.getLength()));
-
 	SlangPathType pathType;
 	SLANG_RETURN_ON_FAIL(fileSystem->getPathType(path, &pathType));
 
@@ -66,6 +66,31 @@ static SlangResult _createAndCheckFile(ISlangMutableFileSystem* fileSystem, cons
 	{
 		return SLANG_FAIL;
 	}
+	return SLANG_OK;
+}
+
+static SlangResult _createAndCheckFile(ISlangMutableFileSystem* fileSystem, const char* path, const char* contents)
+{
+	UnownedStringSlice contentsSlice(contents);
+
+	SLANG_RETURN_ON_FAIL(fileSystem->saveFile(path, contentsSlice.begin(), contentsSlice.getLength()));
+	SLANG_RETURN_ON_FAIL(_checkFile(fileSystem, path, contentsSlice));
+
+	// Delete it
+	SLANG_RETURN_ON_FAIL(fileSystem->remove(path));
+
+	// Check it's gone
+	SlangPathType pathType;
+	if (SLANG_SUCCEEDED(fileSystem->getPathType(path, &pathType)))
+	{
+		return SLANG_FAIL;
+	}
+
+	// Save as a blob
+	ComPtr<ISlangBlob> blob = RawBlob::create(contentsSlice.begin(), contentsSlice.getLength());
+
+	SLANG_RETURN_ON_FAIL(fileSystem->saveFileBlob(path, blob));
+	SLANG_RETURN_ON_FAIL(_checkFile(fileSystem, path, contentsSlice));
 
 	return SLANG_OK;
 }
@@ -91,7 +116,7 @@ static void _entryCallback(SlangPathType pathType, const char* name, void* userD
 	out.add(Entry{pathType, name});
 }
 
-static SlangResult _enumeratePath(ISlangMutableFileSystem* fileSystem, const char* path, const ConstArrayView<Entry>& entries)
+static SlangResult _enumeratePath(ISlangFileSystemExt* fileSystem, const char* path, const ConstArrayView<Entry>& entries)
 {
 	List<Entry> contents;
 	
@@ -107,10 +132,10 @@ static SlangResult _enumeratePath(ISlangMutableFileSystem* fileSystem, const cha
 	return SLANG_OK;
 }
 
-static SlangResult _checkSimplifiedPath(ISlangMutableFileSystem* fileSystem, const char* path, const char* normalPath)
+static SlangResult _checkSimplifiedPath(ISlangFileSystemExt* fileSystem, const char* path, const char* normalPath)
 {
 	ComPtr<ISlangBlob> simplifiedPathBlob;
-	SLANG_RETURN_ON_FAIL(fileSystem->getSimplifiedPath(path, simplifiedPathBlob.writeRef()));
+	SLANG_RETURN_ON_FAIL(fileSystem->getPath(PathKind::Simplified, path, simplifiedPathBlob.writeRef()));
 
 	auto simplifiedPath = StringUtil::getString(simplifiedPathBlob);
 
@@ -122,47 +147,168 @@ static SlangResult _checkSimplifiedPath(ISlangMutableFileSystem* fileSystem, con
 	return SLANG_OK;
 }
 
-static SlangResult _test(FileSystemType type)
+SlangResult _appendPathEntries(ISlangFileSystemExt* fileSystem, const char* inBasePath, List<Entry>& outEntries)
 {
-	ComPtr<ISlangMutableFileSystem> fileSystem;
-
-	switch (type)
+	const UnownedStringSlice basePath(inBasePath);
+	if (basePath == toSlice(".") || basePath.getLength() == 0)
 	{
-		case FileSystemType::Zip:
+		// We don't need to append path prefixes if we are at the root.
+		SLANG_RETURN_ON_FAIL(fileSystem->enumeratePathContents(inBasePath, _entryCallback, (void*)&outEntries));
+	}
+	else
+	{
+		const Index startIndex = outEntries.getCount();
+		SLANG_RETURN_ON_FAIL(fileSystem->enumeratePathContents(inBasePath, _entryCallback, (void*)&outEntries));
+
+		const String basePathString(basePath);
+
+		// we need to fix all of the added paths to make absolute
+		const Count count = outEntries.getCount();
+		for (Index i = startIndex; i < count; ++i)
 		{
-			SLANG_RETURN_ON_FAIL(ZipFileSystem::create(fileSystem));
-			break;
+			auto& entry = outEntries[i];
+			entry.path = Path::combine(basePathString, entry.path);
 		}
-		case FileSystemType::RiffUncompressed:
+	}
+
+	return SLANG_OK;
+}
+
+static SlangResult _getAllEntries(ISlangFileSystemExt* fileSystem, const char* inBasePath, List<Entry>& outEntries)
+{
+	outEntries.clear();
+
+	// Simplify the base
+	auto basePath = Path::simplify(inBasePath);
+
+	_appendPathEntries(fileSystem, basePath.getBuffer(), outEntries);
+
+	for (Index i = 0; i < outEntries.getCount(); ++i)
+	{
+		// We need to make a copy as outEntries is mutated 
+		const Entry entry = outEntries[i];
+		if (entry.type == SLANG_PATH_TYPE_DIRECTORY)
 		{
-			fileSystem = new RiffFileSystem(nullptr);
-			break;
+			_appendPathEntries(fileSystem, entry.path.getBuffer(), outEntries);
 		}
-		case FileSystemType::RiffDeflate:
+	}
+
+	// Sort to remove issues with traversal ordering
+	outEntries.sort();
+	return SLANG_OK;
+}
+
+static SlangResult _checkEqual(ISlangFileSystemExt* a, ISlangFileSystemExt* b)
+{
+	List<Entry> aEntries, bEntries;
+
+	SLANG_RETURN_ON_FAIL(_getAllEntries(a, ".", aEntries));
+	SLANG_RETURN_ON_FAIL(_getAllEntries(b, ".", bEntries));
+
+	if (aEntries != bEntries)
+	{
+		return SLANG_FAIL;
+	}
+
+	// For all the files check the contents is the same
+
+	for (const auto& entry : aEntries)
+	{
+		if (entry.type != SLANG_PATH_TYPE_FILE)
 		{
-			fileSystem = new RiffFileSystem(DeflateCompressionSystem::getSingleton());
-			break;
+			continue;
 		}
-		case FileSystemType::RiffLZ4:
+
+		ComPtr<ISlangBlob> blobA, blobB;
+
+		SLANG_RETURN_ON_FAIL(a->loadFile(entry.path.getBuffer(), blobA.writeRef()));
+		SLANG_RETURN_ON_FAIL(b->loadFile(entry.path.getBuffer(), blobB.writeRef()));
+
+		if (blobA->getBufferSize() != blobB->getBufferSize())
 		{
-			fileSystem = new RiffFileSystem(LZ4CompressionSystem::getSingleton());
-			break;
+			return SLANG_FAIL;
 		}
-		case FileSystemType::Memory:
+
+		if (::memcmp(blobA->getBufferPointer(), blobB->getBufferPointer(), blobA->getBufferSize()) != 0)
 		{
-			fileSystem = new MemoryFileSystem;
-			break;
+			return SLANG_FAIL;
 		}
+	}
+
+	return SLANG_OK;
+}
+
+static SlangResult _createFileSystem(FileSystemType type, ComPtr<ISlangMutableFileSystem>& outFileSystem)
+{
+	outFileSystem.setNull();
+	switch (type)
+	{
+		case FileSystemType::Zip:				return ZipFileSystem::create(outFileSystem);
+		case FileSystemType::RiffUncompressed:	outFileSystem = new RiffFileSystem(nullptr); break;
+		case FileSystemType::RiffDeflate:		outFileSystem = new RiffFileSystem(DeflateCompressionSystem::getSingleton()); break;
+		case FileSystemType::RiffLZ4:			outFileSystem = new RiffFileSystem(LZ4CompressionSystem::getSingleton()); break;
+		case FileSystemType::Memory:			outFileSystem = new MemoryFileSystem; break;
 		case FileSystemType::Relative:
 		{
 			ComPtr<ISlangMutableFileSystem> memoryFileSystem(new MemoryFileSystem);
 			memoryFileSystem->createDirectory("base");
 
-			fileSystem = new RelativeFileSystem(memoryFileSystem, "base");
+			outFileSystem = new RelativeFileSystem(memoryFileSystem, "base");
 			break;
 		}
 	}
 
+	return outFileSystem ? SLANG_OK : SLANG_FAIL;
+}
+
+static SlangResult _testImplicitDirectory(FileSystemType type)
+{
+	ComPtr<ISlangMutableFileSystem> fileSystem;
+	SLANG_RETURN_ON_FAIL(_createFileSystem(type, fileSystem));
+
+	const char contents3[] = "Some text....";
+
+	SLANG_RETURN_ON_FAIL(fileSystem->saveFile("implicit-path/file2.txt", contents3, SLANG_COUNT_OF(contents3)));
+
+	{
+		SlangPathType pathType;
+		SLANG_RETURN_ON_FAIL(fileSystem->getPathType("implicit-path", &pathType));
+
+		SLANG_CHECK(pathType == SLANG_PATH_TYPE_DIRECTORY);
+
+		auto checkEntries = [&]() -> SlangResult
+		{
+			List<Entry> entries;
+			SLANG_RETURN_ON_FAIL(_getAllEntries(fileSystem, "implicit-path", entries));
+
+			// It contains a file
+			SLANG_CHECK(entries.getCount() == 1);
+
+			for (const auto& entry : entries)
+			{
+				// All of these should exist
+				SlangPathType pathType;
+				SLANG_RETURN_ON_FAIL(fileSystem->getPathType(entry.path.getBuffer(), &pathType));
+			}
+			return SLANG_OK;
+		};
+
+		SLANG_RETURN_ON_FAIL(checkEntries());
+
+		// Make an explicit path, and see whe have the same results
+		fileSystem->createDirectory("implicit-path");
+
+		SLANG_RETURN_ON_FAIL(checkEntries());
+	}
+
+	return SLANG_OK;
+}
+
+static SlangResult _test(FileSystemType type)
+{
+	ComPtr<ISlangMutableFileSystem> fileSystem;
+	SLANG_RETURN_ON_FAIL(_createFileSystem(type, fileSystem));
+
 	SLANG_RETURN_ON_FAIL(_createAndCheckFile(fileSystem, "a", "someText"));
 	SLANG_RETURN_ON_FAIL(_createAndCheckFile(fileSystem, "b", "A longer bit of text...."));
 
@@ -186,6 +332,22 @@ static SlangResult _test(FileSystemType type)
 		SLANG_RETURN_ON_FAIL(_checkSimplifiedPath(fileSystem, "d/../a", "a"));
 	}
 	
+
+	// If we have an archive file system check out it's behavior
+	if (IArchiveFileSystem* archiveFileSystem = as<IArchiveFileSystem>(fileSystem))
+	{
+		// Load and check its okay
+
+		ComPtr<ISlangBlob> archiveBlob;
+		SLANG_RETURN_ON_FAIL(archiveFileSystem->storeArchive(false, archiveBlob.writeRef()));
+
+		ComPtr<ISlangFileSystemExt> loadedFileSystem;
+		SLANG_RETURN_ON_FAIL(loadArchiveFileSystem(archiveBlob->getBufferPointer(), archiveBlob->getBufferSize(), loadedFileSystem));
+
+		// Check the file systems contents are the same
+		SLANG_RETURN_ON_FAIL(_checkEqual(loadedFileSystem, fileSystem));
+	}
+
 	SLANG_RETURN_ON_FAIL(fileSystem->remove("d/a"));
 	{
 		const Entry entries[] = { {SLANG_PATH_TYPE_FILE, "b" } };
@@ -216,9 +378,21 @@ SLANG_UNIT_TEST(fileSystem)
 	{
 		const auto type = FileSystemType(i);
 
-		auto const res = _test(type);
-
-		SLANG_CHECK(SLANG_SUCCEEDED(res));
+		SLANG_CHECK(SLANG_SUCCEEDED(_test(type)));
+
+		// Some file system types support 'implicit directories'. 
+		// This means that if a file is created with a path, the directories 
+		// required to make that path valid are 'implicitly' created. 
+		// 
+		// Currently this behavior is supported by zip, and this test checks
+		// that it is working correctly, as we require the file system to 
+		// behave correctly in other ways irrespectively of if the directory is
+		// implicit or not.
+		const bool hasImplicitDirectory = (type == FileSystemType::Zip);
+		if (hasImplicitDirectory)
+		{
+			SLANG_CHECK(SLANG_SUCCEEDED(_testImplicitDirectory(type)));
+		}
 	}
 }