Artifact Container (#2783)

* #include an absolute path didn't work - because paths were taken to always be relative.

* WIP simplifying artifact interface.

* Use ContainedKind.

* Remove LazyCastableList.
Use ContainedKind for find.

* Remove ICastableList.

* Remove need for ICastableList.

* Remove IArtifactContainer.

* Small fixes.

* Small improvements around Artifact.

* Make explicit find is for *representations* that can cast.
Fix bug in handling casting in lookup.

* Made associated items artifacts too.

* Small fixes.

* Small improvements around writing a container.

* WIP artifact container format.

* Make the root a special case.

* Special case if the artifact doesn't have children/associated.

7 files changed, 534 insertions, 12 deletions

diff --git a/source/compiler-core/slang-artifact-container-util.cpp b/source/compiler-core/slang-artifact-container-util.cpp
new file mode 100644
index 000000000..a5cbf1ff1
--- /dev/null
+++ b/source/compiler-core/slang-artifact-container-util.cpp
@@ -0,0 +1,446 @@
+// slang-artifact-container-util.cpp
+#include "slang-artifact-container-util.h"
+
+#include "slang-artifact-util.h"
+#include "slang-artifact-desc-util.h"
+
+#include "../core/slang-file-system.h"
+#include "../core/slang-io.h"
+#include "../core/slang-zip-file-system.h"
+#include "../core/slang-castable.h"
+
+namespace Slang {
+
+/* What is the structure we want here?
+
+Lets say we have an artifact with "blah.spv". One approach would be to take the base name and so end up with
+
+```
+blah/
+blah/blah.spv
+blah/associated/...
+blah/children/...
+```
+
+This is relatively simple and certainly pretty simple to implement. 
+
+This doesn't hold additional information - such as what might be held in the ArtifactDesc. We could use .dsc 
+to describe that. We could also remove the kernels name, as we already have that encoded in the directory name. 
+So 
+
+```
+blah/
+blah/bin.spv
+blah/desc.json
+blah/associated/...
+blah/children/...
+```
+
+What happens if we have data that *doesn't* have a name? We can just produce names by incrementing some counter.
+
+This works but seems a little weird, in that the name of the output kernel is not reflected. We could just use an extension
+that we use for directories for this additional data. Note that it would be "nice" to make this work with just the basename,
+but there is a problem with files that don't have an extension (as seen with executables on unix-like scenarios).
+
+```
+blah.spv
+blah.artifact/
+blah.artifact/desc.json
+blah.artifact/associated
+blah.artifact/... can just hold children directly
+```
+If we just hold children directly then their names could clash with standard names. 
+
+We might want an extension such that we know it is an artifact. 
+
+Other considerations
+
+* Wanting to be able to easily combine containers (by hand if necessary in a file system)
+  * Doing so implies we don't want some kind of global, or directory manifest.
+* Wanting to easily be able to 
+
+Given these constraints it seems like a good combination is
+
+* Everything associated with an artifact is in a directory on it's own
+* The name of the item should probably be stored as is in the directory
+* Any ArtifactDesc or additional information should be held in a file in the directory
+* Children and associated items should be stored in their own directories (which will follow the same mechanisms recursively)
+
+```
+blah/
+blah/blah.spv
+blah/desc.json
+blah/associated/...
+blah/children/...
+```
+
+Perhaps we want to special case for scenarios where we have artifact "leaves" (ie where they have no associated, or children)
+in that case placing in a directory all on it's own is perhaps not needed.
+
+*/
+
+struct ArtifactContainerWriter
+{
+    struct Entry
+    {
+        String path;
+        Index uniqueIndex = 0;
+    };
+
+    struct Scope
+    {
+        SlangResult pushAndRequireDirectory(ArtifactContainerWriter* writer, const String& name)
+        {
+            SLANG_ASSERT(writer);
+            SLANG_ASSERT(m_writer == nullptr);
+
+            SlangResult res = writer->pushAndRequireDirectory(name);
+
+            if (SLANG_SUCCEEDED(res))
+            {
+                m_writer = writer;
+            }
+
+            return res;
+        }
+
+        ~Scope()
+        {
+            if (m_writer)
+            {
+                m_writer->pop();
+            }
+        }
+
+        ArtifactContainerWriter* m_writer = nullptr;
+    };
+
+    void push(const String& name)
+    {
+        auto path = Path::combine(m_entry.path, name);
+
+        m_entryStack.add(m_entry);
+
+        Entry entry;
+        entry.path = path;
+
+        m_entry = entry;
+    }
+    SlangResult pushAndRequireDirectory(const String& name)
+    {
+        push(name);
+
+        const char*const path = m_entry.path.getBuffer();
+
+        SlangPathType pathType;
+        if (SLANG_SUCCEEDED(m_fileSystem->getPathType(path, &pathType)))
+        {
+            if (pathType != SLANG_PATH_TYPE_DIRECTORY)
+            {
+                return SLANG_FAIL;
+            }
+        }
+        
+        // Make sure there is a path to this
+        return m_fileSystem->createDirectory(m_entry.path.getBuffer());
+    }
+    void pop()
+    {
+        SLANG_ASSERT(m_entryStack.getCount() > 0);
+        m_entry = m_entryStack.getLast();
+        m_entryStack.removeLast();
+    }
+
+    SlangResult getBaseName(IArtifact* artifact, String& out);
+
+        /// Write the artifact in the current scope
+    SlangResult write(IArtifact* artifact);
+    SlangResult writeInDirectory(IArtifact* artifact, const String& baseName);
+
+    ArtifactContainerWriter(ISlangMutableFileSystem* fileSystem):
+        m_fileSystem(fileSystem)
+    {
+    }
+
+    List<Entry> m_entryStack;
+    Entry m_entry;
+
+    ISlangMutableFileSystem* m_fileSystem;
+};
+
+SlangResult ArtifactContainerWriter::getBaseName(IArtifact* artifact, String& out)
+{
+    String baseName;
+
+    const auto artifactDesc = artifact->getDesc();
+
+    {
+        auto artifactName = artifact->getName();
+        if (artifactName && artifactName[0] != 0)
+        {
+            baseName = ArtifactDescUtil::getBaseNameFromPath(artifactDesc, UnownedStringSlice(artifactName));
+        }
+    }
+
+    // If we don't have name, use a generated one 
+    if (baseName.getLength() == 0)
+    {
+        baseName.append(m_entry.uniqueIndex++);
+    }
+
+    out = baseName;
+    return SLANG_OK;
+}
+
+SlangResult ArtifactContainerWriter::writeInDirectory(IArtifact* artifact, const String& baseName)
+{
+    
+    // TODO(JS):
+    // We could now output information about the desc/artifact, say as some json.
+    // For now we assume the extension is good enough for most purposes.
+    
+    {
+        // We can't write it without a blob
+        ComPtr<ISlangBlob> blob;
+        const auto res = artifact->loadBlob(ArtifactKeep::No, blob.writeRef());
+
+        if (SLANG_FAILED(res))
+        {
+            // If it failed and it's significant the whole write fails
+            if (ArtifactUtil::isSignificant(artifact))
+            {
+                return res;
+            }
+        }
+        
+        {
+            // Get the name of the artifact
+            StringBuilder artifactName;
+            SLANG_RETURN_ON_FAIL(ArtifactDescUtil::calcNameForDesc(artifact->getDesc(), baseName.getUnownedSlice(), artifactName));
+
+            const auto combinedPath = Path::combine(m_entry.path, artifactName);
+            // Write out the blob
+            SLANG_RETURN_ON_FAIL(m_fileSystem->saveFileBlob(combinedPath.getBuffer(), blob));
+        }
+    }
+
+    {
+        auto children = artifact->getChildren();
+        if (children.count)
+        {
+            Scope childrenScope;
+            SLANG_RETURN_ON_FAIL(childrenScope.pushAndRequireDirectory(this, "children"));
+
+            for (IArtifact* child : children)
+            {
+                SLANG_RETURN_ON_FAIL(write(child));
+            }
+        }
+    }
+    {
+        auto associatedSlice = artifact->getAssociated();
+        if (associatedSlice.count)
+        {
+            Scope associatedScope;
+            SLANG_RETURN_ON_FAIL(associatedScope.pushAndRequireDirectory(this, "associated"));
+
+            for (IArtifact* associated : associatedSlice)
+            {
+                SLANG_RETURN_ON_FAIL(write(associated));
+            }
+        }
+    }
+
+    return SLANG_OK;
+}
+
+SlangResult ArtifactContainerWriter::write(IArtifact* artifact)
+{
+    String baseName;
+    SLANG_RETURN_ON_FAIL(getBaseName(artifact, baseName));
+
+    // Special case if there are no children/associated, we can just write into the same directory
+    if (artifact->getChildren().count == 0 &&
+        artifact->getAssociated().count == 0)
+    {
+        // I guess this assumes the name doesn't clash with any name we already have
+        SLANG_RETURN_ON_FAIL(writeInDirectory(artifact, baseName));
+    }
+    else
+    {
+        Scope artifactScope;
+        SLANG_RETURN_ON_FAIL(artifactScope.pushAndRequireDirectory(this, baseName));
+        SLANG_RETURN_ON_FAIL(writeInDirectory(artifact, baseName));
+    }
+
+    return SLANG_OK;
+}
+
+/* static */SlangResult ArtifactContainerUtil::writeContainer(IArtifact* artifact, const String& defaultFileName, ISlangMutableFileSystem* fileSystem)
+{
+    ArtifactContainerWriter writer(fileSystem);
+
+    String baseName;
+    
+    {
+        const char* name = artifact->getName();
+        if (name == nullptr || name[0] == 0)
+        {
+            // Try to get the name from the defaultFileName
+            baseName = Path::getFileNameWithoutExt(defaultFileName);
+        }
+    }
+
+    // If it's still not set try generating it.
+    if (baseName.getLength() == 0)
+    {
+        SLANG_RETURN_ON_FAIL(writer.getBaseName(artifact, baseName));            
+    }
+    
+    SLANG_RETURN_ON_FAIL(writer.writeInDirectory(artifact, baseName));
+
+    return SLANG_OK;
+}
+
+/* static */SlangResult ArtifactContainerUtil::writeLegacy(IArtifact* artifact, const String& fileName)
+{
+    ComPtr<ISlangBlob> containerBlob;
+    SLANG_RETURN_ON_FAIL(artifact->loadBlob(ArtifactKeep::Yes, containerBlob.writeRef()));
+
+    {
+        FileStream stream;
+        SLANG_RETURN_ON_FAIL(stream.init(fileName, FileMode::Create, FileAccess::Write, FileShare::ReadWrite));
+        SLANG_RETURN_ON_FAIL(stream.write(containerBlob->getBufferPointer(), containerBlob->getBufferSize()));
+    }
+
+    auto parentPath = Path::getParentDirectory(fileName);
+
+    // Lets look to see if we have any maps
+    {
+        Index nameCount = 0;
+
+        for (auto associatedArtifact : artifact->getAssociated())
+        {
+            auto desc = associatedArtifact->getDesc();
+
+            if (isDerivedFrom(desc.payload, ArtifactPayload::SourceMap))
+            {
+                StringBuilder artifactFilename;
+
+                // Dump out
+                const char* artifactName = associatedArtifact->getName();
+                if (artifactName && artifactName[0] != 0)
+                {
+                    SLANG_RETURN_ON_FAIL(ArtifactUtil::calcName(associatedArtifact, UnownedStringSlice(artifactName), artifactFilename));
+                }
+                else
+                {
+                    // Perhaps we can generate the name from the output basename
+                    StringBuilder baseName;
+
+                    baseName << Path::getFileNameWithoutExt(fileName);
+
+                    if (nameCount != 0)
+                    {
+                        baseName.appendChar('-');
+                        baseName.append(nameCount);
+                    }
+
+                    SLANG_RETURN_ON_FAIL(ArtifactUtil::calcName(associatedArtifact, baseName.getUnownedSlice(), artifactFilename));
+
+                    nameCount ++;
+                }
+
+                ComPtr<ISlangBlob> blob;
+                SLANG_RETURN_ON_FAIL(associatedArtifact->loadBlob(ArtifactKeep::No, blob.writeRef()));
+
+                // Try to write it out
+                {
+                    // Work out the path to the artifact
+                    auto artifactPath = Path::combine(parentPath, artifactFilename);
+
+                    // Write out the map
+                    FileStream stream;
+                    SLANG_RETURN_ON_FAIL(stream.init(artifactPath, FileMode::Create, FileAccess::Write, FileShare::ReadWrite));
+                    SLANG_RETURN_ON_FAIL(stream.write(blob->getBufferPointer(), blob->getBufferSize()));
+                }
+            }
+        }
+    }
+
+    return SLANG_OK;
+}
+
+static SlangResult _remove(ISlangMutableFileSystem* fileSystem, const String& path)
+{
+    SlangPathType pathType;
+    if (SLANG_SUCCEEDED(fileSystem->getPathType(path.getBuffer(), &pathType)))
+    {
+        fileSystem->remove(path.getBuffer());
+    }
+    return SLANG_OK;
+}
+
+/* static */SlangResult ArtifactContainerUtil::writeContainer(IArtifact* artifact, const String& fileName)
+{
+    auto osFileSystem = OSFileSystem::getMutableSingleton();
+
+    const auto ext = Path::getPathExt(fileName);
+
+    if (ext == "zip")
+    {
+        SLANG_RETURN_ON_FAIL(_remove(osFileSystem, fileName));
+        // Create the zip
+
+        ComPtr<ISlangMutableFileSystem> fileSystem;
+        SLANG_RETURN_ON_FAIL(ZipFileSystem::create(fileSystem));
+
+        // Write everything out
+        SLANG_RETURN_ON_FAIL(writeContainer(artifact, fileName, fileSystem));
+
+        // Now write out to the output file
+        IArchiveFileSystem* archiveFileSystem = as<IArchiveFileSystem>(fileSystem);
+        SLANG_ASSERT(archiveFileSystem);
+
+        ComPtr<ISlangBlob> blob;
+        SLANG_RETURN_ON_FAIL(archiveFileSystem->storeArchive(false, blob.writeRef()));
+
+        // Okay we can now write out the zip
+        SLANG_RETURN_ON_FAIL(osFileSystem->saveFileBlob(fileName.getBuffer(), blob));
+
+        return SLANG_OK;
+    }
+    else if (ext == toSlice("dir"))
+    {
+        // We use the special extension "dir" to write out to a directory. 
+        // This is a little hokey, but at list is consistent
+        auto path = Path::getPathWithoutExt(fileName);
+
+        SLANG_RETURN_ON_FAIL(_remove(osFileSystem, path));
+
+        SLANG_RETURN_ON_FAIL(osFileSystem->createDirectory(path.getBuffer()));
+
+        ComPtr<ISlangMutableFileSystem> fileSystem(new RelativeFileSystem(osFileSystem, path));
+
+        SLANG_RETURN_ON_FAIL(writeContainer(artifact, fileName, fileSystem));
+    }
+    else
+    {
+        // This is the legacy way to write out the container artifact
+        SLANG_RETURN_ON_FAIL(ArtifactContainerUtil::writeLegacy(artifact, fileName));
+    }
+
+    return SLANG_OK;
+}
+
+/* static */SlangResult ArtifactContainerUtil::readContainer(ISlangFileSystemExt* fileSystem, ComPtr<IArtifact>& outArtifact)
+{
+    SLANG_UNUSED(fileSystem);
+    SLANG_UNUSED(outArtifact);
+
+    // We traverse the file system creating the artifacts
+
+    return SLANG_E_NOT_IMPLEMENTED;
+}
+
+} // namespace Slang
diff --git a/source/compiler-core/slang-artifact-container-util.h b/source/compiler-core/slang-artifact-container-util.h
new file mode 100644
index 000000000..b3e591797
--- /dev/null
+++ b/source/compiler-core/slang-artifact-container-util.h
@@ -0,0 +1,40 @@
+// slang-artifact-container-util.h
+#ifndef SLANG_ARTIFACT_CONTAINER_UTIL_H
+#define SLANG_ARTIFACT_CONTAINER_UTIL_H
+
+#include "slang-artifact-representation.h"
+
+#include "../../slang-com-helper.h"
+#include "../../slang-com-ptr.h"
+
+namespace Slang
+{
+
+/* Functionality to save of and read artifact hierarchies via the slang 
+artifact container style. This style treats storage as a file system.
+
+Since this represention, is not directly a file system representation 
+some conventions are used to associate data, via names etc.
+
+The use of ISlangMutableFileSystem, allows writing this structure, to memory, zip, riff, directory 
+or other file like representations
+*/
+struct ArtifactContainerUtil 
+{
+        /// Writes using the legacy mechanism
+    static SlangResult writeLegacy(IArtifact* artifact, const String& filename);
+
+        /// Write the container using the specified path.
+        /// Uses the extension of the path to determine how to write
+    static SlangResult writeContainer(IArtifact* artifact, const String& path);
+
+        /// If there isn't a suitable name on artifact, the filename is used to generate a name. If it's not set
+        /// a name may be generated.
+    static SlangResult writeContainer(IArtifact* artifact, const String& defaultFileName, ISlangMutableFileSystem* fileSystem);
+
+    static SlangResult readContainer(ISlangFileSystemExt* fileSystem, ComPtr<IArtifact>& outArtifact);
+};
+
+} // namespace Slang
+
+#endif
diff --git a/source/compiler-core/slang-artifact-desc-util.cpp b/source/compiler-core/slang-artifact-desc-util.cpp
index da97ce3f5..fb1c2dbd2 100644
--- a/source/compiler-core/slang-artifact-desc-util.cpp
+++ b/source/compiler-core/slang-artifact-desc-util.cpp
@@ -732,6 +732,16 @@ SlangResult ArtifactDescUtil::appendDefaultExtension(const ArtifactDesc& desc, S
             out << "json";
             return SLANG_OK;
         }
+        case ArtifactKind::CompileBinary:
+        {
+            if (isDerivedFrom(desc.payload, ArtifactPayload::SlangIR) ||
+                isDerivedFrom(desc.payload, ArtifactPayload::SlangAST))
+            {
+                out << "slang-module";
+                return SLANG_OK;
+            }
+            break;
+        }
         default: break;
     }
 
diff --git a/source/compiler-core/slang-artifact-handler-impl.cpp b/source/compiler-core/slang-artifact-handler-impl.cpp
index 1b71dbf59..eb5bfc64a 100644
--- a/source/compiler-core/slang-artifact-handler-impl.cpp
+++ b/source/compiler-core/slang-artifact-handler-impl.cpp
@@ -105,15 +105,25 @@ SlangResult DefaultArtifactHandler::expandChildren(IArtifact* container)
 
 	// For the generic container type, we just expand as empty
 	const auto desc = container->getDesc();
-	if (desc.kind == ArtifactKind::Container)
+    if (desc.kind == ArtifactKind::Container)
+    {
+        // If it's just a generic container, we can assume it's expanded, and be done
+        return SLANG_OK;
+    }
+
+	if (isDerivedFrom(desc.kind, ArtifactKind::Container))
 	{
+        // TODO(JS):
+        // Proper implementation should (for example) be able to expand a Zip file etc.
+
+        // For now we just set that there are no children, and be done
 		container->setChildren(nullptr, 0);
-		return SLANG_OK;
+		return SLANG_E_NOT_IMPLEMENTED;
 	}
 
-	// TODO(JS):
-	// Proper implementation should (for example) be able to expand a Zip file etc.
-	return SLANG_E_NOT_IMPLEMENTED;
+    // We can't expand non container like types, so we just make sure it's empty
+    container->setChildren(nullptr, 0);
+	return SLANG_OK;
 }
 
 SlangResult DefaultArtifactHandler::getOrCreateRepresentation(IArtifact* artifact, const Guid& guid, ArtifactKeep keep, ICastable** outCastable)
diff --git a/source/compiler-core/slang-artifact-impl.cpp b/source/compiler-core/slang-artifact-impl.cpp
index 9ff67603a..d13421880 100644
--- a/source/compiler-core/slang-artifact-impl.cpp
+++ b/source/compiler-core/slang-artifact-impl.cpp
@@ -64,7 +64,8 @@ void Artifact::_requireChildren()
     {
         const auto res = expandChildren();
         SLANG_UNUSED(res);
-        SLANG_ASSERT(SLANG_SUCCEEDED(res));
+
+        SLANG_ASSERT(SLANG_SUCCEEDED(res) || res == SLANG_E_NOT_IMPLEMENTED);
     }
 }
 
diff --git a/source/compiler-core/slang-artifact-util.cpp b/source/compiler-core/slang-artifact-util.cpp
index 1a920ff18..00884c96f 100644
--- a/source/compiler-core/slang-artifact-util.cpp
+++ b/source/compiler-core/slang-artifact-util.cpp
@@ -47,12 +47,13 @@ static bool _checkRecursive(ArtifactUtil::FindStyle findStyle)
     return createArtifact(desc);
 }
 
-/* static */bool ArtifactUtil::isSignificant(IArtifact* artifact, void* data)
+/* static */bool ArtifactUtil::isSignificant(IArtifact* artifact)
 {
-    SLANG_UNUSED(data);
-
-    const auto desc = artifact->getDesc();
+    return isSignificant(artifact->getDesc());
+}
 
+/* static */bool ArtifactUtil::isSignificant(const ArtifactDesc& desc)
+{
     // Containers are not significant as of themselves, they may contain something tho
     if (isDerivedFrom(desc.kind, ArtifactKind::Container))
     {
@@ -85,6 +86,13 @@ static bool _checkRecursive(ArtifactUtil::FindStyle findStyle)
     return true;
 }
 
+
+/* static */bool ArtifactUtil::isSignificant(IArtifact* artifact, void* data)
+{
+    SLANG_UNUSED(data);
+    return isSignificant(artifact->getDesc());
+}
+
 /* static */IArtifact* ArtifactUtil::findSignificant(IArtifact* artifact) 
 { 
     return findArtifactByPredicate(artifact, FindStyle::SelfOrChildren, &ArtifactUtil::isSignificant, nullptr);
diff --git a/source/compiler-core/slang-artifact-util.h b/source/compiler-core/slang-artifact-util.h
index 78f6da7a9..792e6ff94 100644
--- a/source/compiler-core/slang-artifact-util.h
+++ b/source/compiler-core/slang-artifact-util.h
@@ -40,8 +40,15 @@ struct ArtifactUtil
     static ComPtr<IArtifact> createArtifact(const ArtifactDesc& desc, const char* name);
     static ComPtr<IArtifact> createArtifact(const ArtifactDesc& desc);
 
-        /// Returns true if an artifact is 'significant'
-    static bool isSignificant(IArtifact* artifact, void* data = nullptr);
+        /// True if is significant
+    static bool isSignificant(IArtifact* artifact);
+        /// True if is significant
+    static bool isSignificant(const ArtifactDesc& desc);
+
+        /// Returns true if an artifact is 'significant'. 
+        /// The data parameter is unused and just used to make work as FindFunc
+    static bool isSignificant(IArtifact* artifact, void* data);
+
         /// Find a significant artifact
     static IArtifact* findSignificant(IArtifact* artifact);