Artifact closer to being able to replace CompileResult (#2354)

* #include an absolute path didn't work - because paths were taken to always be relative.

* WIP with hierarchical enums.

* Some small fixes and improvements around artifact desc related types.

* Improvements around hierarchical enum.

* Fixes to get Artifact types refactor to be able to execute tests.

* Attempt to better categorize PTX.

* Work around for potentially unused function warning.

* Typo fix.

* Simplify Artifact header.

* Small improvements around Artifact kind/payload/style.

* Added IDestroyable/ICastable

* Add IArtifactList.

* First impl of IArtifactUtil.

* Use the ICastable interface for IArtifactRepresentation.

* Added IArtifactRepresentation & IArtifactAssociated.

* Add SLANG_OVERRIDE to avoid gcc/clang warning.

* Fix calling convention issue on win32.

* Fix missing SLANG_OVERRIDE.

* First attempt at file abstraction around Artifact.

* Added creation of lock file.

* Move functionality for determining file paths to the IArtifactUtil.
Add casting to ICastable.

* Added some casting/finding mechanisms.

* Simplify IArtifact interface, and use Items for file reps.

* Fix problem with libraries on DXIL.

* Split out ArtifactRepresentation.

* Move ArtifactDesc functionality to ArtifactDescUtil. ArtifactInfoUtil becomes ArtifactDescUtil.

* Split implementations from the interfaces for Artifact.

* Use TypeTextUtil for target name outputting.

* Add artifact impls.

* Add ICastableList

* Added UnknownCastableAdapter

* Make ISlangSharedLibrary derive from ICastable, and remain backwards compatible with slang-llvm.

* Refactor Representation on Artifact.

* Make our ISlangBlobs also derive from ICastable.
Make ISlangBlob atomic ref counted.

* Split out CastableList and related types, and placed in core.

* Small fixes around IArtifact.
Improve IArtifact docs.
First impl of getChildren for IArtifact.

* Documentation improvements for Artifact related types.

* Fix typo.

* Special case adding a ICastableList to a LazyCastableList.

* Small simplification of LazyCastableList, by adding State member.

* Removed the ILockFile interface because IFileArtifactRepresentation can be used.

* Implement DiagnosticsArtifactRepresentation.

* Added PostEmitMetadataArtifactRepresentation

* Add searching by predicate.
Added handling of accessing Artifact as ISharedLibrary

* Fix typo.

* Add find to IArtifacgtList.
Fix some missing SLANG_NO_THROW.

* Small improvements around ArtifactDesc types.

* Another small change around ArtifactKind.

* Some more shuffling of ArtifactDesc.

* Make IArtifact castable
Remove IArtifactList
Made IArtifactContainer derive from IArtifact
Made ModuleLibrary atomic ref counted/given IModuleLibrary interface.

* Must call _requireChildren before any children access.

* Fix missing SLANG_MCALL on castAs.

* Fix missing SLANG_OVERRIDE.

* Added IArtifactHandler

* Use ICastable for basis of scope/lookup.

1 files changed, 264 insertions, 65 deletions

diff --git a/source/compiler-core/slang-artifact-util.cpp b/source/compiler-core/slang-artifact-util.cpp
index 909c9b3d2..366ab4111 100644
--- a/source/compiler-core/slang-artifact-util.cpp
+++ b/source/compiler-core/slang-artifact-util.cpp
@@ -6,120 +6,177 @@
 
 #include "slang-artifact-desc-util.h"
 
+#include "../core/slang-castable-list-impl.h"
+
 #include "../core/slang-file-system.h"
 #include "../core/slang-io.h"
+#include "../core/slang-shared-library.h"
 
 namespace Slang {
 
-/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! ArtifactUtilImpl !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! DefaultArtifactHandler !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
 
-/* static */ArtifactUtilImpl ArtifactUtilImpl::g_singleton;
+/* static */DefaultArtifactHandler DefaultArtifactHandler::g_singleton;
 
-SlangResult ArtifactUtilImpl::queryInterface(SlangUUID const& uuid, void** outObject)
+SlangResult DefaultArtifactHandler::queryInterface(SlangUUID const& uuid, void** outObject)
 {
-	if (auto intf = getInterface(uuid))
+	if (auto ptr = getInterface(uuid))
 	{
-		*outObject = intf;
+		addRef();
+		*outObject = static_cast<IArtifactHandler*>(this);
 		return SLANG_OK;
 	}
 	return SLANG_E_NO_INTERFACE;
 }
 
-void* ArtifactUtilImpl::getInterface(const Guid& guid)
+void* DefaultArtifactHandler::castAs(const Guid& guid)
 {
-	if (guid == ISlangUnknown::getTypeGuid() || guid == IArtifactUtil::getTypeGuid())
+	if (auto ptr = getInterface(guid))
 	{
-		return static_cast<IArtifactUtil*>(this);
+		return ptr;
 	}
-	return nullptr;
+	return getObject(guid);
 }
 
-SlangResult ArtifactUtilImpl::createArtifact(const ArtifactDesc& desc, const char* inName, IArtifact** outArtifact)
+void* DefaultArtifactHandler::getInterface(const Guid& uuid)
 {
-	String name;
-	if (inName)
+	if (uuid == ISlangUnknown::getTypeGuid() ||
+		uuid == ICastable::getTypeGuid() ||
+		uuid == IArtifactHandler::getTypeGuid())
 	{
-		name = inName;
+		return static_cast<IArtifactHandler*>(this);
 	}
 
-	ComPtr<IArtifact> artifact(new Artifact(desc, name));
+	return nullptr;
+}
 
-	*outArtifact = artifact.detach();
-	return SLANG_OK;
+void* DefaultArtifactHandler::getObject(const Guid& uuid)
+{
+	SLANG_UNUSED(uuid);
+	return nullptr;
 }
 
-SlangResult ArtifactUtilImpl::createArtifactList(IArtifact* parent, IArtifactList** outArtifactList)
+SlangResult DefaultArtifactHandler::_addRepresentation(IArtifact* artifact, ArtifactKeep keep, ISlangUnknown* rep, ICastable** outCastable)
 {
-	ComPtr<IArtifactList> artifactList(new ArtifactList(parent));
-	*outArtifactList = artifactList.detach();
-	return SLANG_OK;
+	SLANG_ASSERT(rep);
+
+	// See if it implements ICastable
+	{
+		ComPtr<ICastable> castable;
+		if (SLANG_SUCCEEDED(rep->queryInterface(ICastable::getTypeGuid(), (void**)castable.writeRef())) && castable)
+		{
+			return _addRepresentation(artifact, keep, castable, outCastable);
+		}
+	}
+
+	// We have to wrap 
+	ComPtr<IUnknownCastableAdapter> adapter(new UnknownCastableAdapter(rep));
+	return _addRepresentation(artifact, keep, adapter, outCastable);
 }
 
-ArtifactKind ArtifactUtilImpl::getKindParent(ArtifactKind kind) { return getParent(kind); }
-UnownedStringSlice ArtifactUtilImpl::getKindName(ArtifactKind kind) { return getName(kind); }
-bool ArtifactUtilImpl::isKindDerivedFrom(ArtifactKind kind, ArtifactKind base) { return isDerivedFrom(kind, base); }
+SlangResult DefaultArtifactHandler::_addRepresentation(IArtifact* artifact, ArtifactKeep keep, ICastable* castable, ICastable** outCastable)
+{
+	SLANG_ASSERT(castable);
 
-ArtifactPayload ArtifactUtilImpl::getPayloadParent(ArtifactPayload payload) { return getParent(payload); }
-UnownedStringSlice ArtifactUtilImpl::getPayloadName(ArtifactPayload payload) { return getName(payload); }
-bool ArtifactUtilImpl::isPayloadDerivedFrom(ArtifactPayload payload, ArtifactPayload base) { return isDerivedFrom(payload, base); }
+	if (canKeep(keep))
+	{
+		artifact->addRepresentation(castable);
+	}
 
-ArtifactStyle ArtifactUtilImpl::getStyleParent(ArtifactStyle style) { return getParent(style); }
-UnownedStringSlice ArtifactUtilImpl::getStyleName(ArtifactStyle style) { return getName(style); }
-bool ArtifactUtilImpl::isStyleDerivedFrom(ArtifactStyle style, ArtifactStyle base) { return isDerivedFrom(style, base); }
+	castable->addRef();
+	*outCastable = castable;
+	return SLANG_OK;
+}
 
-SlangResult ArtifactUtilImpl::createLockFile(const char* inNameBase, ISlangMutableFileSystem* fileSystem, IFileArtifactRepresentation** outLockFile)
+SlangResult DefaultArtifactHandler::expandChildren(IArtifactContainer* container)
 {
-	if (fileSystem)
+	SlangResult res = container->getExpandChildrenResult();
+	if (res != SLANG_E_UNINITIALIZED)
 	{
-		if (fileSystem != OSFileSystem::getMutableSingleton())
-		{
-			// We can only create lock files, on the global OS file system
-			return SLANG_E_NOT_AVAILABLE;
-		}
-		fileSystem = nullptr;
+		// It's already expanded
+		return res;
 	}
 
-	const UnownedStringSlice nameBase = (inNameBase && inNameBase[0] != 0) ? UnownedStringSlice(inNameBase) : UnownedStringSlice("unknown");
+	// For the generic container type, we just expand as empty
+	const auto desc = container->getDesc();
+	if (desc.kind == ArtifactKind::Container)
+	{
+		container->setChildren(nullptr, 0);
+		return SLANG_OK;
+	}
+	// TODO(JS):
+	// Proper implementation should (for example) be able to expand a Zip file etc.
+	return SLANG_E_NOT_IMPLEMENTED;
+}
 
-	String lockPath;
-	SLANG_RETURN_ON_FAIL(File::generateTemporary(nameBase, lockPath));
+SlangResult DefaultArtifactHandler::getOrCreateRepresentation(IArtifact* artifact, const Guid& guid, ArtifactKeep keep, ICastable** outCastable)
+{
+	
+	// See if we already have a rep of this type
+	{
+		for (ICastable* rep : artifact->getRepresentations())
+		{
+			if (rep->castAs(guid))
+			{
+				rep->addRef();
+				*outCastable = rep;
+				return SLANG_OK;
+			}
+		}
+	}
 
-	ComPtr<IFileArtifactRepresentation> lockFile(new FileArtifactRepresentation(IFileArtifactRepresentation::Kind::Lock, lockPath, nullptr, fileSystem));
+	if (guid == ISlangBlob::getTypeGuid())
+	{
+		ComPtr<ISlangBlob> blob;
+		SLANG_RETURN_ON_FAIL(_loadBlob(artifact, keep, blob.writeRef()));
+		return _addRepresentation(artifact, keep, blob, outCastable);
+	}
+	else if (guid == ISlangSharedLibrary::getTypeGuid())
+	{
+		ComPtr<ISlangSharedLibrary> sharedLib;
+		SLANG_RETURN_ON_FAIL(_loadSharedLibrary(artifact, keep, sharedLib.writeRef()));
+		return _addRepresentation(artifact, keep, sharedLib, outCastable);
+	}
 
-	*outLockFile = lockFile.detach();
-	return SLANG_OK;
+	return SLANG_E_NOT_AVAILABLE;
 }
 
-SlangResult ArtifactUtilImpl::calcArtifactPath(const ArtifactDesc& desc, const char* inBasePath, ISlangBlob** outPath)
+static bool _isFileSystemFile(ICastable* castable, void* data)
 {
-	UnownedStringSlice basePath(inBasePath);
-	StringBuilder path;
-	SLANG_RETURN_ON_FAIL(ArtifactDescUtil::calcPathForDesc(desc, basePath, path));
-	*outPath = StringBlob::create(path).detach();
-	return SLANG_OK;
+	if (auto fileRep = as<IFileArtifactRepresentation>(castable))
+	{
+		ISlangMutableFileSystem* fileSystem = (ISlangMutableFileSystem*)data;
+		if (fileRep->getFileSystem() == fileSystem)
+		{
+			return true;
+		}
+	}
+	return false;
 }
 
-SlangResult ArtifactUtilImpl::requireFileDefaultImpl(IArtifact* artifact, ArtifactKeep keep, IFileArtifactRepresentation** outFile)
+SlangResult DefaultArtifactHandler::getOrCreateFileRepresentation(IArtifact* artifact, ArtifactKeep keep, ISlangMutableFileSystem* fileSystem, IFileArtifactRepresentation** outFileRep)
 {
 	// See if we already have it
-	if (auto fileRep = findRepresentation<IFileArtifactRepresentation>(artifact))
+	if (auto fileRep = as<IFileArtifactRepresentation>(artifact->findRepresentationWithPredicate(&_isFileSystemFile, fileSystem)))
 	{
 		fileRep->addRef();
-		*outFile = fileRep;
+		*outFileRep = fileRep;
 		return SLANG_OK;
 	}
 
+	auto util = ArtifactUtilImpl::getSingleton();
+
 	// If we are going to access as a file we need to be able to write it, and to do that we need a blob
 	ComPtr<ISlangBlob> blob;
-	SLANG_RETURN_ON_FAIL(artifact->loadBlob(keep, blob.writeRef()));
+	SLANG_RETURN_ON_FAIL(artifact->loadBlob(getIntermediateKeep(keep), blob.writeRef()));
 
 	// Okay we need to store as a temporary. Get a lock file.
 	ComPtr<IFileArtifactRepresentation> lockFile;
-	SLANG_RETURN_ON_FAIL(createLockFile(artifact->getName(), nullptr, lockFile.writeRef()));
+	SLANG_RETURN_ON_FAIL(util->createLockFile(artifact->getName(), fileSystem, lockFile.writeRef()));
 
 	// Now we need the appropriate name for this item
 	ComPtr<ISlangBlob> pathBlob;
-	SLANG_RETURN_ON_FAIL(calcArtifactPath(artifact->getDesc(), lockFile->getPath(), pathBlob.writeRef()));
+	SLANG_RETURN_ON_FAIL(util->calcArtifactPath(artifact->getDesc(), lockFile->getPath(), pathBlob.writeRef()));
 
 	const auto path = StringUtil::getString(pathBlob);
 
@@ -141,7 +198,7 @@ SlangResult ArtifactUtilImpl::requireFileDefaultImpl(IArtifact* artifact, Artifa
 	else
 	{
 		// Create a new rep that references the lock file
-		fileRep = new FileArtifactRepresentation(IFileArtifactRepresentation::Kind::Owned, path, lockFile, nullptr);
+		fileRep = new FileArtifactRepresentation(IFileArtifactRepresentation::Kind::Owned, path, lockFile, lockFile->getFileSystem());
 	}
 
 	// Create the rep
@@ -151,28 +208,170 @@ SlangResult ArtifactUtilImpl::requireFileDefaultImpl(IArtifact* artifact, Artifa
 	}
 
 	// Return the file
-	*outFile = fileRep.detach();
+	*outFileRep = fileRep.detach();
 	return SLANG_OK;
 }
 
-ArtifactDesc ArtifactUtilImpl::makeDescFromCompileTarget(SlangCompileTarget target)
+SlangResult DefaultArtifactHandler::_loadSharedLibrary(IArtifact* artifact, ArtifactKeep keep, ISlangSharedLibrary** outSharedLibrary)
 {
-	return ArtifactDescUtil::makeDescFromCompileTarget(target);
+	// If it is 'shared library' for a CPU like thing, we can try and load it
+	const auto desc = artifact->getDesc();
+	if ((isDerivedFrom(desc.kind, ArtifactKind::HostCallable) ||
+		isDerivedFrom(desc.kind, ArtifactKind::SharedLibrary)) &&
+		isDerivedFrom(desc.payload, ArtifactPayload::CPULike))
+	{
+		// Get as a file represenation on the OS file system
+		ComPtr<IFileArtifactRepresentation> fileRep;
+		SLANG_RETURN_ON_FAIL(artifact->requireFile(ArtifactKeep::No, nullptr, fileRep.writeRef()));
+
+		// We requested on the OS file system, just check that's what we got...
+		SLANG_ASSERT(fileRep->getFileSystem() == nullptr);
+
+		// Try loading the shared library
+		SharedLibrary::Handle handle;
+		if (SLANG_FAILED(SharedLibrary::loadWithPlatformPath(fileRep->getPath(), handle)))
+		{
+			return SLANG_FAIL;
+		}
+
+		// The ScopeSharedLibrary will keep the fileRep in scope as long as is needed
+		auto sharedLibrary = new ScopeSharedLibrary(handle, fileRep);
+
+		if (canKeep(keep))
+		{
+			// We want to keep the fileRep, as that is necessary for the sharedLibrary to even work
+			artifact->addRepresentation(fileRep);
+			// Keep the shared library
+			artifact->addRepresentation(sharedLibrary);
+		}
+
+		// Output
+		sharedLibrary->addRef();
+		*outSharedLibrary = sharedLibrary;
+
+		return SLANG_OK;
+	}
+
+	return SLANG_FAIL;
 }
 
-SlangResult ArtifactUtilImpl::getChildrenDefaultImpl(IArtifact* artifact, IArtifactList** outList)
+SlangResult DefaultArtifactHandler::_loadBlob(IArtifact* artifact, ArtifactKeep keep, ISlangBlob** outBlob)
 {
-	auto desc = artifact->getDesc();
+	SLANG_UNUSED(keep);
+
+	ComPtr<ISlangBlob> blob;
+
+	// Look for a representation that we can serialize into a blob
+	for (auto rep : artifact->getRepresentations())
+	{
+		if (auto artifactRep = as<IArtifactRepresentation>(rep))
+		{
+			SlangResult res = artifactRep->writeToBlob(blob.writeRef());
+			if (SLANG_SUCCEEDED(res) && blob)
+			{
+				break;
+			}
+		}
+	}
 
-	// If it's a container type for now, just create empty list of children
-	if (isDerivedFrom(desc.kind, ArtifactKind::Container))
+	// Wasn't able to construct
+	if (!blob)
 	{
-		*outList = ComPtr<IArtifactList>(new ArtifactList(artifact)).detach();
+		return SLANG_E_NOT_FOUND;
+	}
+
+	*outBlob = blob.detach();
+	return SLANG_OK;
+}
+
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! ArtifactUtilImpl !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
+
+/* static */ArtifactUtilImpl ArtifactUtilImpl::g_singleton;
+
+SlangResult ArtifactUtilImpl::queryInterface(SlangUUID const& uuid, void** outObject)
+{
+	if (auto intf = getInterface(uuid))
+	{
+		*outObject = intf;
 		return SLANG_OK;
 	}
+	return SLANG_E_NO_INTERFACE;
+}
+
+void* ArtifactUtilImpl::getInterface(const Guid& guid)
+{
+	if (guid == ISlangUnknown::getTypeGuid() || guid == IArtifactUtil::getTypeGuid())
+	{
+		return static_cast<IArtifactUtil*>(this);
+	}
+	return nullptr;
+}
 
-	return SLANG_E_NOT_AVAILABLE;
+SlangResult ArtifactUtilImpl::createArtifact(const ArtifactDesc& desc, const char* inName, IArtifact** outArtifact)
+{
+	*outArtifact = inName ?
+		Artifact::create(desc, inName).detach() : 
+		Artifact::create(desc).detach();
+
+	return SLANG_OK;
 }
 
+SlangResult ArtifactUtilImpl::createArtifactContainer(const ArtifactDesc& desc, const char* inName, IArtifactContainer** outArtifactContainer)
+{
+	*outArtifactContainer = inName ?
+		ArtifactContainer::create(desc, inName).detach() :
+		ArtifactContainer::create(desc).detach();
+
+	return SLANG_OK;
+}
+
+ArtifactKind ArtifactUtilImpl::getKindParent(ArtifactKind kind) { return getParent(kind); }
+UnownedStringSlice ArtifactUtilImpl::getKindName(ArtifactKind kind) { return getName(kind); }
+bool ArtifactUtilImpl::isKindDerivedFrom(ArtifactKind kind, ArtifactKind base) { return isDerivedFrom(kind, base); }
+
+ArtifactPayload ArtifactUtilImpl::getPayloadParent(ArtifactPayload payload) { return getParent(payload); }
+UnownedStringSlice ArtifactUtilImpl::getPayloadName(ArtifactPayload payload) { return getName(payload); }
+bool ArtifactUtilImpl::isPayloadDerivedFrom(ArtifactPayload payload, ArtifactPayload base) { return isDerivedFrom(payload, base); }
+
+ArtifactStyle ArtifactUtilImpl::getStyleParent(ArtifactStyle style) { return getParent(style); }
+UnownedStringSlice ArtifactUtilImpl::getStyleName(ArtifactStyle style) { return getName(style); }
+bool ArtifactUtilImpl::isStyleDerivedFrom(ArtifactStyle style, ArtifactStyle base) { return isDerivedFrom(style, base); }
+
+SlangResult ArtifactUtilImpl::createLockFile(const char* inNameBase, ISlangMutableFileSystem* fileSystem, IFileArtifactRepresentation** outLockFile)
+{
+	if (fileSystem)
+	{
+		if (fileSystem != OSFileSystem::getMutableSingleton())
+		{
+			// We can only create lock files, on the global OS file system
+			return SLANG_E_NOT_AVAILABLE;
+		}
+		fileSystem = nullptr;
+	}
+
+	const UnownedStringSlice nameBase = (inNameBase && inNameBase[0] != 0) ? UnownedStringSlice(inNameBase) : UnownedStringSlice("unknown");
+
+	String lockPath;
+	SLANG_RETURN_ON_FAIL(File::generateTemporary(nameBase, lockPath));
+
+	ComPtr<IFileArtifactRepresentation> lockFile(new FileArtifactRepresentation(IFileArtifactRepresentation::Kind::Lock, lockPath, nullptr, fileSystem));
+
+	*outLockFile = lockFile.detach();
+	return SLANG_OK;
+}
+
+SlangResult ArtifactUtilImpl::calcArtifactPath(const ArtifactDesc& desc, const char* inBasePath, ISlangBlob** outPath)
+{
+	UnownedStringSlice basePath(inBasePath);
+	StringBuilder path;
+	SLANG_RETURN_ON_FAIL(ArtifactDescUtil::calcPathForDesc(desc, basePath, path));
+	*outPath = StringBlob::create(path).detach();
+	return SLANG_OK;
+}
+
+ArtifactDesc ArtifactUtilImpl::makeDescFromCompileTarget(SlangCompileTarget target)
+{
+	return ArtifactDescUtil::makeDescFromCompileTarget(target);
+}
 
 } // namespace Slang