DXIL library support and Artifact type (#2186)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Compile to a dxil library.

* Added CompileProduct.

* Support handling of ModuleLibrary.

* CacheBehavior -> Cache

* Use CompileProduct for -r references.

* CompileProduct -> Artifact.

* Determining an artifact type on binding.

* Determine binary linkability.

* Added Artifact::exists.

* Added ArtifactKeep.

* Small fixes.

* Small improvements to Artifact.

* Add zip extension.

* Fix some comments.

8 files changed, 1016 insertions, 70 deletions

diff --git a/source/slang/slang-artifact.cpp b/source/slang/slang-artifact.cpp
new file mode 100644
index 000000000..612c2365d
--- /dev/null
+++ b/source/slang/slang-artifact.cpp
@@ -0,0 +1,535 @@
+// slang-artifact.cpp
+#include "slang-artifact.h"
+#include <assert.h>
+
+#include "../core/slang-blob.h"
+#include "../core/slang-riff.h"
+
+#include "../core/slang-type-text-util.h"
+
+// Serialization
+#include "slang-serialize-ir.h"
+#include "slang-serialize-container.h"
+
+namespace Slang {
+
+namespace { // anonymous
+struct KindExtension
+{
+    ArtifactKind kind;
+    UnownedStringSlice ext;
+};
+} // anonymous
+
+#define SLANG_KIND_EXTENSION(kind, ext) \
+    { ArtifactKind::kind, UnownedStringSlice::fromLiteral(ext) },
+
+static const KindExtension g_cpuKindExts[] =
+{
+#if SLANG_WINDOWS_FAMILY
+    SLANG_KIND_EXTENSION(Library, "lib")
+    SLANG_KIND_EXTENSION(ObjectCode, "obj")
+    SLANG_KIND_EXTENSION(Executable, "exe")
+    SLANG_KIND_EXTENSION(SharedLibrary, "dll")
+#else 
+    SLANG_KIND_EXTENSION(Library, "a")
+    SLANG_KIND_EXTENSION(ObjectCode, "o")
+    SLANG_KIND_EXTENSION(Executable, "")
+
+#if __CYGWIN__
+    SLANG_KIND_EXTENSION(SharedLibrary, "dll")
+#elif SLANG_APPLE_FAMILY
+    SLANG_KIND_EXTENSION(SharedLibrary, "dylib")
+#else
+    SLANG_KIND_EXTENSION(SharedLibrary, "so")
+#endif
+
+#endif
+};
+
+/* static */ArtifactDesc ArtifactDesc::fromPath(const UnownedStringSlice& slice)
+{
+    auto extension = Path::getPathExt(slice);
+    return fromExtension(extension);
+}
+
+/* static */ ArtifactDesc ArtifactDesc::fromExtension(const UnownedStringSlice& slice)
+{
+    if (slice == "slang-module" ||
+        slice == "slang-lib")
+    {
+        return make(ArtifactKind::Library, ArtifactPayload::SlangIR, ArtifactStyle::Unknown);
+    }
+
+    for (const auto& kindExt : g_cpuKindExts)
+    {
+        if (slice == kindExt.ext)
+        {
+            // We'll assume it's for the host CPU for now..
+            return make(kindExt.kind, Payload::HostCPU, Style::Unknown);
+        }
+    }
+
+    const auto target = (CodeGenTarget)TypeTextUtil::findCompileTargetFromExtension(slice);
+
+    return make(target);
+}
+
+/* static */ArtifactDesc ArtifactDesc::make(CodeGenTarget target)
+{
+    switch (target)
+    {
+        case CodeGenTarget::Unknown:                return make(Kind::Unknown, Payload::None, Style::Unknown, 0);
+        case CodeGenTarget::None:                   return make(Kind::None, Payload::None, Style::Unknown, 0);
+        case CodeGenTarget::GLSL_Vulkan:
+        case CodeGenTarget::GLSL_Vulkan_OneDesc:
+        case CodeGenTarget::GLSL:
+        {
+            // For the moment we make all just map to GLSL, but we could use flags
+            // or some other mechanism to distinguish the types
+            return make(Kind::Text, Payload::GLSL, Style::Kernel, 0);
+        }
+        case CodeGenTarget::HLSL:                   return make(Kind::Text, Payload::HLSL, Style::Kernel, 0);
+        case CodeGenTarget::SPIRV:                  return make(Kind::Executable, Payload::SPIRV, Style::Kernel, 0);
+        case CodeGenTarget::SPIRVAssembly:          return make(Kind::Text, Payload::SPIRVAssembly, Style::Kernel, 0);
+        case CodeGenTarget::DXBytecode:             return make(Kind::Executable, Payload::DXBC, Style::Kernel, 0);
+        case CodeGenTarget::DXBytecodeAssembly:     return make(Kind::Text, Payload::DXBCAssembly, Style::Kernel, 0);
+        case CodeGenTarget::DXIL:                   return make(Kind::Executable, Payload::DXIL, Style::Kernel, 0);
+        case CodeGenTarget::DXILAssembly:           return make(Kind::Text, Payload::DXILAssembly, Style::Kernel, 0);
+        case CodeGenTarget::CSource:                return make(Kind::Text, Payload::C, Style::Kernel, 0);
+        case CodeGenTarget::CPPSource:              return make(Kind::Text, Payload::CPP, Style::Kernel, 0);
+        case CodeGenTarget::HostCPPSource:          return make(Kind::Text, Payload::CPP, Style::Host, 0);
+        case CodeGenTarget::HostExecutable:         return make(Kind::Executable, Payload::HostCPU, Style::Host, 0);
+        case CodeGenTarget::ShaderSharedLibrary:    return make(Kind::SharedLibrary, Payload::HostCPU, Style::Kernel, 0);
+        case CodeGenTarget::ShaderHostCallable:     return make(Kind::Callable, Payload::HostCPU, Style::Kernel, 0);
+        case CodeGenTarget::CUDASource:             return make(Kind::Text, Payload::CUDA, Style::Kernel, 0);
+        case CodeGenTarget::PTX:                    return make(Kind::Executable, Payload::PTX, Style::Kernel, 0);
+        case CodeGenTarget::ObjectCode:             return make(Kind::ObjectCode, Payload::HostCPU, Style::Kernel, 0);
+        default: break;
+    }
+
+    SLANG_UNEXPECTED("Unhandled type");
+}
+
+/* static */bool ArtifactDesc::isPayloadGpuBinary(Payload payloadType)
+{
+    switch (payloadType)
+    {
+        case Payload::DXIL:
+        case Payload::DXBC:
+        case Payload::SPIRV:
+        case Payload::PTX:
+        {
+            return true;
+        }
+        default: break;
+    }
+    return false;
+}
+
+/* static */bool ArtifactDesc::isPayloadCpuBinary(Payload payloadType)
+{
+    switch (payloadType)
+    {
+        case Payload::X86:
+        case Payload::X86_64:
+        case Payload::AARCH:
+        case Payload::AARCH64:
+        case Payload::HostCPU:
+        {
+            return true;
+        }
+        default: break;
+    }
+    return false;
+}
+
+/* static */bool ArtifactDesc::isPayloadGpuBinaryLinkable(Payload payload)
+{
+    switch (payload)
+    {
+        case Payload::DXIL:
+        case Payload::PTX:
+        case Payload::SPIRV:
+        {
+            // We can't *actually* link PTX or SPIR-V currently but it is in principal possible
+            // so let's say we accept for now
+            return true;
+        }
+        default: break;
+    }
+    return false;
+}
+
+bool ArtifactDesc::isBinaryLinkable() const
+{
+    if (isKindBinaryLinkable(kind))
+    {
+        return isPayloadCpuBinary(payload) || isPayloadGpuBinaryLinkable(payload) ||  payload == ArtifactPayload::SlangIR;
+    }
+    
+    return false;
+}
+
+/* static */bool ArtifactDesc::isKindBinaryLinkable(Kind kind)
+{
+    switch (kind)
+    {
+        case Kind::Library:
+        case Kind::ObjectCode:
+        {
+            return true;
+        }
+        default: break;
+    }
+    return false;
+}
+
+/* static*/ UnownedStringSlice ArtifactDesc::getCpuExtensionForKind(Kind kind)
+{
+    for (const auto& kindExt : g_cpuKindExts)
+    {
+        if (kind == kindExt.kind)
+        {
+            return kindExt.ext;
+        }
+    }
+    return UnownedStringSlice();
+}
+
+UnownedStringSlice ArtifactDesc::getDefaultExtension()
+{
+    if (isPayloadCpuBinary(payload))
+    {
+        return getCpuExtensionForKind(kind);
+    }
+    else
+    {
+        return getDefaultExtensionForPayload(payload);
+    }
+}
+
+/* static */UnownedStringSlice ArtifactDesc::getDefaultExtensionForPayload(Payload payload)
+{
+    switch (payload)
+    {
+        case Payload::None:         return UnownedStringSlice();
+        case Payload::Unknown:      return UnownedStringSlice::fromLiteral("unknown");
+
+        case Payload::DXIL:         return UnownedStringSlice::fromLiteral("dxil");
+        case Payload::DXBC:         return UnownedStringSlice::fromLiteral("dxbc");
+        case Payload::SPIRV:        return UnownedStringSlice::fromLiteral("spirv");
+
+        case Payload::PTX:          return UnownedStringSlice::fromLiteral("ptx");
+
+        case Payload::X86:
+        case Payload::X86_64:
+        case Payload::AARCH:
+        case Payload::AARCH64:
+        case Payload::HostCPU:
+        {
+            return UnownedStringSlice();
+        }
+
+        case Payload::SlangIR:      return UnownedStringSlice::fromLiteral("slang-ir");
+        case Payload::LLVMIR:       return UnownedStringSlice::fromLiteral("llvm-ir");
+
+        case Payload::HLSL:         return UnownedStringSlice::fromLiteral("hlsl");
+        case Payload::GLSL:         return UnownedStringSlice::fromLiteral("glsl");
+
+        case Payload::CPP:          return UnownedStringSlice::fromLiteral("cpp");
+        case Payload::C:            return UnownedStringSlice::fromLiteral("c");
+
+        case Payload::CUDA:         return UnownedStringSlice::fromLiteral("cu");
+
+        case Payload::Slang:        return UnownedStringSlice::fromLiteral("slang");
+
+        case Payload::Zip:          return UnownedStringSlice::fromLiteral("zip");
+
+        default: break;
+    }
+
+    SLANG_UNEXPECTED("Unknown content type");
+}
+
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Artifact !!!!!!!!!!!!!!!!!!!!!!!!!!! */
+
+Artifact::~Artifact()
+{
+    if (m_pathType == PathType::Temporary)
+    {
+        File::remove(m_path);
+    }
+
+    for (const auto& entry : m_entries)
+    {
+        // Release the associated data
+        switch (entry.type)
+        {
+            case Entry::Type::ObjectInstance:
+            {
+                entry.object->releaseReference();
+                break;
+            }
+            case Entry::Type::InterfaceInstance:
+            {
+                entry.intf->release();
+                break;
+            }
+            default: break;
+        }
+    }
+}
+
+Index Artifact::indexOf(Entry::Type type) const
+{
+    return m_entries.findFirstIndex([&](const Entry& entry) -> bool { return entry.type == type; });
+}
+
+void Artifact::add(Entry::Style style, RefObject* obj)
+{
+    SLANG_ASSERT(obj);
+
+    Entry entry;
+    entry.type = Entry::Type::ObjectInstance;
+    entry.style = style;
+    entry.object = obj;
+
+    obj->addReference();
+    
+    m_entries.add(entry);
+}
+
+void Artifact::add(Entry::Style style, ISlangUnknown* intf)
+{
+    // Can't be nullptr
+    SLANG_ASSERT(intf);
+
+    Entry entry;
+    entry.type = Entry::Type::InterfaceInstance;
+    entry.style = style;
+
+    intf->addRef();
+
+    entry.intf = intf;
+    m_entries.add(entry);
+}
+
+bool Artifact::exists() const
+{
+    // If we have a blob it exists
+    if (m_blob)
+    {
+        return true;
+    }
+
+    // If we have an associated entry that represents the artifact it exists
+    for (const auto& entry : m_entries)
+    {
+        if (entry.style == Entry::Style::Artifact)
+        {
+            // There is a representation that 'is' the artifact
+            return true;
+        }
+    }
+
+    // If we don't have a path then it can't exist
+    if (m_pathType == PathType::None)
+    {
+        return false;
+    }
+
+    // If the file exists we assume it exists
+    return File::exists(m_path);
+}
+
+ISlangUnknown* Artifact::findInterfaceInstance(const Guid& guid)
+{
+    for (const auto& entry : m_entries)
+    {
+        if (entry.type == Entry::Type::InterfaceInstance)
+        {
+            ISlangUnknown* intf = nullptr;
+            if (SLANG_SUCCEEDED(entry.intf->queryInterface(guid, (void**)&intf)) && intf)
+            {
+                // NOTE! This assumes we *DONT* need to ref count to keep an interface in scope
+                // (as strict COM requires so as to allow on demand interfaces).
+                intf->release();
+
+                return intf;
+            }
+        }
+    }
+
+    return nullptr;
+}
+
+SlangResult Artifact::requireFilePath(Keep keep, String& outFilePath)
+{
+    if (m_pathType != PathType::None)
+    {
+        outFilePath = m_path; 
+        return SLANG_OK;
+    }
+
+    ComPtr<ISlangBlob> blob;
+
+    // Get the contents as a blob. If we can't do that, then we can't write anything...
+    SLANG_RETURN_ON_FAIL(loadBlob(getIntermediateKeep(keep), blob));
+
+    const UnownedStringSlice ext = m_desc.getDefaultExtension();
+
+    // TODO(JS): NOTE! This isn't strictly correct, as the generated filename is not guarenteed to be unique
+    // if we change it with an extension (or prefix).
+    // This doesn't change the previous behavior though.
+    String path;
+    SLANG_RETURN_ON_FAIL(File::generateTemporary(UnownedStringSlice::fromLiteral("slang-generated"), path));
+
+    if (m_desc.isCpuBinary() && m_desc.kind == ArtifactKind::SharedLibrary)
+    {
+        const bool isSharedLibraryPrefixPlatform = SLANG_LINUX_FAMILY || SLANG_APPLE_FAMILY;
+        if (isSharedLibraryPrefixPlatform)
+        {
+            StringBuilder buf;
+            buf << "lib";
+            buf << Path::getFileName(path);
+
+            auto parentDir = Path::getParentDirectory(path);
+            if (parentDir.getLength())
+            {
+                // Prefix the lib
+                path = Path::combine(parentDir, buf);
+            }
+            else
+            {
+                path = buf;
+            }
+        }
+    }
+
+    if (ext.getLength())
+    {
+        path.appendChar('.');
+        path.append(ext);
+    }
+
+    SLANG_RETURN_ON_FAIL(File::writeAllBytes(path, blob->getBufferPointer(), blob->getBufferSize()));
+
+    // Okay we can now add this as temporary path too
+    setPath(PathType::Temporary, path);
+
+    return SLANG_OK;
+}
+
+SlangResult Artifact::loadBlob(Keep keep, ComPtr<ISlangBlob>& outBlob)
+{
+    if (m_blob)
+    {
+        outBlob = m_blob;
+        return SLANG_OK;
+    }
+
+    // TODO(JS): 
+    // Strictly speaking we could *potentially* convert some other representation into
+    // a blob by serializing it, but we don't worry about any of that here
+    if (m_pathType == PathType::None)
+    {
+        return SLANG_E_NOT_FOUND;
+    }
+
+    // Read into a blob
+    ScopedAllocation alloc;
+    SLANG_RETURN_ON_FAIL(File::readAllBytes(m_path, alloc));
+
+    // Create as a blob
+    RefPtr<RawBlob> blob = RawBlob::moveCreate(alloc);
+
+    // Put in cache 
+    if (canKeep(keep))
+    {
+        setBlob(blob);
+    }
+
+    outBlob = blob;
+    return SLANG_OK;
+}
+
+SlangResult loadModuleLibrary(const Byte* inBytes, size_t bytesCount, EndToEndCompileRequest* req, RefPtr<ModuleLibrary>& outLibrary)
+{
+    RefPtr<ModuleLibrary> library = new ModuleLibrary;
+
+    // Load up the module
+    MemoryStreamBase memoryStream(FileAccess::Read, inBytes, bytesCount);
+
+    RiffContainer riffContainer;
+    SLANG_RETURN_ON_FAIL(RiffUtil::read(&memoryStream, riffContainer));
+
+    auto linkage = req->getLinkage();
+
+    // TODO(JS): May be better to have a ITypeComponent that encapsulates a collection of modules
+    // For now just add to the linkage
+
+    {
+        SerialContainerData containerData;
+
+        SerialContainerUtil::ReadOptions options;
+        options.namePool = req->getNamePool();
+        options.session = req->getSession();
+        options.sharedASTBuilder = linkage->getASTBuilder()->getSharedASTBuilder();
+        options.sourceManager = linkage->getSourceManager();
+        options.linkage = req->getLinkage();
+        options.sink = req->getSink();
+
+        SLANG_RETURN_ON_FAIL(SerialContainerUtil::read(&riffContainer, options, containerData));
+
+        for (const auto& module : containerData.modules)
+        {
+            // If the irModule is set, add it
+            if (module.irModule)
+            {
+                library->m_modules.add(module.irModule);
+            }
+        }
+
+        for (const auto& entryPoint : containerData.entryPoints)
+        {
+            FrontEndCompileRequest::ExtraEntryPointInfo dst;
+            dst.mangledName = entryPoint.mangledName;
+            dst.name = entryPoint.name;
+            dst.profile = entryPoint.profile;
+
+            // Add entry point
+            library->m_entryPoints.add(dst);
+        }
+    }
+
+    outLibrary = library;
+    return SLANG_OK;
+}
+
+SlangResult loadModuleLibrary(Artifact::Keep keep, Artifact* product, EndToEndCompileRequest* req, RefPtr<ModuleLibrary>& outLibrary)
+{
+    if (auto foundLibrary = product->findObjectInstance<ModuleLibrary>())
+    {
+        outLibrary = foundLibrary;
+        return SLANG_OK;
+    }
+
+    // Load the blob
+    ComPtr<ISlangBlob> blob;
+    SLANG_RETURN_ON_FAIL(product->loadBlob(Artifact::getIntermediateKeep(keep), blob));
+
+    // Load the module
+    RefPtr<ModuleLibrary> library;
+    SLANG_RETURN_ON_FAIL(loadModuleLibrary((const Byte*)blob->getBufferPointer(), blob->getBufferSize(), req, library));
+    
+    if (Artifact::canKeep(keep))
+    {
+        product->add(Artifact::Entry::Style::Artifact, library);
+    }
+
+    outLibrary = library;
+    return SLANG_OK;
+}
+
+} // namespace Slang
diff --git a/source/slang/slang-artifact.h b/source/slang/slang-artifact.h
new file mode 100644
index 000000000..0e009e990
--- /dev/null
+++ b/source/slang/slang-artifact.h
@@ -0,0 +1,370 @@
+// slang-artifact.h
+#ifndef SLANG_ARTIFACT_H
+#define SLANG_ARTIFACT_H
+
+#include "slang-compiler.h" 
+
+namespace Slang
+{
+
+enum class ArtifactKind : uint8_t
+{
+    None,                       ///< There is no container
+
+    Unknown,                    ///< There is a container of unknown type
+
+    Library,                    ///< Library of object code (typically made up multiple ObjectCode)
+    ObjectCode,                 ///< Object code (for CPU typically .o or .obj file types)
+
+    Executable,                 ///< Self contained such it can exectuted. On GPU this would be a kernel.
+    SharedLibrary,              ///< Shared library/dll 
+    Callable,                   ///< Callable directly (typically means there isn't a binary artifact)
+
+    Text,                       ///< Text
+
+    Container,                  ///< A container holding other things
+
+    CountOf,
+};
+
+enum class ArtifactPayload : uint8_t
+{
+    None,           ///< There is no payload
+
+    Unknown,        ///< Has payload but its unknown variety
+
+    DXIL,
+    DXBC,
+    SPIRV,
+    PTX,
+
+    DXILAssembly,
+    DXBCAssembly,
+    SPIRVAssembly,
+    PTXAssembly,
+
+    HostCPU,        ///< The host CPU architecture
+
+    SlangIR,        ///< Slang IR
+    LLVMIR,         ///< LLVM IR
+
+    SlangAST,       ///< Slang AST
+
+    X86,
+    X86_64,
+    AARCH,
+    AARCH64,
+
+    HLSL,           ///< HLSL
+    GLSL,           ///< GLSL
+    CPP,            ///< C++
+    C,              ///< C Language
+    CUDA,           ///< CUDA
+    Slang,          ///< Slang
+
+    DebugInfo,      ///< Debug information 
+
+    Zip,            ///< It's a zip 
+
+    CountOf,
+};
+
+enum class ArtifactStyle : uint8_t
+{
+    Unknown,                ///< Unknown
+
+    Kernel,                 ///< Compiled as `GPU kernel` style.        
+    Host,                   ///< Compiled in `host` style
+
+    CountOf,
+};
+
+typedef uint8_t ArtifactFlags;
+struct ArtifactFlag
+{
+    enum Enum : ArtifactFlags
+    {
+        // Don't currently have any flags
+    };
+};
+
+
+/**
+A value type to describe aspects of the contents of an Artifact.
+**/
+struct ArtifactDesc
+{
+public:
+    typedef ArtifactDesc This;
+
+    typedef ArtifactKind Kind;
+    typedef ArtifactPayload Payload;
+    typedef ArtifactStyle Style;
+    typedef ArtifactFlags Flags;
+   
+    typedef uint32_t PackedBacking;
+    enum class Packed : PackedBacking;
+    
+        /// Get in packed format
+    inline Packed getPacked() const;
+
+        /// True if the container appears to be binary linkable
+    bool isBinaryLinkable() const;
+        /// True if is a CPU binary
+    bool isCpuBinary() const { return isPayloadCpuBinary(payload); }
+        /// True if is a GPU binary
+    bool isGpuBinary() const { return isPayloadGpuBinary(payload); }
+
+        /// Gets the default file extension for the artifact type. Returns empty slice if not known
+    UnownedStringSlice getDefaultExtension();
+
+    static UnownedStringSlice getDefaultExtensionForPayload(Payload payload);
+
+        /// Get the extension for CPU/Host for a kind
+    static UnownedStringSlice getCpuExtensionForKind(Kind kind);
+
+        /// Returns true if the kind is binary linkable 
+    static bool isKindBinaryLinkable(Kind kind);
+
+        /// Returns true if the payload type is CPU
+    static bool isPayloadCpuBinary(Payload payload);
+        /// Returns true if the payload type is applicable to the GPU
+    static bool isPayloadGpuBinary(Payload payload);
+
+        /// True if the payload type is in principal binary linkable
+    static bool isPayloadGpuBinaryLinkable(Payload payload);
+
+        /// Try to determine the desc from a path
+    static This fromPath(const UnownedStringSlice& slice);
+        /// Try to determine the desc from just a file extension (passed without .)
+    static This fromExtension(const UnownedStringSlice& slice);
+
+    bool operator==(const This& rhs) const { return kind == rhs.kind && payload == rhs.payload && style == rhs.style && flags == rhs.flags;  }
+    bool operator!=(const This& rhs) const { return !(*this == rhs); }
+
+        /// Given a code gen target, get the equivalent ArtifactDesc
+    static This make(CodeGenTarget target);
+
+        /// Construct from the elements
+    static This make(Kind inKind, Payload inPayload, Style inStyle = Style::Kernel, Flags flags = 0)
+    {
+        return This{ inKind, inPayload, inStyle, flags };
+    }
+        /// Construct from the packed format
+    inline static This make(Packed inPacked);
+
+    Kind kind;
+    Payload payload;
+    Style style;
+    Flags flags;
+};
+
+// --------------------------------------------------------------------------
+inline ArtifactDesc::Packed ArtifactDesc::getPacked() const
+{
+    typedef PackedBacking IntType;
+    return Packed((IntType(kind) << 24) |
+        (IntType(payload) << 16) |
+        (IntType(style) << 8) |
+        flags);
+}
+
+// --------------------------------------------------------------------------
+inline /* static */ArtifactDesc ArtifactDesc::make(Packed inPacked)
+{
+    const PackedBacking packed = PackedBacking(inPacked);
+
+    This r;
+    r.kind = Kind(packed >> 24);
+    r.payload = Payload(uint8_t(packed >> 16));
+    r.style = Style(uint8_t(packed >> 8));
+    r.flags = uint8_t(packed);
+
+    return r;
+}
+
+/* The Artifact type is a type designed to represent some Artifact of compilation. It could be input to or output from a compilation.
+
+An abstraction is desirable here, because depending on the compiler the artifact/s could be
+
+* A file on the file system
+* A blob
+* Multiple files
+* Some other (perhaps multiple) in memory representations 
+
+The artifact uses the Blob as the standard representation of in memory data. 
+
+Some downstream compilers require the artifact to be available as a file system file, or to produce
+artifacts that are files. The Artifact type allows to abstract away this difference, including the
+ability to turn an in memory representation into a temporary file on the system file. 
+
+The mechanism also allows for 'Containers' which allow for Artifacts to contain other Artifacts (amongst other things).
+Those artifacts may be other files. For example a downstream compilation that produces results as well as temporary
+files could be a Container containing artifacts for
+
+* Diagnostics
+* Temporary files (of known and unknown types)
+* Files that contain known types
+* Callable interface (an ISlangSharedLibrary)
+
+A more long term goal would be to
+
+* Make Artifact an interface (such that it can work long term over binary boundaries)
+* Make Diagnostics into an interface (such it can be added to a Artifact result)
+* Use Artifact and related types for downstream compiler 
+*/
+class Artifact : public RefObject
+{
+public:
+
+    typedef ArtifactDesc Desc;
+
+    typedef ArtifactKind Kind;
+    typedef ArtifactPayload Payload;
+    typedef ArtifactStyle Style;
+    typedef ArtifactFlags Flags;
+
+    // Controls what items can be kept. 
+    enum class Keep
+    {
+        No,         ///< Don't keep the item
+        Yes,        ///< Yes keep the final item
+        All,        ///< Keep the final item and any intermediataries
+    };
+
+    enum PathType
+    {
+        None,
+        Temporary,
+        Existing,
+    };
+
+    /* A compile product can be made up of multiple representations.
+    */
+    struct Entry
+    {
+        /// NOTE! Only interface innstances work across dll/shared library boundaries
+        /// because casting other types does not work across those boundaries.
+
+        // The Type of the entry
+        enum class Type : uint8_t
+        {
+            InterfaceInstance,          ///< An interface instance 
+            ObjectInstance,             ///< An object instance
+            RawInstance,
+        };
+        enum class Style : uint8_t
+        {
+            Artifact,       ///< Means this entry *can* represent the whole artifact
+            Child,          ///< Some part of the artifact
+            Info,           ///< Informational
+            Other,          ///< Other
+        };
+
+        Type type;
+        Style style;
+        union
+        {
+            RefObject* object;
+            ISlangUnknown* intf;
+            void* raw;
+        };
+    };
+
+        /// Given a type T find the associated instance
+    template <typename T>
+    T* findObjectInstance();
+
+        /// Finds an instance of that has the guid.
+    ISlangUnknown* findInterfaceInstance(const Guid& guid);
+
+        /// Returns true if the artifact in principal exists (it could be invalid)
+    bool exists() const;
+
+        /// Load as a blob
+    SlangResult loadBlob(Keep keep, ComPtr<ISlangBlob>& outBlob);
+    
+        /// Get as a file. May need to serialize and write as a temporary file.
+    SlangResult requireFilePath(Keep keep, String& outPath);
+
+    SLANG_FORCE_INLINE const Desc& getDesc() { return m_desc; }
+
+        /// Returns the index of the entry
+    Index indexOf(Entry::Type type) const;
+    
+        /// Add items
+    void setPath(PathType pathType, const String& filePath) { m_pathType = pathType; m_path = filePath; }
+    void setBlob(ISlangBlob* blob) { m_blob = blob; }
+
+    void add(Entry::Style style, RefObject* obj);
+    void add(Entry::Style style, ISlangUnknown* intf);
+
+    PathType getPathType() const { return m_pathType;  }
+    const String& getPath() const { return m_path;  }
+
+    const List<Entry>& getEntries() const { return m_entries; }
+
+        /// True if can keep an intermediate item
+    static bool canKeepIntermediate(Keep keep) { return keep == Keep::All; }
+        /// True if can keep
+    static bool canKeep(Keep keep) { return Index(keep) >= Index(Keep::Yes); }
+
+        /// Returns the keep type for an intermediate
+    static Keep getIntermediateKeep(Keep keep) { return (keep == Keep::All) ? Keep::All : Keep::No;  }
+
+        /// Ctor
+    Artifact(const Desc& desc) :m_desc(desc) {}
+        /// Dtor
+    ~Artifact();
+
+protected:
+    Desc m_desc;
+
+    PathType m_pathType = PathType::None;       ///< What the path indicates
+    String m_path;                              ///< The path 
+
+    ComPtr<ISlangBlob> m_blob;                  ///< Blob to store result in memory
+
+    List<Entry> m_entries;
+};
+
+// ----------------------------------------------------------------------
+template <typename T>
+T* Artifact::findObjectInstance()
+{
+    RefObject* check = static_cast<T*>(nullptr);
+    SLANG_UNUSED(check);
+
+    // Check if we already have it
+    for (const auto& entry : m_entries)
+    {
+        if (entry.type == Entry::Type::ObjectInstance)
+        {
+            auto obj = as<T>(entry.object);
+            if (obj)
+            {
+                return obj;
+            }
+        }
+    }
+    return nullptr;
+}
+
+
+// Class to hold information serialized in from a -r slang-lib/slang-module
+class ModuleLibrary : public RefObject
+{
+public:
+
+    List<FrontEndCompileRequest::ExtraEntryPointInfo> m_entryPoints;
+    List<RefPtr<IRModule>> m_modules;
+};
+
+SlangResult loadModuleLibrary(const Byte* inBytes, size_t bytesCount, EndToEndCompileRequest* req, RefPtr<ModuleLibrary>& module);
+
+// Given a product make available as a module
+SlangResult loadModuleLibrary(Artifact::Keep keep, Artifact* artifact, EndToEndCompileRequest* req, RefPtr<ModuleLibrary>& module);
+
+} // namespace Slang
+
+#endif
diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp
index 1b20a869d..80899e69e 100644
--- a/source/slang/slang-compiler.cpp
+++ b/source/slang/slang-compiler.cpp
@@ -1193,24 +1193,46 @@ void printDiagnosticArg(StringBuilder& sb, CodeGenTarget val)
         // Disable exceptions and security checks
         options.flags &= ~(CompileOptions::Flag::EnableExceptionHandling | CompileOptions::Flag::EnableSecurityChecks);
 
+        Profile profile;
+
         if (compilerType == PassThroughMode::Fxc ||
             compilerType == PassThroughMode::Dxc ||
             compilerType == PassThroughMode::Glslang)
         {
-            auto entryPointIndex = getSingleEntryPointIndex();
+            const auto entryPointIndices = getEntryPointIndices();
+            auto targetReq = getTargetReq();
 
-            auto entryPoint = getEntryPoint(entryPointIndex);
-            auto profile = getEffectiveProfile(entryPoint, getTargetReq());
+            const auto entryPointIndicesCount = entryPointIndices.getCount();
+             
+            if (entryPointIndicesCount == 0 && compilerType == PassThroughMode::Dxc)
+            {
+                // Can support no entry points on DXC because we can build libraries
+                profile = targetReq->getTargetProfile();
+            }
+            else if (entryPointIndicesCount == 1)
+            {
+                // All support a single entry point
+                const Index entryPointIndex = entryPointIndices[0];
 
-            options.stage = SlangStage(profile.getStage());
+                auto entryPoint = getEntryPoint(entryPointIndex);
+                profile = getEffectiveProfile(entryPoint, targetReq);
 
-            // Set the entry point name
-            options.entryPointName = getText(entryPoint->getName());
-            auto entryPointNameOverride = getProgram()->getEntryPointNameOverride(entryPointIndex);
-            if (entryPointNameOverride.getLength() != 0)
+                options.entryPointName = getText(entryPoint->getName());
+                auto entryPointNameOverride = getProgram()->getEntryPointNameOverride(entryPointIndex);
+                if (entryPointNameOverride.getLength() != 0)
+                {
+                    options.entryPointName = entryPointNameOverride;
+                }
+            }
+            else 
             {
-                options.entryPointName = entryPointNameOverride;
+                // We only support a single entry point on this target
+                SLANG_ASSERT(!"Can only compile with a single entry point on this target");
+                return SLANG_FAIL;
             }
+            
+            options.stage = SlangStage(profile.getStage());
+
             if (compilerType == PassThroughMode::Dxc)
             {
                 // We will enable the flag to generate proper code for 16 - bit types
@@ -1229,20 +1251,12 @@ void printDiagnosticArg(StringBuilder& sb, CodeGenTarget val)
                     options.flags |= CompileOptions::Flag::EnableFloat16;
                 }
 
-                // Only set the profile if the stage is set
-                if (options.stage != SLANG_STAGE_NONE)
-                {
-                    options.profileName = GetHLSLProfileName(profile);
-                }
-
                 // Set the matrix layout
                 options.matrixLayout = getTargetReq()->getDefaultMatrixLayoutMode();
             }
-            else if (compilerType == PassThroughMode::Fxc)
-            {
-                // Set the profile
-                options.profileName = GetHLSLProfileName(profile);
-            }
+
+            // Set the profile
+            options.profileName = GetHLSLProfileName(profile);
         }
 
         // If we aren't using LLVM 'host callable', we want downstream compile to produce a shared library
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index bf715d5d4..9cd2b0e63 100755
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -36,6 +36,7 @@ namespace Slang
     class TargetProgram;
     class TargetRequest;
     class TypeLayout;
+    class Artifact;
 
     enum class CompilerMode
     {
@@ -1801,7 +1802,7 @@ namespace Slang
         bool m_useFalcorCustomSharedKeywordSemantics = false;
 
         // Modules that have been read in with the -r option
-        List<RefPtr<IRModule>> m_libModules;
+        List<RefPtr<Artifact>> m_libModules;
 
         void _stopRetainingParentSession()
         {
diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h
index 2f18038ac..6500bd1a0 100644
--- a/source/slang/slang-diagnostic-defs.h
+++ b/source/slang/slang-diagnostic-defs.h
@@ -126,6 +126,10 @@ DIAGNOSTIC(    92, Error, unableToCreateDirectory, "unable to create directory '
 DIAGNOSTIC(    93, Error, unableExtractReproToDirectory, "unable to extract repro to directory '$0'")
 DIAGNOSTIC(    94, Error, unableToReadRiff, "unable to read as 'riff'/not a 'riff' file")
 
+DIAGNOSTIC(    95, Error, unknownLibraryKind, "unknown library kind '$0'")
+DIAGNOSTIC(    96, Error, kindNotLinkable, "not a known linkable kind '$0'")
+DIAGNOSTIC(    97, Error, libraryDoesNotExist, "library '$0' does not exist")
+
 //
 // 001xx - Downstream Compilers
 //
diff --git a/source/slang/slang-ir-link.cpp b/source/slang/slang-ir-link.cpp
index 45b9a842f..b03d48764 100644
--- a/source/slang/slang-ir-link.cpp
+++ b/source/slang/slang-ir-link.cpp
@@ -6,6 +6,7 @@
 #include "slang-ir-insts.h"
 #include "slang-mangle.h"
 #include "slang-ir-string-hash.h"
+#include "slang-artifact.h"
 
 namespace Slang
 {
@@ -1387,8 +1388,14 @@ LinkedIR linkIR(
     {
         irModules.add(irModule);
     });
-    irModules.addRange(linkage->m_libModules.getBuffer()->readRef(), linkage->m_libModules.getCount());
-
+    for (Artifact* artifact : linkage->m_libModules)
+    {
+        ModuleLibrary* library = artifact->findObjectInstance<ModuleLibrary>();
+        if (library)
+        {
+            irModules.addRange(library->m_modules.getBuffer()->readRef(), library->m_modules.getCount());
+        }
+    }
     
     // Add any modules that were loaded as libraries
     for (IRModule* irModule : irModules)
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index 982066a33..d4199154f 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -9,6 +9,7 @@
 
 #include "slang-compiler.h"
 #include "slang-profile.h"
+#include "slang-artifact.h"
 
 #include "slang-repro.h"
 #include "slang-serialize-ir.h"
@@ -23,7 +24,7 @@
 
 namespace Slang {
 
-SlangResult _addLibraryReference(EndToEndCompileRequest* req, Stream* stream);
+SlangResult _addLibraryReference(EndToEndCompileRequest* req, Artifact* artifact);
 
 struct OptionsParser
 {
@@ -1398,13 +1399,39 @@ struct OptionsParser
                     CommandLineArg referenceModuleName;
                     SLANG_RETURN_ON_FAIL(reader.expectArg(referenceModuleName));
 
-                    // We need to deserialize and add the modules
-                    FileStream fileStream;
-                    SLANG_RETURN_ON_FAIL(fileStream.init(referenceModuleName.value, FileMode::Open, FileAccess::Read, FileShare::ReadWrite));
+                    auto path = referenceModuleName.value;
 
-                    // TODO: probably near an error when we can't open the file?
+                    auto desc = ArtifactDesc::fromPath(path.getUnownedSlice());
 
-                    _addLibraryReference(requestImpl, &fileStream);
+                    if (desc.kind == ArtifactKind::Unknown)
+                    {
+                        sink->diagnose(referenceModuleName.loc, Diagnostics::unknownLibraryKind, Path::getPathExt(path));
+                        return SLANG_FAIL;
+                    }
+
+                    // If it's a GPU binary, then we'll assume it's a library
+                    if (desc.isGpuBinary())
+                    {
+                        desc.kind = ArtifactKind::Library;
+                    }
+
+                    if (!desc.isBinaryLinkable())
+                    {
+                        sink->diagnose(referenceModuleName.loc, Diagnostics::kindNotLinkable, Path::getPathExt(path));
+                        return SLANG_FAIL;
+                    }
+
+                    // Create the artifact
+                    RefPtr<Artifact> artifact = new Artifact(desc);
+                    // Set the path
+                    artifact->setPath(Artifact::PathType::Existing, referenceModuleName.value);
+
+                    // TODO(JS): We might want to check if the artifact exists.
+                    // If the artifact is a CPU (or downstream compiler) library
+                    // it may be findable by some other mechanism, so we probably don't want to check existance and just let the
+                    // downstream compiler handle.
+                    
+                    SLANG_RETURN_ON_FAIL(_addLibraryReference(requestImpl, artifact));
                 }
                 else if (argValue == "-v" || argValue == "-version")
                 {
@@ -1991,6 +2018,9 @@ struct OptionsParser
                     case CodeGenTarget::ShaderHostCallable:
                     case CodeGenTarget::HostExecutable:
                     case CodeGenTarget::ShaderSharedLibrary:
+
+                    case CodeGenTarget::DXIL:
+
                         rawOutput.isWholeProgram = true;
                         break;
                     default:
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index ab4d0749d..aa3f3ac8c 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -7,6 +7,8 @@
 #include "../core/slang-type-text-util.h"
 #include "../core/slang-type-convert-util.h"
 
+#include "slang-artifact.h"
+
 #include "slang-check.h"
 #include "slang-parameter-binding.h"
 #include "slang-lower-to-ir.h"
@@ -4296,61 +4298,44 @@ void EndToEndCompileRequest::setDefaultModuleName(const char* defaultModuleName)
     frontEndReq->m_defaultModuleName = namePool->getName(defaultModuleName);
 }
 
-SlangResult _addLibraryReference(EndToEndCompileRequest* req, Stream* stream)
+SlangResult _addLibraryReference(EndToEndCompileRequest* req, Artifact* artifact)
 {
-    // Load up the module
-    RiffContainer riffContainer;
-    SLANG_RETURN_ON_FAIL(RiffUtil::read(stream, riffContainer));
-
-    auto linkage = req->getLinkage();
-
-    // TODO(JS): May be better to have a ITypeComponent that encapsulates a collection of modules
-    // For now just add to the linkage
+    auto desc = artifact->getDesc();
 
+    if (desc.kind == ArtifactKind::Library && desc.payload == ArtifactPayload::SlangIR)
     {
-        SerialContainerData containerData;
+        RefPtr<ModuleLibrary> library;
 
-        SerialContainerUtil::ReadOptions options;
-        options.namePool = req->getNamePool();
-        options.session = req->getSession();
-        options.sharedASTBuilder = linkage->getASTBuilder()->getSharedASTBuilder();
-        options.sourceManager = linkage->getSourceManager();
-        options.linkage = req->getLinkage();
-        options.sink = req->getSink();
-
-        SLANG_RETURN_ON_FAIL(SerialContainerUtil::read(&riffContainer, options, containerData));
-
-        for (const auto& module : containerData.modules)
-        {
-            // If the irModule is set, add it
-            if (module.irModule)
-            {
-                linkage->m_libModules.add(module.irModule);
-            }
-        }
+        SLANG_RETURN_ON_FAIL(loadModuleLibrary(Artifact::Keep::Yes, artifact, req, library));
 
         FrontEndCompileRequest* frontEndRequest = req->getFrontEndReq();
-
-        for (const auto& entryPoint : containerData.entryPoints)
-        {
-            FrontEndCompileRequest::ExtraEntryPointInfo dst;
-            dst.mangledName = entryPoint.mangledName;
-            dst.name = entryPoint.name;
-            dst.profile = entryPoint.profile;
-
-            // Add entry point
-            frontEndRequest->m_extraEntryPoints.add(dst);
-        }
+        frontEndRequest->m_extraEntryPoints.addRange(library->m_entryPoints.getBuffer(), library->m_entryPoints.getCount());
+    }
+    else
+    {
+        // TODO(JS): 
+        // Do we want to check the path exists?
     }
 
+    // Add to the m_libModules
+    auto linkage = req->getLinkage();
+    linkage->m_libModules.add(artifact);
+
     return SLANG_OK;
 }
 
 SlangResult EndToEndCompileRequest::addLibraryReference(const void* libData, size_t libDataSize)
 {
     // We need to deserialize and add the modules
-    MemoryStreamBase fileStream(FileAccess::Read, libData, libDataSize);
-    return _addLibraryReference(this, &fileStream);
+    RefPtr<ModuleLibrary> library;
+    SLANG_RETURN_ON_FAIL(loadModuleLibrary((const Byte*)libData, libDataSize, this, library));
+
+    const auto desc = ArtifactDesc::make(ArtifactKind::Library, ArtifactPayload::SlangIR, ArtifactStyle::Unknown);
+    RefPtr<Artifact> artifact = new Artifact(desc);
+
+    artifact->add(Artifact::Entry::Style::Artifact, library);
+
+    return _addLibraryReference(this, artifact);
 }
 
 void EndToEndCompileRequest::addTranslationUnitPreprocessorDefine(int translationUnitIndex, const char* key, const char* value)