Add struct version to DownstreamCompiler::CompileOptions interface (#2692)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Add versioning to CompileOptions for DownstreamCompiler so we can add new options without breaking binary interface.

* Use builtin offset of directly.

7 files changed, 210 insertions, 13 deletions

diff --git a/source/compiler-core/slang-downstream-compiler.cpp b/source/compiler-core/slang-downstream-compiler.cpp
index 104f1e631..b312f9cb8 100644
--- a/source/compiler-core/slang-downstream-compiler.cpp
+++ b/source/compiler-core/slang-downstream-compiler.cpp
@@ -68,11 +68,17 @@ void* DownstreamCompilerBase::getObject(const Guid& guid)
 
 SlangResult CommandLineDownstreamCompiler::compile(const CompileOptions& inOptions, IArtifact** outArtifact)
 {
+    if (!isVersionCompatible(inOptions))
+    {
+        // Not possible to compile with this version of the interface.
+        return SLANG_E_NOT_IMPLEMENTED;
+    }
+
+    CompileOptions options = getCompatibleVersion(&inOptions);
+
     // Copy the command line options
     CommandLine cmdLine(m_cmdLine);
 
-    CompileOptions options(inOptions);
-
     // Work out the ArtifactDesc 
     const auto targetDesc = ArtifactDescUtil::makeDescForCompileTarget(options.targetType);
 
diff --git a/source/compiler-core/slang-downstream-compiler.h b/source/compiler-core/slang-downstream-compiler.h
index e7ab4ddad..b1f1b7b11 100644
--- a/source/compiler-core/slang-downstream-compiler.h
+++ b/source/compiler-core/slang-downstream-compiler.h
@@ -44,8 +44,108 @@ struct DownstreamCompilerDesc
     SemanticVersion version;    ///< The version of the compiler
 };
 
+/* Placed at the start of structs that are versioned. 
+The id uniquely identifies a compatible set of versions. 
+The size indicates the struct size. It should be considered as a kind of version number.
+The larger the number for the target the newer the *compatible* version (assuming the identifiers
+match).
+
+Note that size versioning *only* works, if adding a field *doesn't* use any existing unused "pad" bytes. 
+This implies that any new members *must* take into account padding/alignment. Any additions that have alignment
+*less* than the alignment of struct may need padding. 
+*/
+struct VersionedStruct
+{
+    typedef VersionedStruct ThisType;
+    VersionedStruct(uint32_t inIdentifier, size_t inSize):
+        identifier(inIdentifier),
+        size(uint32_t(inSize))
+    {}
+
+        /// True if the versions are identical
+    bool operator==(const ThisType& rhs) const { return identifier == rhs.identifier && size == rhs.size;  }
+    bool operator!=(const ThisType& rhs) const { return !(*this == rhs); }
+
+    VersionedStruct(const ThisType& rhs) = default;
+    ThisType& operator=(const ThisType& rhs) = default;
+
+    uint32_t identifier;
+    uint32_t size;
+};
+
+template <typename T>
+T getCompatibleVersion(const T* inT)
+{
+    const VersionedStruct* in = &inT->version;
+
+    // It must be at the start of the struct
+    SLANG_ASSERT((void*)in == (void*)inT);
+
+    // Note that the struct is passed in by pointer rather than reference, because
+    // we must ensure that it is not sliced.
+    
+    // Must match
+    SLANG_ASSERT(T::kVersionIdentifier == in->identifier);
+
+    // If the same size we can just use what we have
+    if (in->size == sizeof(T))
+    {
+        return *inT;
+    }
+
+    // Initialize a new T to copy into
+    T t;
+
+    // Keep a copy of the version as will be overwritten
+    const auto currentVersion = t.version;
+
+    // If the size is smaller we just copy the bytes that we have.
+    // NOTE! This only works if care is taken with padding/end bytes of previous versions
+    // see above on VersionedStruct
+    if (in->size < sizeof(T))
+    {
+        // Copy up the size that's stored
+        ::memcpy(&t, in, in->size);
+    }
+    else
+    {
+        t = *inT;
+    }
+
+    t.version = currentVersion;
+    return t;
+}
+
+template <typename T>
+bool isVersionCompatible(const VersionedStruct& ver)
+{
+    return ver.identifier == T::kVersionIdentifier;
+}
+
+template <typename T>
+bool isVersionCompatible(const T& in)
+{
+    return isVersionCompatible<T>(in.version);
+}
+
+/* Downstream compile options
+
+NOTE! This type is trafficed across shared library boundaries and *versioned*. 
+In particular
+
+* The struct can only contain types that can be trivially memcpyd.
+* New fields can only be added to the end of the struct 
+* New fields must take into account alignment/padding such that they do not share bytes in previous version sizes
+*/
 struct DownstreamCompileOptions
 {
+    typedef DownstreamCompileOptions ThisType;
+
+    // A unique identifer for this particular struct kind. If the struct become incompatible 
+    // a new id should be used to identify a specific style. If the change is only to add members
+    // to the end, this should be handled via the version size at use sites. 
+    static const uint32_t kVersionIdentifier = 0x34296897;
+
     typedef uint32_t Flags;
     struct Flag
     {
@@ -105,6 +205,9 @@ struct DownstreamCompileOptions
         SemanticVersion version;
     };
 
+    // These members must be the first members of the struct!
+    VersionedStruct version = VersionedStruct(kVersionIdentifier, sizeof(ThisType));
+
     OptimizationLevel optimizationLevel = OptimizationLevel::Default;
     DebugInfoType debugInfoType = DebugInfoType::Standard;
     SlangCompileTarget targetType = SLANG_HOST_EXECUTABLE;
@@ -151,6 +254,16 @@ struct DownstreamCompileOptions
     SourceManager* sourceManager = nullptr;
 };
 
+#define SLANG_ALIAS_DEPRECIATED_VERSION(name, id, firstField, lastField) \
+struct name##_AliasDepreciated##id \
+{ \
+    static const ptrdiff_t kStart = SLANG_OFFSET_OF(name, firstField); \
+    static const ptrdiff_t kEnd = SLANG_OFFSET_OF(name, lastField) + sizeof(name::lastField); \
+};
+
+// Specifies via kStart/kEnd a slice of a type that is the previous version.
+SLANG_ALIAS_DEPRECIATED_VERSION(DownstreamCompileOptions, 1, optimizationLevel, sourceManager)
+
 /* Used to indicate what kind of products are expected to be produced for a compilation. */
 typedef uint32_t DownstreamProductFlags;
 struct DownstreamProductFlag
diff --git a/source/compiler-core/slang-dxc-compiler.cpp b/source/compiler-core/slang-dxc-compiler.cpp
index 80b1f7a21..9df2a4f3f 100644
--- a/source/compiler-core/slang-dxc-compiler.cpp
+++ b/source/compiler-core/slang-dxc-compiler.cpp
@@ -356,8 +356,16 @@ static SlangResult _handleOperationResult(IDxcOperationResult* dxcResult, IArtif
     return SLANG_OK;
 }
 
-SlangResult DXCDownstreamCompiler::compile(const CompileOptions& options, IArtifact** outArtifact)
+SlangResult DXCDownstreamCompiler::compile(const CompileOptions& inOptions, IArtifact** outArtifact)
 {
+    if (!isVersionCompatible(inOptions)) 
+    {
+        // Not possible to compile with this version of the interface.
+        return SLANG_E_NOT_IMPLEMENTED;
+    }
+
+    CompileOptions options = getCompatibleVersion(&inOptions);
+
     // This compiler can only deal with a single artifact
     if (options.sourceArtifacts.count != 1)
     {
diff --git a/source/compiler-core/slang-fxc-compiler.cpp b/source/compiler-core/slang-fxc-compiler.cpp
index 1706f0fb0..a9e916abf 100644
--- a/source/compiler-core/slang-fxc-compiler.cpp
+++ b/source/compiler-core/slang-fxc-compiler.cpp
@@ -180,8 +180,16 @@ static SlangResult _parseDiagnosticLine(SliceAllocator& allocator, const Unowned
     return SLANG_OK;
 }
 
-SlangResult FXCDownstreamCompiler::compile(const CompileOptions& options, IArtifact** outArtifact)
+SlangResult FXCDownstreamCompiler::compile(const CompileOptions& inOptions, IArtifact** outArtifact)
 {
+    if (!isVersionCompatible(inOptions))
+    {
+        // Not possible to compile with this version of the interface.
+        return SLANG_E_NOT_IMPLEMENTED;
+    }
+
+    CompileOptions options = getCompatibleVersion(&inOptions);
+
     // This compiler can only deal with a single source artifact
     if (options.sourceArtifacts.count != 1)
     {
diff --git a/source/compiler-core/slang-glslang-compiler.cpp b/source/compiler-core/slang-glslang-compiler.cpp
index 0673540dc..17abc469b 100644
--- a/source/compiler-core/slang-glslang-compiler.cpp
+++ b/source/compiler-core/slang-glslang-compiler.cpp
@@ -138,10 +138,16 @@ static SlangResult _parseDiagnosticLine(SliceAllocator& allocator, const Unowned
     return SLANG_OK;
 }
 
+SlangResult GlslangDownstreamCompiler::compile(const CompileOptions& inOptions, IArtifact** outArtifact)
+{
+    if (!isVersionCompatible(inOptions))
+    {
+        // Not possible to compile with this version of the interface.
+        return SLANG_E_NOT_IMPLEMENTED;
+    }
 
+    CompileOptions options = getCompatibleVersion(&inOptions);
 
-SlangResult GlslangDownstreamCompiler::compile(const CompileOptions& options, IArtifact** outArtifact)
-{
     // This compiler can only handle a single artifact
     if (options.sourceArtifacts.count != 1)
     {
diff --git a/source/compiler-core/slang-llvm-compiler.cpp b/source/compiler-core/slang-llvm-compiler.cpp
index 382dd0842..b34c05d42 100644
--- a/source/compiler-core/slang-llvm-compiler.cpp
+++ b/source/compiler-core/slang-llvm-compiler.cpp
@@ -6,6 +6,42 @@
 namespace Slang
 {
 
+class AliasDepreciatedDownstreamCompiler : public DownstreamCompilerBase
+{
+public:
+
+    virtual SLANG_NO_THROW SlangResult SLANG_MCALL compile(const CompileOptions& options, IArtifact** outArtifact) SLANG_OVERRIDE;
+    virtual SLANG_NO_THROW bool SLANG_MCALL canConvert(const ArtifactDesc& from, const ArtifactDesc& to) SLANG_OVERRIDE { return m_inner->canConvert(from, to); }
+    virtual SLANG_NO_THROW SlangResult SLANG_MCALL convert(IArtifact* from, const ArtifactDesc& to, IArtifact** outArtifact) SLANG_OVERRIDE { return m_inner->convert(from, to, outArtifact); }
+    virtual SLANG_NO_THROW SlangResult SLANG_MCALL getVersionString(slang::IBlob** outVersionString) { return m_inner->getVersionString(outVersionString); }
+    virtual SLANG_NO_THROW bool SLANG_MCALL isFileBased() { return m_inner->isFileBased(); }
+
+    template <typename T>
+    void initCompileOptionsDepreciated()
+    {
+        m_compileOptionsOffset = T::kStart;
+    }
+
+    AliasDepreciatedDownstreamCompiler(IDownstreamCompiler* inner) :
+        m_inner(inner)
+    {
+        m_desc = inner->getDesc();
+    }
+
+    ComPtr<IDownstreamCompiler> m_inner;
+    ptrdiff_t m_compileOptionsOffset = 0;
+};
+
+SlangResult AliasDepreciatedDownstreamCompiler::compile(const CompileOptions& options, IArtifact** outArtifact)
+{
+    if (m_compileOptionsOffset == 0)
+    {
+        return m_inner->compile(options, outArtifact);
+    }
+    const uint8_t* ptr = ((const uint8_t*)&options) + m_compileOptionsOffset;
+    return m_inner->compile(*(const CompileOptions*)ptr, outArtifact);
+}
+
 /* static */SlangResult LLVMDownstreamCompilerUtil::locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set)
 {
     ComPtr<ISlangSharedLibrary> library;
@@ -20,15 +56,27 @@ namespace Slang
 
     typedef SlangResult(*CreateDownstreamCompilerFunc)(const Guid& intf, IDownstreamCompiler** outCompiler);
 
-    auto fn = (CreateDownstreamCompilerFunc)library->findFuncByName("createLLVMDownstreamCompiler_V2");
-    if (!fn)
+    ComPtr<IDownstreamCompiler> downstreamCompiler;
+
+    if (auto fnV2 = (CreateDownstreamCompilerFunc)library->findFuncByName("createLLVMDownstreamCompiler_V2"))
     {
-        return SLANG_FAIL;
-    }
+        ComPtr<IDownstreamCompiler> innerDownstreamCompiler;
 
-    ComPtr<IDownstreamCompiler> downstreamCompiler;
+        SLANG_RETURN_ON_FAIL(fnV2(IDownstreamCompiler::getTypeGuid(), innerDownstreamCompiler.writeRef()));
 
-    SLANG_RETURN_ON_FAIL(fn(IDownstreamCompiler::getTypeGuid(), downstreamCompiler.writeRef()));
+        // We then need to wrap
+        AliasDepreciatedDownstreamCompiler* fix = new AliasDepreciatedDownstreamCompiler(innerDownstreamCompiler);
+        downstreamCompiler = fix;
+        fix->initCompileOptionsDepreciated<DownstreamCompileOptions_AliasDepreciated1>();
+    }
+    else if (auto fnV3 = (CreateDownstreamCompilerFunc)library->findFuncByName("createLLVMDownstreamCompiler_V3"))
+    {
+        SLANG_RETURN_ON_FAIL(fnV3(IDownstreamCompiler::getTypeGuid(), downstreamCompiler.writeRef()));
+    }
+    else
+    {
+        return SLANG_FAIL;
+    }
 
     set->addSharedLibrary(library);
     set->addCompiler(downstreamCompiler);
diff --git a/source/compiler-core/slang-nvrtc-compiler.cpp b/source/compiler-core/slang-nvrtc-compiler.cpp
index f650a11f5..e7e90be60 100644
--- a/source/compiler-core/slang-nvrtc-compiler.cpp
+++ b/source/compiler-core/slang-nvrtc-compiler.cpp
@@ -699,8 +699,16 @@ SlangResult NVRTCDownstreamCompiler::_maybeAddHalfSupport(const DownstreamCompil
     return SLANG_OK;
 }
 
-SlangResult NVRTCDownstreamCompiler::compile(const DownstreamCompileOptions& options, IArtifact** outArtifact)
+SlangResult NVRTCDownstreamCompiler::compile(const DownstreamCompileOptions& inOptions, IArtifact** outArtifact)
 {
+    if (!isVersionCompatible(inOptions))
+    {
+        // Not possible to compile with this version of the interface.
+        return SLANG_E_NOT_IMPLEMENTED;
+    }
+
+    CompileOptions options = getCompatibleVersion(&inOptions);
+
     // This compiler can only deal with a single artifact
     if (options.sourceArtifacts.count != 1)
     {