Fix cyclic reference in `ExtendedTypeLayout`. (#1868)

4 files changed, 68 insertions, 3 deletions

diff --git a/source/core/slang-smart-pointer.h b/source/core/slang-smart-pointer.h
index eb9fe7be5..2e5821ffd 100644
--- a/source/core/slang-smart-pointer.h
+++ b/source/core/slang-smart-pointer.h
@@ -272,6 +272,70 @@ namespace Slang
     private:
         T* m_ptr;
     };
-}
 
+    // A pointer that can be transformed to hold either a weak reference or a strong reference.
+    template<typename T>
+    class TransformablePtr
+    {
+    private:
+        T* m_weakPtr = nullptr;
+        RefPtr<T> m_strongPtr;
+
+    public:
+        TransformablePtr() = default;
+        TransformablePtr(T* ptr) { *this = ptr; }
+        TransformablePtr(RefPtr<T> ptr) { *this = ptr; }
+        TransformablePtr(const TransformablePtr<T>& ptr) = default;
+
+        void promoteToStrongReference() { m_strongPtr = m_weakPtr; }
+        void demoteToWeakReference() { m_strongPtr = nullptr; }
+        bool isStrongReference() const { return m_strongPtr != nullptr; }
+
+        T& operator*() const { return *m_weakPtr; }
+
+        T* operator->() const { return m_weakPtr; }
+
+        T* Ptr() const { return m_weakPtr; }
+        T* get() const { return m_weakPtr; }
+
+        operator T*() const { return m_weakPtr; }
+        operator RefPtr<T>() const { return m_weakPtr; }
+
+
+        TransformablePtr<T>& operator=(T* ptr)
+        {
+            m_weakPtr = ptr;
+            m_strongPtr = ptr;
+            return *this;
+        }
+        template<typename U>
+        TransformablePtr<T>& operator=(const RefPtr<U>& ptr)
+        {
+            m_weakPtr = ptr.Ptr();
+            m_strongPtr = ptr;
+            return *this;
+        }
+        
+        HashCode getHashCode() const
+        {
+            // Note: We need a `RefPtr<T>` to hash the same as a `T*`,
+            // so that a `T*` can be used as a key in a dictionary with
+            // `RefPtr<T>` keys, and vice versa.
+            //
+            return Slang::getHashCode(m_weakPtr);
+        }
+
+        bool operator==(const T* ptr) const { return m_weakPtr == ptr; }
+
+        bool operator!=(const T* ptr) const { return m_weakPtr != ptr; }
+
+        bool operator==(RefPtr<T> const& ptr) const { return m_weakPtr == ptr.Ptr(); }
+
+        bool operator!=(RefPtr<T> const& ptr) const { return m_weakPtr != ptr.Ptr(); }
+
+        bool operator==(TransformablePtr<T> const& ptr) const { return m_weakPtr == ptr.m_weakPtr; }
+
+        bool operator!=(TransformablePtr<T> const& ptr) const { return m_weakPtr != ptr.m_weakPtr; }
+    };
+}
 #endif
diff --git a/source/slang/slang-reflection-api.cpp b/source/slang/slang-reflection-api.cpp
index 22f34a41a..059de0be8 100644
--- a/source/slang/slang-reflection-api.cpp
+++ b/source/slang/slang-reflection-api.cpp
@@ -1405,6 +1405,7 @@ namespace Slang
 
             RefPtr<VarLayout> varLayout = new VarLayout();
             varLayout->typeLayout = typeLayout;
+            varLayout->typeLayout.demoteToWeakReference();
 
             for(auto typeResInfo : typeLayout->resourceInfos)
             {
diff --git a/source/slang/slang-type-layout.cpp b/source/slang/slang-type-layout.cpp
index f2869886e..42a93f3f0 100644
--- a/source/slang/slang-type-layout.cpp
+++ b/source/slang/slang-type-layout.cpp
@@ -2819,7 +2819,7 @@ static RefPtr<TypeLayout> maybeAdjustLayoutForArrayElementType(
 
         // If nothing needed to be changed on the inner element type,
         // then we are done.
-        if(adjustedInnerElementTypeLayout == originalInnerElementTypeLayout)
+        if (originalInnerElementTypeLayout == adjustedInnerElementTypeLayout)
             return originalTypeLayout;
 
         // TODO: actually adjust the element type, and create all the required bits and
diff --git a/source/slang/slang-type-layout.h b/source/slang/slang-type-layout.h
index 52b575787..be7ccdf8f 100644
--- a/source/slang/slang-type-layout.h
+++ b/source/slang/slang-type-layout.h
@@ -467,7 +467,7 @@ public:
     Name* getName() { return getVariable()->getName(); }
 
     // The result of laying out the variable's type
-    RefPtr<TypeLayout>      typeLayout;
+    TransformablePtr<TypeLayout> typeLayout;
     TypeLayout* getTypeLayout() { return typeLayout.Ptr(); }
 
     // Additional flags