Handling imported/exporting symbols from serialized modules (#1589)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Fix handling of access modifiers inside type definition.

* Fix access problem for AST node.
Make dumping produce a single function with switch, to potentially make available without Dump specific access.

* WIP on serialization design doc.

* Remove project references to previously generated files.

* More docs on serialization design.

* Improve serialization documentation.
Remove unused function from IRSerialReader.

* Small fixes around naming. Remove long comment from slang-serialize.h - as covered in serialization.md

* Remove long comment in slang-serialize.h as covered in serialization.md

* More information about doing replacements on read for AST and problems surrounding.

* Typo fix.

* Spelling fixes.

* Value serialize.

* Value types with inheritence.

* Use value reflection serial conversion for more AST types

* Use automatic serialization on more of AST.

* Get the types via decltype, simplifies what the extractor has to do.

* Update the serialization.md for the value serialization.

* Small doc improvements.

* Update project.

* Remove ImportExternalDecl type
Added addImportSymbol and ImportSymbol type
Fixed bug in container which meant it wouldn't read back AST module

* Because of change of how imports and handled, store objects as SerialPointers.

* First pass symbol lookup from mangled names.

* Cache current module looked up from mangled name.

* Fix SourceLoc bug.
Improve comments.

* Added diagnostic on mangled symbol not being found

* Fix typo.

Co-authored-by: Tim Foley <tfoleyNV@users.noreply.github.com>

1 files changed, 40 insertions, 7 deletions

diff --git a/source/slang/slang-serialize.h b/source/slang/slang-serialize.h
index 0e7fdd68a..a48a7e216 100644
--- a/source/slang/slang-serialize.h
+++ b/source/slang/slang-serialize.h
@@ -46,6 +46,7 @@ enum class SerialTypeKind : uint8_t
 
     String,             ///< String                         
     Array,              ///< Array
+    ImportSymbol,       ///< Holds the name of the import symbol. Represented in exactly the same way as a string
 
     NodeBase,           ///< NodeBase derived
     RefObject,          ///< RefObject derived types
@@ -160,6 +161,12 @@ struct SerialPointer
     {
     }
 
+        /// True if the ptr is set
+    SLANG_FORCE_INLINE operator bool() const { return m_ptr != nullptr; }
+
+        /// Directly set pointer/kind
+    void set(SerialTypeKind kind, void* ptr) { m_kind = kind; m_ptr = ptr; }
+
     static SerialTypeKind getKind(const RefObject*) { return SerialTypeKind::RefObject; }
     static SerialTypeKind getKind(const NodeBase*) { return SerialTypeKind::NodeBase; }
 
@@ -221,13 +228,23 @@ public:
     Name* getName(SerialIndex index);
     UnownedStringSlice getStringSlice(SerialIndex index);
     
-        /// Load the entries table (without deserializing anything)
-        /// NOTE! data must stay ins scope for outEntries to be valid
-    SlangResult loadEntries(const uint8_t* data, size_t dataCount, List<const SerialInfo::Entry*>& outEntries);
+    SlangResult loadEntries(const uint8_t* data, size_t dataCount) { return loadEntries(data, dataCount, m_classes, m_entries); }
+        /// For each entry construct an object. Does *NOT* deserialize them
+    SlangResult constructObjects(NamePool* namePool);
+        /// Entries must be loaded (with loadEntries), and objects constructed (with constructObjects) before deserializing
+    SlangResult deserializeObjects();
 
         /// NOTE! data must stay ins scope when reading takes place
     SlangResult load(const uint8_t* data, size_t dataCount, NamePool* namePool);
 
+        /// Get the entries list
+    const List<const Entry*>& getEntries() const { return m_entries; }
+
+        /// Access the objects list
+        /// NOTE that if a SerialObject holding a RefObject and needs to be kept in scope, add the RefObject* via addScope
+    List<SerialPointer>& getObjects() { return m_objects; }
+    const List<SerialPointer>& getObjects() const { return m_objects; }
+
         /// Add an object to be kept in scope
     void addScope(const RefObject* obj) { m_scope.add(obj); }
 
@@ -242,9 +259,14 @@ public:
     }
     ~SerialReader();
 
+        /// Load the entries table (without deserializing anything)
+        /// NOTE! data must stay ins scope for outEntries to be valid
+    static SlangResult loadEntries(const uint8_t* data, size_t dataCount, SerialClasses* serialClasses, List<const Entry*>& outEntries);
+
 protected:
     List<const Entry*> m_entries;       ///< The entries
-    List<void*> m_objects;              ///< The constructed objects
+
+    List<SerialPointer> m_objects;      ///< The constructed objects
     NamePool* m_namePool;               ///< Pool names are added to
 
     List<const RefObject*> m_scope;     ///< Keeping objects in scope
@@ -305,10 +327,15 @@ public:
     template <typename T>
     SerialIndex addArray(const T* in, Index count);
 
-    SerialIndex addString(const UnownedStringSlice& slice);
+        /// Add the string
+    SerialIndex addString(const UnownedStringSlice& slice) { return _addStringSlice(SerialTypeKind::String, m_sliceMap, slice); }
     SerialIndex addString(const String& in);
     SerialIndex addName(const Name* name);
-    
+
+        /// Adding import symbols
+    SerialIndex addImportSymbol(const UnownedStringSlice& slice) { return _addStringSlice(SerialTypeKind::ImportSymbol, m_importSymbolMap, slice); }
+    SerialIndex addImportSymbol(const String& string){ return _addStringSlice(SerialTypeKind::ImportSymbol, m_importSymbolMap, string.getUnownedSlice()); }
+
         /// Set a the ptr associated with an index.
         /// NOTE! That there cannot be a pre-existing setting.
     void setPointerIndex(const NodeBase* ptr, SerialIndex index);
@@ -331,6 +358,10 @@ public:
 
 protected:
 
+    typedef Dictionary<UnownedStringSlice, Index> SliceMap;
+
+    SerialIndex _addStringSlice(SerialTypeKind typeKind, SliceMap& sliceMap, const UnownedStringSlice& slice);
+
     SerialIndex _addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount);
 
     SerialIndex _add(const void* nativePtr, SerialInfo::Entry* entry)
@@ -345,8 +376,10 @@ protected:
 
     Dictionary<const void*, Index> m_ptrMap;    // Maps a pointer to an entry index
 
+    
     // NOTE! Assumes the content stays in scope!
-    Dictionary<UnownedStringSlice, Index> m_sliceMap;
+    SliceMap m_sliceMap;
+    SliceMap m_importSymbolMap;
 
     SerialExtraObjects m_extraObjects;      ///< Extra objects