Value type serialization via C++ Extractor (#1588)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Fix handling of access modifiers inside type definition.

* Fix access problem for AST node.
Make dumping produce a single function with switch, to potentially make available without Dump specific access.

* WIP on serialization design doc.

* Remove project references to previously generated files.

* More docs on serialization design.

* Improve serialization documentation.
Remove unused function from IRSerialReader.

* Small fixes around naming. Remove long comment from slang-serialize.h - as covered in serialization.md

* Remove long comment in slang-serialize.h as covered in serialization.md

* More information about doing replacements on read for AST and problems surrounding.

* Typo fix.

* Spelling fixes.

* Value serialize.

* Value types with inheritence.

* Use value reflection serial conversion for more AST types

* Use automatic serialization on more of AST.

* Get the types via decltype, simplifies what the extractor has to do.

* Update the serialization.md for the value serialization.

* Small doc improvements.

* Update project.

8 files changed, 128 insertions, 214 deletions

diff --git a/source/slang/slang-ast-base.h b/source/slang/slang-ast-base.h
index 0d69b8633..e9005212c 100644
--- a/source/slang/slang-ast-base.h
+++ b/source/slang/slang-ast-base.h
@@ -227,23 +227,27 @@ class ThisTypeSubstitution : public Substitutions
     HashCode _getHashCodeOverride() const;
 };
 
+struct GlobalGenericParamSubstitution_ConstraintArg
+{
+    SLANG_VALUE_CLASS(GlobalGenericParamSubstitution_ConstraintArg)
+    Decl*    decl = nullptr;
+    Val*     val = nullptr;
+};
+
 class GlobalGenericParamSubstitution : public Substitutions
 {
     SLANG_AST_CLASS(GlobalGenericParamSubstitution)
+
+    typedef GlobalGenericParamSubstitution_ConstraintArg ConstraintArg;
+
     // the type_param decl to be substituted
     GlobalGenericParamDecl* paramDecl = nullptr;
 
     // the actual type to substitute in
     Type* actualType = nullptr;
 
-    struct ConstraintArg
-    {
-        Decl*    decl = nullptr;
-        Val*     val = nullptr;
-    };
-
     // the values that satisfy any constraints on the type parameter
-    List<ConstraintArg> constraintArgs;
+    List<GlobalGenericParamSubstitution_ConstraintArg> constraintArgs;
 
     // Overrides should be public so base classes can access
     Substitutions* _applySubstitutionsShallowOverride(ASTBuilder* astBuilder, SubstitutionSet substSet, Substitutions* substOuter, int* ioDiff);
diff --git a/source/slang/slang-ast-support-types.h b/source/slang/slang-ast-support-types.h
index a2fa16b53..10bf46b63 100644
--- a/source/slang/slang-ast-support-types.h
+++ b/source/slang/slang-ast-support-types.h
@@ -395,7 +395,8 @@ namespace Slang
         /// A `DeclCheckState` plus a bit to track whether a declaration is currently being checked.
     struct DeclCheckStateExt
     {
-    public:
+        SLANG_VALUE_CLASS(DeclCheckStateExt)
+
         typedef uint8_t RawType;
         DeclCheckStateExt() {}
         DeclCheckStateExt(DeclCheckState state)
@@ -435,7 +436,9 @@ namespace Slang
         RawType getRaw() const { return m_raw; }
         void setRaw(RawType raw) { m_raw = raw; }
 
-    private:
+        // TODO(JS):
+        // Unfortunately for automatic serialization to see this member, it has to be public.
+    //private:
         RawType m_raw = 0;
     };
 
@@ -932,6 +935,7 @@ namespace Slang
     // We store both the original syntax and the resolved type here.
     struct TypeExp
     {
+        SLANG_VALUE_CLASS(TypeExp)
         typedef TypeExp ThisType;
 
         TypeExp() {}
@@ -1101,6 +1105,8 @@ namespace Slang
     // Represents one item found during lookup
     struct LookupResultItem
     {
+        SLANG_VALUE_CLASS(LookupResultItem)
+
         typedef LookupResultItem_Breadcrumb Breadcrumb;
 
         // Sometimes lookup finds an item, but there were additional
@@ -1166,7 +1172,7 @@ namespace Slang
         // This is unused in the simple case where a declaration
         // is being referenced directly (rather than through
         // transparent members).
-        RefPtr<Breadcrumb> breadcrumbs;
+        RefPtr<LookupResultItem_Breadcrumb> breadcrumbs;
 
         LookupResultItem() = default;
         explicit LookupResultItem(DeclRef<Decl> declRef)
@@ -1332,12 +1338,14 @@ namespace Slang
 
     struct SpecializationArg
     {
+        SLANG_VALUE_CLASS(SpecializationArg)
         Val* val = nullptr;
     };
     typedef List<SpecializationArg> SpecializationArgs;
 
     struct ExpandedSpecializationArg : SpecializationArg
     {
+        SLANG_VALUE_CLASS(ExpandedSpecializationArg)
         Val* witness = nullptr;
     };
     typedef List<ExpandedSpecializationArg> ExpandedSpecializationArgs;
diff --git a/source/slang/slang-serialize-ast-type-info.h b/source/slang/slang-serialize-ast-type-info.h
index 03fe61fbb..733694e8a 100644
--- a/source/slang/slang-serialize-ast-type-info.h
+++ b/source/slang/slang-serialize-ast-type-info.h
@@ -8,8 +8,9 @@
 #include "slang-serialize-type-info.h"
 #include "slang-serialize-misc-type-info.h"
 
-namespace Slang {
+#include "slang-serialize-value-type-info.h"
 
+namespace Slang {
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! AST types !!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
 
@@ -38,6 +39,7 @@ struct SerialTypeInfo<SyntaxClass<T>>
     }
 };
 
+// All the templates for DeclRef<T> can use this implementation.
 struct SerialDeclRefBaseTypeInfo
 {
     typedef DeclRefBase NativeType;
@@ -85,104 +87,10 @@ struct SerialTypeInfo<DeclRef<T>> : public SerialDeclRefBaseTypeInfo {};
 template <>
 struct SerialTypeInfo<MatrixCoord> : SerialIdentityTypeInfo<MatrixCoord> {};
 
-
-// QualType
-
-template <>
-struct SerialTypeInfo<QualType>
-{
-    typedef QualType NativeType;
-    struct SerialType
-    {
-        SerialIndex type;
-        uint8_t isLeftValue;
-    };
-    enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) };
-
-    static void toSerial(SerialWriter* writer, const void* native, void* serial)
-    {
-        auto dst = (SerialType*)serial;
-        auto src = (const NativeType*)native;
-        dst->isLeftValue = src->isLeftValue ? 1 : 0;
-        dst->type = writer->addPointer(src->type);
-    }
-    static void toNative(SerialReader* reader, const void* serial, void* native)
-    {
-        auto src = (const SerialType*)serial;
-        auto dst = (NativeType*)native;
-        dst->type = reader->getPointer(src->type).dynamicCast<Type>();
-        dst->isLeftValue = src->isLeftValue != 0;
-    }
-};
-
-
-// LookupResult::Breadcrumb
-template <>
-struct SerialTypeInfo<LookupResultItem::Breadcrumb>
-{
-    typedef LookupResultItem::Breadcrumb NativeType;
-    struct SerialType
-    {
-        NativeType::Kind kind;
-        NativeType::ThisParameterMode thisParameterMode;
-        SerialTypeInfo<DeclRef<Decl>>::SerialType declRef;
-        SerialTypeInfo<RefPtr<NativeType>> next;
-    };
-    enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) };
-
-    static void toSerial(SerialWriter* writer, const void* native, void* serial)
-    {
-        auto& src = *(const NativeType*)native;
-        auto& dst = *(SerialType*)serial;
-
-        dst.kind = src.kind;
-        dst.thisParameterMode = src.thisParameterMode;
-        toSerialValue(writer, src.declRef, dst.declRef);
-        toSerialValue(writer, src.next, dst.next);
-    }
-
-    static void toNative(SerialReader* reader, const void* serial, void* native)
-    {
-        auto& dst = *(NativeType*)native;
-        auto& src = *(const SerialType*)serial;
-
-        dst.kind = src.kind;
-        dst.thisParameterMode = src.thisParameterMode;
-        toNativeValue(reader, src.declRef, dst.declRef);
-        toNativeValue(reader, src.next, dst.next);
-    }
-};
-
 // LookupResultItem
-template <>
-struct SerialTypeInfo<LookupResultItem>
-{
-    typedef LookupResultItem NativeType;
-    struct SerialType
-    {
-        SerialTypeInfo<DeclRef<Decl>>::SerialType declRef;
-        SerialTypeInfo<RefPtr<NativeType::Breadcrumb>> breadcrumbs;
-    };
-    enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) };
-
-    static void toSerial(SerialWriter* writer, const void* native, void* serial)
-    {
-        auto& src = *(const NativeType*)native;
-        auto& dst = *(SerialType*)serial;
-
-        toSerialValue(writer, src.declRef, dst.declRef);
-        toSerialValue(writer, src.breadcrumbs, dst.breadcrumbs);
-    }
-
-    static void toNative(SerialReader* reader, const void* serial, void* native)
-    {
-        auto& dst = *(NativeType*)native;
-        auto& src = *(const SerialType*)serial;
-
-        toNativeValue(reader, src.declRef, dst.declRef);
-        toNativeValue(reader, src.breadcrumbs, dst.breadcrumbs);
-    }
-};
+SLANG_VALUE_TYPE_INFO(LookupResultItem)
+// QualType
+SLANG_VALUE_TYPE_INFO(QualType)
 
 // LookupResult
 template <>
@@ -236,115 +144,16 @@ struct SerialTypeInfo<LookupResult>
 };
 
 // GlobalGenericParamSubstitution::ConstraintArg
-template <>
-struct SerialTypeInfo<GlobalGenericParamSubstitution::ConstraintArg>
-{
-    typedef GlobalGenericParamSubstitution::ConstraintArg NativeType;
-    struct SerialType
-    {
-        SerialIndex decl;
-        SerialIndex val;
-    };
-    enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) };
-
-    static void toSerial(SerialWriter* writer, const void* native, void* serial)
-    {
-        auto& dst = *(SerialType*)serial;
-        auto& src = *(const NativeType*)native;
-
-        dst.decl = writer->addPointer(src.decl);
-        dst.val = writer->addPointer(src.val);
-    }
-    static void toNative(SerialReader* reader, const void* serial, void* native)
-    {
-        auto& src = *(const SerialType*)serial;
-        auto& dst = *(NativeType*)native;
-
-        dst.decl = reader->getPointer(src.decl).dynamicCast<Decl>();
-        dst.val = reader->getPointer(src.val).dynamicCast<Val>();
-    }
-};
+SLANG_VALUE_TYPE_INFO(GlobalGenericParamSubstitution_ConstraintArg)
 
+// SpecializationArg
+SLANG_VALUE_TYPE_INFO(SpecializationArg)
 // ExpandedSpecializationArg
-template <>
-struct SerialTypeInfo<ExpandedSpecializationArg>
-{
-    typedef ExpandedSpecializationArg NativeType;
-    struct SerialType
-    {
-        SerialIndex val;
-        SerialIndex witness;
-    };
-    enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) };
-
-    static void toSerial(SerialWriter* writer, const void* native, void* serial)
-    {
-        auto& dst = *(SerialType*)serial;
-        auto& src = *(const NativeType*)native;
-
-        dst.witness = writer->addPointer(src.witness);
-        dst.val = writer->addPointer(src.val);
-    }
-    static void toNative(SerialReader* reader, const void* serial, void* native)
-    {
-        auto& src = *(const SerialType*)serial;
-        auto& dst = *(NativeType*)native;
-
-        dst.witness = reader->getPointer(src.witness).dynamicCast<Val>();
-        dst.val = reader->getPointer(src.val).dynamicCast<Val>();
-    }
-};
-
+SLANG_VALUE_TYPE_INFO(ExpandedSpecializationArg)
 // TypeExp
-template <>
-struct SerialTypeInfo<TypeExp>
-{
-    typedef TypeExp NativeType;
-    struct SerialType
-    {
-        SerialIndex type;
-        SerialIndex expr;
-    };
-    enum { SerialAlignment = SLANG_ALIGN_OF(SerialIndex) };
-
-    static void toSerial(SerialWriter* writer, const void* native, void* serial)
-    {
-        auto& dst = *(SerialType*)serial;
-        auto& src = *(const NativeType*)native;
-
-        dst.type = writer->addPointer(src.type);
-        dst.expr = writer->addPointer(src.exp);
-    }
-    static void toNative(SerialReader* reader, const void* serial, void* native)
-    {
-        auto& src = *(const SerialType*)serial;
-        auto& dst = *(NativeType*)native;
-
-        dst.type = reader->getPointer(src.type).dynamicCast<Type>();
-        dst.exp = reader->getPointer(src.expr).dynamicCast<Expr>();
-    }
-};
-
+SLANG_VALUE_TYPE_INFO(TypeExp)
 // DeclCheckStateExt
-template <>
-struct SerialTypeInfo<DeclCheckStateExt>
-{
-    typedef DeclCheckStateExt NativeType;
-    typedef DeclCheckStateExt::RawType SerialType;
-
-    enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) };
-
-    static void toSerial(SerialWriter* writer, const void* native, void* serial)
-    {
-        SLANG_UNUSED(writer);
-        *(SerialType*)serial = (*(const NativeType*)native).getRaw();
-    }
-    static void toNative(SerialReader* reader, const void* serial, void* native)
-    {
-        SLANG_UNUSED(reader);
-        (*(NativeType*)serial).setRaw(*(const SerialType*)native);
-    }
-};
+SLANG_VALUE_TYPE_INFO(DeclCheckStateExt)
 
 // Modifiers
 template <>
diff --git a/source/slang/slang-serialize-value-type-info.h b/source/slang/slang-serialize-value-type-info.h
new file mode 100644
index 000000000..55e204d9f
--- /dev/null
+++ b/source/slang/slang-serialize-value-type-info.h
@@ -0,0 +1,77 @@
+// slang-serialize-value-type-info.h
+
+#ifndef SLANG_SERIALIZE_VALUE_TYPE_INFO_H
+#define SLANG_SERIALIZE_VALUE_TYPE_INFO_H
+
+#include "slang-ast-support-types.h"
+
+#include "slang-serialize.h"
+
+#include "slang-serialize-misc-type-info.h"
+#include "slang-serialize-type-info.h"
+
+#include "slang-generated-value.h"
+#include "slang-generated-value-macro.h"
+
+// Create the functions to automatically convert between value types
+
+namespace Slang {
+
+// TODO(JS): We may want to strip const or other modifiers
+// Just strips the brackets.
+#define SLANG_VALUE_GET_TYPE(TYPE)  TYPE
+
+#define SLANG_VALUE_FIELD_TO_SERIAL(FIELD_NAME, TYPE, param) SerialTypeInfo<decltype(src->FIELD_NAME)>::toSerial(writer, &src->FIELD_NAME, &dst->FIELD_NAME);
+#define SLANG_VALUE_FIELD_TO_NATIVE(FIELD_NAME, TYPE, param) SerialTypeInfo<decltype(dst->FIELD_NAME)>::toNative(reader, &src->FIELD_NAME, &dst->FIELD_NAME);
+
+#define SLANG_IF_HAS_SUPER_BASE(x)
+#define SLANG_IF_HAS_SUPER_INNER(x) x
+#define SLANG_IF_HAS_SUPER_LEAF(x) x
+
+#define SLANG_VALUE_TO_SERIAL(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \
+static void toSerial(SerialWriter* writer, const void* native, void* serial) \
+{ \
+    SLANG_IF_HAS_SUPER_##TYPE(SerialTypeInfo<SUPER>::toSerial(writer, native, serial);) \
+    auto dst = (SerialType*)serial; \
+    auto src = (const NativeType*)native; \
+    SLANG_FIELDS_Value_##NAME(SLANG_VALUE_FIELD_TO_SERIAL, param) \
+}
+
+#define SLANG_VALUE_TO_NATIVE(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \
+static void toNative(SerialReader* reader, const void* serial, void* native) \
+{ \
+    SLANG_IF_HAS_SUPER_##TYPE(SerialTypeInfo<SUPER>::toNative(reader, serial, native);) \
+    auto src = (const SerialType*)serial; \
+    auto dst = (NativeType*)native; \
+    SLANG_FIELDS_Value_##NAME(SLANG_VALUE_FIELD_TO_NATIVE, param) \
+}
+
+//#define SLANG_VALUE_SERIAL_FIELD(FIELD_NAME, TYPE, param) SerialTypeInfo<SLANG_VALUE_GET_TYPE TYPE>::SerialType FIELD_NAME;
+#define SLANG_VALUE_SERIAL_FIELD(FIELD_NAME, TYPE, param) SerialTypeInfo<decltype(((param*)nullptr)->FIELD_NAME)>::SerialType FIELD_NAME;
+
+#define SLANG_VALUE_SERIAL_STRUCT(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \
+struct SerialType SLANG_IF_HAS_SUPER_##TYPE( : SerialTypeInfo<SUPER>::SerialType) \
+{ \
+    SLANG_FIELDS_Value_##NAME(SLANG_VALUE_SERIAL_FIELD, NAME) \
+}; 
+
+#define SLANG_VALUE_TYPE_INFO_IMPL(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \
+template <> \
+struct SerialTypeInfo<NAME> \
+{ \
+    typedef NAME NativeType; \
+    SLANG_VALUE_SERIAL_STRUCT(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \
+    \
+    enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; \
+    \
+    SLANG_VALUE_TO_NATIVE(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \
+    SLANG_VALUE_TO_SERIAL(NAME, SUPER, ORIGIN, LAST, MARKER, TYPE, param) \
+};
+    
+#define SLANG_VALUE_TYPE_INFO(NAME) \
+    SLANG_Value_##NAME(SLANG_VALUE_TYPE_INFO_IMPL, _)
+
+
+} // namespace Slang
+
+#endif // SLANG_SERIALIZE_VALUE_TYPE_INFO_H
diff --git a/source/slang/slang-value-reflect.cpp b/source/slang/slang-value-reflect.cpp
index 9a7f1feb3..3c19deba4 100644
--- a/source/slang/slang-value-reflect.cpp
+++ b/source/slang/slang-value-reflect.cpp
@@ -2,7 +2,12 @@
 
 #include "slang-value-reflect.h"
 
+#include "slang-generated-value.h"
+#include "slang-generated-value-macro.h"
+
 namespace Slang
 {
 
+
+
 } // namespace Slang
diff --git a/source/slang/slang-value-reflect.h b/source/slang/slang-value-reflect.h
index 4d6ec5582..eded586c3 100644
--- a/source/slang/slang-value-reflect.h
+++ b/source/slang/slang-value-reflect.h
@@ -3,4 +3,11 @@
 #ifndef SLANG_VALUE_REFLECT_H
 #define SLANG_VALUE_REFLECT_H
 
+#include "slang-generated-value.h"
+#include "slang-generated-value-macro.h"
+
+// Create the functions to automatically convert between value types
+
+
+
 #endif // SLANG_VALUE_REFLECT_H
diff --git a/source/slang/slang.vcxproj b/source/slang/slang.vcxproj
index 2490b9caa..17590a21a 100644
--- a/source/slang/slang.vcxproj
+++ b/source/slang/slang.vcxproj
@@ -300,6 +300,7 @@
     <ClInclude Include="slang-serialize-source-loc.h" />
     <ClInclude Include="slang-serialize-type-info.h" />
     <ClInclude Include="slang-serialize-types.h" />
+    <ClInclude Include="slang-serialize-value-type-info.h" />
     <ClInclude Include="slang-serialize.h" />
     <ClInclude Include="slang-source-loc.h" />
     <ClInclude Include="slang-syntax.h" />
@@ -445,4 +446,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project> 
\ No newline at end of file
+</Project>
\ No newline at end of file
diff --git a/source/slang/slang.vcxproj.filters b/source/slang/slang.vcxproj.filters
index 9abc2b21d..5664b1f66 100644
--- a/source/slang/slang.vcxproj.filters
+++ b/source/slang/slang.vcxproj.filters
@@ -351,6 +351,9 @@
     <ClInclude Include="slang-serialize-types.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="slang-serialize-value-type-info.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="slang-serialize.h">
       <Filter>Header Files</Filter>
     </ClInclude>