Test for serializing out and reading back Stdlib (#1605)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Mangling/module name extraction for GenericDecl

* Add comment on SerialFilter to explain re-enabling Stmt.

* Support setting up SyntaxDecl when reconstructed after deserialization.

* Improvements to setup SyntaxDecl.

* Fix typo so can read compressed SourceLocs.

* Fix issue with SourceManger.

* Simple test for serializing out stdlib and reading back in.

* Fix calling convention.

* Add override to StdLib impls.

* Fix typo.

* Apply testing to an actual compute test when using load-stdlib
Make -load/compile-stdlib processable by Slang
Move out testing into util into TestToolUtil so can be shared.

* Slightly more concise setup of session.

* Fix some errors introduced with session handling.

* Made setup for compile same across slangc and slangc-tool.

11 files changed, 243 insertions, 65 deletions

diff --git a/slang.h b/slang.h
index 2936f179a..10dfb2b18 100644
--- a/slang.h
+++ b/slang.h
@@ -2988,6 +2988,22 @@ namespace slang
         virtual SLANG_NO_THROW void SLANG_MCALL getLanguagePrelude(
             SlangSourceLanguage sourceLanguage,
             ISlangBlob** outPrelude) = 0;
+
+        
+            /** Compile from (embedded source) the StdLib on the session.
+            Will return a failure if there is already a StdLib available
+            NOTE! API is experimental and not ready for production code 
+            */
+        virtual SLANG_NO_THROW SlangResult SLANG_MCALL compileStdLib() = 0;
+
+            /** Load the StdLib. Currently loads modules from the file system
+            NOTE! API is experimental and not ready for production code 
+            */
+        virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadStdLib() = 0;
+
+            /** Save the StdLib modules to the file system
+            NOTE! API is experimental and not ready for production code  */
+        virtual SLANG_NO_THROW SlangResult SLANG_MCALL saveStdLib() = 0;
     };
 
     #define SLANG_UUID_IGlobalSession { 0xc140b5fd, 0xc78, 0x452e, { 0xba, 0x7c, 0x1a, 0x1e, 0x70, 0xc7, 0xf7, 0x1c } };
@@ -3378,12 +3394,31 @@ namespace slang
     };
 }
 
+// Passed into functions to create globalSession to identify the API version client code is
+// using. 
 #define SLANG_API_VERSION 0
 
+/* Create a global session, with built in StdLib.
+
+@param apiVersion Pass in SLANG_API_VERSION
+@param outGlobalSession (out)The created global session. 
+*/
 SLANG_API SlangResult slang_createGlobalSession(
     SlangInt                apiVersion,
     slang::IGlobalSession** outGlobalSession);
 
+/* Create a global session, but do not set up the stdlib. The stdlib can
+then be loaded via loadStdLib or compileStdLib
+
+@param apiVersion Pass in SLANG_API_VERSION
+@param outGlobalSession (out)The created global session that doesn't have a StdLib setup.
+
+NOTE! API is experimental and not ready for production code 
+*/
+SLANG_API SlangResult slang_createGlobalSessionWithoutStdLib(
+    SlangInt                apiVersion,
+    slang::IGlobalSession** outGlobalSession);
+
 namespace slang
 {
     inline SlangResult createGlobalSession(
diff --git a/source/core/slang-test-tool-util.cpp b/source/core/slang-test-tool-util.cpp
index f06095ada..68673a29f 100644
--- a/source/core/slang-test-tool-util.cpp
+++ b/source/core/slang-test-tool-util.cpp
@@ -37,6 +37,19 @@ namespace Slang
     }
 }
 
+/* static */bool TestToolUtil::hasDeferredStdLib(Index argc, const char*const* argv)
+{
+    for (Index i = 0; i < argc; ++i)
+    {
+        UnownedStringSlice option(argv[i]);
+        if (option == "-load-stdlib" || option == "-compile-stdlib")
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
 /* static */SlangResult TestToolUtil::getIncludePath(const String& parentPath, const char* path, String& outIncludePath)
 {
     String includePath;
diff --git a/source/core/slang-test-tool-util.h b/source/core/slang-test-tool-util.h
index 07532682b..186003a62 100644
--- a/source/core/slang-test-tool-util.h
+++ b/source/core/slang-test-tool-util.h
@@ -63,6 +63,9 @@ struct TestToolUtil
     
         /// Sets the default preludes on the session based on the executable path
     static SlangResult setSessionDefaultPreludeFromExePath(const char* exePath, slang::IGlobalSession* session);
+
+        /// Returns true if the StdLib should not be initialized immediately (eg when doing a -load-stdlib).
+    static bool hasDeferredStdLib(Index numArgs, const char*const* args);
 };
 
 } // namespace Slang
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index 649283f35..45ebc2909 100755
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -2077,6 +2077,10 @@ namespace Slang
         SLANG_NO_THROW void SLANG_MCALL setLanguagePrelude(SlangSourceLanguage inSourceLanguage, char const* prelude) override;
         SLANG_NO_THROW void SLANG_MCALL getLanguagePrelude(SlangSourceLanguage inSourceLanguage, ISlangBlob** outPrelude) override;
 
+        SLANG_NO_THROW SlangResult SLANG_MCALL compileStdLib() override;
+        SLANG_NO_THROW SlangResult SLANG_MCALL loadStdLib() override;
+        SLANG_NO_THROW SlangResult SLANG_MCALL saveStdLib() override;
+
             /// Get the default compiler for a language
         DownstreamCompiler* getDefaultDownstreamCompiler(SourceLanguage sourceLanguage);
 
@@ -2132,7 +2136,7 @@ namespace Slang
         String getCoreLibraryCode();
         String getHLSLLibraryCode();
 
-        
+     
         RefPtr<SharedASTBuilder> m_sharedASTBuilder;
 
 
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index 8897a6549..6a5b43876 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -447,6 +447,18 @@ struct OptionsParser
                 {
                     flags |= SLANG_COMPILE_FLAG_NO_MANGLING;
                 }
+                else if (argStr == "-load-stdlib")
+                {
+                    SLANG_RETURN_ON_FAIL(session->loadStdLib());
+                }
+                else if (argStr == "-compile-stdlib")
+                {
+                    SLANG_RETURN_ON_FAIL(session->compileStdLib());
+                }
+                else if (argStr == "-save-stdlib")
+                {
+                    SLANG_RETURN_ON_FAIL(session->saveStdLib());
+                }
                 else if (argStr == "-no-codegen")
                 {
                     flags |= SLANG_COMPILE_FLAG_NO_CODEGEN;
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index 6f670167d..d61814136 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -152,7 +152,6 @@ void Session::init()
     //
     m_builtinLinkage->_stopRetainingParentSession();
 
-  
     // Create scopes for various language builtins.
     //
     // TODO: load these on-demand to avoid parsing
@@ -174,44 +173,6 @@ void Session::init()
     slangLanguageScope = new Scope();
     slangLanguageScope->nextSibling = hlslLanguageScope;
 
-    if (false)
-    {
-        // Let's try loading serialized modules and adding them
-        _readBuiltinModule(coreLanguageScope, "core");
-        _readBuiltinModule(hlslLanguageScope, "hlsl");
-    }
-    else
-    {
-        addBuiltinSource(coreLanguageScope, "core", getCoreLibraryCode());
-        addBuiltinSource(hlslLanguageScope, "hlsl", getHLSLLibraryCode());
-
-        // Write out
-        if (false)
-        {
-            for (auto& pair : m_builtinLinkage->mapNameToLoadedModules)
-            {
-                const Name* moduleName = pair.Key;
-                Module* module = pair.Value;
-
-                // Set up options
-                SerialContainerUtil::WriteOptions options;
-
-                options.optionFlags |= SerialOptionFlag::SourceLocation;
-                // TODO(JS): Should this be the Session::getBuiltinSourceManager()
-                options.sourceManager = m_builtinLinkage->getSourceManager();
-
-                StringBuilder builder;
-                builder << moduleName->text << ".slang-module";
-
-                FileStream stream(builder.ProduceString(), FileMode::Create, FileAccess::Write, FileShare::ReadWrite);
-                if (SLANG_FAILED(SerialContainerUtil::write(module, options, &stream)))
-                {
-                    SLANG_UNEXPECTED("Unable to load stdlib");
-                }
-            }
-        }
-    }
-
     {
         for (Index i = 0; i < Index(SourceLanguage::CountOf); ++i)
         {
@@ -228,6 +189,66 @@ void Session::init()
     m_languagePreludes[Index(SourceLanguage::HLSL)] = get_slang_hlsl_prelude();
 }
 
+SlangResult Session::compileStdLib()
+{
+    if (m_builtinLinkage->mapNameToLoadedModules.Count())
+    {
+        // Already have a StdLib loaded
+        return SLANG_FAIL;
+    }
+
+    // TODO(JS): Could make this return a SlangResult as opposed to exception
+    addBuiltinSource(coreLanguageScope, "core", getCoreLibraryCode());
+    addBuiltinSource(hlslLanguageScope, "hlsl", getHLSLLibraryCode());
+    return SLANG_OK;
+}
+
+SlangResult Session::loadStdLib()
+{
+    if (m_builtinLinkage->mapNameToLoadedModules.Count())
+    {
+        // Already have a StdLib loaded
+        return SLANG_FAIL;
+    }
+
+    // Let's try loading serialized modules and adding them
+    SLANG_RETURN_ON_FAIL(_readBuiltinModule(coreLanguageScope, "core"));
+    SLANG_RETURN_ON_FAIL(_readBuiltinModule(hlslLanguageScope, "hlsl"));
+    return SLANG_OK;
+}
+
+SlangResult Session::saveStdLib()
+{
+    if (m_builtinLinkage->mapNameToLoadedModules.Count() == 0)
+    {
+        // There is no standard lib loaded
+        return SLANG_FAIL;
+    }
+
+    for (auto& pair : m_builtinLinkage->mapNameToLoadedModules)
+    {
+        const Name* moduleName = pair.Key;
+        Module* module = pair.Value;
+
+        // Set up options
+        SerialContainerUtil::WriteOptions options;
+
+        // Save with SourceLocation information
+        options.optionFlags |= SerialOptionFlag::SourceLocation;
+
+        // TODO(JS): Should this be the Session::getBuiltinSourceManager()?
+        options.sourceManager = m_builtinLinkage->getSourceManager();
+
+        StringBuilder builder;
+        builder << moduleName->text << ".slang-module";
+
+        FileStream stream(builder.ProduceString(), FileMode::Create, FileAccess::Write, FileShare::ReadWrite);
+        SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(module, options, &stream));
+    }
+
+    return SLANG_OK;
+}
+
 SlangResult Session::_readBuiltinModule(Scope* scope, String moduleName)
 {
     StringBuilder moduleFilename;
@@ -3163,22 +3184,41 @@ Session::~Session()
 
 SLANG_API SlangSession* spCreateSession(const char*)
 {
-    Slang::RefPtr<Slang::Session> session(new Slang::Session());
-    session->init();
+    Slang::ComPtr<slang::IGlobalSession> globalSession;
+    if (SLANG_FAILED(slang_createGlobalSession(SLANG_API_VERSION, globalSession.writeRef())))
+    {
+        return nullptr;
+    }
     // Will be returned with a refcount of 1
-    return asExternal(session.detach());
+    return globalSession.detach();
 }
 
 SLANG_API SlangResult slang_createGlobalSession(
     SlangInt                apiVersion,
     slang::IGlobalSession** outGlobalSession)
 {
-    if(apiVersion != 0)
+    Slang::ComPtr<slang::IGlobalSession> globalSession;
+    SLANG_RETURN_ON_FAIL(slang_createGlobalSessionWithoutStdLib(apiVersion, globalSession.writeRef()));
+    SLANG_RETURN_ON_FAIL(globalSession->compileStdLib());
+    *outGlobalSession = globalSession.detach();
+    return SLANG_OK;
+}
+
+SLANG_API SlangResult slang_createGlobalSessionWithoutStdLib(
+    SlangInt                apiVersion,
+    slang::IGlobalSession** outGlobalSession)
+{
+    if (apiVersion != 0)
         return SLANG_E_NOT_IMPLEMENTED;
 
-    Slang::RefPtr<Slang::Session> globalSession(new Slang::Session());
+    // Create the session
+    Slang::Session* globalSession = new Slang::Session();
+    // Put an interface ref on it
+    Slang::ComPtr<slang::IGlobalSession> result(globalSession);
+
+    // Initialize it
     globalSession->init();
-    Slang::ComPtr<slang::IGlobalSession> result(Slang::asExternal(globalSession));
+
     *outGlobalSession = result.detach();
     return SLANG_OK;
 }
diff --git a/source/slangc/main.cpp b/source/slangc/main.cpp
index 9f4ffcaff..887eca77a 100644
--- a/source/slangc/main.cpp
+++ b/source/slangc/main.cpp
@@ -66,10 +66,28 @@ static SlangResult _compile(SlangCompileRequest* compileRequest, int argc, const
     return res;
 }
 
-SLANG_TEST_TOOL_API SlangResult innerMain(StdWriters* stdWriters, SlangSession* session, int argc, const char*const* argv)
+SLANG_TEST_TOOL_API SlangResult innerMain(StdWriters* stdWriters, slang::IGlobalSession* sharedSession, int argc, const char*const* argv)
 {
     StdWriters::setSingleton(stdWriters);
 
+    // Assume we will used the shared session
+    ComPtr<slang::IGlobalSession> session(sharedSession);
+
+    // The sharedSession always has a pre-loaded stdlib, is sharedSession is not nullptr.
+    // This differed test checks if the command line has an option to setup the stdlib.
+    // If so we *don't* use the sharedSession, and create a new stdlib-less session just for this compilation. 
+    if (TestToolUtil::hasDeferredStdLib(Index(argc - 1), argv + 1))
+    {
+        SLANG_RETURN_ON_FAIL(slang_createGlobalSessionWithoutStdLib(SLANG_API_VERSION, session.writeRef()));
+        TestToolUtil::setSessionDefaultPreludeFromExePath(argv[0], session);
+    }
+    else if (!session)
+    {
+        // Just create the global session in the regular way if there isn't one set
+        SLANG_RETURN_ON_FAIL(slang_createGlobalSession(SLANG_API_VERSION, session.writeRef()));
+        TestToolUtil::setSessionDefaultPreludeFromExePath(argv[0], session);
+    }
+
     SlangCompileRequest* compileRequest = spCreateCompileRequest(session);
     SlangResult res = _compile(compileRequest, argc, argv);
     // Now that we are done, clean up after ourselves
@@ -80,16 +98,8 @@ SLANG_TEST_TOOL_API SlangResult innerMain(StdWriters* stdWriters, SlangSession*
 
 int MAIN(int argc, char** argv)
 {
-    SlangResult res;
-    {
-        SlangSession* session = spCreateSession(nullptr);
-        TestToolUtil::setSessionDefaultPreludeFromExePath(argv[0], session);
-
-        auto stdWriters = StdWriters::initDefaultSingleton();
-        
-        res = innerMain(stdWriters, session, argc, argv);
-        spDestroySession(session);
-    }
+    auto stdWriters = StdWriters::initDefaultSingleton();
+    SlangResult res = innerMain(stdWriters, nullptr, argc, argv);
     return (int)TestToolUtil::getReturnCode(res);
 }
 
diff --git a/tests/serialization/std-lib-serialize.slang b/tests/serialization/std-lib-serialize.slang
new file mode 100644
index 000000000..e55a2cd69
--- /dev/null
+++ b/tests/serialization/std-lib-serialize.slang
@@ -0,0 +1,32 @@
+//TEST:COMPILE: -save-stdlib
+//TEST:COMPARE_COMPUTE: -compile-arg -load-stdlib 
+
+struct A
+{
+	float x;
+
+	float addWith(float y)
+	{
+		return this.x + y;
+	}
+};
+
+//TEST_INPUT:ubuffer(data=[0 0 0 0], stride=4):out,name=outputBuffer
+RWStructuredBuffer<float> outputBuffer : register(u0);
+
+
+float test(float inVal)
+{
+	A a;
+	a.x = inVal;
+	return a.addWith(inVal*inVal);
+}
+
+[numthreads(4, 1, 1)]
+void computeMain(uint3 dispatchThreadID : SV_DispatchThreadID)
+{
+	uint tid = dispatchThreadID.x;
+	float inVal = float(tid);
+	float outVal = test(inVal);
+	outputBuffer[tid] = outVal;
+}
\ No newline at end of file
diff --git a/tests/serialization/std-lib-serialize.slang.expected.txt b/tests/serialization/std-lib-serialize.slang.expected.txt
new file mode 100644
index 000000000..f73cfe6c3
--- /dev/null
+++ b/tests/serialization/std-lib-serialize.slang.expected.txt
@@ -0,0 +1,4 @@
+0
+40000000
+40C00000
+41400000
diff --git a/tools/render-test/render-test-main.cpp b/tools/render-test/render-test-main.cpp
index 85e038e5a..f6f3e4ce6 100644
--- a/tools/render-test/render-test-main.cpp
+++ b/tools/render-test/render-test-main.cpp
@@ -787,14 +787,25 @@ static SlangResult _innerMain(Slang::StdWriters* stdWriters, SlangSession* sessi
 	}
 }
 
-SLANG_TEST_TOOL_API SlangResult innerMain(Slang::StdWriters* stdWriters, SlangSession* session, int argcIn, const char*const* argvIn)
+SLANG_TEST_TOOL_API SlangResult innerMain(Slang::StdWriters* stdWriters, SlangSession* sharedSession, int inArgc, const char*const* inArgv)
 {
     using namespace Slang;
 
+    // Assume we will used the shared session
+    ComPtr<slang::IGlobalSession> session(sharedSession);
+
+    // The sharedSession always has a pre-loaded stdlib.
+    // This differed test checks if the command line has an option to setup the stdlib.
+    // If so we *don't* use the sharedSession, and create a new stdlib-less session just for this compilation. 
+    if (TestToolUtil::hasDeferredStdLib(Index(inArgc - 1), inArgv + 1))
+    {
+        SLANG_RETURN_ON_FAIL(slang_createGlobalSessionWithoutStdLib(SLANG_API_VERSION, session.writeRef()));
+    }
+
     SlangResult res = SLANG_FAIL;
     try
     {
-        res = _innerMain(stdWriters, session, argcIn, argvIn);
+        res = _innerMain(stdWriters, session, inArgc, inArgv);
     }
     catch (const Slang::Exception& exception)
     {
diff --git a/tools/slang-test/slangc-tool.cpp b/tools/slang-test/slangc-tool.cpp
index d7043001a..4757f2fff 100644
--- a/tools/slang-test/slangc-tool.cpp
+++ b/tools/slang-test/slangc-tool.cpp
@@ -2,6 +2,7 @@
 #include "slangc-tool.h"
 
 #include "../../source/core/slang-exception.h"
+#include "../../source/core/slang-test-tool-util.h"
 
 using namespace Slang;
 
@@ -16,6 +17,9 @@ static void _diagnosticCallback(char const* message, void* /*userData*/)
 
 static SlangResult _compile(SlangCompileRequest* compileRequest, int argc, const char*const* argv)
 {
+    spSetDiagnosticCallback(compileRequest, &_diagnosticCallback, nullptr);
+    spSetCommandLineCompilerMode(compileRequest);
+
     {
         const SlangResult res = spProcessCommandLineArguments(compileRequest, &argv[1], argc - 1);
         if (SLANG_FAILED(res))
@@ -48,22 +52,32 @@ static SlangResult _compile(SlangCompileRequest* compileRequest, int argc, const
     return res;
 }
 
-SlangResult SlangCTool::innerMain(StdWriters* stdWriters, SlangSession* session, int argc, const char*const* argv)
+SlangResult SlangCTool::innerMain(StdWriters* stdWriters, slang::IGlobalSession* sharedSession, int argc, const char*const* argv)
 {
+    StdWriters::setSingleton(stdWriters);
+
+    // Assume we will used the shared session
+    ComPtr<slang::IGlobalSession> session(sharedSession);
+
+    // The sharedSession always has a pre-loaded stdlib.
+    // This differed test checks if the command line has an option to setup the stdlib.
+    // If so we *don't* use the sharedSession, and create a new stdlib-less session just for this compilation. 
+    if (TestToolUtil::hasDeferredStdLib(Index(argc - 1), argv + 1))
+    {
+        SLANG_RETURN_ON_FAIL(slang_createGlobalSessionWithoutStdLib(SLANG_API_VERSION, session.writeRef()));
+    }
+
     SlangCompileRequest* compileRequest = spCreateCompileRequest(session);
-    spSetDiagnosticCallback(compileRequest, &_diagnosticCallback, nullptr);
 
-    spSetCommandLineCompilerMode(compileRequest);
     // Do any app specific configuration
     for (int i = 0; i < SLANG_WRITER_CHANNEL_COUNT_OF; ++i)
     {
         spSetWriter(compileRequest, SlangWriterChannel(i), stdWriters->getWriter(i));
     }
-    
-    SlangResult res = _compile(compileRequest, argc, argv);
 
+    SlangResult res = _compile(compileRequest, argc, argv);
     // Now that we are done, clean up after ourselves
     spDestroyCompileRequest(compileRequest);
+
     return res;
 }
-