Feature/capture (#4397)

* Add the function tailer for appending the output

- The basic format for the capture encode is as follow:

Header:
4 bytes: magic number ('H' 'E' 'A' 'D' )
4 bytes: call id - specify the method name
8 bytes: handle id - specify 'this' pointer
8 bytes: payload size in bytes - specify the data size of parameters
8 bytes: thread id

Payload:
Encode for all the parameters.

Tailer (optional):
Tailer is an optional, it only used when the output of the method is
also stored in the method. Usually it just the opaque handle allocated
by slang.

4 bytes: magic number ('T' 'A' 'I' 'L')
4 bytes: payload size in bytes.

- Fix some issues in checking the result of write of output stream.

* Encoding methods of IGlobalSession

Add encoding logic for all the member functions of IGlobalSession,
except those query functions that do not impact the internal state
of slang.

Because some get functions will invoke allocations by slang, these
functions are account for the "query functions". Therefore those
functions are still captured.

All the allocations are stored by using their address, because those
allocations are opaque and will finally be used as inputs for other
APIs, there is no need to store the data. We just need to track
those address and know which APIs will consume them.

* Add SLANG_CAPTURE_CHECK macro

Add SLANG_CAPTURE_CHECK macro to check SLANG_OK is returned.

* Fix build error

1 files changed, 207 insertions, 2 deletions

diff --git a/source/slang-capture-replay/slang-global-session.cpp b/source/slang-capture-replay/slang-global-session.cpp
index 45ea0b58d..b90d7611d 100644
--- a/source/slang-capture-replay/slang-global-session.cpp
+++ b/source/slang-capture-replay/slang-global-session.cpp
@@ -12,8 +12,8 @@ namespace SlangCapture
     {
         SLANG_CAPTURE_ASSERT(m_actualGlobalSession != nullptr);
 
-        m_thisHandle = reinterpret_cast<SlangCapture::AddressFormat>(this);
-        m_captureManager = std::make_unique<CaptureManager>(m_thisHandle);
+        m_globalSessionHandle = reinterpret_cast<SlangCapture::AddressFormat>(m_actualGlobalSession.get());
+        m_captureManager = std::make_unique<CaptureManager>(m_globalSessionHandle);
 
         // We will use the address of the global session as the filename for the capture manager
         // to make it unique for each global session.
@@ -41,8 +41,22 @@ namespace SlangCapture
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
 
         slang::ISession* actualSession = nullptr;
+
+        ParameterEncoder* encoder{};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_createSession, m_globalSessionHandle);
+            encoder->encodeStruct(desc);
+            encoder->encodeAddress(nullptr);
+            encoder = m_captureManager->endMethodCapture();
+        }
+
         SlangResult res = m_actualGlobalSession->createSession(desc, &actualSession);
 
+        {   // capture output
+            encoder->encodeAddress(actualSession);
+            m_captureManager->endMethodCaptureAppendOutput();
+        }
+
         if (actualSession != nullptr)
         {
             // reset the file system to our capture file system. After createSession() call,
@@ -65,6 +79,14 @@ namespace SlangCapture
     SLANG_NO_THROW SlangProfileID SLANG_MCALL GlobalSessionCapture::findProfile(char const* name)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_findProfile, m_globalSessionHandle);
+            encoder->encodeString(name);
+            encoder = m_captureManager->endMethodCapture();
+        }
+
         SlangProfileID profileId = m_actualGlobalSession->findProfile(name);
         return profileId;
     }
@@ -72,24 +94,58 @@ namespace SlangCapture
     SLANG_NO_THROW void SLANG_MCALL GlobalSessionCapture::setDownstreamCompilerPath(SlangPassThrough passThrough, char const* path)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_setDownstreamCompilerPath, m_globalSessionHandle);
+            encoder->encodeEnumValue(passThrough);
+            encoder->encodeString(path);
+            m_captureManager->endMethodCapture();
+        }
+
         m_actualGlobalSession->setDownstreamCompilerPath(passThrough, path);
     }
 
     SLANG_NO_THROW void SLANG_MCALL GlobalSessionCapture::setDownstreamCompilerPrelude(SlangPassThrough inPassThrough, char const* prelude)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_setDownstreamCompilerPrelude, m_globalSessionHandle);
+            encoder->encodeEnumValue(inPassThrough);
+            encoder->encodeString(prelude);
+            m_captureManager->endMethodCapture();
+        }
+
         m_actualGlobalSession->setDownstreamCompilerPrelude(inPassThrough, prelude);
     }
 
     SLANG_NO_THROW void SLANG_MCALL GlobalSessionCapture::getDownstreamCompilerPrelude(SlangPassThrough inPassThrough, ISlangBlob** outPrelude)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_getDownstreamCompilerPrelude, m_globalSessionHandle);
+            encoder->encodeEnumValue(inPassThrough);
+            encoder->encodeAddress(nullptr);
+            encoder = m_captureManager->endMethodCapture();
+        }
+
         m_actualGlobalSession->getDownstreamCompilerPrelude(inPassThrough, outPrelude);
+
+        {
+            encoder->encodeAddress(*outPrelude);
+            m_captureManager->endMethodCaptureAppendOutput();
+        }
     }
 
     SLANG_NO_THROW const char* SLANG_MCALL GlobalSessionCapture::getBuildTagString()
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        // No need to capture this function. It's just a query function and it won't impact the internal state.
         const char* resStr = m_actualGlobalSession->getBuildTagString();
         return resStr;
     }
@@ -97,6 +153,15 @@ namespace SlangCapture
     SLANG_NO_THROW SlangResult SLANG_MCALL GlobalSessionCapture::setDefaultDownstreamCompiler(SlangSourceLanguage sourceLanguage, SlangPassThrough defaultCompiler)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_setDefaultDownstreamCompiler, m_globalSessionHandle);
+            encoder->encodeEnumValue(sourceLanguage);
+            encoder->encodeEnumValue(defaultCompiler);
+            encoder = m_captureManager->endMethodCapture();
+        }
+
         SlangResult res = m_actualGlobalSession->setDefaultDownstreamCompiler(sourceLanguage, defaultCompiler);
         return res;
     }
@@ -104,6 +169,14 @@ namespace SlangCapture
     SLANG_NO_THROW SlangPassThrough SLANG_MCALL GlobalSessionCapture::getDefaultDownstreamCompiler(SlangSourceLanguage sourceLanguage)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_getDefaultDownstreamCompiler, m_globalSessionHandle);
+            encoder->encodeEnumValue(sourceLanguage);
+            encoder = m_captureManager->endMethodCapture();
+        }
+
         SlangPassThrough passThrough = m_actualGlobalSession->getDefaultDownstreamCompiler(sourceLanguage);
         return passThrough;
     }
@@ -111,43 +184,103 @@ namespace SlangCapture
     SLANG_NO_THROW void SLANG_MCALL GlobalSessionCapture::setLanguagePrelude(SlangSourceLanguage inSourceLanguage, char const* prelude)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_setLanguagePrelude, m_globalSessionHandle);
+            encoder->encodeEnumValue(inSourceLanguage);
+            encoder->encodeString(prelude);
+            encoder = m_captureManager->endMethodCapture();
+        }
+
         m_actualGlobalSession->setLanguagePrelude(inSourceLanguage, prelude);
     }
 
     SLANG_NO_THROW void SLANG_MCALL GlobalSessionCapture::getLanguagePrelude(SlangSourceLanguage inSourceLanguage, ISlangBlob** outPrelude)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_getLanguagePrelude, m_globalSessionHandle);
+            encoder->encodeEnumValue(inSourceLanguage);
+            encoder->encodeAddress(nullptr);
+            encoder = m_captureManager->endMethodCapture();
+        }
+
         m_actualGlobalSession->getLanguagePrelude(inSourceLanguage, outPrelude);
+
+        {
+            encoder->encodeAddress(*outPrelude);
+            m_captureManager->endMethodCaptureAppendOutput();
+        }
     }
 
     SLANG_NO_THROW SlangResult SLANG_MCALL GlobalSessionCapture::createCompileRequest(slang::ICompileRequest** outCompileRequest)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_createCompileRequest, m_globalSessionHandle);
+            encoder->encodeAddress(nullptr);
+            encoder = m_captureManager->endMethodCapture();
+        }
+
         SlangResult res = m_actualGlobalSession->createCompileRequest(outCompileRequest);
+
+        {
+            encoder->encodeAddress(*outCompileRequest);
+            m_captureManager->endMethodCaptureAppendOutput();
+        }
+
         return res;
     }
 
     SLANG_NO_THROW void SLANG_MCALL GlobalSessionCapture::addBuiltins(char const* sourcePath, char const* sourceString)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_addBuiltins, m_globalSessionHandle);
+            encoder->encodeString(sourcePath);
+            encoder->encodeString(sourceString);
+            encoder = m_captureManager->endMethodCapture();
+        }
+
         m_actualGlobalSession->addBuiltins(sourcePath, sourceString);
     }
 
     SLANG_NO_THROW void SLANG_MCALL GlobalSessionCapture::setSharedLibraryLoader(ISlangSharedLibraryLoader* loader)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+        // TODO: Not sure if we need to capture this function. Because this functions is something like the file system
+        // override, it's provided by user code. So capturing it makes no sense. The only way is to wrapper this interface
+        // by our own implementation, and capture it there.
         m_actualGlobalSession->setSharedLibraryLoader(loader);
     }
 
     SLANG_NO_THROW ISlangSharedLibraryLoader* SLANG_MCALL GlobalSessionCapture::getSharedLibraryLoader()
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_getSharedLibraryLoader, m_globalSessionHandle);
+            encoder->encodeAddress(nullptr);
+        }
+
         ISlangSharedLibraryLoader* loader = m_actualGlobalSession->getSharedLibraryLoader();
+
+        {
+            encoder->encodeAddress(loader);
+        }
         return loader;
     }
 
     SLANG_NO_THROW SlangResult SLANG_MCALL GlobalSessionCapture::checkCompileTargetSupport(SlangCompileTarget target)
     {
+        // No need to capture this function. It's just a query function and it won't impact the internal state.
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
         SlangResult res = m_actualGlobalSession->checkCompileTargetSupport(target);
         return res;
@@ -155,6 +288,7 @@ namespace SlangCapture
 
     SLANG_NO_THROW SlangResult SLANG_MCALL GlobalSessionCapture::checkPassThroughSupport(SlangPassThrough passThrough)
     {
+        // No need to capture this function. It's just a query function and it won't impact the internal state.
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
         SlangResult res = m_actualGlobalSession->checkPassThroughSupport(passThrough);
         return res;
@@ -163,6 +297,13 @@ namespace SlangCapture
     SLANG_NO_THROW SlangResult SLANG_MCALL GlobalSessionCapture::compileStdLib(slang::CompileStdLibFlags flags)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_compileStdLib, m_globalSessionHandle);
+            encoder->encodeEnumValue(flags);
+        }
+
         SlangResult res = m_actualGlobalSession->compileStdLib(flags);
         return res;
     }
@@ -170,6 +311,14 @@ namespace SlangCapture
     SLANG_NO_THROW SlangResult SLANG_MCALL GlobalSessionCapture::loadStdLib(const void* stdLib, size_t stdLibSizeInBytes)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_loadStdLib, m_globalSessionHandle);
+            encoder->encodePointer(stdLib, false, stdLibSizeInBytes);
+            m_captureManager->endMethodCapture();
+        }
+
         SlangResult res = m_actualGlobalSession->loadStdLib(stdLib, stdLibSizeInBytes);
         return res;
     }
@@ -177,12 +326,22 @@ namespace SlangCapture
     SLANG_NO_THROW SlangResult SLANG_MCALL GlobalSessionCapture::saveStdLib(SlangArchiveType archiveType, ISlangBlob** outBlob)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_saveStdLib, m_globalSessionHandle);
+            encoder->encodeEnumValue(archiveType);
+            encoder->encodeAddress(outBlob);
+            m_captureManager->endMethodCapture();
+        }
+
         SlangResult res = m_actualGlobalSession->saveStdLib(archiveType, outBlob);
         return res;
     }
 
     SLANG_NO_THROW SlangCapabilityID SLANG_MCALL GlobalSessionCapture::findCapability(char const* name)
     {
+        // No need to capture this function. It's just a query function and it won't impact the internal state.
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
         SlangCapabilityID capId = m_actualGlobalSession->findCapability(name);
         return capId;
@@ -191,11 +350,22 @@ namespace SlangCapture
     SLANG_NO_THROW void SLANG_MCALL GlobalSessionCapture::setDownstreamCompilerForTransition(SlangCompileTarget source, SlangCompileTarget target, SlangPassThrough compiler)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_setDownstreamCompilerForTransition, m_globalSessionHandle);
+            encoder->encodeEnumValue(source);
+            encoder->encodeEnumValue(target);
+            encoder->encodeEnumValue(compiler);
+            m_captureManager->endMethodCapture();
+        }
+
         m_actualGlobalSession->setDownstreamCompilerForTransition(source, target, compiler);
     }
 
     SLANG_NO_THROW SlangPassThrough SLANG_MCALL GlobalSessionCapture::getDownstreamCompilerForTransition(SlangCompileTarget source, SlangCompileTarget target)
     {
+        // No need to capture this function. It's just a query function and it won't impact the internal state.
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
         SlangPassThrough passThrough = m_actualGlobalSession->getDownstreamCompilerForTransition(source, target);
         return passThrough;
@@ -203,6 +373,7 @@ namespace SlangCapture
 
     SLANG_NO_THROW void SLANG_MCALL GlobalSessionCapture::getCompilerElapsedTime(double* outTotalTime, double* outDownstreamTime)
     {
+        // No need to capture this function. It's just a query function and it won't impact the internal state.
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
         m_actualGlobalSession->getCompilerElapsedTime(outTotalTime, outDownstreamTime);
     }
@@ -210,6 +381,14 @@ namespace SlangCapture
     SLANG_NO_THROW SlangResult SLANG_MCALL GlobalSessionCapture::setSPIRVCoreGrammar(char const* jsonPath)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_setSPIRVCoreGrammar, m_globalSessionHandle);
+            encoder->encodeString(jsonPath);
+            m_captureManager->endMethodCapture();
+        }
+
         SlangResult res = m_actualGlobalSession->setSPIRVCoreGrammar(jsonPath);
         return res;
     }
@@ -218,14 +397,40 @@ namespace SlangCapture
         int argc, const char* const* argv, slang::SessionDesc* outSessionDesc, ISlangUnknown** outAllocation)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_parseCommandLineArguments, m_globalSessionHandle);
+            encoder->encodeStringArray(argv, argc);
+            encoder = m_captureManager->endMethodCapture();
+        }
+
         SlangResult res = m_actualGlobalSession->parseCommandLineArguments(argc, argv, outSessionDesc, outAllocation);
+
+        {
+            encoder->encodeStruct(*outSessionDesc);
+            encoder->encodeAddress(*outAllocation);
+        }
         return res;
     }
 
     SLANG_NO_THROW SlangResult SLANG_MCALL GlobalSessionCapture::getSessionDescDigest(slang::SessionDesc* sessionDesc, ISlangBlob** outBlob)
     {
         slangCaptureLog(LogLevel::Verbose, "%p: %s\n", m_actualGlobalSession.get(), __PRETTY_FUNCTION__);
+
+        ParameterEncoder* encoder {};
+        {
+            encoder = m_captureManager->beginMethodCapture(ApiCallId::IGlobalSession_getSessionDescDigest, m_globalSessionHandle);
+            encoder->encodeStruct(*sessionDesc);
+            encoder = m_captureManager->endMethodCapture();
+        }
+
         SlangResult res = m_actualGlobalSession->getSessionDescDigest(sessionDesc, outBlob);
+
+        {
+            encoder->encodeAddress(*outBlob);
+            m_captureManager->endMethodCaptureAppendOutput();
+        }
         return res;
     }
 }