Feature/record unit test (#4910)

* Fix the slang-test bug

Since we reorganize the build directory, now the libraries are
located at different directory with executables in non-Windows
platform, we have to change the code on how to find the dll directory.

* Integrate the record/replay test into slang-unit-test
We create a unit-test-record-replay.cpp to run the converted slang
examples in child process as our tests for the record-replay layer.

* Disable the test on Apple
Due to the limitation of current examples, we temporarily disable them
on apples.

Change the ci to make this test only be run on the gpu-equipped runners,
for other runners we add a white-list file
"expected-failure-record-replay-tests.txt".

* Remove 'hello-world' example from unit test
"hello-world" doesn't use gfx abstract library, instead it uses vk directly, it's
not a preferable way. So we will drop this test, instead, we will use cpu-hello-world
example.

9 files changed, 483 insertions, 2 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b60556568..b80f41423 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -54,6 +54,8 @@ jobs:
           - { config: release, test-category: full }
           # default not full gpu tests
           - full-gpu-tests: false
+          # The runners don't have a GPU by default except for the self-hosted ones
+          - has-gpu: false
           # Self-hosted aarch64 build
           - os: linux
             config: release
@@ -62,6 +64,7 @@ jobs:
             test-category: smoke
             full-gpu-tests: false
             runs-on: [self-hosted, Linux, ARM64]
+            has-gpu: true
           # Self-hosted full gpu build
           - os: windows
             config: release
@@ -70,6 +73,7 @@ jobs:
             test-category: full
             full-gpu-tests: true
             runs-on: [Windows, self-hosted]
+            has-gpu: true
       fail-fast: false
     runs-on: ${{ matrix.runs-on }}
 
@@ -121,12 +125,19 @@ jobs:
               -server-count 8 \
               -category ${{ matrix.test-category }} \
               -api all-cpu
-          else
+          elif [[ "${{matrix.has-gpu}}" == "true" ]]; then
             "$bin_dir/slang-test" \
               -use-test-server \
               -category ${{ matrix.test-category }} \
               -api all-dx12 \
               -expected-failure-list tests/expected-failure-github.txt
+          else
+            "$bin_dir/slang-test" \
+              -use-test-server \
+              -category ${{ matrix.test-category }} \
+              -api all-dx12 \
+              -expected-failure-list tests/expected-failure-github.txt \
+              -expected-failure-list tests/expected-failure-record-replay-tests.txt
           fi
       - name: Test Slang via glsl
         if: ${{matrix.full-gpu-tests}}
diff --git a/examples/hello-world/main.cpp b/examples/hello-world/main.cpp
index 87d440901..526ab1a2a 100644
--- a/examples/hello-world/main.cpp
+++ b/examples/hello-world/main.cpp
@@ -70,6 +70,7 @@ struct HelloWorldExample : public TestBase
 
 int main(int argc, char* argv[])
 {
+    fprintf(stdout, "Hello, world! Entry Point\n");
     initDebugCallback();
     HelloWorldExample example;
     example.parseOption(argc, argv);
@@ -83,10 +84,19 @@ int main(int argc, char* argv[])
 int HelloWorldExample::run()
 {
     RETURN_ON_FAIL(initVulkanInstanceAndDevice());
+    fprintf(stdout, "initVulkanInstanceAndDevice done\n");
+
     RETURN_ON_FAIL(createComputePipelineFromShader());
+    fprintf(stdout, "createComputePipelineFromShader done\n");
+
     RETURN_ON_FAIL(createInOutBuffers());
+    fprintf(stdout, "createInOutBuffers done\n");
+
     RETURN_ON_FAIL(dispatchCompute());
+    fprintf(stdout, "dispatchCompute done\n");
+
     RETURN_ON_FAIL(printComputeResults());
+    fprintf(stdout, "printComputeResults done\n");
     return 0;
 }
 
diff --git a/source/core/slang-test-tool-util.cpp b/source/core/slang-test-tool-util.cpp
index dad06effe..9a0ebe4f1 100644
--- a/source/core/slang-test-tool-util.cpp
+++ b/source/core/slang-test-tool-util.cpp
@@ -107,6 +107,24 @@ static SlangResult _addCUDAPrelude(const String& rootPath, slang::IGlobalSession
     return SLANG_OK;
 }
 
+/* static */SlangResult TestToolUtil::getDllDirectoryPath(const char* exePath, String& outDllDirectoryPath)
+{
+    String canonicalPath;
+    SLANG_RETURN_ON_FAIL(Path::getCanonical(exePath, canonicalPath));
+
+    // Get the directory
+    String binPath = Path::getParentDirectory(canonicalPath);
+
+    // Windows puts the dlls in the same directory as the exe, while on other platforms they are in a 'lib' directory
+#ifdef _WIN32
+    outDllDirectoryPath = binPath;
+#else
+    String binaryRootPath = Path::getParentDirectory(binPath);
+    outDllDirectoryPath = Path::combine(binaryRootPath, "lib");
+#endif
+    return SLANG_OK;
+}
+
 /* static */SlangResult TestToolUtil::getRootPath(const char* inExePath, String& outExePath)
 {
     // Get the directory holding the exe
diff --git a/source/core/slang-test-tool-util.h b/source/core/slang-test-tool-util.h
index 1e56500d2..03f3b768e 100644
--- a/source/core/slang-test-tool-util.h
+++ b/source/core/slang-test-tool-util.h
@@ -66,6 +66,8 @@ struct TestToolUtil
 
         /// Returns true if the StdLib should not be initialized immediately (eg when doing a -load-stdlib).
     static bool hasDeferredStdLib(Index numArgs, const char*const* args);
+
+    static SlangResult getDllDirectoryPath(const char* exePath, String& outDllDirectoryPath);
 };
 
 } // namespace Slang
diff --git a/tests/expected-failure-record-replay-tests.txt b/tests/expected-failure-record-replay-tests.txt
new file mode 100644
index 000000000..9b2f264f5
--- /dev/null
+++ b/tests/expected-failure-record-replay-tests.txt
@@ -0,0 +1 @@
+slang-unit-test-tool/RecordReplay.internal
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index 06eae7047..7616ab614 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -4256,7 +4256,7 @@ static SlangResult runUnitTestModule(TestContext* context, TestOptions& testOpti
     ComPtr<ISlangSharedLibrary> moduleLibrary;
 
     SLANG_RETURN_ON_FAIL(loader->loadSharedLibrary(
-        Path::combine(context->exeDirectoryPath, moduleName).getBuffer(),
+        Path::combine(context->dllDirectoryPath, moduleName).getBuffer(),
         moduleLibrary.writeRef()));
 
     UnitTestGetModuleFunc getModuleFunc =
diff --git a/tools/slang-test/test-context.cpp b/tools/slang-test/test-context.cpp
index b1be005cb..ed25831e3 100644
--- a/tools/slang-test/test-context.cpp
+++ b/tools/slang-test/test-context.cpp
@@ -88,6 +88,7 @@ Result TestContext::init(const char* inExePath)
     }
     exePath = inExePath;
     SLANG_RETURN_ON_FAIL(TestToolUtil::getExeDirectoryPath(inExePath, exeDirectoryPath));
+    SLANG_RETURN_ON_FAIL(TestToolUtil::getDllDirectoryPath(inExePath, dllDirectoryPath));
 
     SLANG_RETURN_ON_FAIL(locateFileCheck());
 
diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h
index b42ac1ec9..28d39b064 100644
--- a/tools/slang-test/test-context.h
+++ b/tools/slang-test/test-context.h
@@ -148,6 +148,7 @@ class TestContext
     Slang::RefPtr<Slang::DownstreamCompilerSet> compilerSet;
 
     Slang::String exeDirectoryPath;
+    Slang::String dllDirectoryPath;
     Slang::String exePath;
 
         /// Timeout time for communication over connection.
diff --git a/tools/slang-unit-test/unit-test-record-replay.cpp b/tools/slang-unit-test/unit-test-record-replay.cpp
new file mode 100644
index 000000000..2d5619f71
--- /dev/null
+++ b/tools/slang-unit-test/unit-test-record-replay.cpp
@@ -0,0 +1,437 @@
+// unit-test-record-replay.cpp
+
+#include "../../source/core/slang-string-util.h"
+#include "../../source/core/slang-process-util.h"
+
+#include "../../source/core/slang-io.h"
+#include "../../source/core/slang-http.h"
+#include "../../source/core/slang-random-generator.h"
+
+#include "tools/unit-test/slang-unit-test.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#include <shellapi.h>
+#else
+#include <ftw.h>
+#endif
+
+#include <chrono>
+#include <thread>
+
+using namespace Slang;
+
+static SlangResult createProcess(UnitTestContext* context, const char* processName, const List<String>* optArgs, RefPtr<Process>& outProcess)
+{
+    CommandLine cmdLine;
+    cmdLine.setExecutableLocation(ExecutableLocation(context->executableDirectory, processName));
+    if (optArgs)
+    {
+        cmdLine.m_args.addRange(optArgs->getBuffer(), optArgs->getCount());
+    }
+
+    SLANG_RETURN_ON_FAIL(Process::create(cmdLine, Process::Flag::AttachDebugger, outProcess));
+
+    return SLANG_OK;
+}
+
+struct entryHashInfo
+{
+    int64_t targetIndex = -1;
+    int64_t entryPointIndex = -1;
+    String hash;
+};
+
+static SlangResult parseHashes(List<String> const& lines, List<entryHashInfo>& outHashes)
+{
+    SlangResult res = SLANG_OK;
+
+    for (const auto& line : lines)
+    {
+        List<UnownedStringSlice> tokens;
+        Index skipCharacters = line.indexOf(UnownedStringSlice("[slang-record-replay]:"));
+        if (skipCharacters == -1)
+        {
+            skipCharacters = 0;
+        }
+        else
+        {
+            skipCharacters += strlen("[slang-record-replay]:");
+        }
+        StringUtil::split(UnownedStringSlice(line.getBuffer() + skipCharacters), ',', tokens);
+
+        if (tokens.getCount() != 3)
+        {
+            return SLANG_FAIL;
+        }
+
+        entryHashInfo hashInfo;
+        auto extractToken = [](const UnownedStringSlice& token, const char splitChar, UnownedStringSlice& outToken) -> SlangResult
+        {
+            List<UnownedStringSlice> subTokens;
+            StringUtil::split(token, splitChar, subTokens);
+            if (subTokens.getCount() != 2)
+            {
+                return SLANG_FAIL;
+            }
+            outToken = subTokens[1];
+            return SLANG_OK;
+        };
+
+        {
+            UnownedStringSlice subToken;
+            SLANG_RETURN_ON_FAIL(extractToken(tokens[0], ':', subToken));
+            int64_t outNumer = 0;
+            StringUtil::parseInt64(subToken, outNumer);
+            hashInfo.entryPointIndex = outNumer;
+        }
+
+        {
+            UnownedStringSlice subToken;
+            SLANG_RETURN_ON_FAIL(extractToken(tokens[1], ':', subToken));
+            int64_t outNumer = 0;
+            StringUtil::parseInt64(subToken, outNumer);
+            hashInfo.targetIndex = outNumer;
+        }
+
+        {
+            UnownedStringSlice subToken;
+            SLANG_RETURN_ON_FAIL(extractToken(tokens[2], ':', subToken));
+            hashInfo.hash = subToken.begin() + 1;
+        }
+
+        outHashes.add(hashInfo);
+    }
+    return res;
+}
+
+static int writeEnvironmentVariable(const char* key, const char* val)
+{
+#ifdef _WIN32
+    String var = String(key) + "=" + val;
+    return _putenv(var.getBuffer());
+#else
+    return setenv(key, val, 1);
+#endif
+}
+
+static bool enableRecordLayer()
+{
+    int retCode = writeEnvironmentVariable("SLANG_RECORD_LAYER", "1");
+    return retCode == 0;
+}
+
+static bool disableRecordLayer()
+{
+    int retCode = writeEnvironmentVariable("SLANG_RECORD_LAYER", "0");
+    return retCode == 0;
+}
+
+static bool enableLogInReplayer()
+{
+    int retCode = writeEnvironmentVariable("SLANG_RECORD_LOG_LEVEL", "3");
+    return retCode == 0;
+}
+
+static bool disableLogInReplayer()
+{
+    int retCode = writeEnvironmentVariable("SLANG_RECORD_LOG_LEVEL", "0");
+    return retCode == 0;
+}
+
+static void findRecordFileName(List<String>* fileNames)
+{
+    struct Visitor : Path::Visitor
+    {
+        void accept(Path::Type type, const UnownedStringSlice& filename) SLANG_OVERRIDE
+        {
+            if (type == Path::Type::File)
+            {
+                m_fileNames->add(filename);
+            }
+        }
+        Visitor(List<String>* fileNames) : m_fileNames(fileNames) {}
+        List<String>* m_fileNames;
+    };
+
+    Visitor visitor(fileNames);
+    Path::find("slang-record", "*.cap", &visitor);
+}
+
+static SlangResult launchProcessAndReadStdout(UnitTestContext* context, const List<String>& optArgs,
+        const char* exampleName, RefPtr<Process>& process, ExecuteResult& exeRes)
+{
+    StringBuilder msgBuilder;
+    SlangResult res = createProcess(context, exampleName, &optArgs, process);
+    if (SLANG_FAILED(res))
+    {
+        msgBuilder << "Failed to launch process of '"<< exampleName << "'\n";
+        getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+        return res;
+    }
+
+    res = ProcessUtil::readUntilTermination(process, exeRes);
+    if (SLANG_FAILED(res))
+    {
+        msgBuilder << "Failed to read stdout from '" << exampleName << "'\n";
+        msgBuilder << "process ret code: " << exeRes.resultCode;
+        getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+        return res;
+    }
+
+    if (exeRes.standardOutput.getLength() == 0)
+    {
+        msgBuilder << "No stdout found in '" << exampleName << "'\n";
+        msgBuilder << "Standard error: " << exeRes.standardError;
+        getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+        return SLANG_FAIL;
+    }
+    return SLANG_OK;
+}
+
+static SlangResult runExample(UnitTestContext* context, const char* exampleName, List<entryHashInfo>& outHashes)
+{
+    SlangResult finalRes = SLANG_OK;
+
+    RefPtr<Process> process;
+    ExecuteResult exeRes;
+    List<String> optArgs;
+    optArgs.add("--test-mode");
+
+    StringBuilder msgBuilder;
+    SlangResult res = SLANG_OK;
+
+    enableRecordLayer();
+    res = launchProcessAndReadStdout(context, optArgs, exampleName, process, exeRes);
+    disableRecordLayer();
+
+    if (SLANG_FAILED(res))
+    {
+        return res;
+    }
+
+    List<String> hashLines;
+    for (auto line : LineParser(exeRes.standardOutput.getUnownedSlice()))
+    {
+        if (line.getLength() == 0)
+        {
+            continue;
+        }
+
+        if (line.indexOf(UnownedStringSlice("hash:")) == -1)
+        {
+            continue;
+        }
+
+        hashLines.add(line);
+    }
+
+    res = parseHashes(hashLines, outHashes);
+    if (SLANG_FAILED(res))
+    {
+        msgBuilder << "Failed to parse hash from stdout of '" << exampleName << "'\n";
+        getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+        return res;
+    }
+
+    return SLANG_OK;
+}
+
+static SlangResult replayExample(UnitTestContext* context, List<entryHashInfo>& outHashes)
+{
+    List<String> fileNames;
+    findRecordFileName(&fileNames);
+
+    List<String> optArgs;
+    String recordFileName = Path::combine("slang-record", fileNames[0]);
+    optArgs.add(recordFileName.getBuffer());
+
+    RefPtr<Process> process;
+    ExecuteResult exeRes;
+
+    StringBuilder msgBuilder;
+    msgBuilder << "replay the test\n";
+
+    enableLogInReplayer();
+    SlangResult res = launchProcessAndReadStdout(context, optArgs, "slang-replay", process, exeRes);
+    disableLogInReplayer();
+
+    if (SLANG_FAILED(res))
+    {
+        return res;
+    }
+
+    List<String> hashLines;
+    for (auto line : LineParser(exeRes.standardOutput.getUnownedSlice()))
+    {
+        if (line.getLength() == 0)
+        {
+            continue;
+        }
+
+        if (line.indexOf(UnownedStringSlice("hash:")) == -1)
+        {
+            continue;
+        }
+
+        hashLines.add(line);
+    }
+
+    res = parseHashes(hashLines, outHashes);
+    if (SLANG_FAILED(res))
+    {
+        msgBuilder << "Failed to parse hash from stdout of 'slang-replay'\n";
+        getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+        return SLANG_FAIL;
+    }
+
+    return SLANG_OK;
+}
+
+static SlangResult resultCompare(List<entryHashInfo> const& expectHashes, List<entryHashInfo> const& resultHashes)
+{
+    if (expectHashes.getCount() != resultHashes.getCount())
+    {
+        return SLANG_FAIL;
+    }
+
+    StringBuilder msgBuilder;
+    for (Index i = 0; i < expectHashes.getCount(); i++)
+    {
+        if (expectHashes[i].targetIndex != resultHashes[i].targetIndex)
+        {
+            msgBuilder << "Failed to match 'targetIndex' at index " << i << "\n";
+            msgBuilder << "Expect: " << expectHashes[i].targetIndex << ", actual: " << resultHashes[i].targetIndex << "\n";
+            getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+            return SLANG_FAIL;
+        }
+        if (expectHashes[i].entryPointIndex != resultHashes[i].entryPointIndex)
+        {
+            msgBuilder << "Failed to match 'entryPointIndex' at index " << i << "\n";
+            msgBuilder << "Expect: " << expectHashes[i].entryPointIndex << ", actual: " << resultHashes[i].entryPointIndex << "\n";
+            getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+            return SLANG_FAIL;
+        }
+
+        if (expectHashes[i].hash != resultHashes[i].hash)
+        {
+            msgBuilder << "Failed to match 'hash' at index " << i << "\n";
+            msgBuilder << "Expect: " << expectHashes[i].hash << ", actual: " << resultHashes[i].hash << "\n";
+            getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+            return SLANG_FAIL;
+        }
+    }
+
+    return SLANG_OK;
+}
+
+static SlangResult cleanupRecordFiles()
+{
+    if (File::exists("slang-record") == false)
+    {
+        return SLANG_OK;
+    }
+
+    StringBuilder msgBuilder;
+    // Path::remove() doesn't support remove a non-empty directory, so we need to implement
+    // a simple function to remove the directory recursively.
+#ifdef _WIN32
+    // https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-shfileoperationa
+    SHFILEOPSTRUCTA file_op = {
+        NULL,
+        FO_DELETE,
+        "slang-record",
+        "",
+        FOF_NOCONFIRMATION |
+        FOF_NOERRORUI |
+        FOF_SILENT,
+        false,
+        0,
+        "" };
+    int ret = SHFileOperationA(&file_op);
+    if (ret)
+    {
+        msgBuilder << "fail to remove 'slang-record' dir, error: " << ret << "\n";
+        getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+        return SLANG_FAIL;
+    }
+#else
+    auto unlink_cb = [](const char* fpath, const struct stat* sb, int typeflag, struct FTW* ftwbuf) -> int
+    {
+        int rv = ::remove(fpath);
+        if (rv)
+        {
+            perror(fpath);
+        }
+        return rv;
+    };
+    // https://linux.die.net/man/3/nftw
+    int ret = nftw("slang-record", unlink_cb, 64, FTW_DEPTH | FTW_PHYS);
+    if (ret)
+    {
+        msgBuilder << "fail to remove 'slang-record' dir, error: " << ret << ", " << strerror(errno) << "\n";
+        getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+        return SLANG_FAIL;
+    }
+#endif
+
+    return SLANG_OK;
+}
+
+static SlangResult runTest(UnitTestContext* context, const char* testName)
+{
+    List<entryHashInfo> expectHashes;
+    List<entryHashInfo> resultHashes;
+    SlangResult res = SLANG_OK;
+    if((res = runExample(context, testName, expectHashes)) != SLANG_OK)
+    {
+        goto error;
+    }
+
+    if((res = replayExample(context, resultHashes)) != SLANG_OK)
+    {
+        goto error;
+    }
+
+    if((res = resultCompare(expectHashes, resultHashes)) != SLANG_OK)
+    {
+        goto error;
+    }
+
+error:
+    cleanupRecordFiles();
+    return res;
+}
+
+static SlangResult runTests(UnitTestContext* context)
+{
+ const char* testBinaryNames[] = {
+        "triangle",
+    };
+
+    SlangResult finalRes = SLANG_OK;
+    for (const auto& testBinaryName : testBinaryNames)
+    {
+        SlangResult res = runTest(context, testBinaryName);
+        if (SLANG_FAILED(res))
+        {
+            StringBuilder msgBuilder;
+            msgBuilder << "Failed subtest: '" << testBinaryName << "'\n\n\n";
+            getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+            finalRes = res;
+        }
+    }
+
+    return SLANG_OK;
+}
+
+// Those examples all depend on the Vulkan, so we only run them on non-Apple platforms.
+// In the future, we may be able to modify the examples further to remove all the render APIs
+// such that it can be ran on Apple platforms.
+#if !(SLANG_APPLE_FAMILY)
+SLANG_UNIT_TEST(RecordReplay)
+{
+    SLANG_CHECK(SLANG_SUCCEEDED(runTests(unitTestContext)));
+}
+
+#endif