extend fiddle to allow custom lua splices in more places (#7559)

* Add fkYAML submodule

* Generate slang-ir-inst-defs.h from slang-ir-inst-defs.yaml

* generate ir-inst-defs.h

* neaten things

* neaten inst def parser

* add rapidyaml submodule

* remove fkyaml

* remove fkyaml submodule

* remove use of ir-inst-defs.h

* format and warnings

* fix wasm build

* tidy

* remove rapidyaml

* Extend fiddle to allow custom splices in more places

* Use lua to describe ir insts

* fix

* neaten

* neaten

* neaten

* spelling

* neaten

* comment comment out assert

* merge

5 files changed, 381 insertions, 61 deletions

diff --git a/tools/slang-fiddle/README.md b/tools/slang-fiddle/README.md
index 19bda6e8d..105a9a8e1 100644
--- a/tools/slang-fiddle/README.md
+++ b/tools/slang-fiddle/README.md
@@ -153,3 +153,17 @@ For example, given the input above, the generated output for the template might
 #include "my-class-forward-decls.h.fiddle"
 #endif
 ```
+
+Inside templates there is a global defined `THIS_FILE` which is the file path
+of the C++ source containing the splice. This can be used to load adjacent lua
+files for example.
+
+Lua Function Calls
+------------------
+
+A more lightweight alternative to templates is to call
+`FIDDLE(myLuaFunction(my, parameters))`. This will load a lua file at
+`current_cpp_source_path.cpp.lua` and call the `myLuaFunction` function in the
+table returned by that lua file. Inside such splices a `fiddle` global table is
+available, containing a `current_decl` member if available, pointing to the
+namespace or class/struct containing this splice.
diff --git a/tools/slang-fiddle/slang-fiddle-scrape.cpp b/tools/slang-fiddle/slang-fiddle-scrape.cpp
index 763ef8f48..a96183130 100644
--- a/tools/slang-fiddle/slang-fiddle-scrape.cpp
+++ b/tools/slang-fiddle/slang-fiddle-scrape.cpp
@@ -1,6 +1,7 @@
 // slang-fiddle-scrape.cpp
 #include "slang-fiddle-scrape.h"
 
+#include "core/slang-string-util.h"
 #include "slang-fiddle-script.h"
 
 namespace fiddle
@@ -238,6 +239,8 @@ public:
 
     RefPtr<Expr> parseCppSimpleTypeSpecififer()
     {
+        while (advanceIf("const") || advanceIf("static"))
+            ;
 
         switch (peekType())
         {
@@ -397,14 +400,14 @@ public:
         addDecl(decl);
         WithParentDecl withParent(this, decl);
 
-        // We expect any `FIDDLE()`-marked aggregate type
-        // declaration to start with a `FIDDLE(...)` invocation,
-        // so that there is a suitable insertion point for
-        // the expansion step.
+        // We expect any `FIDDLE()`-marked aggregate type declaration to start
+        // with a `FIDDLE(...)` or `FIDDLE(myFunc(a,b,c))` invocation, so that
+        // there is a suitable insertion point for the expansion step or the
+        // user has specific a custom step
         //
         {
             auto saved = _cursor;
-            bool found = peekFiddleEllipsisInvocation();
+            bool found = peekFiddleEllipsisInvocation() || peekFiddleLuaCall();
             _cursor = saved;
             if (!found)
             {
@@ -433,6 +436,15 @@ public:
         return true;
     }
 
+    bool peekFiddleLuaCall()
+    {
+        auto saved = _cursor;
+        const bool found = advanceIf(TokenType::Identifier) &&
+                           (peekType() == TokenType::LParent || peekType() == TokenType::LBrace);
+        _cursor = saved;
+        return found;
+    }
+
     RefPtr<Declarator> parseCppSimpleDeclarator()
     {
         switch (peekType())
@@ -468,7 +480,8 @@ public:
 
     RefPtr<Declarator> parseCppDeclarator()
     {
-        advanceIf("const");
+        while (advanceIf("const") || advanceIf("static"))
+            ;
 
         if (advanceIf(TokenType::OpMul))
         {
@@ -559,20 +572,30 @@ public:
                 return modifiers;
 
             case TokenType::Identifier:
+                if (advanceIf("abstract"))
+                {
+                    modifiers.add(new AbstractModifier());
+                }
+                else if (advanceIf("hidden"))
+                {
+                    modifiers.add(new HiddenModifier());
+                }
+                else
+                {
+                    return modifiers;
+                }
                 break;
-            }
 
-            if (advanceIf("abstract"))
-            {
-                modifiers.add(new AbstractModifier());
-            }
-            else if (advanceIf("hidden"))
-            {
-                modifiers.add(new HiddenModifier());
-            }
-            else
-            {
-                return modifiers;
+            case TokenType::LBrace:
+                {
+                    const auto b = read();
+                    StringBuilder sb;
+                    sb << b.getContent();
+                    for (int i = 0; i < b.getSkipCount(); ++i)
+                        sb << read().getContent() << " ";
+                    modifiers.add(new TableModifier(std::move(sb)));
+                }
+                break;
             }
         }
 
@@ -772,20 +795,41 @@ public:
 
             if (peekType() != TokenType::RParent)
             {
-                // In this case we are expecting a fiddle-mode declaration
-                // to appear, in which case we will allow any number of full
-                // fiddle-mode declarations, but won't expect a C++-mode
-                // declaration to follow.
-
-                // TODO: We should associate these declarations
-                // as children of the `FiddleMacroInvocation`,
-                // so that they can be emitted as part of its
-                // expansion (if we decide to make more use
-                // of the `FIDDLE()` approach...).
-
-                parseFiddleModeDecls(fiddleModifiers);
-                expect(TokenType::RParent);
-                return;
+                if (peekFiddleLuaCall())
+                {
+                    StringBuilder sb;
+                    const auto f = expect(TokenType::Identifier);
+                    const auto b = read();
+                    sb << f.getContent() << b.getContent();
+                    for (int i = 0; i < b.getSkipCount(); ++i)
+                        sb << read().getContent() << " ";
+                    auto fiddleLuaCall = RefPtr(new FiddleLuaCallInvocation());
+                    fiddleLuaCall->fiddleToken = fiddleToken;
+                    fiddleLuaCall->parentDecl = _currentParentDecl;
+                    fiddleLuaCall->callString = std::move(sb);
+                    addDecl(fiddleLuaCall);
+
+                    expect(TokenType::RParent);
+                    return;
+                }
+                else
+                {
+
+                    // In this case we are expecting a fiddle-mode declaration
+                    // to appear, in which case we will allow any number of full
+                    // fiddle-mode declarations, but won't expect a C++-mode
+                    // declaration to follow.
+
+                    // TODO: We should associate these declarations
+                    // as children of the `FiddleMacroInvocation`,
+                    // so that they can be emitted as part of its
+                    // expansion (if we decide to make more use
+                    // of the `FIDDLE()` approach...).
+
+                    parseFiddleModeDecls(fiddleModifiers);
+                    expect(TokenType::RParent);
+                    return;
+                }
             }
             expect(TokenType::RParent);
         }
@@ -1037,6 +1081,9 @@ private:
         else if (as<FiddleMacroInvocation>(decl))
         {
         }
+        else if (as<FiddleLuaCallInvocation>(decl))
+        {
+        }
         else
         {
             sink.diagnose(SourceLoc(), Diagnostics::unexpected, "case in checkDecl", "known type");
@@ -1093,6 +1140,7 @@ private:
     }
 };
 
+void push(lua_State* L, Val* val);
 
 // Emit
 
@@ -1102,6 +1150,7 @@ private:
     SourceManager& _sourceManager;
     RefPtr<LogicalModule> _module;
     StringBuilder& _builder;
+    DiagnosticSink& _sink;
 
 public:
     EmitContext(
@@ -1109,9 +1158,8 @@ public:
         DiagnosticSink& sink,
         SourceManager& sourceManager,
         LogicalModule* module)
-        : _builder(builder), _sourceManager(sourceManager), _module(module)
+        : _builder(builder), _sourceManager(sourceManager), _module(module), _sink(sink)
     {
-        SLANG_UNUSED(sink);
     }
 
     void emitMacrosRec(Decl* decl)
@@ -1131,20 +1179,23 @@ private:
         {
             emitMacroForFiddleInvocation(fiddleMacroInvocation);
         }
+        else if (const auto fiddleLuaCallInvocation = as<FiddleLuaCallInvocation>(decl))
+        {
+            emitMacroForFiddleLuaCallInvocation(fiddleLuaCallInvocation);
+        }
         else
         {
             // do nothing with most decls
         }
     }
 
-    void emitMacroForFiddleInvocation(FiddleMacroInvocation* fiddleInvocation)
-    {
-        SourceLoc loc = fiddleInvocation->fiddleToken.getLoc();
-        auto humaneLoc = _sourceManager.getHumaneLoc(loc);
-        auto lineNumber = humaneLoc.line;
-
 #define MACRO_LINE_ENDING " \\\n"
 
+    void emitMacroForFiddleInvocationPreamble(const TokenWithTrivia& fiddleToken)
+    {
+        const auto loc = fiddleToken.getLoc();
+        const auto humaneLoc = _sourceManager.getHumaneLoc(loc);
+        const auto lineNumber = humaneLoc.line;
         // Un-define the old `FIDDLE_#` macro for the
         // given line number, since this file might
         // be pulling in another generated header
@@ -1160,6 +1211,52 @@ private:
         _builder.append(lineNumber);
         _builder.append("(...)");
         _builder.append(MACRO_LINE_ENDING);
+    }
+
+    void emitMacroForFiddleInvocationPostamble() { _builder.append("/* end */\n\n"); }
+
+    void emitMacroForFiddleLuaCallInvocation(FiddleLuaCallInvocation* fiddleInvocation)
+    {
+        _builder.append("/*\n");
+        _builder.append(fiddleInvocation->callString);
+        _builder.append("\n*/\n");
+
+        emitMacroForFiddleInvocationPreamble(fiddleInvocation->fiddleToken);
+
+        const auto file =
+            _sourceManager.getHumaneLoc(fiddleInvocation->fiddleToken.getLoc()).pathInfo.getName();
+        StringBuilder sb;
+        sb << "require(\"" << file << ".lua\")." << fiddleInvocation->callString;
+
+        // Create the fiddle table
+        const auto L = getLuaState();
+        lua_newtable(L);
+        push(L, fiddleInvocation->parentDecl);
+        lua_setfield(L, -2, "current_decl");
+        lua_setglobal(L, "fiddle");
+
+        const auto output = evaluateLuaExpression(
+            fiddleInvocation->fiddleToken.getLoc(),
+            file,
+            sb.produceString(),
+            &_sink);
+
+        // Deregister the fiddle table
+        lua_pushnil(L);
+        lua_setglobal(L, "fiddle");
+
+        _builder.append(StringUtil::replaceAll(
+            output.getUnownedSlice(),
+            UnownedStringSlice("\n"),
+            UnownedStringSlice(MACRO_LINE_ENDING)));
+
+        _builder.append(MACRO_LINE_ENDING);
+        emitMacroForFiddleInvocationPostamble();
+    }
+
+    void emitMacroForFiddleInvocation(FiddleMacroInvocation* fiddleInvocation)
+    {
+        emitMacroForFiddleInvocationPreamble(fiddleInvocation->fiddleToken);
 
         auto decl = as<AggTypeDecl>(fiddleInvocation->node);
         if (decl)
@@ -1196,7 +1293,7 @@ private:
             }
             _builder.append("public:" MACRO_LINE_ENDING);
         }
-        _builder.append("/* end */\n\n");
+        emitMacroForFiddleInvocationPostamble();
     }
 
     void emitTypedDecl(Expr* expr, const char* name)
@@ -1567,6 +1664,14 @@ void push(lua_State* L, List<T> const& values)
     }
 }
 
+List<RefPtr<AggTypeDecl>> getDirectSubclasses(AggTypeDecl* decl)
+{
+    List<RefPtr<AggTypeDecl>> result;
+    for (auto subclass : decl->directSubTypeDecls)
+        result.add(subclass);
+    return result;
+}
+
 void getAllSubclasses(AggTypeDecl* decl, List<RefPtr<AggTypeDecl>>& ioSubclasses)
 {
     ioSubclasses.add(decl);
@@ -1605,6 +1710,57 @@ int _indexVal(lua_State* L)
     Val* val = (Val*)lua_touserdata(L, 1);
     char const* name = lua_tostring(L, 2);
 
+    // If we have some user data attached to this declaration, index that
+    if (auto decl = as<Decl>(val))
+    {
+        if (auto tableModifier = decl->findModifier<TableModifier>())
+        {
+            // Check if we have a cached table
+            if (tableModifier->tableRef == LUA_NOREF)
+            {
+                // Evaluate the table string and cache it
+                std::string tableCode =
+                    "return " + std::string(tableModifier->tableSource.getBuffer());
+
+                if (luaL_dostring(L, tableCode.c_str()) == LUA_OK)
+                {
+                    // Store the table in the registry
+                    tableModifier->tableRef = luaL_ref(L, LUA_REGISTRYINDEX);
+                }
+                else
+                {
+                    // Handle error - pop error message and continue
+                    lua_pop(L, 1);
+                }
+            }
+
+            // If we have a cached table, try to index it
+            if (tableModifier->tableRef != LUA_NOREF)
+            {
+                // Get the cached table from registry
+                lua_rawgeti(L, LUA_REGISTRYINDEX, tableModifier->tableRef);
+
+                // Index the table with the requested name
+                lua_pushstring(L, name);
+                lua_gettable(L, -2);
+
+                // Remove the table from stack, leaving just the result
+                lua_remove(L, -2);
+
+                // Check if we found something
+                if (!lua_isnil(L, -1))
+                {
+                    return 1;
+                }
+                else
+                {
+                    lua_pop(L, 1); // Pop the nil
+                    // Fall through to check other properties
+                }
+            }
+        }
+    }
+
     if (auto containerDecl = as<ContainerDecl>(val))
     {
         for (auto m : containerDecl->members)
@@ -1617,18 +1773,25 @@ int _indexVal(lua_State* L)
         }
     }
 
-    if (auto classDecl = as<ClassDecl>(val))
+    if (auto aggTypeDecl = as<AggTypeDecl>(val))
     {
+        if (strcmp(name, "directSubclasses") == 0)
+        {
+            auto value = getDirectSubclasses(aggTypeDecl);
+            push(L, value);
+            return 1;
+        }
+
         if (strcmp(name, "subclasses") == 0)
         {
-            auto value = getAllSubclasses(classDecl);
+            auto value = getAllSubclasses(aggTypeDecl);
             push(L, value);
             return 1;
         }
 
         if (strcmp(name, "directSuperClass") == 0)
         {
-            push(L, classDecl->directBaseType);
+            push(L, aggTypeDecl->directBaseType);
             return 1;
         }
     }
@@ -1657,6 +1820,23 @@ int _indexVal(lua_State* L)
         }
     }
 
+    if (auto varDecl = as<VarDecl>(val))
+    {
+        if (strcmp(name, "initExpr") == 0)
+        {
+            // TODO: do any expression here
+            if (const auto literalExpr = as<LiteralExpr>(varDecl->initExpr))
+            {
+                lua_pushlstring(
+                    L,
+                    literalExpr->token.getContent().begin(),
+                    literalExpr->token.getContent().getLength());
+                return 1;
+            }
+            return 0;
+        }
+    }
+
     return 0;
 }
 
diff --git a/tools/slang-fiddle/slang-fiddle-scrape.h b/tools/slang-fiddle/slang-fiddle-scrape.h
index 5cd0b6d05..050576674 100644
--- a/tools/slang-fiddle/slang-fiddle-scrape.h
+++ b/tools/slang-fiddle/slang-fiddle-scrape.h
@@ -2,6 +2,7 @@
 #pragma once
 
 #include "compiler-core/slang-lexer.h"
+#include "lua/lauxlib.h"
 #include "slang-fiddle-diagnostics.h"
 
 namespace fiddle
@@ -109,6 +110,16 @@ class AbstractModifier : public ModifierNode
 class HiddenModifier : public ModifierNode
 {
 };
+class TableModifier : public ModifierNode
+{
+public:
+    TableModifier(String s)
+        : tableSource(s)
+    {
+    }
+    String tableSource;
+    int tableRef = LUA_NOREF;
+};
 
 enum class Mode
 {
@@ -207,6 +218,14 @@ public:
     RefPtr<Node> node; // the node whose generated content should get emitted...
 };
 
+class FiddleLuaCallInvocation : public Decl
+{
+public:
+    TokenWithTrivia fiddleToken; // the actual `FIDDLE` identifier
+    String callString;
+    RefPtr<Node> parentDecl;
+};
+
 class UncheckedExpr : public Expr
 {
 };
diff --git a/tools/slang-fiddle/slang-fiddle-script.cpp b/tools/slang-fiddle/slang-fiddle-script.cpp
index 246488199..e713a3798 100644
--- a/tools/slang-fiddle/slang-fiddle-script.cpp
+++ b/tools/slang-fiddle/slang-fiddle-script.cpp
@@ -1,10 +1,13 @@
 // slang-fiddle-script.cpp
 #include "slang-fiddle-script.h"
 
+#include "compiler-core/slang-diagnostic-sink.h"
 #include "lua/lapi.h"
 #include "lua/lauxlib.h"
 #include "lua/lualib.h"
 
+#include <cstdio>
+
 namespace fiddle
 {
 DiagnosticSink* _sink = nullptr;
@@ -114,6 +117,35 @@ int _template(lua_State* L)
 
 lua_State* L = nullptr;
 
+// Add a custom searcher that handles relative paths
+// So we can do things like require("source/slang/foo.lua")
+static int path_searcher(lua_State* L)
+{
+    const char* modname = luaL_checkstring(L, 1);
+
+    if (luaL_loadfile(L, modname) == LUA_OK)
+    {
+        lua_pushstring(L, modname); // Push filename as second return
+        return 2;
+    }
+
+    // Not found
+    lua_pushfstring(L, "\n\tno file '%s'", modname);
+    return 1;
+}
+
+void install_path_searcher(lua_State* L)
+{
+    lua_getglobal(L, "package");
+    lua_getfield(L, -1, "searchers");
+
+    // Insert at position 2 (after preload)
+    lua_pushcfunction(L, path_searcher);
+    lua_rawseti(L, -2, 2);
+
+    lua_pop(L, 2);
+}
+
 void ensureLuaInitialized()
 {
     if (L)
@@ -137,6 +169,8 @@ void ensureLuaInitialized()
     lua_pushcclosure(L, &_template, 0);
     lua_setglobal(L, "TEMPLATE");
 
+    install_path_searcher(L);
+
     // TODO: register custom stuff here...
 }
 
@@ -146,6 +180,33 @@ lua_State* getLuaState()
     return L;
 }
 
+static void setupLuaEnvironment(const String& originalFileName)
+{
+    ensureLuaInitialized();
+
+    lua_pushstring(L, originalFileName.getBuffer());
+    lua_setglobal(L, "THIS_FILE");
+
+    lua_pushcfunction(L, &_handleLuaErrorRaised);
+}
+
+static void handleLuaError(
+    SourceLoc loc,
+    DiagnosticSink* sink,
+    const char* errorType,
+    DiagnosticInfo diagnosticID)
+{
+    size_t size = 0;
+    char const* buffer = lua_tolstring(L, -1, &size);
+    String message = UnownedStringSlice(buffer, size);
+    message = message + "\n";
+
+    sink->diagnose(loc, diagnosticID, message);
+
+    String abortMessage = "fiddle failed during Lua ";
+    abortMessage = abortMessage + errorType;
+    SLANG_ABORT_COMPILATION(abortMessage.getBuffer());
+}
 
 String evaluateScriptCode(
     SourceLoc loc,
@@ -157,39 +218,79 @@ String evaluateScriptCode(
     _builder = &builder;
     _templateCounter = 0;
 
-    ensureLuaInitialized();
+    setupLuaEnvironment(originalFileName);
 
     String luaChunkName = "@" + originalFileName;
 
-    lua_pushcfunction(L, &_handleLuaErrorRaised);
-
     if (LUA_OK != luaL_loadbuffer(
                       L,
                       scriptSource.getBuffer(),
                       scriptSource.getLength(),
                       luaChunkName.getBuffer()))
     {
-        size_t size = 0;
-        char const* buffer = lua_tolstring(L, -1, &size);
-        String message = UnownedStringSlice(buffer, size);
-        message = message + "\n";
-
-        sink->diagnose(loc, fiddle::Diagnostics::scriptLoadError, message);
-        SLANG_ABORT_COMPILATION("fiddle failed during Lua script loading");
+        handleLuaError(loc, sink, "script loading", fiddle::Diagnostics::scriptLoadError);
     }
 
     if (LUA_OK != lua_pcall(L, 0, 0, -2))
     {
-        size_t size = 0;
-        char const* buffer = lua_tolstring(L, -1, &size);
-        String message = UnownedStringSlice(buffer, size);
-        message = message + "\n";
-
-        sink->diagnose(loc, fiddle::Diagnostics::scriptExecutionError, message);
-        SLANG_ABORT_COMPILATION("fiddle failed during Lua script execution");
+        handleLuaError(loc, sink, "script execution", fiddle::Diagnostics::scriptExecutionError);
     }
 
     _builder = nullptr;
     return builder.produceString();
 }
+
+String evaluateLuaExpression(
+    SourceLoc loc,
+    String originalFileName,
+    String luaExpression,
+    DiagnosticSink* sink)
+{
+    setupLuaEnvironment(originalFileName);
+
+    String luaChunkName = "@" + originalFileName;
+
+    // Wrap expression in return statement to get its value
+    String wrappedExpression = "return " + luaExpression;
+
+    if (LUA_OK != luaL_loadbuffer(
+                      L,
+                      wrappedExpression.getBuffer(),
+                      wrappedExpression.getLength(),
+                      luaChunkName.getBuffer()))
+    {
+        handleLuaError(loc, sink, "expression loading", fiddle::Diagnostics::scriptLoadError);
+    }
+
+    // Execute and expect 1 return value
+    if (LUA_OK != lua_pcall(L, 0, 1, -2))
+    {
+        handleLuaError(
+            loc,
+            sink,
+            "expression evaluation",
+            fiddle::Diagnostics::scriptExecutionError);
+    }
+
+    // Convert the result to string
+    size_t resultSize = 0;
+    const char* resultBuffer = lua_tolstring(L, -1, &resultSize);
+
+    if (!resultBuffer)
+    {
+        sink->diagnose(
+            loc,
+            fiddle::Diagnostics::scriptExecutionError,
+            "Lua expression did not return a string value\n");
+        SLANG_ABORT_COMPILATION("fiddle failed: non-string expression result");
+    }
+
+    String result;
+    result.append(resultBuffer, resultSize);
+
+    // Pop the result and error handler
+    lua_pop(L, 2);
+
+    return result;
+}
 } // namespace fiddle
diff --git a/tools/slang-fiddle/slang-fiddle-script.h b/tools/slang-fiddle/slang-fiddle-script.h
index 98cdfa148..94bda453b 100644
--- a/tools/slang-fiddle/slang-fiddle-script.h
+++ b/tools/slang-fiddle/slang-fiddle-script.h
@@ -20,4 +20,10 @@ String evaluateScriptCode(
     String originalFileName,
     String scriptSource,
     DiagnosticSink* sink);
+
+String evaluateLuaExpression(
+    SourceLoc loc,
+    String originalFileName,
+    String scriptSource,
+    DiagnosticSink* sink);
 } // namespace fiddle