From 1e86f5657d38ae5bab0ced7dc17aeda48198fdd5 Mon Sep 17 00:00:00 2001
From: Theresa Foley <10618364+tangent-vector@users.noreply.github.com>
Date: Thu, 17 Apr 2025 12:40:53 -0700
Subject: Add Yet Another Source Code Generator (#6844)

* Add Yet Another Source Code Generator

This change introduces an offline source code generation tool,
provisionally called `fiddle`. More information about the design of
the tool can be found in `tools/slang-fiddle/README.md`.

Yes... this is yet another code generator in a project that already
has too many. Yes, this could easily be a very obvious instnace of
[XKCD 927](https://xkcd.com/927/).

This change is part of a larger effort to change how the AST
types are being serialized, and the way code generation for them
is implemented.

Right now, the source code for the new tool is being checked in and
the relevant build step is enabled, just to make sure everything is
working as intended, but please note that this change does *not*
introduce any code in the repository that actually makes use of
the new generator. All of the AST-related reflection information that
feeds the current serialization system is still being generated using
`slang-cpp-extractor`.

The design of the new tool is primarily motivated by the new approach
to serialization that I'm implementing, and once that new approach
lands we should be able to deprecate the `slang-cpp-extractor`.

In addition, the new tool should in principle be able to handle
many of the kinds of code generation tasks that are currently being
implemented with other tools like `slang-generate` (used for the core
and glsl libraries). This tool should also be well suited to the task
of generating more of the code related to the IR instructions.

* format code

* Build fixes caught by CI

* Fix another warning coming from CI

* Another CI-caught fix

* Change bare 	hrows over to more proper abort execptions

* format code

---------

Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>
---
 source/compiler-core/slang-lexer.cpp    | 12 +++++++++
 source/compiler-core/slang-lexer.h      |  3 +++
 source/slang-core-module/CMakeLists.txt |  1 +
 source/slang/CMakeLists.txt             | 46 +++++++++++++++++++++++++++++++++
 4 files changed, 62 insertions(+)

(limited to 'source')

diff --git a/source/compiler-core/slang-lexer.cpp b/source/compiler-core/slang-lexer.cpp
index 84a4df93b..048c266ca 100644
--- a/source/compiler-core/slang-lexer.cpp
+++ b/source/compiler-core/slang-lexer.cpp
@@ -1810,6 +1810,18 @@ TokenList Lexer::lexAllMarkupTokens()
     }
 }
 
+TokenList Lexer::lexAllTokens()
+{
+    TokenList tokenList;
+    for (;;)
+    {
+        Token token = lexToken();
+        tokenList.add(token);
+        if (token.type == TokenType::EndOfFile)
+            return tokenList;
+    }
+}
+
 /* static */ UnownedStringSlice Lexer::sourceLocationLexer(const UnownedStringSlice& in)
 {
     Lexer lexer;
diff --git a/source/compiler-core/slang-lexer.h b/source/compiler-core/slang-lexer.h
index a36719ab7..0152ff6f5 100644
--- a/source/compiler-core/slang-lexer.h
+++ b/source/compiler-core/slang-lexer.h
@@ -139,6 +139,9 @@ struct Lexer
     /// Lex all tokens (up to the end of the stream) that are relevant to things like markup
     TokenList lexAllMarkupTokens();
 
+    /// Lex all tokens (up to the end of the stream) whether relevant or not.
+    TokenList lexAllTokens();
+
     /// Get the diagnostic sink, taking into account flags. Will return null if suppressing
     /// diagnostics.
     DiagnosticSink* getDiagnosticSink()
diff --git a/source/slang-core-module/CMakeLists.txt b/source/slang-core-module/CMakeLists.txt
index 50b45c90c..ba70d77b9 100644
--- a/source/slang-core-module/CMakeLists.txt
+++ b/source/slang-core-module/CMakeLists.txt
@@ -60,6 +60,7 @@ set(core_module_source_common_args
     LINK_WITH_PRIVATE
     core
     slang-capability-defs
+    slang-fiddle-output
     slang-reflect-headers
     SPIRV-Headers
     INCLUDE_DIRECTORIES_PRIVATE
diff --git a/source/slang/CMakeLists.txt b/source/slang/CMakeLists.txt
index 9c51ed767..2adc96939 100644
--- a/source/slang/CMakeLists.txt
+++ b/source/slang/CMakeLists.txt
@@ -1,3 +1,48 @@
+#
+# Invoke the "fiddle" tool to generate source
+#
+
+set(SLANG_FIDDLE_INPUT_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+set(SLANG_FIDDLE_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/fiddle")
+
+file(GLOB SLANG_FIDDLE_INPUT_FILE_NAMES
+    CONFIGURE_DEPENDS
+    RELATIVE "${SLANG_FIDDLE_INPUT_DIR}"
+    "*.h"
+    "*.cpp")
+
+list(TRANSFORM SLANG_FIDDLE_INPUT_FILE_NAMES
+    PREPEND "${SLANG_FIDDLE_INPUT_DIR}/"
+    OUTPUT_VARIABLE SLANG_FIDDLE_INPUTS)
+
+list(TRANSFORM SLANG_FIDDLE_INPUT_FILE_NAMES
+    APPEND ".fiddle"
+    OUTPUT_VARIABLE SLANG_FIDDLE_OUTPUTS)
+list(TRANSFORM SLANG_FIDDLE_OUTPUTS
+    PREPEND "${SLANG_FIDDLE_OUTPUT_DIR}/")
+
+add_custom_command(
+    OUTPUT ${SLANG_FIDDLE_OUTPUTS}
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${SLANG_FIDDLE_OUTPUT_DIR}
+    COMMAND slang-fiddle
+        -i "${SLANG_FIDDLE_INPUT_DIR}/"
+        -o "${SLANG_FIDDLE_OUTPUT_DIR}/"
+        ${SLANG_FIDDLE_INPUT_FILE_NAMES}
+    DEPENDS ${SLANG_FIDDLE_INPUTS} slang-fiddle
+    VERBATIM
+)
+add_library(
+    slang-fiddle-output
+    INTERFACE
+    EXCLUDE_FROM_ALL
+    ${SLANG_FIDDLE_OUTPUTS}
+)
+set_target_properties(slang-fiddle-output PROPERTIES FOLDER generated)
+target_include_directories(
+    slang-fiddle-output
+    INTERFACE ${SLANG_FIDDLE_OUTPUT_DIR}
+)
+
 #
 # generate capability files
 #
@@ -233,6 +278,7 @@ set(slang_link_args
     compiler-core
     slang-capability-defs
     slang-capability-lookup
+    slang-fiddle-output
     slang-reflect-headers
     slang-lookup-tables
     SPIRV-Headers
-- 
cgit v1.2.3