yum-mirror/slang

Making it easier to work with shaders

git clone https://git.yummers.dev/yum-mirror/slang

Theresa FoleyAdd support for on-demand AST deserialization (#7482)3ed776159

master
branchesmasterall refs →
16.0 KiB559 linesraw
1// slang-fiddle-template.cpp
2#include "slang-fiddle-template.h"
3
4#include "slang-fiddle-script.h"
5
6namespace fiddle
7{
8struct TextTemplateParserBase
9{
10protected:
11    TextTemplateParserBase(
12        SourceView* inputSourceView,
13        DiagnosticSink* sink,
14        UnownedStringSlice source)
15        : _inputSourceView(inputSourceView)
16        , _sink(sink)
17        , _cursor(source.begin())
18        , _end(source.end())
19    {
20    }
21
22    SourceView* _inputSourceView = nullptr;
23    DiagnosticSink* _sink = nullptr;
24    char const* _cursor = nullptr;
25    char const* _end = nullptr;
26
27    bool atEnd() { return _cursor == _end; }
28
29    UnownedStringSlice readLine()
30    {
31        auto lineBegin = _cursor;
32
33        while (!atEnd())
34        {
35            char const* lineEnd = _cursor;
36            switch (*_cursor)
37            {
38            default:
39                _cursor++;
40                continue;
41
42            case '\r':
43                _cursor++;
44                if (*_cursor == '\n')
45                    _cursor++;
46                break;
47
48            case '\n':
49                _cursor++;
50                break;
51            }
52
53            return UnownedStringSlice(lineBegin, lineEnd);
54        }
55
56        return UnownedStringSlice(lineBegin, _end);
57    }
58};
59
60struct TextTemplateParser : TextTemplateParserBase
61{
62public:
63    TextTemplateParser(
64        SourceView* inputSourceView,
65        DiagnosticSink* sink,
66        UnownedStringSlice templateSource)
67        : TextTemplateParserBase(inputSourceView, sink, templateSource)
68    {
69    }
70
71    char const* findScriptStmtLine(UnownedStringSlice line)
72    {
73        char const* lineCursor = line.begin();
74        char const* lineEnd = line.end();
75        while (lineCursor != lineEnd)
76        {
77            switch (*lineCursor)
78            {
79            default:
80                return nullptr;
81
82            case ' ':
83            case '\t':
84                lineCursor++;
85                continue;
86
87            case '%':
88                return lineCursor;
89            }
90        }
91        return nullptr;
92    }
93
94    List<RefPtr<TextTemplateStmt>> stmts;
95
96    void addRaw(char const* rawBegin, char const* rawEnd)
97    {
98        if (rawBegin == rawEnd)
99            return;
100
101        auto stmt = RefPtr(new TextTemplateRawStmt());
102        stmt->text = UnownedStringSlice(rawBegin, rawEnd);
103        stmts.add(stmt);
104    }
105
106    void addScriptStmtLine(char const* sourceBegin, char const* sourceEnd)
107    {
108        auto stmt = RefPtr(new TextTemplateScriptStmt());
109        stmt->scriptSource = UnownedStringSlice(sourceBegin, sourceEnd);
110        stmts.add(stmt);
111    }
112
113    void addScriptSpliceExpr(char const* sourceBegin, char const* sourceEnd)
114    {
115        auto stmt = RefPtr(new TextTemplateSpliceStmt());
116        stmt->scriptExprSource = UnownedStringSlice(sourceBegin, sourceEnd);
117        stmts.add(stmt);
118    }
119
120    bool isIdentifierStartChar(int c)
121    {
122        return (('a' <= c) && (c <= 'z')) || (('A' <= c) && (c <= 'Z')) || (c == '_');
123    }
124
125    bool isIdentifierChar(int c) { return isIdentifierStartChar(c) || (('0' <= c) && (c <= '9')); }
126
127    RefPtr<TextTemplateStmt> parseTextTemplateBody()
128    {
129        bool isAtStartOfLine = true;
130        bool isInScriptLine = false;
131        int depthInSplice = 0;
132
133        char const* currentLineBegin = _cursor;
134        char const* currentSpanBegin = _cursor;
135        while (!atEnd())
136        {
137            char const* currentSpanEnd = _cursor;
138
139            bool wasAtStartOfLine = isAtStartOfLine;
140            isAtStartOfLine = false;
141
142            int c = *_cursor++;
143            switch (c)
144            {
145            default:
146                break;
147
148            case '\r':
149                if (*_cursor == '\n')
150                {
151                    _cursor++;
152                }
153            case '\n':
154                isAtStartOfLine = true;
155                currentLineBegin = _cursor;
156                if (isInScriptLine)
157                {
158                    addScriptStmtLine(currentSpanBegin, currentSpanEnd);
159                    isInScriptLine = false;
160                    currentSpanBegin = currentSpanEnd;
161                }
162                break;
163
164            case ' ':
165            case '\t':
166                isAtStartOfLine = wasAtStartOfLine;
167                break;
168
169            case '%':
170                if (wasAtStartOfLine && !depthInSplice)
171                {
172                    addRaw(currentSpanBegin, currentLineBegin);
173                    isInScriptLine = true;
174                    currentSpanBegin = _cursor;
175                }
176                break;
177
178            case '$':
179                if (isInScriptLine)
180                    continue;
181                if (depthInSplice)
182                    SLANG_ABORT_COMPILATION("fiddle encountered a '$' nested inside a splice");
183
184                if (*_cursor == '(')
185                {
186                    _cursor++;
187                    addRaw(currentSpanBegin, currentSpanEnd);
188                    depthInSplice = 1;
189                    currentSpanBegin = _cursor;
190                    break;
191                }
192                else if (isIdentifierStartChar(*_cursor))
193                {
194                    addRaw(currentSpanBegin, currentSpanEnd);
195
196                    auto spliceExprBegin = _cursor;
197                    while (isIdentifierChar(*_cursor))
198                        _cursor++;
199                    auto spliceExprEnd = _cursor;
200                    addScriptSpliceExpr(spliceExprBegin, spliceExprEnd);
201                    currentSpanBegin = _cursor;
202                    break;
203                }
204                break;
205
206            case '(':
207                if (!depthInSplice)
208                    continue;
209                depthInSplice++;
210                break;
211
212            case ')':
213                if (!depthInSplice)
214                    continue;
215                depthInSplice--;
216                if (depthInSplice == 0)
217                {
218                    addScriptSpliceExpr(currentSpanBegin, currentSpanEnd);
219                    currentSpanBegin = _cursor;
220                }
221                break;
222            }
223        }
224        addRaw(currentSpanBegin, _end);
225
226        if (stmts.getCount() == 1)
227            return stmts[0];
228        else
229        {
230            auto stmt = RefPtr(new TextTemplateSeqStmt());
231            stmt->stmts = stmts;
232            return stmt;
233        }
234    }
235
236private:
237};
238
239
240char const* templateStartMarker = "FIDDLE TEMPLATE";
241char const* outputStartMarker = "FIDDLE OUTPUT";
242char const* endMarker = "FIDDLE END";
243
244struct TextTemplateFileParser : TextTemplateParserBase
245{
246public:
247    TextTemplateFileParser(SourceView* inputSourceView, DiagnosticSink* sink)
248        : TextTemplateParserBase(inputSourceView, sink, inputSourceView->getContent())
249    {
250    }
251
252    RefPtr<TextTemplateFile> parseTextTemplateFile()
253    {
254        auto textTemplateFile = RefPtr(new TextTemplateFile());
255        textTemplateFile->loc = _inputSourceView->getRange().begin;
256        textTemplateFile->originalFileContent = _inputSourceView->getContent();
257        while (!atEnd())
258        {
259            auto textTemplate = parseOptionalTextTemplate();
260            if (textTemplate)
261                textTemplateFile->textTemplates.add(textTemplate);
262        }
263        return textTemplateFile;
264    }
265
266private:
267    Count _templateCounter = 0;
268
269    bool matches(UnownedStringSlice const& line, char const* marker)
270    {
271        auto index = line.indexOf(UnownedTerminatedStringSlice(marker));
272        return index >= 0;
273    }
274
275    bool findMatchingLine(char const* marker, UnownedStringSlice& outMatchingLine)
276    {
277        while (!atEnd())
278        {
279            auto line = readLine();
280            if (!matches(line, marker))
281            {
282                // TODO: If the line doesn't match the expected marker,
283                // but it *does* match one of the other markers, then
284                // we should consider it a probable error.
285
286                continue;
287            }
288
289            outMatchingLine = line;
290            return true;
291        }
292        return false;
293    }
294
295    SourceLoc getLoc(char const* ptr)
296    {
297        auto offset = ptr - _inputSourceView->getContent().begin();
298        auto startLoc = _inputSourceView->getRange().begin;
299        auto loc = SourceLoc::fromRaw(startLoc.getRaw() + offset);
300        return loc;
301    }
302
303    SourceLoc getLoc(UnownedStringSlice text) { return getLoc(text.begin()); }
304
305    RefPtr<TextTemplateStmt> parseTextTemplateBody(UnownedStringSlice const& source)
306    {
307        TextTemplateParser parser(_inputSourceView, _sink, source);
308        return parser.parseTextTemplateBody();
309    }
310
311    RefPtr<TextTemplate> parseOptionalTextTemplate()
312    {
313        // The idea is pretty simple; we scan through the source, one line at
314        // a time, until we find a line that matches our template start pattern.
315        //
316        // If we *don't* find the start marker, then there must not be any
317        // templates left.
318        //
319        UnownedStringSlice templateStartLine;
320        if (!findMatchingLine(templateStartMarker, templateStartLine))
321            return nullptr;
322
323        char const* templateSourceBegin = _cursor;
324
325        // If we *do* find a start line for a template, then we will expect
326        // to find the other two kinds of lines, to round things out.
327
328        UnownedStringSlice outputStartLine;
329        if (!findMatchingLine(outputStartMarker, outputStartLine))
330        {
331            // TODO: need to diagnose a problem here...
332            _sink->diagnose(
333                getLoc(templateStartLine),
334                fiddle::Diagnostics::expectedOutputStartMarker,
335                outputStartMarker);
336        }
337
338        char const* templateSourceEnd = outputStartLine.begin();
339
340        char const* existingOutputBegin = _cursor;
341
342        UnownedStringSlice endLine;
343        if (!findMatchingLine(endMarker, endLine))
344        {
345            // TODO: need to diagnose a problem here...
346            _sink->diagnose(
347                getLoc(templateStartLine),
348                fiddle::Diagnostics::expectedEndMarker,
349                endMarker);
350        }
351        char const* existingOutputEnd = endLine.begin();
352
353        auto templateSource = UnownedStringSlice(templateSourceBegin, templateSourceEnd);
354        auto templateBody = parseTextTemplateBody(templateSource);
355
356        auto textTemplate = RefPtr(new TextTemplate());
357        textTemplate->id = _templateCounter++;
358        textTemplate->templateStartLine = templateStartLine;
359        textTemplate->templateSource = templateSource;
360        textTemplate->body = templateBody;
361        textTemplate->outputStartLine = outputStartLine;
362        textTemplate->existingOutputContent =
363            UnownedStringSlice(existingOutputBegin, existingOutputEnd);
364        textTemplate->endLine = endLine;
365        return textTemplate;
366    }
367};
368
369struct TextTemplateScriptCodeEmitter
370{
371public:
372    TextTemplateScriptCodeEmitter(TextTemplateFile* templateFile)
373        : _templateFile(templateFile)
374    {
375    }
376
377    String emitScriptCodeForTextTemplateFile()
378    {
379        // We start by emitting the content of the template
380        // file out as Lua code, so that we can evaluate
381        // it all using the Lua VM.
382        //
383        // We go to some effort to make sure that the line
384        // numbers in the generated Lua will match those
385        // in the input.
386        //
387
388        char const* originalFileRawSpanStart = _templateFile->originalFileContent.begin();
389        for (auto t : _templateFile->textTemplates)
390        {
391            flushOriginalFileRawSpan(originalFileRawSpanStart, t->templateSource.begin());
392
393            evaluateTextTemplate(t);
394
395            originalFileRawSpanStart = t->outputStartLine.begin();
396        }
397        flushOriginalFileRawSpan(
398            originalFileRawSpanStart,
399            _templateFile->originalFileContent.end());
400
401        return _builder.produceString();
402    }
403
404private:
405    TextTemplateFile* _templateFile = nullptr;
406    StringBuilder _builder;
407
408    void flushOriginalFileRawSpan(char const* begin, char const* end)
409    {
410        if (begin == end)
411            return;
412
413        // TODO: implement the important stuff...
414        _builder.append("ORIGINAL [==[");
415        _builder.append(UnownedStringSlice(begin, end));
416        _builder.append("]==]");
417    }
418
419    void evaluateTextTemplate(TextTemplate* textTemplate)
420    {
421        // TODO: there really needs to be some framing around this...
422        _builder.append("TEMPLATE(function() ");
423        evaluateTextTemplateStmt(textTemplate->body);
424        _builder.append(" end)");
425    }
426
427    bool isEntirelyWhitespace(UnownedStringSlice const& text)
428    {
429        for (auto c : text)
430        {
431            switch (c)
432            {
433            default:
434                return false;
435
436            case ' ':
437            case '\t':
438            case '\r':
439            case '\n':
440                continue;
441            }
442        }
443        return true;
444    }
445
446    void evaluateTextTemplateStmt(TextTemplateStmt* stmt)
447    {
448        if (auto seqStmt = as<TextTemplateSeqStmt>(stmt))
449        {
450            for (auto s : seqStmt->stmts)
451                evaluateTextTemplateStmt(s);
452        }
453        else if (auto rawStmt = as<TextTemplateRawStmt>(stmt))
454        {
455            auto rawContent = rawStmt->text;
456            if (isEntirelyWhitespace(rawContent))
457            {
458                _builder.append(rawContent);
459            }
460            else
461            {
462                _builder.append("RAW [==[");
463                _builder.append(rawContent);
464                _builder.append("]==]");
465            }
466        }
467        else if (auto scriptStmt = as<TextTemplateScriptStmt>(stmt))
468        {
469            _builder.append(scriptStmt->scriptSource);
470            _builder.append(" ");
471        }
472        else if (auto spliceStmt = as<TextTemplateSpliceStmt>(stmt))
473        {
474            _builder.append("SPLICE(function()return(");
475            _builder.append(spliceStmt->scriptExprSource);
476            _builder.append(")end)");
477        }
478        else
479        {
480            SLANG_ABORT_COMPILATION(
481                "fiddle encountered an unknown construct when converting a text template to Lua");
482        }
483    }
484};
485
486
487RefPtr<TextTemplateFile> parseTextTemplateFile(SourceView* inputSourceView, DiagnosticSink* sink)
488{
489    TextTemplateFileParser parser(inputSourceView, sink);
490    return parser.parseTextTemplateFile();
491}
492
493void generateTextTemplateOutputs(
494    String originalFileName,
495    TextTemplateFile* file,
496    StringBuilder& builder,
497    DiagnosticSink* sink)
498{
499    TextTemplateScriptCodeEmitter emitter(file);
500    String scriptCode = emitter.emitScriptCodeForTextTemplateFile();
501
502    String output = evaluateScriptCode(file->loc, originalFileName, scriptCode, sink);
503
504    builder.append(output);
505    builder.append("\n");
506}
507
508String generateModifiedInputFileForTextTemplates(
509    String templateOutputFileName,
510    TextTemplateFile* file,
511    DiagnosticSink* sink)
512{
513    // The basic idea here is that we need to emit most of
514    // the body of the file exactly as it originally
515    // appeared, and then only modifify the few lines
516    // that represent the text template output.
517    //
518    // TODO(tfoley): We could also use this as an opportunity
519    // to insert the `FIDDLE(...)` markers that the scraping
520    // tool needs, but that is more work than makes sense
521    // right now.
522
523    StringBuilder builder;
524
525
526    char const* originalFileRawSpanStart = file->originalFileContent.begin();
527    for (auto t : file->textTemplates)
528    {
529        builder.append(
530            UnownedStringSlice(originalFileRawSpanStart, t->existingOutputContent.begin()));
531
532        builder.append("#define FIDDLE_GENERATED_OUTPUT_ID ");
533        builder.append(t->id);
534        builder.append("\n");
535        builder.append("#include \"");
536        for (auto c : templateOutputFileName)
537        {
538            switch (c)
539            {
540            case '"':
541            case '\\':
542                builder.appendChar('\\');
543                builder.appendChar(c);
544                break;
545
546            default:
547                builder.appendChar(c);
548                break;
549            }
550        }
551        builder.append("\"\n");
552        originalFileRawSpanStart = t->existingOutputContent.end();
553    }
554    builder.append(UnownedStringSlice(originalFileRawSpanStart, file->originalFileContent.end()));
555
556    return builder.produceString();
557}
558
559} // namespace fiddle