yum-mirror/slang

Making it easier to work with shaders

git clone https://git.yummers.dev/yum-mirror/slang

Sai Praveen BangaruImprove embed tool to search all include directories as determined by CMake (#6675)b9300bae0

master
branchesmasterall refs →
14.3 KiB425 linesraw
1// slang-embed.cpp
2
3// This file implements a simple utility for taking an input file
4// and embedding into a C++ source file as a `static const` array.
5
6// For now this utility uses plain C stdlib functionality rather
7// than depending on any of the utiltiies from the Slang project
8// libraries.
9//
10#ifdef _MSC_VER
11#pragma warning(disable : 4996)
12#endif
13#include "../../source/core/slang-dictionary.h"
14#include "../../source/core/slang-io.h"
15#include "../../source/core/slang-list.h"
16#include "../../source/core/slang-string-util.h"
17#include "../../source/core/slang-string.h"
18
19#include <stdio.h>
20#include <stdlib.h>
21#include <string.h>
22
23// Utility to free pointers on scope exit
24struct ScopedMemory
25{
26    ScopedMemory(void* ptr)
27        : ptr(ptr)
28    {
29    }
30
31    ~ScopedMemory()
32    {
33        if (ptr)
34            free(ptr);
35    }
36
37    void* ptr;
38};
39
40// Utility to close file on scope exit
41struct ScopedFile
42{
43    ScopedFile(FILE* file)
44        : file(file)
45    {
46    }
47
48    ~ScopedFile()
49    {
50        if (file)
51            fclose(file);
52    }
53
54    FILE* file;
55};
56
57// The utility is implemented as a single `struct` type
58// that provides a context for the code. We do this as
59// an alternative to using global variables for passing
60// around options easily.
61//
62struct App
63{
64    char const* appName = "slang-embed";
65    char const* inputPath = nullptr;
66    char const* outputPath = nullptr;
67    Slang::List<Slang::String> includeDirs;
68    Slang::HashSet<Slang::String> includedFiles;
69    size_t charCount = 0;
70    bool useNewStringLit = true;
71
72    void parseOptions(int argc, char** argv)
73    {
74        // First, get the program name
75        if (argc > 0)
76        {
77            appName = *argv++;
78            argc--;
79        }
80
81        // Parse remaining arguments - we need at least inputPath
82        if (argc < 1)
83        {
84            fprintf(stderr, "usage: %s inputPath [outputPath] [-I<includeDir> ...]\n", appName);
85            exit(1);
86        }
87
88        // Get input path (first positional argument)
89        inputPath = *argv++;
90        argc--;
91
92        // Process remaining arguments
93        while (argc > 0)
94        {
95            char* arg = *argv++;
96            argc--;
97
98            // Check for -I prefix for include directories
99            if (strncmp(arg, "-I", 2) == 0)
100            {
101                // Check if this is a concatenated string of include directories
102                char* startPtr = arg;
103
104                // Process the entire string as potentially multiple -I directives
105                while (startPtr && *startPtr)
106                {
107                    // Find the -I prefix
108                    char* iPos = strstr(startPtr, "-I");
109                    if (!iPos)
110                        break;
111
112                    // Move past the -I
113                    char* dirStart = iPos + 2;
114
115                    // Find the next -I or end of string
116                    char* nextIPos = strstr(dirStart, "-I");
117
118                    // Determine end of current include dir
119                    char* dirEnd = nextIPos ? nextIPos : (startPtr + strlen(startPtr));
120
121                    // Check if the directory has a semicolon or quotes at the end
122                    if (dirEnd > dirStart && (*(dirEnd - 1) == ';' || *(dirEnd - 1) == '"'))
123                        dirEnd--;
124
125                    // Save the current directory by creating a substring
126                    if (dirEnd > dirStart)
127                    {
128                        // Create a null-terminated copy
129                        size_t dirLen = dirEnd - dirStart;
130                        Slang::String tempDir(Slang::UnownedStringSlice(dirStart, dirLen));
131
132                        // Remove any quotes
133                        if (tempDir[0] == '"')
134                            tempDir = tempDir.subString(1, tempDir.getLength() - 1);
135                        if (tempDir.endsWith("\"") && tempDir.getLength() > 0)
136                            tempDir = tempDir.subString(0, tempDir.getLength() - 1);
137
138                        // Remove trailing whitespace
139                        tempDir = tempDir.trimEnd();
140
141                        // Add to include dirs
142                        includeDirs.add(tempDir);
143                    }
144
145                    // Move to next position (if any)
146                    startPtr = nextIPos;
147                }
148            }
149            // Otherwise treat as output path if not already set
150            else if (!outputPath)
151            {
152                outputPath = arg;
153            }
154            else
155            {
156                fprintf(stderr, "unexpected argument: %s\n", arg);
157                fprintf(stderr, "usage: %s inputPath [outputPath] [-I<includeDir> ...]\n", appName);
158                exit(1);
159            }
160        }
161
162        // Validate we have the required arguments
163        if (!inputPath)
164        {
165            fprintf(stderr, "usage: %s inputPath [outputPath] [-I<includeDir> ...]\n", appName);
166            exit(1);
167        }
168    }
169
170    void processInputFile(FILE* outputFile, Slang::String inputPath)
171    {
172        using namespace Slang;
173
174        String canonicalPath;
175        if (SLANG_SUCCEEDED(Slang::Path::getCanonical(inputPath, canonicalPath)))
176        {
177            if (!includedFiles.add(canonicalPath))
178                return;
179        }
180
181        // We open the input file in text mode because we are currently
182        // embedding textual source files. If/when this utility gets
183        // used for binary files another mode could be called for.
184        //
185        // (Alternatively, we might always use binary mode, but this
186        // could lead to a difference in the embedded bytes based on
187        // the line ending convention of the host platform)
188        //
189
190        String contents;
191        {
192            auto res = File::readAllText(inputPath, contents);
193            SLANG_ASSERT(SLANG_SUCCEEDED(res));
194        }
195
196        LineParser lineReader(contents.getUnownedSlice());
197
198        for (auto line : lineReader)
199        {
200            auto trimedLine = line.trimStart();
201            if (trimedLine.startsWith("#include"))
202            {
203                auto fileName = Slang::StringUtil::getAtInSplit(trimedLine, ' ', 1);
204                bool isSystemInclude = false;
205
206                // Handle both quoted and angle-bracket includes
207                if (fileName[0] == '<')
208                {
209                    // Handle <filename> format
210                    isSystemInclude = true;
211                    // Extract filename between < and >
212                    if (fileName.getLength() >= 2 && fileName[fileName.getLength() - 1] == '>')
213                    {
214                        fileName =
215                            Slang::UnownedStringSlice(fileName.begin() + 1, fileName.end() - 1);
216                    }
217                    else
218                    {
219                        goto normalProcess; // Malformed include, skip it
220                    }
221                }
222                else if (fileName[0] == '"' && fileName[fileName.getLength() - 1] == '"')
223                {
224                    // Handle "filename" format
225                    fileName = Slang::UnownedStringSlice(fileName.begin() + 1, fileName.end() - 1);
226                }
227                else
228                {
229                    // Malformed include, skip it
230                    goto normalProcess;
231                }
232
233                // For system includes, only look in include dirs, not relative to current file
234                auto path = isSystemInclude ? Slang::String()
235                                            : Slang::Path::combine(
236                                                  Slang::Path::getParentDirectory(inputPath),
237                                                  fileName);
238
239                bool foundInclude = false;
240                if (isSystemInclude || !Slang::File::exists(path))
241                {
242                    // Try looking in each of the include directories
243                    for (auto& includeDir : includeDirs)
244                    {
245                        path = Slang::Path::combine(includeDir, fileName);
246                        if (Slang::File::exists(path))
247                        {
248                            foundInclude = true;
249                            break;
250                        }
251                    }
252                }
253                else
254                {
255                    foundInclude = true;
256                }
257
258                if (!foundInclude)
259                    goto normalProcess;
260                processInputFile(outputFile, path.getUnownedSlice());
261                continue;
262            }
263        normalProcess:;
264            if (!useNewStringLit && charCount + line.getLength() > 0x4000)
265            {
266                charCount = 0;
267                useNewStringLit = true;
268                fprintf(outputFile, ";\n");
269            }
270            if (useNewStringLit)
271            {
272                fprintf(outputFile, "sb << \n\"");
273                useNewStringLit = false;
274            }
275            else
276            {
277                fprintf(outputFile, "\"");
278            }
279            charCount += line.getLength();
280            for (auto c : line)
281            {
282                // Based on the byte that we are trying to emit,
283                // we may need to emit an escape sequence.
284                //
285                switch (c)
286                {
287                // The common C escape sequencs are handled directly.
288                //
289                case '"':
290                    fprintf(outputFile, "\\\"");
291                    break;
292                case '\n':
293                    fprintf(outputFile, "\\n");
294                    break;
295                case '\t':
296                    fprintf(outputFile, "\\t");
297                    break;
298                case '\\':
299                    fprintf(outputFile, "\\\\");
300                    break;
301                default:
302                    // For all other cases, we detect if the byte
303                    // is in the printable ASCII range, and emit
304                    // it directly if sco.
305                    //
306                    if (c >= 32 && c <= 126)
307                    {
308                        fputc(c, outputFile);
309                    }
310                    else
311                    {
312                        // Otherwise, we emit the byte as an octal
313                        // escape sequence, being sure to emit a
314                        // full three digits to avoid errorneous
315                        // encoding if the following byte might
316                        // represent a digit.
317                        //
318                        fprintf(outputFile, "\\%03o", c);
319                    }
320                    break;
321                }
322            }
323            fprintf(outputFile, "\\n\"\n");
324        }
325    }
326
327    void processInputFile()
328    {
329        // Note: Eventually we might support multiple input files in a
330        // single invocation of the tool, but for now we only have
331        // a single file to process.
332
333        // We derive an output path simply by appending `.cpp` to the input
334        // path, if not otherwise specified
335        char* defaultOutputPath = (char*)malloc(strlen(inputPath) + strlen(".cpp") + 1);
336        ScopedMemory outputPathCleanup(defaultOutputPath);
337        strcpy(defaultOutputPath, inputPath);
338        strcat(defaultOutputPath, ".cpp");
339        if (!outputPath)
340            outputPath = defaultOutputPath;
341
342        FILE* outputFile = fopen(outputPath, "w");
343        ScopedFile outputFileCleanup(outputFile);
344        if (!outputFile)
345        {
346            fprintf(stderr, "%s: error: failed to open '%s' for reading\n", appName, outputPath);
347            exit(1);
348        }
349
350        // We want to derive a variable name based on the name of
351        // the input file we are mbedded. Toward this end, we
352        // start by trying to strip off any leading directories
353        // in the path. This logic is ad hoc but should suffice,
354        // given that we don't plan to give the files we embed
355        // unconventional names.
356        //
357        char const* fileName = inputPath;
358        if (auto pos = strrchr(fileName, '\\'))
359            fileName = pos + 1;
360        if (auto pos = strrchr(fileName, '/'))
361            fileName = pos + 1;
362
363        // The variable name will start as a copy of the file
364        // name, although we will immediately drop any extension
365        // that comes after a `.` to trim the name further.
366        //
367        char* variableName = (char*)malloc(strlen(fileName) + 1);
368        ScopedMemory variableNameCleanup(variableName);
369        strcpy(variableName, fileName);
370        if (auto pos = strchr(variableName, '.'))
371            *pos = 0;
372
373        // We will also replace any `-` in the file name with
374        // a `_` in the generate variable name, so that the
375        // tool will be compatible with our current naming
376        // convention of using `-` as the separator in file names.
377        //
378        for (auto cursor = variableName; *cursor; ++cursor)
379        {
380            switch (*cursor)
381            {
382            default:
383                break;
384            case '-':
385                *cursor = '_';
386            }
387        }
388
389        // With all the preliminaries out of the way, the actual
390        // task of outputting the generated source file is simple.
391        //
392        fprintf(outputFile, "// generated code; do not edit\n");
393        fprintf(outputFile, "#include \"../source/core/slang-basic.h\"\n");
394
395        fprintf(outputFile, "Slang::String get_%s()\n", variableName);
396        fprintf(outputFile, "{\n");
397        fprintf(outputFile, "Slang::StringBuilder sb;\n");
398
399        // Note: For now we are embedding the file as a string
400        // literal, with full knowledge that this strategy
401        // will run into limitations in certain compilers
402        // (e.g., some versions of the Visual C++ compiler
403        // don't handle string literals larger than 64KB).
404        //
405        // TODO: Eventually we should replace this logic with
406        // code to emit a plain array of `unsigned char` with
407        // an array initializer list `{ ... }`. While some
408        // compilers have limitations or performance issues
409        // with large array literals, the practical limits
410        // appear to be higher than they are for string literals.
411
412        processInputFile(outputFile, Slang::UnownedStringSlice(inputPath));
413
414        fprintf(outputFile, ";\n");
415        fprintf(outputFile, "return sb.produceString();\n}\n");
416    }
417};
418
419int main(int argc, char** argv)
420{
421    App app;
422    app.parseOptions(argc, argv);
423    app.processInputFile();
424    return 0;
425}