diff options
| author | Tim Foley <tfoleyNV@users.noreply.github.com> | 2018-03-08 12:35:00 -0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2018-03-08 12:35:00 -0800 |
| commit | a22b7520745334ecef3cbd920a0331f01c905935 (patch) | |
| tree | f5ecf29f957f6b40f3fb5a4f987b2a1d00dd3bed /tools/slang-generate/main.cpp | |
| parent | ed718ba1048ef856efbf0d02902e4d60f173b207 (diff) | |
Cleanups on slang-generate (#437)
* Cleanups on slang-generate
There is nothing too significant in these changes, but I'm trying to get
things in place so that we can:
- Clean up the stdlib code to do less explicit `StringBuilder`
operations and instead to use more of the "template engine" approach
- Start using slang-generate for code other than the slang stdlib, so
that we can generate more of our boilerplate.
The main new functionality here is that in a template/meta file, you can
now enclose an expression in `$(...)` to indicate that is should be
spliced into the result. E.g. instead of:
class ${{ sb << someClassName; }}
{
...
}
We can now write:
class $(someClassName)
{
...
}
The other bit of new functionality is support for a whole-line statement
escape, so that instead of:
${{ for( auto a : someCollection ) { }}
void $(a)() { ... }
${{ } }}
We can instead write:
$: for(auto a : someCollection) {
void $(a)() { ... }
$: }
I haven't yet tried to use that functionality in the stdlib meta-code,
but doing so would be an obvious next step.
* Fixup: change some $P to $p
The capitalization on some of the GLSL intrinsic mappings got messed up during a find-and-replace operation when removing the double `$` that used to be required to escape things.
Diffstat (limited to 'tools/slang-generate/main.cpp')
| -rw-r--r-- | tools/slang-generate/main.cpp | 822 |
1 files changed, 587 insertions, 235 deletions
diff --git a/tools/slang-generate/main.cpp b/tools/slang-generate/main.cpp index 475849993..6b51f0e47 100644 --- a/tools/slang-generate/main.cpp +++ b/tools/slang-generate/main.cpp @@ -6,270 +6,419 @@ #include <string.h> #include "../../source/core/secure-crt.h" -struct StringSpan +#include "../../source/core/list.h" +#include "../../source/core/slang-string.h" + +using namespace Slang; + +typedef Slang::UnownedStringSlice StringSpan; + +struct Node { - char const* begin; - char const* end; + enum class Flavor + { + text, // Ordinary text to write to output + escape, // Meta-level code (statements) + splice, // Meta-level expression to splice into output + }; + + // What sort of node is this? + Flavor flavor; + + // The text of this node for `Flavor::text` + StringSpan span; + + // The body of this node for other flavors + Node* body; + + // The next node in the document + Node* next; }; -StringSpan makeEmptySpan() +void addNode( + Node**& ioLink, + Node::Flavor flavor, + char const* spanBegin, + char const* spanEnd) { - StringSpan span = { 0, 0 }; - return span; + Node* node = new Node(); + node->flavor = flavor; + node->span = StringSpan(spanBegin, spanEnd); + node->next = nullptr; + + *ioLink = node; + ioLink = &node->next; } -StringSpan makeSpan(char const* begin, char const* end) +void addNode( + Node**& ioLink, + Node::Flavor flavor, + Node* body) { - StringSpan span; - span.begin = begin; - span.end = end; - return span; + Node* node = new Node(); + node->flavor = flavor; + node->body = body; + node->next = nullptr; + + *ioLink = node; + ioLink = &node->next; } -struct Node +bool isAlpha(int c) { - // The textual range covered by this node - // (does not including the opening sigil) - StringSpan span; + return ((c >= 'a') && (c <= 'z')) + || ((c >= 'A') && (c <= 'Z')) + || (c == '_'); +} + +void addTextSpan( + Node**& ioLink, + char const* spanBegin, + char const* spanEnd) +{ + // Don't add an empty text span. + if (spanBegin == spanEnd) + return; - // The textual range of the identifier - // part of this node (if any) - StringSpan id; + addNode(ioLink, Node::Flavor::text, spanBegin, spanEnd); +} - // The textual range of the body part of - // this node (if any) - StringSpan body; +void addSpliceSpan( + Node**& ioLink, + Node* body) +{ + addNode(ioLink, Node::Flavor::splice, body); +} - // The parent of this node - Node* parent; +void addEscapeSpan( + Node**& ioLink, + Node* body) +{ + addNode(ioLink, Node::Flavor::escape, body); +} - // The first child node of this node - Node* firstChild; +void addEscapeSpan( + Node**& ioLink, + char const* spanBegin, + char const* spanEnd) +{ + Node* body = nullptr; + Node** link = &body; - // The next node belonging to the same parent - Node* nextSibling; -}; + addTextSpan(link, spanBegin, spanEnd); -struct NodeBuilder + return addEscapeSpan(ioLink, body); +} + +bool isIdentifierChar(int c) +{ + if (c >= 'a' && c <= 'z') return true; + if (c >= 'A' && c <= 'Z') return true; + if (c == '_') return true; + + return false; + +} + +struct Reader { - Node* node; - Node** childLink; - unsigned int curlyCount; - unsigned int nestedCurlyCount; + char const* cursor; + char const* end; }; -Node* createNode() +int peek(Reader const& reader) { - Node* result = (Node*) malloc(sizeof(Node)); - memset(result, 0, sizeof(Node)); - return result; + if (reader.cursor == reader.end) + return EOF; + + return *reader.cursor; } -void addNode( - NodeBuilder* builder, - Node* node) +int get(Reader& reader) { - node->parent = builder->node; + if (reader.cursor == reader.end) + return -1; - *builder->childLink = node; - builder->childLink = &node->nextSibling; + return *reader.cursor++; } -bool isAlpha(int c) +void handleNewline(Reader& reader, int c) { - return ((c >= 'a') && (c <= 'z')) - || ((c >= 'A') && (c <= 'Z')) - || (c == '_'); + int d = peek(reader); + if ((c ^ d) == ('\r' ^ '\n')) + { + get(reader); + } } -Node* readInput( - char const* inputBegin, - char const* inputEnd) +bool isHorizontalSpace(int c) { - static const int kMaxDepth = 16; - NodeBuilder nodeStack[kMaxDepth]; - NodeBuilder* nodeStackEnd = &nodeStack[kMaxDepth]; + return (c == ' ') || (c == '\t'); +} - Node* root = createNode(); - root->span.begin = inputBegin; - root->span.end = inputEnd; - root->body = root->span; +void skipHorizontalSpace(Reader& reader) +{ + while (isHorizontalSpace(peek(reader))) + get(reader); +} - NodeBuilder* builder = &nodeStack[0]; +void skipOptionalNewline(Reader& reader) +{ + switch (peek(reader)) + { + default: + break; - builder->node = root; - builder->childLink = &root->firstChild; - builder->curlyCount = (unsigned int)(-1); - builder->nestedCurlyCount = 0; + case '\r': case '\n': + { + int c = get(reader); + handleNewline(reader, c); + } + break; + } +} - char const* cursor = inputBegin; +typedef unsigned int NodeReadFlags; +enum +{ + kNodeReadFlag_AllowEscape = 1 << 0, +}; - for(;;) +Node* readBody( + Reader& reader, + NodeReadFlags flags, + char openChar, + int openCount, + char closeChar) +{ + while (peek(reader) == openChar) { - int c = *cursor; - switch(c) + get(reader); + openCount++; + } + + Node* nodes = nullptr; + Node** link = &nodes; + + bool atStartOfLine = true; + int depth = 0; + + char const* spanBegin = reader.cursor; + char const* lineBegin = reader.cursor; + for (;;) + { + int c = get(reader); + + switch (c) { default: - // ordinary text, so we continue the current span - cursor++; - continue; + atStartOfLine = false; + break; - case 0: - // possible end of input - if(cursor == inputEnd) + case EOF: { - return root; + addTextSpan(link, spanBegin, reader.cursor); + return nodes; } - // Otherwise it is just an embedded NULL - cursor++; - continue; - case '$': - // We've hit our dedicated meta-character, which means - // we are being asked to do some kind of splicing. + case '{': case '(': + if (c == openChar) + { + depth++; + } + atStartOfLine = false; + break; + + case ')': case '}': + if (c == closeChar) { - cursor++; + char const* spanEnd = reader.cursor - 1; - switch(*cursor) + if (openCount == 1) { - case '$': - // This is an escaped single `$`. - // We need to create an empty node to - // represent it + if (depth == 0) { - Node* node = createNode(); - addNode(builder, node); - node->span.begin = cursor; - cursor++; - node->span.end = cursor; - continue; + // We are at the end of the body. + addTextSpan(link, spanBegin, spanEnd); + return nodes; } - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - case '\'': - case '\"': - case ')': - // HACK: allow existing usage through - cursor++; - continue; - - default: - break; + depth--; } - - Node* node = createNode(); - addNode(builder, node); - - node->span.begin = cursor-1; - - char const* nodeBegin = cursor; - - // Piece one is an optional "identifier" section - node->id.begin = cursor; - while( isAlpha(*cursor) ) + else { - cursor++; - } - node->id.end = cursor; + // Count how many closing chars are stacked up - // Next we have an optional `{}`-delimeted span - if( *cursor == '{' ) - { - unsigned int count = 0; - while( *cursor == '{' ) + int closeCount = 1; + while (peek(reader) == closeChar) { - count++; - cursor++; + get(reader); + closeCount++; } - node->body.begin = cursor; - - assert(builder != nodeStackEnd); - builder++; - builder->node = node; - builder->childLink = &node->firstChild; - builder->curlyCount = count; - builder->nestedCurlyCount = 0; - } - else - { - node->body.begin = cursor; - node->body.end = cursor; - node->span.end = cursor; + if (closeCount == openCount) + { + // We are at the end of the body. + addTextSpan(link, spanBegin, spanEnd); + return nodes; + } } - - continue; } + atStartOfLine = false; break; - case '{': - builder->nestedCurlyCount++; - cursor++; - continue; - case '}': + case ' ': case '\t': + break; + + case '\r': case '\n': { - // Possible end of an open span + addTextSpan(link, spanBegin, reader.cursor); - unsigned int count = 0; - char const* cc = cursor; - while( *cc == '}' ) + handleNewline(reader, c); + + lineBegin = reader.cursor; + spanBegin = reader.cursor; + atStartOfLine = true; + } + break; + + case '$': + { + // If this is the start of a splice, then + // the end of the preceding raw-text space + // will be the byte before `$` + char const* spanEnd = reader.cursor - 1; + + if (peek(reader) == '(') { - count++; - cc++; + // This appears to be an expression splice. + // + // We must end the preceding span. + // + addTextSpan(link, spanBegin, spanEnd); + + Node* body = readBody( + reader, + 0, + '(', + 0, + ')'); + + addSpliceSpan(link, body); + + spanBegin = reader.cursor; + atStartOfLine = false; } + else if (peek(reader) == '{') + { + // This is the start of a block-structured escape, which will + // end at a matching `}`. - unsigned int expected = builder->curlyCount; + addTextSpan(link, spanBegin, lineBegin); - if( expected == 1 ) + Node* body = readBody( + reader, + 0, + '{', + 0, + '}'); + + addEscapeSpan(link, body); + + spanBegin = reader.cursor; + atStartOfLine = false; + } + else if (atStartOfLine && peek(reader) == ':') { - unsigned int nested = builder->nestedCurlyCount; + // This is a statement escape, which will + // continue to the end of the line. + // + // The spliced text begins *after* the `:` + get(reader); + char const* spliceBegin = reader.cursor; - // The user isn't guarding for unmatched braces, - // so some of these braces might go to cancel - // out any open braces inside this scope: - if( count > nested ) - { - // There are more available braces than our - // nesting depth, so we need to close them - // out and move on. - cursor += builder->nestedCurlyCount; - count -= builder->nestedCurlyCount; - builder->nestedCurlyCount = 0; - } - else + // The preceding text span will end at the + // start of this line. + addTextSpan(link, spanBegin, lineBegin); + + // Any indentation on this line will be ignored. + + // Read up to end of line. + for (;;) { - // These braces are only being used to close out - // nested constructs that were already opened. - builder->nestedCurlyCount -= count; - cursor += count; - continue; + int c = get(reader); + switch (c) + { + default: + continue; + + case EOF: + break; + + case '\r': + case '\n': + handleNewline(reader, c); + break; + } + + break; } - } - if(count >= expected) + addEscapeSpan(link, spliceBegin, reader.cursor); + + spanBegin = reader.cursor; + lineBegin = reader.cursor; + } + else if (atStartOfLine && isIdentifierChar(peek(reader))) { - // There are enough braces there to close out this construct + // This is a statement splice, which will use a {}-enclosed + // body for the template to generate. + + // Consume an optional identifier + while (isIdentifierChar(peek(reader))) + get(reader); + + // Consume optional horizontal space + skipHorizontalSpace(reader); - Node* node = builder->node; - node->body.end = cursor; + // Consume an optional `()`-enclosed block (strip + // all but the outer-most `()`. - cursor += expected; - node->span.end = cursor; + // optional space/newline/space before `{` + skipHorizontalSpace(reader); + skipOptionalNewline(reader); + skipHorizontalSpace(reader); - builder--; - continue; + throw 99; } else { - cursor += count; - continue; + // Doesn't seem to be a splice at all, just + // a literal `$` in the output. + atStartOfLine = false; } } + break; } - } + +} + +Node* readInput( + char const* inputBegin, + char const* inputEnd) +{ + Reader reader; + reader.cursor = inputBegin; + reader.end = inputEnd; + + return readBody( + reader, + kNodeReadFlag_AllowEscape, + -2, + 0, + -2); } void emitRaw( @@ -328,41 +477,147 @@ void emitCode( } } -void emitNode( - FILE* stream, - Node* node) +void emit( + FILE* stream, + char const* text) +{ + fprintf(stream, "%s", text); +} + +void emit( + FILE* stream, + StringSpan const& span) +{ + fprintf(stream, "%.*s", int(span.end() - span.begin()), span.begin()); +} + +bool isASCIIPrintable(int c) { - // TODO: need to look at the identifier part of the node in case - // there are custom instructions there... + return (c >= 0x20) && (c <= 0x7E); +} - char const* cursor = node->body.begin; +void emitStringLiteralText( + FILE* stream, + StringSpan const& span) +{ + char const* cursor = span.begin(); + char const* end = span.end(); - for( auto nn = node->firstChild; nn; nn = nn->nextSibling ) + while (cursor != end) { - emitCode(stream, cursor, nn->span.begin); + int c = *cursor++; + switch (c) + { + case '\r': case '\n': + fprintf(stream, "\\n"); + break; + + case '\t': + fprintf(stream, "\\t"); + break; + + case ' ': + fprintf(stream, " "); + break; + + case '"': + fprintf(stream, "\\\""); + break; + + case '\\': + fprintf(stream, "\\\\"); + break; - cursor = nn->span.end; + default: + if (isASCIIPrintable(c)) + { + fprintf(stream, "%c", c); + } + else + { + fprintf(stream, "%03u", c); + } + break; + } } +} + +void emitSimpleText( + FILE* stream, + StringSpan const& span) +{ + char const* cursor = span.begin(); + char const* end = span.end(); - emitCode(stream, cursor, node->body.end); + while (cursor != end) + { + int c = *cursor++; + switch (c) + { + default: + fprintf(stream, "%c", c); + break; + + case '\r': case '\n': + if (cursor != end) + { + int d = *cursor; + if ((c ^ d) == ('\r' ^ '\n')) + { + cursor++; + } + fprintf(stream, "\n"); + } + break; + } + } } -void emitBody( +void emitCodeNodes( FILE* stream, Node* node) { - char const* cursor = node->body.begin; + for (auto nn = node; nn; nn = nn->next) + { + switch (nn->flavor) + { + case Node::Flavor::text: + emitSimpleText(stream, nn->span); + emit(stream, "\n"); + break; - for( auto nn = node->firstChild; nn; nn = nn->nextSibling ) + default: + throw "unexpected"; + break; + } + } +} + +void emitTemplateNodes( + FILE* stream, + Node* node) +{ + for (auto nn = node; nn; nn = nn->next) { - emitRaw(stream, cursor, nn->span.begin); + switch (nn->flavor) + { + case Node::Flavor::text: + emit(stream, "SLANG_RAW(\""); + emitStringLiteralText(stream, nn->span); + emit(stream, "\")\n"); + break; - emitNode(stream, nn); + case Node::Flavor::splice: + emit(stream, "SLANG_SPLICE("); + emitCodeNodes(stream, nn->body); + emit(stream, ")\n"); + break; - cursor = nn->span.end; + case Node::Flavor::escape: + emitCodeNodes(stream, nn->body); + break; + } } - - emitRaw(stream, cursor, node->body.end); } void usage(char const* appName) @@ -406,10 +661,99 @@ void writeAllText(char const *srcFileName, char const* fileName, char* content) } } +#define PARSE_HANDLER(NAME) \ + Node* NAME(StringSpan const& text) + +typedef PARSE_HANDLER((*ParseHandler)); + +PARSE_HANDLER(parseTemplateFile) +{ + // Read a template node! + return readInput(text.begin(), text.end()); +} + +PARSE_HANDLER(parseCxxFile) +{ + // TODO: "scrape" the source file for metadata + return nullptr; +} + +PARSE_HANDLER(parseUnknownFile) +{ + // Don't process files we don't know how to handle. + return nullptr; +} + +// Information about a source file +struct SourceFile +{ + char const* inputPath; + StringSpan text; + Node* node; +}; + +Node* parseSourceFile(SourceFile* file) +{ + auto path = file->inputPath; + auto text = file->text; + + static const struct + { + char const* extension; + ParseHandler handler; + } kHandlers[] = + { + { ".meta.slang", &parseTemplateFile }, + { ".meta.cpp", &parseTemplateFile }, + { ".cpp", &parseCxxFile }, + { "", &parseUnknownFile }, + }; + + for (auto hh : kHandlers) + { + if (UnownedTerminatedStringSlice(path).endsWith(hh.extension)) + { + return hh.handler(text); + } + } + + return nullptr; +} + + + +SourceFile* parseSourceFile(char const* path) +{ + FILE* inputStream; + fopen_s(&inputStream, path, "rb"); + fseek(inputStream, 0, SEEK_END); + size_t inputSize = ftell(inputStream); + fseek(inputStream, 0, SEEK_SET); + + char* input = (char*)malloc(inputSize + 1); + fread(input, inputSize, 1, inputStream); + input[inputSize] = 0; + + char const* inputEnd = input + inputSize; + StringSpan span = StringSpan(input, inputEnd); + + SourceFile* sourceFile = new SourceFile(); + sourceFile->inputPath = path; + sourceFile->text = span; + + Node* node = parseSourceFile(sourceFile); + + sourceFile->node = node; + return sourceFile; +} + +List<SourceFile*> gSourceFiles; + int main( int argc, char** argv) { + // Parse command-line arguments. char** argCursor = argv; char** argEnd = argv + argc; @@ -419,12 +763,16 @@ int main( appName = *argCursor++; } - char const* inputPath = nullptr; - if( argCursor != argEnd ) + char** writeCursor = argv; + char const* const* inputPaths = writeCursor; + + while(argCursor != argEnd) { - inputPath = *argCursor++; + *writeCursor++ = *argCursor++; } - else + + size_t inputPathCount = writeCursor - inputPaths; + if(inputPathCount == 0) { usage(appName); exit(1); @@ -436,44 +784,48 @@ int main( exit(1); } - // Read the contents o the file and translate it into a "template" file - - FILE* inputStream; - fopen_s(&inputStream, inputPath, "rb"); - fseek(inputStream, 0, SEEK_END); - size_t inputSize = ftell(inputStream); - fseek(inputStream, 0, SEEK_SET); - - char* input = (char*) malloc(inputSize + 1); - fread(input, inputSize, 1, inputStream); - input[inputSize] = 0; - - char const* inputEnd = input + inputSize; + // Read each input file and process it according + // to the type of treatment it requires. + for (size_t ii = 0; ii < inputPathCount; ++ii) + { + char const* inputPath = inputPaths[ii]; + SourceFile* sourceFile = parseSourceFile(inputPath); + if (sourceFile) + { + gSourceFiles.Add(sourceFile); + } + } - Node* node = readInput(input, inputEnd); + // Once all inputs have been read, we can start + // to produce output files by expanding templates. + for (auto sourceFile : gSourceFiles) + { + auto inputPath = sourceFile->inputPath; + auto node = sourceFile->node; - // write output to a temporary file first - char outputPath[1024]; - sprintf_s(outputPath, "%s.temp.h", inputPath); + // write output to a temporary file first + char outputPath[1024]; + sprintf_s(outputPath, "%s.temp.h", inputPath); - FILE* outputStream; - fopen_s(&outputStream, outputPath, "w"); + FILE* outputStream; + fopen_s(&outputStream, outputPath, "w"); - emitBody(outputStream, node); + emitTemplateNodes(outputStream, node); - fclose(outputStream); + fclose(outputStream); - // update final output only when content has changed - char outputPathFinal[1024]; - sprintf_s(outputPathFinal, "%s.h", inputPath); + // update final output only when content has changed + char outputPathFinal[1024]; + sprintf_s(outputPathFinal, "%s.h", inputPath); - char * allTextOld = readAllText(outputPathFinal); - char * allTextNew = readAllText(outputPath); - if (strcmp(allTextNew, allTextOld) != 0) - { - writeAllText(inputPath, outputPathFinal, allTextNew); + char * allTextOld = readAllText(outputPathFinal); + char * allTextNew = readAllText(outputPath); + if (strcmp(allTextNew, allTextOld) != 0) + { + writeAllText(inputPath, outputPathFinal, allTextNew); + } + remove(outputPath); } - remove(outputPath); return 0; } |
