Initial work on boilerplate code generator

The goal here is to get the Slang "standard library" code out of string literals and into something a bit more like an actual code file.
This is handled by having a `slang-generate` tool that can translate a "template" file that mixes raw Slang code (or any language we want to generate...) with generation logic that is implemented in C++ (currently).

This work isn't final by any stretch of the imagination, but it moves a lot of code and not merging it ASAP will complicate other changes.
My expectation is that the generator tool will be beefed up on an as-needed basis, to get our stdlib code working.

Similarly, the stdlib code does not really take advantage of the new approach as much as it could. That is something we can clean up along the way as we do modifications of the stdlib.

4 files changed, 607 insertions, 0 deletions

diff --git a/tools/slang-generate/README.md b/tools/slang-generate/README.md
new file mode 100644
index 000000000..1daf3ec09
--- /dev/null
+++ b/tools/slang-generate/README.md
@@ -0,0 +1,4 @@
+# Boilerplate Code Generation Tool
+
+This directory contains the implementation of the `slang-generate` tool used to generate boilerplate code inside of the compiler implementation.
+
diff --git a/tools/slang-generate/main.cpp b/tools/slang-generate/main.cpp
new file mode 100644
index 000000000..17fa4edae
--- /dev/null
+++ b/tools/slang-generate/main.cpp
@@ -0,0 +1,427 @@
+// main.cpp
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct StringSpan
+{
+    char const* begin;
+    char const* end;
+};
+
+StringSpan makeEmptySpan()
+{
+    StringSpan span = { 0, 0 };
+    return span;
+}
+
+StringSpan makeSpan(char const* begin, char const* end)
+{
+    StringSpan span;
+    span.begin = begin;
+    span.end = end;
+    return span;
+}
+
+struct Node
+{
+    // The textual range covered by this node
+    // (does not including the opening sigil)
+    StringSpan  span;
+
+    // The textual range of the identifier
+    // part of this node (if any)
+    StringSpan  id;
+
+    // The textual range of the body part of
+    // this node (if any)
+    StringSpan  body;
+
+    // The parent of this node
+    Node*       parent;
+
+    // The first child node of this node
+    Node*       firstChild;
+
+    // The next node belonging to the same parent
+    Node*       nextSibling;
+};
+
+struct NodeBuilder
+{
+    Node*           node;
+    Node**          childLink;
+    unsigned int    curlyCount;
+    unsigned int    nestedCurlyCount;
+};
+
+Node* createNode()
+{
+    Node* result = (Node*) malloc(sizeof(Node));
+    memset(result, 0, sizeof(Node));
+    return result;
+}
+
+void addNode(
+    NodeBuilder*    builder,
+    Node*           node)
+{
+    node->parent = builder->node;
+
+    *builder->childLink = node;
+    builder->childLink = &node->nextSibling;
+}
+
+bool isAlpha(int c)
+{
+    return ((c >= 'a') && (c <= 'z'))
+        || ((c >= 'A') && (c <= 'Z'))
+        || (c == '_');
+}
+
+Node* readInput(
+    char const* inputBegin,
+    char const* inputEnd)
+{
+    static const int kMaxDepth = 16;
+    NodeBuilder nodeStack[kMaxDepth];
+    NodeBuilder* nodeStackEnd = &nodeStack[kMaxDepth];
+
+    Node* root = createNode();
+    root->span.begin = inputBegin;
+    root->span.end = inputEnd;
+    root->body = root->span;
+
+    NodeBuilder* builder = &nodeStack[0];
+
+    builder->node = root;
+    builder->childLink = &root->firstChild;
+    builder->curlyCount = (unsigned int)(-1);
+    builder->nestedCurlyCount = 0;
+
+    char const* cursor = inputBegin;
+
+    for(;;)
+    {
+        int c = *cursor;
+        switch(c)
+        {
+        default:
+            // ordinary text, so we continue the current span
+            cursor++;
+            continue;
+
+        case 0:
+            // possible end of input
+            if(cursor == inputEnd)
+            {
+                return root;
+            }
+            // Otherwise it is just an embedded NULL
+            cursor++;
+            continue;
+
+        case '$':
+            // We've hit our dedicated meta-character, which means
+            // we are being asked to do some kind of splicing.
+            {
+                cursor++;
+
+                switch(*cursor)
+                {
+                case '$':
+                    // This is an escaped single `$`.
+                    // We need to create an empty node to
+                    // represent it
+                    {
+                        Node* node = createNode();
+                        addNode(builder, node);
+                        node->span.begin = cursor;
+                        cursor++;
+                        node->span.end = cursor;
+                        continue;
+                    }
+
+                case '0': case '1': case '2': case '3': case '4':
+                case '5': case '6': case '7': case '8': case '9':
+                case '\'':
+                case '\"':
+                case ')':
+                    // HACK: allow existing usage through
+                    cursor++;
+                    continue;
+
+                default:
+                    break;
+                }
+
+                Node* node = createNode();
+                addNode(builder, node);
+
+                node->span.begin = cursor-1;
+
+                char const* nodeBegin = cursor;
+
+                // Piece one is an optional "identifier" section
+                node->id.begin = cursor;
+                while( isAlpha(*cursor) )
+                {
+                    cursor++;
+                }
+                node->id.end = cursor;
+
+                // Next we have an optional `{}`-delimeted span
+                if( *cursor == '{' )
+                {
+                    unsigned int count = 0;
+                    while( *cursor == '{' )
+                    {
+                        count++;
+                        cursor++;
+                    }
+                    node->body.begin = cursor;
+
+                    assert(builder != nodeStackEnd);
+
+                    builder++;
+                    builder->node = node;
+                    builder->childLink = &node->firstChild;
+                    builder->curlyCount = count;
+                    builder->nestedCurlyCount = 0;
+                }
+                else
+                {
+                    node->body.begin = cursor;
+                    node->body.end = cursor;
+                    node->span.end = cursor;
+                }
+
+                continue;
+            }
+            break;
+
+        case '{':
+            builder->nestedCurlyCount++;
+            cursor++;
+            continue;
+
+        case '}':
+            {
+                // Possible end of an open span
+
+                unsigned int count = 0;
+                char const* cc = cursor;
+                while( *cc == '}' )
+                {
+                    count++;
+                    cc++;
+                }
+
+                unsigned int expected = builder->curlyCount;
+
+                if( expected == 1 )
+                {
+                    unsigned int nested = builder->nestedCurlyCount;
+
+                    // The user isn't guarding for unmatched braces,
+                    // so some of these braces might go to cancel
+                    // out any open braces inside this scope:
+                    if( count > nested )
+                    {
+                        // There are more available braces than our
+                        // nesting depth, so we need to close them
+                        // out and move on.
+                        cursor += builder->nestedCurlyCount;
+                        count -= builder->nestedCurlyCount;
+                        builder->nestedCurlyCount = 0;
+                    }
+                    else
+                    {
+                        // These braces are only being used to close out
+                        // nested constructs that were already opened.
+                        builder->nestedCurlyCount -= count;
+                        cursor += count;
+                        continue;
+                    }
+                }
+
+                if(count >= expected)
+                {
+                    // There are enough braces there to close out this construct
+
+                    Node* node = builder->node;
+                    node->body.end = cursor;
+
+                    cursor += expected;
+                    node->span.end = cursor;
+
+                    builder--;
+                    continue;
+                }
+                else
+                {
+                    cursor += count;
+                    continue;
+                }
+            }
+        }
+
+    }
+}
+
+void emitRaw(
+    FILE* stream,
+    char const* begin,
+    char const* end)
+{
+    // We will write the raw text to our output file.
+
+    // TODO: need to output `#line` directives as well
+
+    fputs("sb << \"", stream);
+    for( char const* cc = begin; cc != end; ++cc )
+    {
+        int c = *cc;
+        switch( c )
+        {
+        case '\\':
+            fputs("\\\\", stream);
+            break;
+
+        case '\r': break;
+        case '\t': fputs("\\t", stream); break;
+        case '\"': fputs("\\\"", stream); break;
+        case '\n':
+            fputs("\\n\";\n", stream);
+            fputs("sb << \"", stream);
+            break;
+
+        default:
+            if((c >= 32) && (c <= 126))
+            {
+                fputc(c, stream);
+            }
+            else
+            {
+                assert(false);
+            }
+        }
+
+    }
+    fprintf(stream, "\";\n");
+}
+
+void emitCode(
+    FILE*   stream,
+    char const* begin,
+    char const* end)
+{
+    for( auto cc = begin; cc != end; ++cc )
+    {
+        if(*cc == '\r')
+            continue;
+
+        fputc(*cc, stream);
+    }
+}
+
+void emitNode(
+    FILE*   stream,
+    Node*   node)
+{
+    // TODO: need to look at the identifier part of the node in case
+    // there are custom instructions there...
+
+    char const* cursor = node->body.begin;
+
+    for( auto nn = node->firstChild; nn; nn = nn->nextSibling )
+    {
+        emitCode(stream, cursor, nn->span.begin);
+
+        cursor = nn->span.end;
+    }
+
+    emitCode(stream, cursor, node->body.end);
+}
+
+void emitBody(
+    FILE*   stream,
+    Node*   node)
+{
+    char const* cursor = node->body.begin;
+
+    for( auto nn = node->firstChild; nn; nn = nn->nextSibling )
+    {
+        emitRaw(stream, cursor, nn->span.begin);
+
+        emitNode(stream, nn);
+
+        cursor = nn->span.end;
+    }
+
+    emitRaw(stream, cursor, node->body.end);
+}
+
+void usage(char const* appName)
+{
+    fprintf(stderr, "usage: %s <input>\n", appName);
+}
+
+int main(
+    int     argc,
+    char**  argv)
+{
+    char** argCursor = argv;
+    char** argEnd = argv + argc;
+
+    char const* appName = "slang-generate";
+    if( argCursor != argEnd )
+    {
+        appName = *argCursor++;
+    }
+
+    char const* inputPath = nullptr;
+    if( argCursor != argEnd )
+    {
+        inputPath = *argCursor++;
+    }
+    else
+    {
+        usage(appName);
+        exit(1);
+    }
+
+    if( argCursor != argEnd )
+    {
+        usage(appName);
+        exit(1);
+    }
+
+    // Read the contents o the file and translate it into a "template" file
+
+    FILE* inputStream = fopen(inputPath, "rb");
+    fseek(inputStream, 0, SEEK_END);
+    size_t inputSize = ftell(inputStream);
+    fseek(inputStream, 0, SEEK_SET);
+
+    char* input = (char*) malloc(inputSize + 1);
+    fread(input, inputSize, 1, inputStream);
+    input[inputSize] = 0;
+
+    char const* inputEnd = input + inputSize;
+
+    Node* node = readInput(input, inputEnd);
+
+    char outputPath[1024];
+    sprintf(outputPath, "%s.cpp", inputPath);
+
+    FILE* outputStream = fopen(outputPath, "w");
+
+    emitBody(outputStream, node);
+
+    fclose(outputStream);
+
+    return 0;
+}
diff --git a/tools/slang-generate/slang-generate.vcxproj b/tools/slang-generate/slang-generate.vcxproj
new file mode 100644
index 000000000..0ba6b8fef
--- /dev/null
+++ b/tools/slang-generate/slang-generate.vcxproj
@@ -0,0 +1,154 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{66174227-8541-41FC-A6DF-4764FC66F78E}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>slanggenerate</RootNamespace>
+    <WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v140</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v140</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v140</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v140</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="Shared">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+    <Import Project="..\..\build\slang-build.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+    <Import Project="..\..\build\slang-build.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+    <Import Project="..\..\build\slang-build.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+    <Import Project="..\..\build\slang-build.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="main.cpp" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/tools/slang-generate/slang-generate.vcxproj.filters b/tools/slang-generate/slang-generate.vcxproj.filters
new file mode 100644
index 000000000..0d8d9e457
--- /dev/null
+++ b/tools/slang-generate/slang-generate.vcxproj.filters
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Source Files">
+      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+      <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    </Filter>
+    <Filter Include="Header Files">
+      <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+      <Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
+    </Filter>
+    <Filter Include="Resource Files">
+      <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+      <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="main.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+  </ItemGroup>
+</Project>
\ No newline at end of file