Added compiler-core project (#1775)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Split out compiler-core initially with just slang-source-loc.cpp

* More lexer, name, token to compiler-core.

* Split Lexer and Core diagnostics.

* Move slang-file-system to core.

* Add slang-file-system to core.

* More DownstreamCompiler into compiler-core

* Fix typo.

* Add compiler-core to bootstrap proj.

* Small fixes to premake

* For linux try with compiler-core

* Remove compiler-core from examples.

* Added NameConventionUtil to compiler-core

* Add global function to CharUtil to *hopefully* avoid linking issue.

* Hack to make linkage of CharUtil work on linux.

58 files changed, 1130 insertions, 898 deletions

diff --git a/source/compiler-core/slang-core-diagnostics.cpp b/source/compiler-core/slang-core-diagnostics.cpp
new file mode 100644
index 000000000..7aa6c5c14
--- /dev/null
+++ b/source/compiler-core/slang-core-diagnostics.cpp
@@ -0,0 +1,50 @@
+// slang-core-diagnostics.cpp
+#include "slang-core-diagnostics.h"
+
+namespace Slang {
+
+namespace MiscDiagnostics
+{
+#define DIAGNOSTIC(id, severity, name, messageFormat) const DiagnosticInfo name = { id, Severity::severity, #name, messageFormat };
+#include "slang-misc-diagnostic-defs.h"
+#undef DIAGNOSTIC
+}
+
+static const DiagnosticInfo* const kMiscDiagnostics[] =
+{
+#define DIAGNOSTIC(id, severity, name, messageFormat) &MiscDiagnostics::name, 
+#include "slang-misc-diagnostic-defs.h"
+#undef DIAGNOSTIC
+};
+
+
+namespace LexerDiagnostics
+{
+#define DIAGNOSTIC(id, severity, name, messageFormat) const DiagnosticInfo name = { id, Severity::severity, #name, messageFormat };
+#include "slang-lexer-diagnostic-defs.h"
+#undef DIAGNOSTIC
+}
+
+static const DiagnosticInfo* const kLexerDiagnostics[] =
+{
+#define DIAGNOSTIC(id, severity, name, messageFormat) &LexerDiagnostics::name, 
+#include "slang-lexer-diagnostic-defs.h"
+#undef DIAGNOSTIC
+};
+
+static DiagnosticsLookup* _newCoreDiagnosticsLookup()
+{
+    auto lookup = new DiagnosticsLookup;
+    lookup->add(kMiscDiagnostics, SLANG_COUNT_OF(kMiscDiagnostics));
+    lookup->add(kLexerDiagnostics, SLANG_COUNT_OF(kLexerDiagnostics));
+
+    return lookup;
+}
+
+DiagnosticsLookup* getCoreDiagnosticsLookup()
+{
+    static RefPtr<DiagnosticsLookup> s_lookup = _newCoreDiagnosticsLookup();
+    return s_lookup;
+}
+
+} // namespace Slang
diff --git a/source/compiler-core/slang-core-diagnostics.h b/source/compiler-core/slang-core-diagnostics.h
new file mode 100644
index 000000000..2d56591fd
--- /dev/null
+++ b/source/compiler-core/slang-core-diagnostics.h
@@ -0,0 +1,32 @@
+#ifndef SLANG_CORE_DIAGNOSTICS_H
+#define SLANG_CORE_DIAGNOSTICS_H
+
+#include "../core/slang-basic.h"
+#include "../core/slang-writer.h"
+
+#include "slang-source-loc.h"
+#include "slang-diagnostic-sink.h"
+#include "slang-token.h"
+
+#include "../../slang.h"
+
+namespace Slang
+{
+
+DiagnosticsLookup* getCoreDiagnosticsLookup();
+
+namespace MiscDiagnostics
+{
+#define DIAGNOSTIC(id, severity, name, messageFormat) extern const DiagnosticInfo name;
+#include "slang-misc-diagnostic-defs.h"
+}
+
+namespace LexerDiagnostics
+{
+#define DIAGNOSTIC(id, severity, name, messageFormat) extern const DiagnosticInfo name;
+#include "slang-lexer-diagnostic-defs.h"
+}
+
+}
+
+#endif
diff --git a/source/compiler-core/slang-diagnostic-sink.cpp b/source/compiler-core/slang-diagnostic-sink.cpp
new file mode 100644
index 000000000..2adf31f69
--- /dev/null
+++ b/source/compiler-core/slang-diagnostic-sink.cpp
@@ -0,0 +1,537 @@
+// slang-diagnostic-sink.cpp
+#include "slang-diagnostic-sink.h"
+
+#include "slang-name.h"
+#include "slang-core-diagnostics.h"
+#include "slang-name-convention-util.h"
+
+#include "../core/slang-memory-arena.h"
+#include "../core/slang-dictionary.h"
+#include "../core/slang-string-util.h"
+#include "../core/slang-char-util.h"
+
+namespace Slang {
+
+void printDiagnosticArg(StringBuilder& sb, char const* str)
+{
+    sb << str;
+}
+
+void printDiagnosticArg(StringBuilder& sb, int32_t val)
+{
+    sb << val;
+}
+
+void printDiagnosticArg(StringBuilder& sb, uint32_t val)
+{
+    sb << val;
+}
+
+void printDiagnosticArg(StringBuilder& sb, int64_t val)
+{
+    sb << val;
+}
+
+void printDiagnosticArg(StringBuilder& sb, uint64_t val)
+{
+    sb << val;
+}
+
+void printDiagnosticArg(StringBuilder& sb, double val)
+{
+    sb << val;
+}
+
+void printDiagnosticArg(StringBuilder& sb, Slang::String const& str)
+{
+    sb << str;
+}
+
+void printDiagnosticArg(StringBuilder& sb, Slang::UnownedStringSlice const& str)
+{
+    sb.append(str);
+}
+
+
+void printDiagnosticArg(StringBuilder& sb, Name* name)
+{
+    sb << getText(name);
+}
+
+
+void printDiagnosticArg(StringBuilder& sb, TokenType tokenType)
+{
+    sb << TokenTypeToString(tokenType);
+}
+
+void printDiagnosticArg(StringBuilder& sb, Token const& token)
+{
+    sb << token.getContent();
+}
+
+SourceLoc const& getDiagnosticPos(Token const& token)
+{
+    return token.loc;
+}
+
+// Take the format string for a diagnostic message, along with its arguments, and turn it into a
+static void formatDiagnosticMessage(StringBuilder& sb, char const* format, int argCount, DiagnosticArg const* const* args)
+{
+    char const* spanBegin = format;
+    for(;;)
+    {
+        char const* spanEnd = spanBegin;
+        while (int c = *spanEnd)
+        {
+            if (c == '$')
+                break;
+            spanEnd++;
+        }
+
+        sb.Append(spanBegin, int(spanEnd - spanBegin));
+        if (!*spanEnd)
+            return;
+
+        SLANG_ASSERT(*spanEnd == '$');
+        spanEnd++;
+        int d = *spanEnd++;
+        switch (d)
+        {
+        // A double dollar sign `$$` is used to emit a single `$` 
+        case '$':
+            sb.Append('$');
+            break;
+
+        // A single digit means to emit the corresponding argument.
+        // TODO: support more than 10 arguments, and add options
+        // to control formatting, etc.
+        case '0': case '1': case '2': case '3': case '4':
+        case '5': case '6': case '7': case '8': case '9':
+            {
+                int index = d - '0';
+                if (index >= argCount)
+                {
+                    // TODO(tfoley): figure out what a good policy will be for "panic" situations like this
+                    throw InvalidOperationException("too few arguments for diagnostic message");
+                }
+                else
+                {
+                    DiagnosticArg const* arg = args[index];
+                    arg->printFunc(sb, arg->data);
+                }
+            }
+            break;
+
+        default:
+            throw InvalidOperationException("invalid diagnostic message format");
+            break;
+        }
+
+        spanBegin = spanEnd;
+    }
+}
+
+static void formatDiagnostic(const HumaneSourceLoc& humaneLoc, Diagnostic const& diagnostic, StringBuilder& outBuilder)
+{
+    outBuilder << humaneLoc.pathInfo.foundPath;
+    outBuilder << "(";
+    outBuilder << Int32(humaneLoc.line);
+    outBuilder << "): ";
+
+    outBuilder << getSeverityName(diagnostic.severity);
+
+    if (diagnostic.ErrorID >= 0)
+    {
+        outBuilder << " ";
+        outBuilder << diagnostic.ErrorID;
+    }
+
+    outBuilder << ": ";
+    outBuilder << diagnostic.Message;
+    outBuilder << "\n";
+}
+
+static void _replaceTabWithSpaces(const UnownedStringSlice& slice, Int tabSize, StringBuilder& out)
+{
+    const char* start = slice.begin();
+    const char*const end = slice.end();
+
+    const Index startLength = out.getLength();
+
+    for (const char* cur = start; cur < end; cur++)
+    {
+        if (*cur == '\t')
+        {
+            if (start < cur)
+            {
+                out.append(start, cur);
+            }
+
+            // The amount of spaces we add depends on the current position.
+            const Index lastPosition = out.getLength() - startLength;
+            Index tabPosition = lastPosition;
+
+            // Strip the tabPosition so it's back to the tab stop
+            // Special case if tabSize is a power of 2
+            if ((tabSize & (tabSize - 1)) == 0)
+            {
+                tabPosition = tabPosition & ~Index(tabSize - 1);
+            }
+            else
+            {
+                tabPosition -= tabPosition % tabSize;
+            }
+
+            // Move to next tab
+            tabPosition += tabSize;
+
+            // The amount of spaces to simulate the tab
+            const Index spacesCount = tabPosition - lastPosition;
+
+            // Add the spaces
+            out.appendRepeatedChar(' ', spacesCount);
+
+            // Set the start at the first character past
+            start = cur + 1;
+        }
+    }
+
+    if (start < end)
+    {
+        out.append(start, end);
+    }
+}
+
+// Given multi-line text, and a position within the text (as a pointer into the memory of text)
+// extract the line that contains pos
+static UnownedStringSlice _extractLineContainingPosition(const UnownedStringSlice& text, const char* pos)
+{
+    SLANG_ASSERT(text.isMemoryContained(pos));
+
+    const char*const contentStart = text.begin();
+    const char*const contentEnd = text.end();
+
+    // We want to determine the start of the line, and the end of the line
+    const char* start = pos;
+    for (; start > contentStart; --start)
+    {
+        const char c = *start;
+        if (c == '\n' || c == '\r')
+        {
+            // We want the character after, but we can only do this if not already at pos
+            start += int(start < pos);
+            break;
+        }
+    }
+    const char* end = pos;
+    for (; end < contentEnd; ++end)
+    {
+        const char c = *end;
+        if (c == '\n' || c == '\r')
+        {
+            break;
+        }
+    }
+
+    return UnownedStringSlice(start, end);
+}
+
+static void _sourceLocationNoteDiagnostic(SourceView* sourceView, SourceLoc sourceLoc, DiagnosticSink::SourceLocationLexer lexer, StringBuilder& sb)
+{
+    SourceFile* sourceFile = sourceView->getSourceFile();
+    if (!sourceFile)
+    {
+        return;
+    }
+
+    UnownedStringSlice content = sourceFile->getContent();
+
+    // Make sure the offset is within content.
+    // This is important because it's possible to have a 'SourceFile' that doesn't contain any content
+    // (for example when reconstructed via serialization with just line offsets, the actual source text 'content' isn't available).
+    const int offset = sourceView->getRange().getOffset(sourceLoc);
+    if (offset < 0 || offset >= content.getLength())
+    {
+        return;
+    }
+
+    // Work out the position of the SourceLoc in the source
+    const char*const pos = content.begin() + offset;
+
+    UnownedStringSlice line = _extractLineContainingPosition(content, pos);
+
+    // Trim any trailing white space
+    line = UnownedStringSlice(line.begin(), line.trim().end());
+
+    // TODO(JS): The tab size should ideally be configurable from command line.
+    // For now just go with 4.
+    const Index tabSize = 4;
+    
+    StringBuilder sourceLine;
+    StringBuilder caretLine;
+
+    // First work out the sourceLine
+    _replaceTabWithSpaces(line, tabSize, sourceLine);
+    
+    // Now the caretLine which appears underneath the sourceLine
+    {
+        // Produce the text up to the caret position (at pos), taking into account tabs
+        _replaceTabWithSpaces(UnownedStringSlice(line.begin(), pos), tabSize, caretLine);
+
+        // Now make all spaces
+        const Index length = caretLine.getLength();
+        caretLine.Clear();
+        caretLine.appendRepeatedChar(' ', length);
+        
+        // Add caret
+        caretLine << "^";
+
+        if (lexer)
+        {
+            UnownedStringSlice token = lexer(UnownedStringSlice(pos, line.end()));
+
+            if (token.getLength() > 1)
+            {
+                caretLine.appendRepeatedChar('~', token.getLength() - 1);
+            }
+        }
+    }
+
+    // We could have handling here for if the line is too long, that we surround the important section
+    // will ellipsis for example.
+    // For now we just output.
+
+    sb << sourceLine << "\n";
+    sb << caretLine << "\n";
+}
+
+
+static void formatDiagnostic(
+    DiagnosticSink*     sink,
+    Diagnostic const&   diagnostic,
+    StringBuilder&      sb)
+{
+    auto sourceManager = sink->getSourceManager();
+
+    SourceView* sourceView = nullptr;
+    HumaneSourceLoc humaneLoc;
+    const auto sourceLoc = diagnostic.loc;
+    {
+        sourceView = sourceManager->findSourceViewRecursively(sourceLoc);
+        if (sourceView)
+        {
+            humaneLoc = sourceView->getHumaneLoc(sourceLoc);
+        }
+        formatDiagnostic(humaneLoc, diagnostic, sb);
+
+        {
+            SourceView* currentView = sourceView;
+
+            while (currentView && currentView->getInitiatingSourceLoc().isValid() && currentView->getSourceFile()->getPathInfo().type == PathInfo::Type::TokenPaste)
+            {
+                SourceView* initiatingView = sourceManager->findSourceView(currentView->getInitiatingSourceLoc());
+                if (initiatingView == nullptr)
+                {
+                    break;
+                }
+
+                const DiagnosticInfo& diagnosticInfo = MiscDiagnostics::seeTokenPasteLocation;
+
+                // Turn the message format into a message. For the moment it assumes no parameters.
+                StringBuilder msg;
+                formatDiagnosticMessage(msg, diagnosticInfo.messageFormat, 0, nullptr);
+
+                // Set up the diagnostic.
+                Diagnostic initiationDiagnostic;
+                initiationDiagnostic.ErrorID = diagnosticInfo.id;
+                initiationDiagnostic.Message = msg.ProduceString();
+                initiationDiagnostic.loc = sourceView->getInitiatingSourceLoc();
+                initiationDiagnostic.severity = diagnosticInfo.severity;
+
+                // TODO(JS):
+                // Not 100%  clear what the best sourceLoc type is most useful here - we will go with default for now
+                HumaneSourceLoc pasteHumaneLoc = initiatingView->getHumaneLoc(sourceView->getInitiatingSourceLoc());
+
+                // Okay we should output where the token paste took place
+                formatDiagnostic(pasteHumaneLoc, initiationDiagnostic, sb);
+
+                // Make the initiatingView the current view
+                currentView = initiatingView;
+            }
+        }
+    }
+
+    // We don't don't output source line information if this is a 'note' as a note is extra information for one
+    // of the other main severity types, and so the information should already be output on the initial line
+    if (sourceView && sink->isFlagSet(DiagnosticSink::Flag::SourceLocationLine) && diagnostic.severity != Severity::Note)
+    {
+       _sourceLocationNoteDiagnostic(sourceView, sourceLoc, sink->getSourceLocationLexer(), sb);
+    }
+
+    if (sourceView && sink->isFlagSet(DiagnosticSink::Flag::VerbosePath))
+    {
+        auto actualHumaneLoc = sourceView->getHumaneLoc(diagnostic.loc, SourceLocType::Actual);
+
+        // Look up the path verbosely (will get the canonical path if necessary)
+        actualHumaneLoc.pathInfo.foundPath = sourceView->getSourceFile()->calcVerbosePath();
+
+        // Only output if it's actually different
+        if (actualHumaneLoc.pathInfo.foundPath != humaneLoc.pathInfo.foundPath ||
+            actualHumaneLoc.line != humaneLoc.line ||
+            actualHumaneLoc.column != humaneLoc.column)
+        { 
+            formatDiagnostic(actualHumaneLoc, diagnostic, sb);
+        }
+    }
+}
+
+void DiagnosticSink::diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args)
+{
+    StringBuilder sb;
+    formatDiagnosticMessage(sb, info.messageFormat, argCount, args);
+
+    Diagnostic diagnostic;
+    diagnostic.ErrorID = info.id;
+    diagnostic.Message = sb.ProduceString();
+    diagnostic.loc = pos;
+    diagnostic.severity = info.severity;
+
+    if (diagnostic.severity >= Severity::Error)
+    {
+        m_errorCount++;
+    }
+
+    // Did the client supply a callback for us to use?
+    if( writer )
+    {
+        // If so, pass the error string along to them
+        StringBuilder messageBuilder;
+        formatDiagnostic(this, diagnostic, messageBuilder);
+
+        writer->write(messageBuilder.getBuffer(), messageBuilder.getLength());
+    }
+    else
+    {
+        // If the user doesn't have a callback, then just
+        // collect our diagnostic messages into a buffer
+        formatDiagnostic(this, diagnostic, outputBuffer);
+    }
+
+    if (diagnostic.severity >= Severity::Fatal)
+    {
+        // TODO: figure out a better policy for aborting compilation
+        throw AbortCompilationException();
+    }
+}
+
+void DiagnosticSink::diagnoseRaw(
+    Severity    severity,
+    char const* message)
+{
+    return diagnoseRaw(severity, UnownedStringSlice(message));
+}
+
+void DiagnosticSink::diagnoseRaw(
+    Severity    severity,
+    const UnownedStringSlice& message)
+{
+    if (severity >= Severity::Error)
+    {
+        m_errorCount++;
+    }
+
+    // Did the client supply a callback for us to use?
+    if(writer)
+    {
+        // If so, pass the error string along to them
+        writer->write(message.begin(), message.getLength());
+    }
+    else
+    {
+        // If the user doesn't have a callback, then just
+        // collect our diagnostic messages into a buffer
+        outputBuffer.append(message);
+    }
+
+    if (severity >= Severity::Fatal)
+    {
+        // TODO: figure out a better policy for aborting compilation
+        throw InvalidOperationException();
+    }
+}
+
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DiagnosticLookup !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
+
+void DiagnosticsLookup::_add(const char* name, Index index)
+{
+    UnownedStringSlice nameSlice(name);
+    m_map.Add(nameSlice, index);
+
+    // Add a dashed version (KababCase)
+    {
+        m_work.Clear();
+
+        NameConventionUtil::convert(NameConvention::Camel, nameSlice, CharCase::Lower, NameConvention::Kabab, m_work);
+
+        UnownedStringSlice dashSlice(m_arena.allocateString(m_work.getBuffer(), m_work.getLength()), m_work.getLength());
+
+        m_map.AddIfNotExists(dashSlice, index);
+    }
+}
+void DiagnosticsLookup::addAlias(const char* name, const char* diagnosticName)
+{
+    const Index index = _findDiagnosticIndex(UnownedStringSlice(diagnosticName));
+    SLANG_ASSERT(index >= 0);
+    if (index >= 0)
+    {
+        _add(name, index);
+    }
+}
+
+Index DiagnosticsLookup::add(const DiagnosticInfo* info)
+{
+    // Check it's not already added
+    SLANG_ASSERT(m_diagnostics.indexOf(info) < 0);
+
+    const Index index = m_diagnostics.getCount();
+
+    _add(info->name, index);
+
+    m_diagnostics.add(info);
+    return index;
+}
+
+void DiagnosticsLookup::add(const DiagnosticInfo*const* infos, Index infosCount)
+{
+    for (Index i = 0; i < infosCount; ++i)
+    {
+        add(infos[i]);
+    }
+}
+
+DiagnosticsLookup::DiagnosticsLookup():
+    m_arena(kArenaInitialSize)
+{
+}
+
+DiagnosticsLookup::DiagnosticsLookup(const DiagnosticInfo*const* diagnostics, Index diagnosticsCount) :
+    m_arena(kArenaInitialSize)
+{
+    m_diagnostics.addRange(diagnostics, diagnosticsCount);
+
+    // TODO: We should eventually have a more formal system for associating individual
+    // diagnostics, or groups of diagnostics, with user-exposed names for use when
+    // enabling/disabling warnings (or turning warnings into errors, etc.).
+    //
+    // For now we build a map from diagnostic name to it's entry. Two entries are typically
+    // added - the 'original name' as associated with the diagnostic in lowerCamel, and
+    // a dashified version.
+
+    for (Index i = 0; i < diagnosticsCount; ++i)
+    {
+        const DiagnosticInfo* diagnostic = diagnostics[i];
+        _add(diagnostic->name, i);
+    }
+}
+
+} // namespace Slang
diff --git a/source/compiler-core/slang-diagnostic-sink.h b/source/compiler-core/slang-diagnostic-sink.h
new file mode 100644
index 000000000..84001e6f5
--- /dev/null
+++ b/source/compiler-core/slang-diagnostic-sink.h
@@ -0,0 +1,316 @@
+#ifndef SLANG_DIAGNOSTIC_SINK_H
+#define SLANG_DIAGNOSTIC_SINK_H
+
+#include "../core/slang-basic.h"
+#include "../core/slang-writer.h"
+#include "../core/slang-memory-arena.h"
+
+#include "slang-source-loc.h"
+#include "slang-token.h"
+
+#include "../../slang.h"
+
+namespace Slang
+{
+
+enum class Severity
+{
+    Note,
+    Warning,
+    Error,
+    Fatal,
+    Internal,
+};
+
+// TODO(tfoley): move this into a source file...
+inline const char* getSeverityName(Severity severity)
+{
+    switch (severity)
+    {
+    case Severity::Note:        return "note";
+    case Severity::Warning:     return "warning";
+    case Severity::Error:       return "error";
+    case Severity::Fatal:       return "fatal error";
+    case Severity::Internal:    return "internal error";
+    default:                    return "unknown error";
+    }
+}
+
+// A structure to be used in static data describing different
+// diagnostic messages.
+struct DiagnosticInfo
+{
+    int id;
+    Severity severity;
+    char const* name;               ///< Unique name
+    char const* messageFormat;
+};
+
+class Diagnostic
+{
+public:
+    String Message;
+    SourceLoc loc;
+    int ErrorID;
+    Severity severity;
+
+    Diagnostic()
+    {
+        ErrorID = -1;
+    }
+    Diagnostic(
+        const String & msg,
+        int id,
+        const SourceLoc & pos,
+        Severity severity)
+        : severity(severity)
+    {
+        Message = msg;
+        ErrorID = id;
+        loc = pos;
+    }
+};
+
+class Name;
+
+void printDiagnosticArg(StringBuilder& sb, char const* str);
+
+void printDiagnosticArg(StringBuilder& sb, int32_t val);
+void printDiagnosticArg(StringBuilder& sb, uint32_t val);
+
+void printDiagnosticArg(StringBuilder& sb, int64_t val);
+void printDiagnosticArg(StringBuilder& sb, uint64_t val);
+
+void printDiagnosticArg(StringBuilder& sb, double val);
+
+void printDiagnosticArg(StringBuilder& sb, Slang::String const& str);
+void printDiagnosticArg(StringBuilder& sb, Slang::UnownedStringSlice const& str);
+void printDiagnosticArg(StringBuilder& sb, Name* name);
+
+void printDiagnosticArg(StringBuilder& sb, TokenType tokenType);
+void printDiagnosticArg(StringBuilder& sb, Token const& token);
+
+struct IRInst;
+void printDiagnosticArg(StringBuilder& sb, IRInst* irObject);
+    
+template<typename T>
+void printDiagnosticArg(StringBuilder& sb, RefPtr<T> ptr)
+{
+    printDiagnosticArg(sb, ptr.Ptr());
+}
+
+inline SourceLoc const& getDiagnosticPos(SourceLoc const& pos) { return pos;  }
+
+SourceLoc const& getDiagnosticPos(Token const& token);
+    
+
+template<typename T>
+SourceLoc getDiagnosticPos(RefPtr<T> const& ptr)
+{
+    return getDiagnosticPos(ptr.Ptr());
+}
+
+struct DiagnosticArg
+{
+    void* data;
+    void (*printFunc)(StringBuilder&, void*);
+
+    template<typename T>
+    struct Helper
+    {
+        static void printFunc(StringBuilder& sb, void* data) { printDiagnosticArg(sb, *(T*)data); }
+    };
+
+    template<typename T>
+    DiagnosticArg(T const& arg)
+        : data((void*)&arg)
+        , printFunc(&Helper<T>::printFunc)
+    {}
+};
+
+    
+class DiagnosticSink
+{
+public:
+        /// Flags to control some aspects of Diagnostic sink behavior
+    typedef uint32_t Flags;
+    struct Flag 
+    {
+        enum Enum: Flags
+        {
+            VerbosePath = 0x1,                  ///< Will display a more verbose path (if available) - such as a canonical or absolute path
+            SourceLocationLine = 0x2,           ///< If set will display the location line if source is available
+        };
+    };
+
+        /// Used by diagnostic sink to be able to underline tokens. If not defined on the DiagnosticSink,
+        /// will only display a caret at the SourceLoc
+    typedef UnownedStringSlice(*SourceLocationLexer)(const UnownedStringSlice& text);
+
+        /// Get the total amount of errors that have taken place on this DiagnosticSink
+    SLANG_FORCE_INLINE int getErrorCount() { return m_errorCount; }
+
+    void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info)
+    {
+        diagnoseImpl(pos, info, 0, nullptr);
+    }
+
+    void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0)
+    {
+        DiagnosticArg const* args[] = { &arg0 };
+        diagnoseImpl(pos, info, 1, args);
+    }
+
+    void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1)
+    {
+        DiagnosticArg const* args[] = { &arg0, &arg1 };
+        diagnoseImpl(pos, info, 2, args);
+    }
+
+    void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1, DiagnosticArg const& arg2)
+    {
+        DiagnosticArg const* args[] = { &arg0, &arg1, &arg2 };
+        diagnoseImpl(pos, info, 3, args);
+    }
+
+    void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1, DiagnosticArg const& arg2, DiagnosticArg const& arg3)
+    {
+        DiagnosticArg const* args[] = { &arg0, &arg1, &arg2, &arg3 };
+        diagnoseImpl(pos, info, 4, args);
+    }
+
+    template<typename P, typename... Args>
+    void diagnose(P const& pos, DiagnosticInfo const& info, Args const&... args )
+    {
+        diagnoseDispatch(getDiagnosticPos(pos), info, args...);
+    }
+
+        // Add a diagnostic with raw text
+        // (used when we get errors from a downstream compiler)
+    void diagnoseRaw(Severity severity, char const* message);
+    void diagnoseRaw(Severity severity, const UnownedStringSlice& message);
+
+        /// During propagation of an exception for an internal
+        /// error, note that this source location was involved
+    void noteInternalErrorLoc(SourceLoc const& loc);
+
+        /// Create a blob containing diagnostics if there were any errors.
+        /// *note* only works if writer is not set, the blob is created from outputBuffer
+    SlangResult getBlobIfNeeded(ISlangBlob** outBlob);
+
+        /// Get the source manager used 
+    SourceManager* getSourceManager() const { return m_sourceManager; }
+        /// Set the source manager used for lookup of source locs
+    void setSourceManager(SourceManager* inSourceManager) { m_sourceManager = inSourceManager; }
+
+        /// Get the flags
+    Flags getFlags() const { return m_flags; }
+        /// Set a flag
+    void setFlag(Flag::Enum flag) { m_flags |= Flags(flag); }
+        /// Reset a flag
+    void resetFlag(Flag::Enum flag) { m_flags &= ~Flags(flag); }
+        /// Test if flag is set
+    bool isFlagSet(Flag::Enum flag) { return (m_flags & Flags(flag)) != 0; }
+
+        /// Get the (optional) diagnostic sink lexer. This is used to
+        /// improve quality of highlighting a locations token. If not set, will just have a single
+        /// character caret at location
+    SourceLocationLexer getSourceLocationLexer() const { return m_sourceLocationLexer; }
+
+        /// Ctor
+    DiagnosticSink(SourceManager* sourceManager, SourceLocationLexer sourceLocationLexer)
+        : m_sourceManager(sourceManager),
+            m_sourceLocationLexer(sourceLocationLexer)
+    {
+        // If we have a source location lexer, we'll by default enable source location output
+        if (sourceLocationLexer)
+        {
+            setFlag(Flag::SourceLocationLine);
+        }
+    }
+
+    // Public members
+
+        /// The outputBuffer will contain any diagnostics *iff* the writer is *not* set
+    StringBuilder outputBuffer;
+        /// If a writer is set output will *not* be written to the outputBuffer
+    ISlangWriter* writer = nullptr;
+
+protected:
+    void diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args);
+
+    int m_errorCount = 0;
+    int m_internalErrorLocsNoted = 0;
+
+    Flags m_flags = 0;
+
+    // The source manager to use when mapping source locations to file+line info
+    SourceManager* m_sourceManager = nullptr;
+
+    SourceLocationLexer m_sourceLocationLexer;
+};
+
+    /// An `ISlangWriter` that writes directly to a diagnostic sink.
+class DiagnosticSinkWriter : public AppendBufferWriter
+{
+public:
+    typedef AppendBufferWriter Super;
+
+    DiagnosticSinkWriter(DiagnosticSink* sink)
+        : Super(WriterFlag::IsStatic)
+        , m_sink(sink)
+    {}
+
+    // ISlangWriter
+    SLANG_NO_THROW virtual SlangResult SLANG_MCALL write(const char* chars, size_t numChars) SLANG_OVERRIDE
+    {
+        m_sink->diagnoseRaw(Severity::Note, UnownedStringSlice(chars, chars+numChars));
+        return SLANG_OK;
+    }
+
+private:
+    DiagnosticSink* m_sink = nullptr;
+};
+
+class DiagnosticsLookup : public RefObject
+{
+public:
+    static const Index kArenaInitialSize = 2048;
+
+    const DiagnosticInfo* findDiagostic(const UnownedStringSlice& slice) const
+    {
+        const Index* indexPtr = m_map.TryGetValue(slice);
+        return indexPtr ? m_diagnostics[*indexPtr] : nullptr;
+    }
+    Index _findDiagnosticIndex(const UnownedStringSlice& slice) const
+    {
+        const Index* indexPtr = m_map.TryGetValue(slice);
+        return indexPtr ? *indexPtr : 0;
+    }
+
+        /// info must stay in scope
+    Index add(const DiagnosticInfo* info);
+    void add(const DiagnosticInfo*const* infos, Index infosCount);
+
+    void addAlias(const char* name, const char* diagnosticName);
+
+        /// Get the diagnostics held in this lookup
+    const List<const DiagnosticInfo*>& getDiagnostics() const { return m_diagnostics; }
+
+        /// NOTE! diagnostics must stay in scope for lifetime of lookup
+    DiagnosticsLookup(const DiagnosticInfo*const* diagnostics, Index diagnosticsCount);
+    DiagnosticsLookup();
+
+protected:  
+    void _add(const char* name, Index index);
+
+    List<const DiagnosticInfo*> m_diagnostics;
+
+    StringBuilder m_work;
+    Dictionary<UnownedStringSlice, Index> m_map;
+    MemoryArena m_arena;
+};
+
+}
+
+#endif
diff --git a/source/core/slang-downstream-compiler.cpp b/source/compiler-core/slang-downstream-compiler.cpp
index 1a81fcac8..696376b49 100644
--- a/source/core/slang-downstream-compiler.cpp
+++ b/source/compiler-core/slang-downstream-compiler.cpp
@@ -1,15 +1,15 @@
 // slang-downstream-compiler.cpp
 #include "slang-downstream-compiler.h"
 
-#include "slang-common.h"
+#include "../core/slang-common.h"
 #include "../../slang-com-helper.h"
-#include "slang-string-util.h"
+#include "../core/slang-string-util.h"
 
-#include "slang-type-text-util.h"
+#include "../core/slang-type-text-util.h"
 
-#include "slang-io.h"
-#include "slang-shared-library.h"
-#include "slang-blob.h"
+#include "../core/slang-io.h"
+#include "../core/slang-shared-library.h"
+#include "../core/slang-blob.h"
 
 #ifdef SLANG_VC
 #   include "windows/slang-win-visual-studio-util.h"
diff --git a/source/core/slang-downstream-compiler.h b/source/compiler-core/slang-downstream-compiler.h
index 9c3b9055f..35acc820e 100644
--- a/source/core/slang-downstream-compiler.h
+++ b/source/compiler-core/slang-downstream-compiler.h
@@ -1,15 +1,15 @@
 #ifndef SLANG_DOWNSTREAM_COMPILER_H
 #define SLANG_DOWNSTREAM_COMPILER_H
 
-#include "slang-common.h"
-#include "slang-string.h"
+#include "../core/slang-common.h"
+#include "../core/slang-string.h"
 
-#include "slang-process-util.h"
+#include "../core/slang-process-util.h"
 
-#include "slang-platform.h"
-#include "slang-semantic-version.h"
+#include "../core/slang-platform.h"
+#include "../core/slang-semantic-version.h"
 
-#include "slang-io.h"
+#include "../core/slang-io.h"
 
 #include "../../slang-com-ptr.h"
 
diff --git a/source/core/slang-gcc-compiler-util.cpp b/source/compiler-core/slang-gcc-compiler-util.cpp
index 56955f1da..a51d5735f 100644
--- a/source/core/slang-gcc-compiler-util.cpp
+++ b/source/compiler-core/slang-gcc-compiler-util.cpp
@@ -1,12 +1,12 @@
 // slang-gcc-compiler-util.cpp
 #include "slang-gcc-compiler-util.h"
 
-#include "slang-common.h"
+#include "../core/slang-common.h"
 #include "../../slang-com-helper.h"
-#include "slang-string-util.h"
+#include "../core/slang-string-util.h"
 
-#include "slang-io.h"
-#include "slang-shared-library.h"
+#include "../core/slang-io.h"
+#include "../core/slang-shared-library.h"
 
 namespace Slang
 {
diff --git a/source/core/slang-gcc-compiler-util.h b/source/compiler-core/slang-gcc-compiler-util.h
index b97144e35..b97144e35 100644
--- a/source/core/slang-gcc-compiler-util.h
+++ b/source/compiler-core/slang-gcc-compiler-util.h
diff --git a/source/compiler-core/slang-lexer-diagnostic-defs.h b/source/compiler-core/slang-lexer-diagnostic-defs.h
new file mode 100644
index 000000000..666ab057f
--- /dev/null
+++ b/source/compiler-core/slang-lexer-diagnostic-defs.h
@@ -0,0 +1,33 @@
+// The file is meant to be included multiple times, to produce different
+// pieces of declaration/definition code related to diagnostic messages
+//
+// Each diagnostic is declared here with:
+//
+//     DIAGNOSTIC(id, severity, name, messageFormat)
+//
+// Where `id` is the unique diagnostic ID, `severity` is the default
+// severity (from the `Severity` enum), `name` is a name used to refer
+// to this diagnostic from code, and `messageFormat` is the default
+// (non-localized) message for the diagnostic, with placeholders
+// for any arguments.
+
+#ifndef DIAGNOSTIC
+#error Need to #define DIAGNOSTIC(...) before including 
+#define DIAGNOSTIC(id, severity, name, messageFormat) /* */
+#endif
+
+//
+// 1xxxx - Lexical analysis
+//
+
+DIAGNOSTIC(10000, Error, illegalCharacterPrint, "illegal character '$0'")
+DIAGNOSTIC(10000, Error, illegalCharacterHex, "illegal character (0x$0)")
+DIAGNOSTIC(10001, Error, illegalCharacterLiteral, "illegal character literal")
+
+DIAGNOSTIC(10002, Warning, octalLiteral, "'0' prefix indicates octal literal")
+DIAGNOSTIC(10003, Error, invalidDigitForBase, "invalid digit for base-$1 literal: '$0'")
+
+DIAGNOSTIC(10004, Error, endOfFileInLiteral, "end of file in literal")
+DIAGNOSTIC(10005, Error, newlineInLiteral, "newline in literal")
+
+#undef DIAGNOSTIC
diff --git a/source/slang/slang-lexer.cpp b/source/compiler-core/slang-lexer.cpp
index 6c8e9474a..83b8e0eec 100644
--- a/source/slang/slang-lexer.cpp
+++ b/source/compiler-core/slang-lexer.cpp
@@ -8,7 +8,7 @@
 #include "slang-name.h"
 #include "slang-source-loc.h"
 
-#include <assert.h>
+#include "slang-core-diagnostics.h"
 
 namespace Slang
 {
@@ -363,7 +363,7 @@ namespace Slang
             if(digitVal >= base)
             {
                 char buffer[] = { (char) c, 0 };
-                lexer->m_sink->diagnose(_getSourceLoc(lexer), Diagnostics::invalidDigitForBase, buffer, base);
+                lexer->m_sink->diagnose(_getSourceLoc(lexer), LexerDiagnostics::invalidDigitForBase, buffer, base);
             }
 
             _advance(lexer);
@@ -680,11 +680,11 @@ namespace Slang
             switch(c)
             {
             case kEOF:
-                lexer->m_sink->diagnose(_getSourceLoc(lexer), Diagnostics::endOfFileInLiteral);
+                lexer->m_sink->diagnose(_getSourceLoc(lexer), LexerDiagnostics::endOfFileInLiteral);
                 return;
 
             case '\n': case '\r':
-                lexer->m_sink->diagnose(_getSourceLoc(lexer), Diagnostics::newlineInLiteral);
+                lexer->m_sink->diagnose(_getSourceLoc(lexer), LexerDiagnostics::newlineInLiteral);
                 return;
 
             case '\\':
@@ -981,7 +981,7 @@ namespace Slang
 
                 case '0': case '1': case '2': case '3': case '4':
                 case '5': case '6': case '7': case '8': case '9':
-                    lexer->m_sink->diagnose(loc, Diagnostics::octalLiteral);
+                    lexer->m_sink->diagnose(loc, LexerDiagnostics::octalLiteral);
                     return _lexNumber(lexer, 8);
                 }
             }
@@ -1195,12 +1195,12 @@ namespace Slang
                 if(c >= 0x20 && c <=  0x7E)
                 {
                     char buffer[] = { (char) c, 0 };
-                    sink->diagnose(loc, Diagnostics::illegalCharacterPrint, buffer);
+                    sink->diagnose(loc, LexerDiagnostics::illegalCharacterPrint, buffer);
                 }
                 else
                 {
                     // Fallback: print as hexadecimal
-                    sink->diagnose(loc, Diagnostics::illegalCharacterHex, String((unsigned char)c, 16));
+                    sink->diagnose(loc, LexerDiagnostics::illegalCharacterHex, String((unsigned char)c, 16));
                 }
             }
 
diff --git a/source/slang/slang-lexer.h b/source/compiler-core/slang-lexer.h
index f1fe89516..f9a698568 100644
--- a/source/slang/slang-lexer.h
+++ b/source/compiler-core/slang-lexer.h
@@ -2,7 +2,7 @@
 #define SLANG_LEXER_H
 
 #include "../core/slang-basic.h"
-#include "slang-diagnostics.h"
+#include "slang-diagnostic-sink.h"
 
 namespace Slang
 {
diff --git a/source/compiler-core/slang-misc-diagnostic-defs.h b/source/compiler-core/slang-misc-diagnostic-defs.h
new file mode 100644
index 000000000..a1fd01475
--- /dev/null
+++ b/source/compiler-core/slang-misc-diagnostic-defs.h
@@ -0,0 +1,27 @@
+//
+
+// The file is meant to be included multiple times, to produce different
+// pieces of declaration/definition code related to diagnostic messages
+//
+// Each diagnostic is declared here with:
+//
+//     DIAGNOSTIC(id, severity, name, messageFormat)
+//
+// Where `id` is the unique diagnostic ID, `severity` is the default
+// severity (from the `Severity` enum), `name` is a name used to refer
+// to this diagnostic from code, and `messageFormat` is the default
+// (non-localized) message for the diagnostic, with placeholders
+// for any arguments.
+
+#ifndef DIAGNOSTIC
+#error Need to #define DIAGNOSTIC(...) before including 
+#define DIAGNOSTIC(id, severity, name, messageFormat) /* */
+#endif
+
+//
+// -1 - Notes that decorate another diagnostic.
+//
+
+DIAGNOSTIC(-1, Note, seeTokenPasteLocation, "see token pasted location")
+
+#undef DIAGNOSTIC
diff --git a/source/core/slang-name-convention-util.cpp b/source/compiler-core/slang-name-convention-util.cpp
index a5acc6370..930e00e18 100644
--- a/source/core/slang-name-convention-util.cpp
+++ b/source/compiler-core/slang-name-convention-util.cpp
@@ -1,8 +1,8 @@
 
 #include "slang-name-convention-util.h"
 
-#include "slang-char-util.h"
-#include "slang-string-util.h"
+#include "../core/slang-char-util.h"
+#include "../core/slang-string-util.h"
 
 namespace Slang
 {
diff --git a/source/core/slang-name-convention-util.h b/source/compiler-core/slang-name-convention-util.h
index d4a984ca0..d84ffd3b7 100644
--- a/source/core/slang-name-convention-util.h
+++ b/source/compiler-core/slang-name-convention-util.h
@@ -1,8 +1,8 @@
-#ifndef SLANG_CORE_NAME_CONVENTION_UTIL_H
-#define SLANG_CORE_NAME_CONVENTION_UTIL_H
+#ifndef SLANG_COMPILER_CORE_NAME_CONVENTION_UTIL_H
+#define SLANG_COMPILER_CORE_NAME_CONVENTION_UTIL_H
 
-#include "slang-string.h"
-#include "slang-list.h"
+#include "../core/slang-string.h"
+#include "../core/slang-list.h"
 
 namespace Slang
 {
diff --git a/source/slang/slang-name.cpp b/source/compiler-core/slang-name.cpp
index b6035982b..b6035982b 100644
--- a/source/slang/slang-name.cpp
+++ b/source/compiler-core/slang-name.cpp
diff --git a/source/slang/slang-name.h b/source/compiler-core/slang-name.h
index cf702686b..cf702686b 100644
--- a/source/slang/slang-name.h
+++ b/source/compiler-core/slang-name.h
diff --git a/source/core/slang-nvrtc-compiler.cpp b/source/compiler-core/slang-nvrtc-compiler.cpp
index eb117379f..b581d21fd 100644
--- a/source/core/slang-nvrtc-compiler.cpp
+++ b/source/compiler-core/slang-nvrtc-compiler.cpp
@@ -1,17 +1,17 @@
 // slang-nvrtc-compiler.cpp
 #include "slang-nvrtc-compiler.h"
 
-#include "slang-common.h"
+#include "../core/slang-common.h"
 #include "../../slang-com-helper.h"
 
 #include "../core/slang-blob.h"
 
-#include "slang-string-util.h"
-#include "slang-string-slice-pool.h"
+#include "../core/slang-string-util.h"
+#include "../core/slang-string-slice-pool.h"
 
-#include "slang-io.h"
-#include "slang-shared-library.h"
-#include "slang-semantic-version.h"
+#include "../core/slang-io.h"
+#include "../core/slang-shared-library.h"
+#include "../core/slang-semantic-version.h"
 
 
 namespace nvrtc
diff --git a/source/core/slang-nvrtc-compiler.h b/source/compiler-core/slang-nvrtc-compiler.h
index 48c6d4da6..48c6d4da6 100644
--- a/source/core/slang-nvrtc-compiler.h
+++ b/source/compiler-core/slang-nvrtc-compiler.h
diff --git a/source/slang/slang-source-loc.cpp b/source/compiler-core/slang-source-loc.cpp
index 4b589fbf3..4b589fbf3 100644
--- a/source/slang/slang-source-loc.cpp
+++ b/source/compiler-core/slang-source-loc.cpp
diff --git a/source/slang/slang-source-loc.h b/source/compiler-core/slang-source-loc.h
index 54811918f..54811918f 100644
--- a/source/slang/slang-source-loc.h
+++ b/source/compiler-core/slang-source-loc.h
diff --git a/source/slang/slang-token-defs.h b/source/compiler-core/slang-token-defs.h
index 6cece330e..6cece330e 100644
--- a/source/slang/slang-token-defs.h
+++ b/source/compiler-core/slang-token-defs.h
diff --git a/source/slang/slang-token.cpp b/source/compiler-core/slang-token.cpp
index a7f6d7d62..576b08d66 100644
--- a/source/slang/slang-token.cpp
+++ b/source/compiler-core/slang-token.cpp
@@ -1,13 +1,10 @@
 // slang-token.cpp
 #include "slang-token.h"
 
-#include <assert.h>
+//#include <assert.h>
 
 namespace Slang {
 
-
-
-
 char const* TokenTypeToString(TokenType type)
 {
     switch( type )
diff --git a/source/slang/slang-token.h b/source/compiler-core/slang-token.h
index 9697a5c2d..9697a5c2d 100644
--- a/source/slang/slang-token.h
+++ b/source/compiler-core/slang-token.h
diff --git a/source/core/slang-visual-studio-compiler-util.cpp b/source/compiler-core/slang-visual-studio-compiler-util.cpp
index a3578483f..fd879e63f 100644
--- a/source/core/slang-visual-studio-compiler-util.cpp
+++ b/source/compiler-core/slang-visual-studio-compiler-util.cpp
@@ -1,16 +1,16 @@
 // slang-visual-studio-compiler-util.cpp
 #include "slang-visual-studio-compiler-util.h"
 
-#include "slang-common.h"
+#include "../core/slang-common.h"
 #include "../../slang-com-helper.h"
-#include "slang-string-util.h"
+#include "../core/slang-string-util.h"
 
 // if Visual Studio import the visual studio platform specific header
 #if SLANG_VC
 #   include "windows/slang-win-visual-studio-util.h"
 #endif
 
-#include "slang-io.h"
+#include "../core/slang-io.h"
 
 namespace Slang
 {
diff --git a/source/core/slang-visual-studio-compiler-util.h b/source/compiler-core/slang-visual-studio-compiler-util.h
index 018dde212..018dde212 100644
--- a/source/core/slang-visual-studio-compiler-util.h
+++ b/source/compiler-core/slang-visual-studio-compiler-util.h
diff --git a/source/core/windows/slang-win-visual-studio-util.cpp b/source/compiler-core/windows/slang-win-visual-studio-util.cpp
index 078406d99..0dc8ea764 100644
--- a/source/core/windows/slang-win-visual-studio-util.cpp
+++ b/source/compiler-core/windows/slang-win-visual-studio-util.cpp
@@ -1,8 +1,8 @@
 #include "slang-win-visual-studio-util.h"
 
-#include "../slang-common.h"
-#include "../slang-process-util.h"
-#include "../slang-string-util.h"
+#include "../../core/slang-common.h"
+#include "../../core/slang-process-util.h"
+#include "../../core/slang-string-util.h"
 
 #include "../slang-visual-studio-compiler-util.h"
 
diff --git a/source/core/windows/slang-win-visual-studio-util.h b/source/compiler-core/windows/slang-win-visual-studio-util.h
index 530d18582..05a3ea061 100644
--- a/source/core/windows/slang-win-visual-studio-util.h
+++ b/source/compiler-core/windows/slang-win-visual-studio-util.h
@@ -1,10 +1,10 @@
 #ifndef SLANG_WIN_VISUAL_STUDIO_UTIL_H
 #define SLANG_WIN_VISUAL_STUDIO_UTIL_H
 
-#include "../slang-list.h"
-#include "../slang-string.h"
+#include "../../core/slang-list.h"
+#include "../../core/slang-string.h"
 
-#include "../slang-process-util.h"
+#include "../../core/slang-process-util.h"
 
 #include "../slang-downstream-compiler.h"
 
diff --git a/source/core/slang-char-util.cpp b/source/core/slang-char-util.cpp
index 298f9b75f..ea9e6dbf2 100644
--- a/source/core/slang-char-util.cpp
+++ b/source/core/slang-char-util.cpp
@@ -2,10 +2,8 @@
 
 namespace Slang {
 
-static const CharUtil::CharFlagMap _calcCharFlagsMap()
+/* static */CharUtil::CharFlagMap CharUtil::makeCharFlagMap()
 {
-    typedef CharUtil::Flag Flag;
-
     CharUtil::CharFlagMap map;
     memset(&map, 0, sizeof(map));
 
@@ -48,6 +46,11 @@ static const CharUtil::CharFlagMap _calcCharFlagsMap()
     return map;
 }
 
-/* static */const CharUtil::CharFlagMap CharUtil::g_charFlagMap = _calcCharFlagsMap();
+/* static */int CharUtil::_ensureLink()
+{
+    return makeCharFlagMap().flags[0];
+}
+
+/* static */const CharUtil::CharFlagMap CharUtil::g_charFlagMap = makeCharFlagMap();
 
 } // namespace Slang
diff --git a/source/core/slang-char-util.h b/source/core/slang-char-util.h
index 2434697d4..8f7f69c90 100644
--- a/source/core/slang-char-util.h
+++ b/source/core/slang-char-util.h
@@ -41,12 +41,19 @@ struct CharUtil
         /// Given a character return the upper case equivalent
     SLANG_FORCE_INLINE static char toUpper(char c) { return (c >= 'a' && c <= 'z') ? (c -'a' + 'A') : c; }
 
-    
+
     struct CharFlagMap
     {
         Flags flags[0x100];
     };
 
+    static CharFlagMap makeCharFlagMap();
+
+        // HACK!
+        // JS: Many of the inlined functions of CharUtil just access a global map. That referencing this global is *NOT* enough to
+        // link correctly with CharUtil on linux for a shared library. Caling this function can force linkage.
+    static int _ensureLink();
+
     static const CharFlagMap g_charFlagMap;
 };
     
diff --git a/source/slang/slang-file-system.cpp b/source/core/slang-file-system.cpp
index 992ae4155..992ae4155 100644
--- a/source/slang/slang-file-system.cpp
+++ b/source/core/slang-file-system.cpp
diff --git a/source/slang/slang-file-system.h b/source/core/slang-file-system.h
index d5145404d..d5145404d 100644
--- a/source/slang/slang-file-system.h
+++ b/source/core/slang-file-system.h
diff --git a/source/core/slang-string.cpp b/source/core/slang-string.cpp
index 62bc19754..0a5b7d260 100644
--- a/source/core/slang-string.cpp
+++ b/source/core/slang-string.cpp
@@ -5,6 +5,12 @@
 
 namespace Slang
 {
+    // HACK!
+    // JS: Many of the inlined functions of CharUtil just access a global map. That referencing this global is *NOT* enough to
+    // link correctly with CharUtil on linux for a shared library. The following call exists to try and force linkage of CharUtil
+    // for anything that uses core
+    static const auto s_charUtilLink = CharUtil::_ensureLink();
+
     // TODO: this belongs in a different file:
 
     SLANG_RETURN_NEVER void signalUnexpectedError(char const* message)
diff --git a/source/slang/slang-ast-support-types.h b/source/slang/slang-ast-support-types.h
index 1921a101f..168d358d6 100644
--- a/source/slang/slang-ast-support-types.h
+++ b/source/slang/slang-ast-support-types.h
@@ -2,7 +2,10 @@
 #define SLANG_AST_SUPPORT_TYPES_H
 
 #include "../core/slang-basic.h"
-#include "slang-lexer.h"
+
+#include "../compiler-core/slang-lexer.h"
+#include "../compiler-core/slang-name.h"
+
 #include "slang-profile.h"
 #include "slang-type-system-shared.h"
 #include "../../slang.h"
@@ -16,7 +19,6 @@
 #include "slang-ast-reflect.h"
 #include "slang-ref-object-reflect.h"
 
-#include "slang-name.h"
 
 #include <assert.h>
 
diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp
index 2d0287613..c034f5f7d 100755
--- a/source/slang/slang-compiler.cpp
+++ b/source/slang/slang-compiler.cpp
@@ -10,7 +10,9 @@
 
 #include "slang-check.h"
 #include "slang-compiler.h"
-#include "slang-lexer.h"
+
+#include "../compiler-core/slang-lexer.h"
+
 #include "slang-lower-to-ir.h"
 #include "slang-mangle.h"
 #include "slang-parameter-binding.h"
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index 9ef75ab86..b1e3c3a70 100755
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -3,9 +3,11 @@
 
 #include "../core/slang-basic.h"
 #include "../core/slang-shared-library.h"
-
-#include "../core/slang-downstream-compiler.h"
 #include "../core/slang-archive-file-system.h"
+#include "../core/slang-file-system.h"
+
+#include "../compiler-core/slang-downstream-compiler.h"
+#include "../compiler-core/slang-name.h"
 
 #include "../core/slang-std-writers.h"
 
@@ -13,12 +15,11 @@
 
 #include "slang-capability.h"
 #include "slang-diagnostics.h"
-#include "slang-name.h"
+
 #include "slang-preprocessor.h"
 #include "slang-profile.h"
 #include "slang-syntax.h"
 
-#include "slang-file-system.h"
 
 #include "slang-include-system.h"
 
diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h
index 6387ff8b2..ef5a94501 100644
--- a/source/slang/slang-diagnostic-defs.h
+++ b/source/slang/slang-diagnostic-defs.h
@@ -43,7 +43,6 @@ DIAGNOSTIC(-1, Note, doYouForgetToMakeComponentAccessible, "do you forget to mak
 DIAGNOSTIC(-1, Note, seeDeclarationOf, "see declaration of '$0'")
 DIAGNOSTIC(-1, Note, seeOtherDeclarationOf, "see other declaration of '$0'")
 DIAGNOSTIC(-1, Note, seePreviousDeclarationOf, "see previous declaration of '$0'")
-DIAGNOSTIC(-1, Note, seeTokenPasteLocation, "see token pasted location")
 DIAGNOSTIC(-1, Note, includeOutput, "include $0")
 
 //
@@ -129,21 +128,6 @@ DIAGNOSTIC(    88, Error, unknownArchiveType, "archive type '%0' is unknown")
 DIAGNOSTIC(  100, Error, failedToLoadDownstreamCompiler, "failed to load downstream compiler '$0'")
 DIAGNOSTIC(99999, Note, noteFailedToLoadDynamicLibrary, "failed to load dynamic library '$0'")
 
-
-//
-// 1xxxx - Lexical analysis
-//
-
-DIAGNOSTIC(10000, Error, illegalCharacterPrint, "illegal character '$0'")
-DIAGNOSTIC(10000, Error, illegalCharacterHex, "illegal character (0x$0)")
-DIAGNOSTIC(10001, Error, illegalCharacterLiteral, "illegal character literal")
-
-DIAGNOSTIC(10002, Warning, octalLiteral, "'0' prefix indicates octal literal")
-DIAGNOSTIC(10003, Error, invalidDigitForBase, "invalid digit for base-$1 literal: '$0'")
-
-DIAGNOSTIC(10004, Error, endOfFileInLiteral, "end of file in literal")
-DIAGNOSTIC(10005, Error, newlineInLiteral, "newline in literal")
-
 //
 // 15xxx - Preprocessing
 //
diff --git a/source/slang/slang-diagnostics.cpp b/source/slang/slang-diagnostics.cpp
index 4b450c9a9..4a831aacf 100644
--- a/source/slang/slang-diagnostics.cpp
+++ b/source/slang/slang-diagnostics.cpp
@@ -1,473 +1,15 @@
 // slang-diagnostics.cpp
 #include "slang-diagnostics.h"
 
-#include "slang-name.h"
-
 #include "../core/slang-memory-arena.h"
 #include "../core/slang-dictionary.h"
 #include "../core/slang-string-util.h"
-#include "../core/slang-char-util.h"
-#include "../core/slang-name-convention-util.h"
-
-#include <assert.h>
-
-#ifdef _WIN32
-#define WIN32_LEAN_AND_MEAN
-#define NOMINMAX
-#include <Windows.h>
-#undef WIN32_LEAN_AND_MEAN
-#undef NOMINMAX
-#include <d3dcompiler.h>
-#endif
-
-namespace Slang {
-
-void printDiagnosticArg(StringBuilder& sb, char const* str)
-{
-    sb << str;
-}
-
-void printDiagnosticArg(StringBuilder& sb, int32_t val)
-{
-    sb << val;
-}
-
-void printDiagnosticArg(StringBuilder& sb, uint32_t val)
-{
-    sb << val;
-}
-
-void printDiagnosticArg(StringBuilder& sb, int64_t val)
-{
-    sb << val;
-}
-
-void printDiagnosticArg(StringBuilder& sb, uint64_t val)
-{
-    sb << val;
-}
-
-void printDiagnosticArg(StringBuilder& sb, double val)
-{
-    sb << val;
-}
-
-void printDiagnosticArg(StringBuilder& sb, Slang::String const& str)
-{
-    sb << str;
-}
-
-void printDiagnosticArg(StringBuilder& sb, Slang::UnownedStringSlice const& str)
-{
-    sb.append(str);
-}
-
-
-void printDiagnosticArg(StringBuilder& sb, Name* name)
-{
-    sb << getText(name);
-}
-
-
-void printDiagnosticArg(StringBuilder& sb, TokenType tokenType)
-{
-    sb << TokenTypeToString(tokenType);
-}
-
-void printDiagnosticArg(StringBuilder& sb, Token const& token)
-{
-    sb << token.getContent();
-}
-
-SourceLoc const& getDiagnosticPos(Token const& token)
-{
-    return token.loc;
-}
-
-// Take the format string for a diagnostic message, along with its arguments, and turn it into a
-static void formatDiagnosticMessage(StringBuilder& sb, char const* format, int argCount, DiagnosticArg const* const* args)
-{
-    char const* spanBegin = format;
-    for(;;)
-    {
-        char const* spanEnd = spanBegin;
-        while (int c = *spanEnd)
-        {
-            if (c == '$')
-                break;
-            spanEnd++;
-        }
-
-        sb.Append(spanBegin, int(spanEnd - spanBegin));
-        if (!*spanEnd)
-            return;
-
-        SLANG_ASSERT(*spanEnd == '$');
-        spanEnd++;
-        int d = *spanEnd++;
-        switch (d)
-        {
-        // A double dollar sign `$$` is used to emit a single `$` 
-        case '$':
-            sb.Append('$');
-            break;
-
-        // A single digit means to emit the corresponding argument.
-        // TODO: support more than 10 arguments, and add options
-        // to control formatting, etc.
-        case '0': case '1': case '2': case '3': case '4':
-        case '5': case '6': case '7': case '8': case '9':
-            {
-                int index = d - '0';
-                if (index >= argCount)
-                {
-                    // TODO(tfoley): figure out what a good policy will be for "panic" situations like this
-                    throw InvalidOperationException("too few arguments for diagnostic message");
-                }
-                else
-                {
-                    DiagnosticArg const* arg = args[index];
-                    arg->printFunc(sb, arg->data);
-                }
-            }
-            break;
-
-        default:
-            throw InvalidOperationException("invalid diagnostic message format");
-            break;
-        }
-
-        spanBegin = spanEnd;
-    }
-}
-
-static void formatDiagnostic(const HumaneSourceLoc& humaneLoc, Diagnostic const& diagnostic, StringBuilder& outBuilder)
-{
-    outBuilder << humaneLoc.pathInfo.foundPath;
-    outBuilder << "(";
-    outBuilder << Int32(humaneLoc.line);
-    outBuilder << "): ";
-
-    outBuilder << getSeverityName(diagnostic.severity);
-
-    if (diagnostic.ErrorID >= 0)
-    {
-        outBuilder << " ";
-        outBuilder << diagnostic.ErrorID;
-    }
-
-    outBuilder << ": ";
-    outBuilder << diagnostic.Message;
-    outBuilder << "\n";
-}
-
-static void _replaceTabWithSpaces(const UnownedStringSlice& slice, Int tabSize, StringBuilder& out)
-{
-    const char* start = slice.begin();
-    const char*const end = slice.end();
-
-    const Index startLength = out.getLength();
-
-    for (const char* cur = start; cur < end; cur++)
-    {
-        if (*cur == '\t')
-        {
-            if (start < cur)
-            {
-                out.append(start, cur);
-            }
-
-            // The amount of spaces we add depends on the current position.
-            const Index lastPosition = out.getLength() - startLength;
-            Index tabPosition = lastPosition;
-
-            // Strip the tabPosition so it's back to the tab stop
-            // Special case if tabSize is a power of 2
-            if ((tabSize & (tabSize - 1)) == 0)
-            {
-                tabPosition = tabPosition & ~Index(tabSize - 1);
-            }
-            else
-            {
-                tabPosition -= tabPosition % tabSize;
-            }
-
-            // Move to next tab
-            tabPosition += tabSize;
-
-            // The amount of spaces to simulate the tab
-            const Index spacesCount = tabPosition - lastPosition;
-
-            // Add the spaces
-            out.appendRepeatedChar(' ', spacesCount);
-
-            // Set the start at the first character past
-            start = cur + 1;
-        }
-    }
-
-    if (start < end)
-    {
-        out.append(start, end);
-    }
-}
-
-// Given multi-line text, and a position within the text (as a pointer into the memory of text)
-// extract the line that contains pos
-static UnownedStringSlice _extractLineContainingPosition(const UnownedStringSlice& text, const char* pos)
-{
-    SLANG_ASSERT(text.isMemoryContained(pos));
-
-    const char*const contentStart = text.begin();
-    const char*const contentEnd = text.end();
-
-    // We want to determine the start of the line, and the end of the line
-    const char* start = pos;
-    for (; start > contentStart; --start)
-    {
-        const char c = *start;
-        if (c == '\n' || c == '\r')
-        {
-            // We want the character after, but we can only do this if not already at pos
-            start += int(start < pos);
-            break;
-        }
-    }
-    const char* end = pos;
-    for (; end < contentEnd; ++end)
-    {
-        const char c = *end;
-        if (c == '\n' || c == '\r')
-        {
-            break;
-        }
-    }
-
-    return UnownedStringSlice(start, end);
-}
-
-static void _sourceLocationNoteDiagnostic(SourceView* sourceView, SourceLoc sourceLoc, DiagnosticSink::SourceLocationLexer lexer, StringBuilder& sb)
-{
-    SourceFile* sourceFile = sourceView->getSourceFile();
-    if (!sourceFile)
-    {
-        return;
-    }
-
-    UnownedStringSlice content = sourceFile->getContent();
-
-    // Make sure the offset is within content.
-    // This is important because it's possible to have a 'SourceFile' that doesn't contain any content
-    // (for example when reconstructed via serialization with just line offsets, the actual source text 'content' isn't available).
-    const int offset = sourceView->getRange().getOffset(sourceLoc);
-    if (offset < 0 || offset >= content.getLength())
-    {
-        return;
-    }
-
-    // Work out the position of the SourceLoc in the source
-    const char*const pos = content.begin() + offset;
-
-    UnownedStringSlice line = _extractLineContainingPosition(content, pos);
-
-    // Trim any trailing white space
-    line = UnownedStringSlice(line.begin(), line.trim().end());
-
-    // TODO(JS): The tab size should ideally be configurable from command line.
-    // For now just go with 4.
-    const Index tabSize = 4;
-    
-    StringBuilder sourceLine;
-    StringBuilder caretLine;
-
-    // First work out the sourceLine
-    _replaceTabWithSpaces(line, tabSize, sourceLine);
-    
-    // Now the caretLine which appears underneath the sourceLine
-    {
-        // Produce the text up to the caret position (at pos), taking into account tabs
-        _replaceTabWithSpaces(UnownedStringSlice(line.begin(), pos), tabSize, caretLine);
 
-        // Now make all spaces
-        const Index length = caretLine.getLength();
-        caretLine.Clear();
-        caretLine.appendRepeatedChar(' ', length);
-        
-        // Add caret
-        caretLine << "^";
+#include "../compiler-core/slang-name.h"
+#include "../compiler-core/slang-core-diagnostics.h"
 
-        if (lexer)
-        {
-            UnownedStringSlice token = lexer(UnownedStringSlice(pos, line.end()));
-
-            if (token.getLength() > 1)
-            {
-                caretLine.appendRepeatedChar('~', token.getLength() - 1);
-            }
-        }
-    }
-
-    // We could have handling here for if the line is too long, that we surround the important section
-    // will ellipsis for example.
-    // For now we just output.
-
-    sb << sourceLine << "\n";
-    sb << caretLine << "\n";
-}
-
-static void formatDiagnostic(
-    DiagnosticSink*     sink,
-    Diagnostic const&   diagnostic,
-    StringBuilder&      sb)
+namespace Slang
 {
-    auto sourceManager = sink->getSourceManager();
-
-    SourceView* sourceView = nullptr;
-    HumaneSourceLoc humaneLoc;
-    const auto sourceLoc = diagnostic.loc;
-    {
-        sourceView = sourceManager->findSourceViewRecursively(sourceLoc);
-        if (sourceView)
-        {
-            humaneLoc = sourceView->getHumaneLoc(sourceLoc);
-        }
-        formatDiagnostic(humaneLoc, diagnostic, sb);
-
-        {
-            SourceView* currentView = sourceView;
-
-            while (currentView && currentView->getInitiatingSourceLoc().isValid() && currentView->getSourceFile()->getPathInfo().type == PathInfo::Type::TokenPaste)
-            {
-                SourceView* initiatingView = sourceManager->findSourceView(currentView->getInitiatingSourceLoc());
-                if (initiatingView == nullptr)
-                {
-                    break;
-                }
-
-                const DiagnosticInfo& diagnosticInfo = Diagnostics::seeTokenPasteLocation;
-
-                // Turn the message format into a message. For the moment it assumes no parameters.
-                StringBuilder msg;
-                formatDiagnosticMessage(msg, diagnosticInfo.messageFormat, 0, nullptr);
-
-                // Set up the diagnostic.
-                Diagnostic initiationDiagnostic;
-                initiationDiagnostic.ErrorID = diagnosticInfo.id;
-                initiationDiagnostic.Message = msg.ProduceString();
-                initiationDiagnostic.loc = sourceView->getInitiatingSourceLoc();
-                initiationDiagnostic.severity = diagnosticInfo.severity;
-
-                // TODO(JS):
-                // Not 100%  clear what the best sourceLoc type is most useful here - we will go with default for now
-                HumaneSourceLoc pasteHumaneLoc = initiatingView->getHumaneLoc(sourceView->getInitiatingSourceLoc());
-
-                // Okay we should output where the token paste took place
-                formatDiagnostic(pasteHumaneLoc, initiationDiagnostic, sb);
-
-                // Make the initiatingView the current view
-                currentView = initiatingView;
-            }
-        }
-    }
-
-    // We don't don't output source line information if this is a 'note' as a note is extra information for one
-    // of the other main severity types, and so the information should already be output on the initial line
-    if (sourceView && sink->isFlagSet(DiagnosticSink::Flag::SourceLocationLine) && diagnostic.severity != Severity::Note)
-    {
-       _sourceLocationNoteDiagnostic(sourceView, sourceLoc, sink->getSourceLocationLexer(), sb);
-    }
-
-    if (sourceView && sink->isFlagSet(DiagnosticSink::Flag::VerbosePath))
-    {
-        auto actualHumaneLoc = sourceView->getHumaneLoc(diagnostic.loc, SourceLocType::Actual);
-
-        // Look up the path verbosely (will get the canonical path if necessary)
-        actualHumaneLoc.pathInfo.foundPath = sourceView->getSourceFile()->calcVerbosePath();
-
-        // Only output if it's actually different
-        if (actualHumaneLoc.pathInfo.foundPath != humaneLoc.pathInfo.foundPath ||
-            actualHumaneLoc.line != humaneLoc.line ||
-            actualHumaneLoc.column != humaneLoc.column)
-        { 
-            formatDiagnostic(actualHumaneLoc, diagnostic, sb);
-        }
-    }
-}
-
-void DiagnosticSink::diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args)
-{
-    StringBuilder sb;
-    formatDiagnosticMessage(sb, info.messageFormat, argCount, args);
-
-    Diagnostic diagnostic;
-    diagnostic.ErrorID = info.id;
-    diagnostic.Message = sb.ProduceString();
-    diagnostic.loc = pos;
-    diagnostic.severity = info.severity;
-
-    if (diagnostic.severity >= Severity::Error)
-    {
-        m_errorCount++;
-    }
-
-    // Did the client supply a callback for us to use?
-    if( writer )
-    {
-        // If so, pass the error string along to them
-        StringBuilder messageBuilder;
-        formatDiagnostic(this, diagnostic, messageBuilder);
-
-        writer->write(messageBuilder.getBuffer(), messageBuilder.getLength());
-    }
-    else
-    {
-        // If the user doesn't have a callback, then just
-        // collect our diagnostic messages into a buffer
-        formatDiagnostic(this, diagnostic, outputBuffer);
-    }
-
-    if (diagnostic.severity >= Severity::Fatal)
-    {
-        // TODO: figure out a better policy for aborting compilation
-        throw AbortCompilationException();
-    }
-}
-
-void DiagnosticSink::diagnoseRaw(
-    Severity    severity,
-    char const* message)
-{
-    return diagnoseRaw(severity, UnownedStringSlice(message));
-}
-
-void DiagnosticSink::diagnoseRaw(
-    Severity    severity,
-    const UnownedStringSlice& message)
-{
-    if (severity >= Severity::Error)
-    {
-        m_errorCount++;
-    }
-
-    // Did the client supply a callback for us to use?
-    if(writer)
-    {
-        // If so, pass the error string along to them
-        writer->write(message.begin(), message.getLength());
-    }
-    else
-    {
-        // If the user doesn't have a callback, then just
-        // collect our diagnostic messages into a buffer
-        outputBuffer.append(message);
-    }
-
-    if (severity >= Severity::Fatal)
-    {
-        // TODO: figure out a better policy for aborting compilation
-        throw InvalidOperationException();
-    }
-}
 
 namespace Diagnostics
 {
@@ -476,89 +18,41 @@ namespace Diagnostics
 #undef DIAGNOSTIC
 }
 
-static const DiagnosticInfo* const kAllDiagnostics[] =
+static const DiagnosticInfo* const kCompilerDiagnostics[] =
 {
 #define DIAGNOSTIC(id, severity, name, messageFormat) &Diagnostics::name, 
 #include "slang-diagnostic-defs.h"
 #undef DIAGNOSTIC
 };
 
-class DiagnosticsLookup : public RefObject
+static DiagnosticsLookup* _newDiagnosticsLookup()
 {
-public:
-    const DiagnosticInfo* findDiagostic(const UnownedStringSlice& slice) const
-    {
-        const Index* indexPtr = m_map.TryGetValue(slice);
-        return indexPtr ? kAllDiagnostics[*indexPtr] : nullptr;
-    }
-    Index _findDiagnosticIndex(const UnownedStringSlice& slice) const
-    {
-        const Index* indexPtr = m_map.TryGetValue(slice);
-        return indexPtr ? *indexPtr : 0;
-    }
-    static DiagnosticsLookup* getSingleton()
-    {
-        static RefPtr<DiagnosticsLookup> singleton = new DiagnosticsLookup;
-        return singleton;
-    }
-
-protected:
-    void _add(const char* name, Index index)
-    {
-        UnownedStringSlice nameSlice(name);
-        m_map.Add(nameSlice, index);
-
-        // Add a dashed version (KababCase)
-        {
-            m_work.Clear();
+    DiagnosticsLookup* lookup = new DiagnosticsLookup(kCompilerDiagnostics, SLANG_COUNT_OF(kCompilerDiagnostics));
 
-            NameConventionUtil::convert(NameConvention::Camel, nameSlice, CharCase::Lower, NameConvention::Kabab, m_work);
-
-            UnownedStringSlice dashSlice(m_arena.allocateString(m_work.getBuffer(), m_work.getLength()), m_work.getLength());
-            m_map.AddIfNotExists(dashSlice, index);
-        }
-    }
-    void _addAlias(const char* name, const char* diagnosticName)
+    // Add all the diagnostics in 'core'
+    DiagnosticsLookup* coreLookup = getCoreDiagnosticsLookup();
+    if (coreLookup)
     {
-        const Index index = _findDiagnosticIndex(UnownedStringSlice(diagnosticName));
-        SLANG_ASSERT(index >= 0);
-        if (index >= 0)
+        for (auto diagnostic : coreLookup->getDiagnostics())
         {
-            _add(name, index);
+            lookup->add(diagnostic);
         }
     }
-    DiagnosticsLookup();
 
-    StringBuilder m_work;
-    Dictionary<UnownedStringSlice, Index> m_map;
-    MemoryArena m_arena;
-};
+    // Add the alias
+    lookup->addAlias("overlappingBindings", "parameterBindingsOverlap");
+    return lookup;
+}
 
-DiagnosticsLookup::DiagnosticsLookup():
-    m_arena(2048)
+static DiagnosticsLookup* _getDiagnosticLookupSingleton()
 {
-    // TODO: We should eventually have a more formal system for associating individual
-    // diagnostics, or groups of diagnostics, with user-exposed names for use when
-    // enabling/disabling warnings (or turning warnings into errors, etc.).
-    //
-    // For now we build a map from diagnostic name to it's entry. Two entries are typically
-    // added - the 'original name' as associated with the diagnostic in lowerCamel, and
-    // a dashified version.
-    
-    for (Index i = 0; i < SLANG_COUNT_OF(kAllDiagnostics); ++i)
-    {
-        const DiagnosticInfo* diagnostic = kAllDiagnostics[i];
-        _add(diagnostic->name, i);
-    }
-
-    // Add any aliases
-    _addAlias("overlappingBindings", "parameterBindingsOverlap");
+    static RefPtr<DiagnosticsLookup> s_lookup = _newDiagnosticsLookup();
+    return s_lookup;
 }
 
 DiagnosticInfo const* findDiagnosticByName(UnownedStringSlice const& name)
 {
-    return DiagnosticsLookup::getSingleton()->findDiagostic(name);
+    return _getDiagnosticLookupSingleton()->findDiagostic(name);
 }
 
-
 } // namespace Slang
diff --git a/source/slang/slang-diagnostics.h b/source/slang/slang-diagnostics.h
index 05080989a..05411a90a 100644
--- a/source/slang/slang-diagnostics.h
+++ b/source/slang/slang-diagnostics.h
@@ -1,280 +1,17 @@
-#ifndef RASTER_RENDERER_COMPILE_ERROR_H
-#define RASTER_RENDERER_COMPILE_ERROR_H
+#ifndef SLANG_DIAGNOSTICS_H
+#define SLANG_DIAGNOSTICS_H
 
 #include "../core/slang-basic.h"
 #include "../core/slang-writer.h"
 
-#include "slang-source-loc.h"
-#include "slang-token.h"
+#include "../compiler-core/slang-source-loc.h"
+#include "../compiler-core/slang-diagnostic-sink.h"
+#include "../compiler-core/slang-token.h"
 
 #include "../../slang.h"
 
 namespace Slang
 {
-    enum class Severity
-    {
-        Note,
-        Warning,
-        Error,
-        Fatal,
-        Internal,
-    };
-
-    // TODO(tfoley): move this into a source file...
-    inline const char* getSeverityName(Severity severity)
-    {
-        switch (severity)
-        {
-        case Severity::Note:        return "note";
-        case Severity::Warning:     return "warning";
-        case Severity::Error:       return "error";
-        case Severity::Fatal:       return "fatal error";
-        case Severity::Internal:    return "internal error";
-        default:                    return "unknown error";
-        }
-    }
-
-    // A structure to be used in static data describing different
-    // diagnostic messages.
-    struct DiagnosticInfo
-    {
-        int id;
-        Severity severity;
-        char const* name;               ///< Unique name
-        char const* messageFormat;
-    };
-
-    class Diagnostic
-    {
-    public:
-        String Message;
-        SourceLoc loc;
-        int ErrorID;
-        Severity severity;
-
-        Diagnostic()
-        {
-            ErrorID = -1;
-        }
-        Diagnostic(
-            const String & msg,
-            int id,
-            const SourceLoc & pos,
-            Severity severity)
-            : severity(severity)
-        {
-            Message = msg;
-            ErrorID = id;
-            loc = pos;
-        }
-    };
-
-    class Name;
-
-    //enum class CodeGenTarget;
-
-    //enum class Stage : SlangStage;
-    //enum class ProfileVersion;
-
-    void printDiagnosticArg(StringBuilder& sb, char const* str);
-
-    void printDiagnosticArg(StringBuilder& sb, int32_t val);
-    void printDiagnosticArg(StringBuilder& sb, uint32_t val);
-
-    void printDiagnosticArg(StringBuilder& sb, int64_t val);
-    void printDiagnosticArg(StringBuilder& sb, uint64_t val);
-
-    void printDiagnosticArg(StringBuilder& sb, double val);
-
-    void printDiagnosticArg(StringBuilder& sb, Slang::String const& str);
-    void printDiagnosticArg(StringBuilder& sb, Slang::UnownedStringSlice const& str);
-    void printDiagnosticArg(StringBuilder& sb, Name* name);
-
-    void printDiagnosticArg(StringBuilder& sb, TokenType tokenType);
-    void printDiagnosticArg(StringBuilder& sb, Token const& token);
-
-    struct IRInst;
-    void printDiagnosticArg(StringBuilder& sb, IRInst* irObject);
-    
-    template<typename T>
-    void printDiagnosticArg(StringBuilder& sb, RefPtr<T> ptr)
-    {
-        printDiagnosticArg(sb, ptr.Ptr());
-    }
-
-    inline SourceLoc const& getDiagnosticPos(SourceLoc const& pos) { return pos;  }
-
-    SourceLoc const& getDiagnosticPos(Token const& token);
-    
-
-    template<typename T>
-    SourceLoc getDiagnosticPos(RefPtr<T> const& ptr)
-    {
-        return getDiagnosticPos(ptr.Ptr());
-    }
-
-    struct DiagnosticArg
-    {
-        void* data;
-        void (*printFunc)(StringBuilder&, void*);
-
-        template<typename T>
-        struct Helper
-        {
-            static void printFunc(StringBuilder& sb, void* data) { printDiagnosticArg(sb, *(T*)data); }
-        };
-
-        template<typename T>
-        DiagnosticArg(T const& arg)
-            : data((void*)&arg)
-            , printFunc(&Helper<T>::printFunc)
-        {}
-    };
-
-    
-    class DiagnosticSink
-    {
-    public:
-            /// Flags to control some aspects of Diagnostic sink behavior
-        typedef uint32_t Flags;
-        struct Flag 
-        {
-            enum Enum: Flags
-            {
-                VerbosePath = 0x1,                  ///< Will display a more verbose path (if available) - such as a canonical or absolute path
-                SourceLocationLine = 0x2,           ///< If set will display the location line if source is available
-            };
-        };
-
-            /// Used by diagnostic sink to be able to underline tokens. If not defined on the DiagnosticSink,
-            /// will only display a caret at the SourceLoc
-        typedef UnownedStringSlice(*SourceLocationLexer)(const UnownedStringSlice& text);
-
-            /// Get the total amount of errors that have taken place on this DiagnosticSink
-        SLANG_FORCE_INLINE int getErrorCount() { return m_errorCount; }
-
-        void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info)
-        {
-            diagnoseImpl(pos, info, 0, nullptr);
-        }
-
-        void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0)
-        {
-            DiagnosticArg const* args[] = { &arg0 };
-            diagnoseImpl(pos, info, 1, args);
-        }
-
-        void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1)
-        {
-            DiagnosticArg const* args[] = { &arg0, &arg1 };
-            diagnoseImpl(pos, info, 2, args);
-        }
-
-        void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1, DiagnosticArg const& arg2)
-        {
-            DiagnosticArg const* args[] = { &arg0, &arg1, &arg2 };
-            diagnoseImpl(pos, info, 3, args);
-        }
-
-        void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1, DiagnosticArg const& arg2, DiagnosticArg const& arg3)
-        {
-            DiagnosticArg const* args[] = { &arg0, &arg1, &arg2, &arg3 };
-            diagnoseImpl(pos, info, 4, args);
-        }
-
-        template<typename P, typename... Args>
-        void diagnose(P const& pos, DiagnosticInfo const& info, Args const&... args )
-        {
-            diagnoseDispatch(getDiagnosticPos(pos), info, args...);
-        }
-
-            // Add a diagnostic with raw text
-            // (used when we get errors from a downstream compiler)
-        void diagnoseRaw(Severity severity, char const* message);
-        void diagnoseRaw(Severity severity, const UnownedStringSlice& message);
-
-            /// During propagation of an exception for an internal
-            /// error, note that this source location was involved
-        void noteInternalErrorLoc(SourceLoc const& loc);
-
-            /// Create a blob containing diagnostics if there were any errors.
-            /// *note* only works if writer is not set, the blob is created from outputBuffer
-        SlangResult getBlobIfNeeded(ISlangBlob** outBlob);
-
-            /// Get the source manager used 
-        SourceManager* getSourceManager() const { return m_sourceManager; }
-            /// Set the source manager used for lookup of source locs
-        void setSourceManager(SourceManager* inSourceManager) { m_sourceManager = inSourceManager; }
-
-            /// Get the flags
-        Flags getFlags() const { return m_flags; }
-            /// Set a flag
-        void setFlag(Flag::Enum flag) { m_flags |= Flags(flag); }
-            /// Reset a flag
-        void resetFlag(Flag::Enum flag) { m_flags &= ~Flags(flag); }
-            /// Test if flag is set
-        bool isFlagSet(Flag::Enum flag) { return (m_flags & Flags(flag)) != 0; }
-
-            /// Get the (optional) diagnostic sink lexer. This is used to
-            /// improve quality of highlighting a locations token. If not set, will just have a single
-            /// character caret at location
-        SourceLocationLexer getSourceLocationLexer() const { return m_sourceLocationLexer; }
-
-            /// Ctor
-        DiagnosticSink(SourceManager* sourceManager, SourceLocationLexer sourceLocationLexer)
-            : m_sourceManager(sourceManager),
-              m_sourceLocationLexer(sourceLocationLexer)
-        {
-            // If we have a source location lexer, we'll by default enable source location output
-            if (sourceLocationLexer)
-            {
-                setFlag(Flag::SourceLocationLine);
-            }
-        }
-
-        // Public members
-
-            /// The outputBuffer will contain any diagnostics *iff* the writer is *not* set
-        StringBuilder outputBuffer;
-            /// If a writer is set output will *not* be written to the outputBuffer
-        ISlangWriter* writer = nullptr;
-
-    protected:
-        void diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args);
-
-        int m_errorCount = 0;
-        int m_internalErrorLocsNoted = 0;
-
-        Flags m_flags = 0;
-
-        // The source manager to use when mapping source locations to file+line info
-        SourceManager* m_sourceManager = nullptr;
-
-        SourceLocationLexer m_sourceLocationLexer;
-    };
-
-        /// An `ISlangWriter` that writes directly to a diagnostic sink.
-    class DiagnosticSinkWriter : public AppendBufferWriter
-    {
-    public:
-        typedef AppendBufferWriter Super;
-
-        DiagnosticSinkWriter(DiagnosticSink* sink)
-            : Super(WriterFlag::IsStatic)
-            , m_sink(sink)
-        {}
-
-        // ISlangWriter
-        SLANG_NO_THROW virtual SlangResult SLANG_MCALL write(const char* chars, size_t numChars) SLANG_OVERRIDE
-        {
-            m_sink->diagnoseRaw(Severity::Note, UnownedStringSlice(chars, chars+numChars));
-            return SLANG_OK;
-        }
-
-    private:
-        DiagnosticSink* m_sink = nullptr;
-    };
-
     DiagnosticInfo const* findDiagnosticByName(UnownedStringSlice const& name);
 
     namespace Diagnostics
diff --git a/source/slang/slang-emit-c-like.cpp b/source/slang/slang-emit-c-like.cpp
index a0fa71fee..1b0fe7c44 100644
--- a/source/slang/slang-emit-c-like.cpp
+++ b/source/slang/slang-emit-c-like.cpp
@@ -2,6 +2,8 @@
 #include "slang-emit-c-like.h"
 
 #include "../core/slang-writer.h"
+#include "../compiler-core/slang-name.h"
+
 #include "slang-ir-bind-existentials.h"
 #include "slang-ir-dce.h"
 #include "slang-ir-entry-point-uniforms.h"
@@ -17,7 +19,7 @@
 #include "slang-legalize-types.h"
 #include "slang-lower-to-ir.h"
 #include "slang-mangle.h"
-#include "slang-name.h"
+
 #include "slang-syntax.h"
 #include "slang-type-layout.h"
 #include "slang-visitor.h"
diff --git a/source/slang/slang-emit.cpp b/source/slang/slang-emit.cpp
index c97412cf0..01b682a39 100644
--- a/source/slang/slang-emit.cpp
+++ b/source/slang/slang-emit.cpp
@@ -4,6 +4,8 @@
 #include "../core/slang-writer.h"
 #include "../core/slang-type-text-util.h"
 
+#include "../compiler-core/slang-name.h"
+
 #include "slang-ir-bind-existentials.h"
 #include "slang-ir-byte-address-legalize.h"
 #include "slang-ir-collect-global-uniforms.h"
@@ -33,7 +35,7 @@
 #include "slang-legalize-types.h"
 #include "slang-lower-to-ir.h"
 #include "slang-mangle.h"
-#include "slang-name.h"
+
 #include "slang-syntax.h"
 #include "slang-type-layout.h"
 #include "slang-visitor.h"
diff --git a/source/slang/slang-include-system.h b/source/slang/slang-include-system.h
index 3b368b70d..70f6dd81e 100644
--- a/source/slang/slang-include-system.h
+++ b/source/slang/slang-include-system.h
@@ -2,7 +2,7 @@
 #define SLANG_INCLUDE_SYSTEM_H
 // slang-include-system.h
 
-#include "slang-source-loc.h"
+#include "../compiler-core/slang-source-loc.h"
 
 namespace Slang
 {
diff --git a/source/slang/slang-ir-legalize-types.cpp b/source/slang/slang-ir-legalize-types.cpp
index 5d26741c0..9b95e069d 100644
--- a/source/slang/slang-ir-legalize-types.cpp
+++ b/source/slang/slang-ir-legalize-types.cpp
@@ -10,12 +10,13 @@
 // fully specialized (no more generics/interfaces), so
 // that the concrete type of everything is known.
 
+#include "../compiler-core/slang-name.h"
+
 #include "slang-ir.h"
 #include "slang-ir-clone.h"
 #include "slang-ir-insts.h"
 #include "slang-legalize-types.h"
 #include "slang-mangle.h"
-#include "slang-name.h"
 
 namespace Slang
 {
diff --git a/source/slang/slang-ir.h b/source/slang/slang-ir.h
index ce3a5c29a..b27542424 100644
--- a/source/slang/slang-ir.h
+++ b/source/slang/slang-ir.h
@@ -8,11 +8,10 @@
 //
 
 #include "../core/slang-basic.h"
-
-#include "slang-source-loc.h"
-
 #include "../core/slang-memory-arena.h"
 
+#include "../compiler-core/slang-source-loc.h"
+
 #include "slang-type-system-shared.h"
 
 namespace Slang {
diff --git a/source/slang/slang-legalize-types.h b/source/slang/slang-legalize-types.h
index 1201f669d..8f2a7572f 100644
--- a/source/slang/slang-legalize-types.h
+++ b/source/slang/slang-legalize-types.h
@@ -27,7 +27,8 @@
 #include "slang-ir-insts.h"
 #include "slang-syntax.h"
 #include "slang-type-layout.h"
-#include "slang-name.h"
+
+#include "../compiler-core/slang-name.h"
 
 namespace Slang
 {
diff --git a/source/slang/slang-lookup.cpp b/source/slang/slang-lookup.cpp
index c50364201..f41d8ff3b 100644
--- a/source/slang/slang-lookup.cpp
+++ b/source/slang/slang-lookup.cpp
@@ -1,6 +1,7 @@
 // slang-lookup.cpp
 #include "slang-lookup.h"
-#include "slang-name.h"
+
+#include "../compiler-core/slang-name.h"
 
 namespace Slang {
 
diff --git a/source/slang/slang-mangle.cpp b/source/slang/slang-mangle.cpp
index a08b05a5d..ed38a1261 100644
--- a/source/slang/slang-mangle.cpp
+++ b/source/slang/slang-mangle.cpp
@@ -1,6 +1,6 @@
 #include "slang-mangle.h"
 
-#include "slang-name.h"
+#include "../compiler-core/slang-name.h"
 #include "slang-syntax.h"
 
 namespace Slang
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index fcc2e0e4c..b88fa9bca 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -10,11 +10,10 @@
 #include "slang-compiler.h"
 #include "slang-profile.h"
 
-#include "slang-file-system.h"
-
 #include "slang-repro.h"
 #include "slang-serialize-ir.h"
 
+#include "../core/slang-file-system.h"
 #include "../core/slang-type-text-util.h"
 #include "../core/slang-hex-dump-util.h"
 
diff --git a/source/slang/slang-parser.h b/source/slang/slang-parser.h
index a1077d4a7..0ec2dcb8a 100644
--- a/source/slang/slang-parser.h
+++ b/source/slang/slang-parser.h
@@ -1,7 +1,8 @@
 #ifndef SLANG_PARSER_H
 #define SLANG_PARSER_H
 
-#include "slang-lexer.h"
+#include "../compiler-core/slang-lexer.h"
+
 #include "slang-compiler.h"
 #include "slang-syntax.h"
 
diff --git a/source/slang/slang-preprocessor.cpp b/source/slang/slang-preprocessor.cpp
index f9d18332d..bc78e4d31 100644
--- a/source/slang/slang-preprocessor.cpp
+++ b/source/slang/slang-preprocessor.cpp
@@ -3,7 +3,7 @@
 
 #include "slang-compiler.h"
 #include "slang-diagnostics.h"
-#include "slang-lexer.h"
+#include "../compiler-core/slang-lexer.h"
 // Needed so that we can construct modifier syntax to represent GLSL directives
 #include "slang-syntax.h"
 
diff --git a/source/slang/slang-preprocessor.h b/source/slang/slang-preprocessor.h
index efecb912b..9de82b9f2 100644
--- a/source/slang/slang-preprocessor.h
+++ b/source/slang/slang-preprocessor.h
@@ -4,7 +4,7 @@
 
 #include "../core/slang-basic.h"
 
-#include "slang-lexer.h"
+#include "../compiler-core/slang-lexer.h"
 
 #include "slang-include-system.h"
 
diff --git a/source/slang/slang-repro.cpp b/source/slang/slang-repro.cpp
index 61ab3b75d..3618568c1 100644
--- a/source/slang/slang-repro.cpp
+++ b/source/slang/slang-repro.cpp
@@ -10,7 +10,7 @@
 
 #include "slang-options.h"
 
-#include "slang-source-loc.h"
+#include "../compiler-core/slang-source-loc.h"
 
 namespace Slang {
 
diff --git a/source/slang/slang-repro.h b/source/slang/slang-repro.h
index dd9984395..38a76a50a 100644
--- a/source/slang/slang-repro.h
+++ b/source/slang/slang-repro.h
@@ -9,8 +9,7 @@
 #include "slang-compiler.h"
 
 #include "../core/slang-offset-container.h"
-
-#include "slang-file-system.h"
+#include "../core/slang-file-system.h"
 
 namespace Slang {
 
diff --git a/source/slang/slang-serialize-ir-types.h b/source/slang/slang-serialize-ir-types.h
index 326e41f6e..41759d378 100644
--- a/source/slang/slang-serialize-ir-types.h
+++ b/source/slang/slang-serialize-ir-types.h
@@ -9,8 +9,8 @@
 #include "slang-serialize-types.h"
 #include "slang-serialize-source-loc.h"
 
-#include "slang-name.h"
-#include "slang-source-loc.h"
+#include "../compiler-core/slang-name.h"
+#include "../compiler-core/slang-source-loc.h"
 
 #include "slang-ir.h"
 
diff --git a/source/slang/slang-serialize-misc-type-info.h b/source/slang/slang-serialize-misc-type-info.h
index 08fd1269d..191514785 100644
--- a/source/slang/slang-serialize-misc-type-info.h
+++ b/source/slang/slang-serialize-misc-type-info.h
@@ -4,7 +4,7 @@
 
 #include "slang-serialize-type-info.h"
 
-#include "slang-source-loc.h"
+#include "../compiler-core/slang-source-loc.h"
 #include "slang-compiler.h"
 
 namespace Slang {
diff --git a/source/slang/slang-serialize-reflection.h b/source/slang/slang-serialize-reflection.h
index 7eaf8543c..5ae87877e 100644
--- a/source/slang/slang-serialize-reflection.h
+++ b/source/slang/slang-serialize-reflection.h
@@ -2,7 +2,7 @@
 #ifndef SLANG_SERIALIZE_REFLECTION_H
 #define SLANG_SERIALIZE_REFLECTION_H
 
-#include "slang-name.h"
+#include "../compiler-core/slang-name.h"
 
 namespace Slang
 {
diff --git a/source/slang/slang-serialize-source-loc.h b/source/slang/slang-serialize-source-loc.h
index 5ebd264cc..595a55ea6 100644
--- a/source/slang/slang-serialize-source-loc.h
+++ b/source/slang/slang-serialize-source-loc.h
@@ -8,8 +8,8 @@
 
 #include "slang-serialize-types.h"
 
-#include "slang-name.h"
-#include "slang-source-loc.h"
+#include "../compiler-core/slang-name.h"
+#include "../compiler-core/slang-source-loc.h"
 
 namespace Slang {
 
diff --git a/source/slang/slang-serialize.h b/source/slang/slang-serialize.h
index ff402b35c..990a36adc 100644
--- a/source/slang/slang-serialize.h
+++ b/source/slang/slang-serialize.h
@@ -11,7 +11,7 @@
 
 #include "slang-serialize-types.h"
 
-#include "slang-name.h"
+#include "../compiler-core/slang-name.h"
 
 namespace Slang
 {
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index bcc3d1c67..5c04777de 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -18,11 +18,10 @@
 
 #include "slang-repro.h"
 
-#include "slang-file-system.h"
-
+#include "../core/slang-file-system.h"
 #include "../core/slang-writer.h"
 
-#include "slang-source-loc.h"
+#include "../compiler-core/slang-source-loc.h"
 
 #include "slang-ast-dump.h"