Feature/lex memory reduction (#762)

* Only do scrubbing if needed. When allocating content try to limit size (with scrubbing each token takes up 1k), now it's 16 bytes min size.

* Don't allocate for every call to write on the CallbackWriter - use the m_appendBuffer.

* Don't allocate memory for CallbackWriter use m_appendBuffer.

* Use UnownedStringSlice for suffix output for parsing float/int literals.
Fix typo in invalidFloatingPointLiteralSuffix

* Using memory arena to hold tokens that are not in SourceManager.

* Improve comment on lexing.

* Make UnownedStringSlice allocation simpler on SourceManager.

* Fix error on gcc around UnownedStringSlice - because VC converted string + UnownedStringSlice automatically into a String.

* Fix generateName needing concat string for gcc.

* When constructing a Token in parseAttributeName - because it's a Identifier, we have to set the Name.

* Remove translation through String on getIntrinsicOp

* Make func-cbuffer-param disablable with -exclude compatibility-issue

* Move memory leak in render-test.

* From review - can just use "?:" instead of performing a concat.

17 files changed, 158 insertions, 78 deletions

diff --git a/source/core/slang-string.h b/source/core/slang-string.h
index 5a20d8b75..4c87ce81e 100644
--- a/source/core/slang-string.h
+++ b/source/core/slang-string.h
@@ -111,9 +111,9 @@ namespace Slang
             return -1;
         }
 
-        const char& operator[](int i) const
+        const char& operator[](UInt i) const
         {
-            assert(i >= 0 && i < int(endData - beginData));
+            assert(i < UInt(endData - beginData));
             return beginData[i];
         }
 
diff --git a/source/core/slang-writer.cpp b/source/core/slang-writer.cpp
index 21c8c6209..433cc6992 100644
--- a/source/core/slang-writer.cpp
+++ b/source/core/slang-writer.cpp
@@ -80,15 +80,35 @@ SLANG_NO_THROW SlangResult SLANG_MCALL AppendBufferWriter::endAppendBuffer(char*
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!! CallbackWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/
 
+SLANG_NO_THROW char* SLANG_MCALL CallbackWriter::beginAppendBuffer(size_t maxNumChars)
+{
+    // Add one so there is always space for end termination, we need for the callback.
+    m_appendBuffer.SetSize(maxNumChars + 1);
+    return m_appendBuffer.Buffer();
+}
+
 SlangResult CallbackWriter::write(const char* chars, size_t numChars)
 {
     if (numChars > 0)
     {
-        // Make sure zero terminated
-        StringBuilder builder;
-        builder.Append(chars, numChars);
-
-        m_callback(builder.Buffer(), (void*)m_data);
+        char* appendBuffer = m_appendBuffer.Buffer();
+        // See if it's from an append buffer
+        if (chars >= appendBuffer && (chars + numChars) < (appendBuffer + m_appendBuffer.Count()))
+        {
+            // Set terminating 0
+            appendBuffer[(chars + numChars) - appendBuffer] = 0;
+
+            m_callback(chars, (void*)m_data);
+        }
+        else
+        {
+            // Use the append buffer to add the terminating 0
+            m_appendBuffer.SetSize(numChars + 1);
+            ::memcpy(m_appendBuffer.Buffer(), chars, numChars);
+            m_appendBuffer[numChars] = 0;
+
+            m_callback(m_appendBuffer.Buffer(), (void*)m_data);
+        }
     }
 
     return SLANG_OK;
diff --git a/source/core/slang-writer.h b/source/core/slang-writer.h
index 0e67684ec..b18af373f 100644
--- a/source/core/slang-writer.h
+++ b/source/core/slang-writer.h
@@ -85,6 +85,7 @@ class CallbackWriter : public AppendBufferWriter
 public:
     typedef AppendBufferWriter Parent;
     // ISlangWriter
+    SLANG_NO_THROW char* SLANG_MCALL beginAppendBuffer(size_t maxNumChars) SLANG_OVERRIDE;
     SLANG_NO_THROW virtual SlangResult SLANG_MCALL write(const char* chars, size_t numChars) SLANG_OVERRIDE;
     
     CallbackWriter(SlangDiagnosticCallback callback, const void* data, WriterFlags flags) :
diff --git a/source/slang/diagnostic-defs.h b/source/slang/diagnostic-defs.h
index 2a08dfb6a..a967fca98 100644
--- a/source/slang/diagnostic-defs.h
+++ b/source/slang/diagnostic-defs.h
@@ -318,7 +318,7 @@ DIAGNOSTIC(39999, Error, notEnoughArguments, "not enough arguments to call (got
 DIAGNOSTIC(39999, Error, tooManyArguments, "too many arguments to call (got $0, expected $1)")
 
 DIAGNOSTIC(39999, Error, invalidIntegerLiteralSuffix, "invalid suffix '$0' on integer literal")
-DIAGNOSTIC(39999, Error, invalidFloatingPOintLiteralSuffix, "invalid suffix '$0' on floating-point literal")
+DIAGNOSTIC(39999, Error, invalidFloatingPointLiteralSuffix, "invalid suffix '$0' on floating-point literal")
 
 
 
diff --git a/source/slang/ir.cpp b/source/slang/ir.cpp
index ca424b4a4..02a6e9f6a 100644
--- a/source/slang/ir.cpp
+++ b/source/slang/ir.cpp
@@ -75,11 +75,11 @@ namespace Slang
         return kIROps[kIROpCount].info;
     }
 
-    IROp findIROp(char const* name)
+    IROp findIROp(const UnownedStringSlice& name)
     {
         for (auto ee : kIROps)
         {
-            if (strcmp(name, ee.info.name) == 0)
+            if (name == ee.info.name)
                 return ee.op;
         }
 
diff --git a/source/slang/ir.h b/source/slang/ir.h
index 86d1d1eb4..de644a709 100644
--- a/source/slang/ir.h
+++ b/source/slang/ir.h
@@ -92,7 +92,7 @@ enum IROpMeta
 // True if op is pseudo (or invalid which is 'pseudo-like' at least in as so far as current behavior)
 SLANG_FORCE_INLINE bool isPseudoOp(IROp op) { return (op & kIROpMeta_IsPseudoOp) != 0; }
 
-IROp findIROp(char const* name);
+IROp findIROp(const UnownedStringSlice& name);
 
 // A logical operation/opcode in the IR
 struct IROpInfo
diff --git a/source/slang/lexer.cpp b/source/slang/lexer.cpp
index 918668296..3d87dd7ea 100644
--- a/source/slang/lexer.cpp
+++ b/source/slang/lexer.cpp
@@ -14,7 +14,7 @@ namespace Slang
 {
     static Token GetEndOfFileToken()
     {
-        return Token(TokenType::EndOfFile, "", SourceLoc());
+        return Token(TokenType::EndOfFile, UnownedStringSlice::fromLiteral(""), SourceLoc());
     }
 
     Token* TokenList::begin() const
@@ -86,11 +86,13 @@ namespace Slang
     void Lexer::initialize(
         SourceView*     inSourceView,
         DiagnosticSink* inSink,
-        NamePool*       inNamePool)
+        NamePool*       inNamePool,
+        MemoryArena*    inMemoryArena)
     {
         sourceView  = inSourceView;
         sink        = inSink;
         namePool    = inNamePool;
+        memoryArena = inMemoryArena;
 
         auto content = inSourceView->getContent();
         
@@ -548,7 +550,7 @@ namespace Slang
 
 
 
-    IntegerLiteralValue getIntegerLiteralValue(Token const& token, String* outSuffix)
+    IntegerLiteralValue getIntegerLiteralValue(Token const& token, UnownedStringSlice* outSuffix)
     {
         IntegerLiteralValue value = 0;
 
@@ -568,13 +570,13 @@ namespace Slang
 
         if(outSuffix)
         {
-            *outSuffix = String(cursor, end);
+            *outSuffix = UnownedStringSlice(cursor, end);
         }
 
         return value;
     }
 
-    FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, String* outSuffix)
+    FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, UnownedStringSlice* outSuffix)
     {
         FloatingPointLiteralValue value = 0;
 
@@ -661,7 +663,7 @@ namespace Slang
 
         if(outSuffix)
         {
-            *outSuffix = String(cursor, end);
+            *outSuffix = UnownedStringSlice(cursor, end);
         }
 
         return value;
@@ -784,7 +786,7 @@ namespace Slang
                 return valueBuilder.ProduceString();
             }
 
-            // Charcters that don't being escape sequences are easy;
+            // Characters that don't being escape sequences are easy;
             // just append them to the buffer and move on.
             if(c != '\\')
             {
@@ -888,11 +890,11 @@ namespace Slang
     {
         // A file name usually doesn't process escape sequences
         // (this is import on Windows, where `\\` is a valid
-        // path separator cahracter).
+        // path separator character).
 
         // Just trim off the first and last characters to remove the quotes
         // (whether they were `""` or `<>`.
-        return token.Content.SubString(1, token.Content.Length()-2);
+        return String(token.Content.begin() + 1, token.Content.end() - 1); 
     }
 
 
@@ -1268,40 +1270,49 @@ namespace Slang
             // Note(tfoley): `StringBuilder::Append()` seems to crash when appending zero bytes
             if(textEnd != textBegin)
             {
-                // HACK(tfoley): "scrubbing" token value here to remove escaped newlines...
+                // "scrubbing" token value here to remove escaped newlines...
                 //
-                // TODO: Only perform this work if we encountered an escaped newline
+                // Only perform this work if we encountered an escaped newline
                 // while lexing this token (e.g., keep a flag on the lexer), or
                 // do it on-demand when the actual value of the token is needed.
-
-                StringBuilder valueBuilder;
-                auto tt = textBegin;
-                while(tt != textEnd)
+                if (tokenFlags & TokenFlag::ScrubbingNeeded)
                 {
-                    char c = *tt++;
-                    if(c == '\\')
+                    // Allocate space that will always be more than enough for stripped contents
+                    char* startDst = (char*)memoryArena->allocateUnaligned(textEnd - textBegin);
+                    char* dst = startDst;
+
+                    auto tt = textBegin;
+                    while (tt != textEnd)
                     {
-                        char d = *tt;
-                        switch(d)
+                        char c = *tt++;
+                        if (c == '\\')
                         {
-                        case '\r': case '\n':
+                            char d = *tt;
+                            switch (d)
+                            {
+                            case '\r': case '\n':
                             {
                                 tt++;
                                 char e = *tt;
-                                if((d ^ e) == ('\r' ^ '\n'))
+                                if ((d ^ e) == ('\r' ^ '\n'))
                                 {
                                     tt++;
                                 }
                             }
                             continue;
 
-                        default:
-                            break;
+                            default:
+                                break;
+                            }
                         }
+                        *dst++ = c;
                     }
-                    valueBuilder.Append(c);
+                    token.Content = UnownedStringSlice(startDst, dst);
+                }
+                else
+                {
+                    token.Content = UnownedStringSlice(textBegin, textEnd);
                 }
-                token.Content = valueBuilder.ProduceString();
             }
 
             token.flags = flags;
diff --git a/source/slang/lexer.h b/source/slang/lexer.h
index 85aca77dd..593e07c22 100644
--- a/source/slang/lexer.h
+++ b/source/slang/lexer.h
@@ -75,7 +75,8 @@ namespace Slang
         void initialize(
             SourceView*     sourceView,
             DiagnosticSink* sink,
-            NamePool*       namePool);
+            NamePool*       namePool,
+            MemoryArena*    memoryArena);
 
         ~Lexer();
 
@@ -97,6 +98,8 @@ namespace Slang
 
         TokenFlags      tokenFlags;
         LexerFlags      lexerFlags;
+
+        MemoryArena*    memoryArena;
     };
 
     // Helper routines for extracting values from tokens
@@ -106,8 +109,8 @@ namespace Slang
     typedef int64_t IntegerLiteralValue;
     typedef double FloatingPointLiteralValue;
 
-    IntegerLiteralValue getIntegerLiteralValue(Token const& token, String* outSuffix = 0);
-    FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, String* outSuffix = 0);
+    IntegerLiteralValue getIntegerLiteralValue(Token const& token, UnownedStringSlice* outSuffix = 0);
+    FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, UnownedStringSlice* outSuffix = 0);
 }
 
 #endif
diff --git a/source/slang/lower-to-ir.cpp b/source/slang/lower-to-ir.cpp
index 0d39664c8..2242619d0 100644
--- a/source/slang/lower-to-ir.cpp
+++ b/source/slang/lower-to-ir.cpp
@@ -458,9 +458,9 @@ IROp getIntrinsicOp(
     // based on the name of the declaration...
 
     auto name = decl->getName();
-    auto nameText = getText(name);
+    auto nameText = getUnownedStringSliceText(name);
 
-    IROp op = findIROp(nameText.Buffer());
+    IROp op = findIROp(nameText);
     SLANG_ASSERT(op != kIROp_Invalid);
     return op;
 }
@@ -4909,7 +4909,7 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
                 definition = definitionToken.Content;
             }
 
-            builder->addTargetIntrinsicDecoration(irInst, targetMod->targetToken.Content.getUnownedSlice(), definition.getUnownedSlice());
+            builder->addTargetIntrinsicDecoration(irInst, targetMod->targetToken.Content, definition.getUnownedSlice());
         }
     }
 
@@ -5240,7 +5240,7 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
             // a specialized definition of the particular function for the given
             // target, and we need to reflect that at the IR level.
 
-            getBuilder()->addTargetDecoration(irFunc, targetMod->targetToken.Content.getUnownedSlice());
+            getBuilder()->addTargetDecoration(irFunc, targetMod->targetToken.Content);
         }
 
         // If this declaration was marked as having a target-specific lowering
@@ -5255,7 +5255,7 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
         //
         for(auto extensionMod : decl->GetModifiersOfType<RequiredGLSLExtensionModifier>())
         {
-            getBuilder()->addRequireGLSLExtensionDecoration(irFunc, extensionMod->extensionNameToken.Content.getUnownedSlice());
+            getBuilder()->addRequireGLSLExtensionDecoration(irFunc, extensionMod->extensionNameToken.Content);
         }
         for(auto versionMod : decl->GetModifiersOfType<RequiredGLSLVersionModifier>())
         {
diff --git a/source/slang/name.cpp b/source/slang/name.cpp
index 995bce19f..a5cbb6541 100644
--- a/source/slang/name.cpp
+++ b/source/slang/name.cpp
@@ -9,6 +9,11 @@ String getText(Name* name)
     return name->text;
 }
 
+UnownedStringSlice getUnownedStringSliceText(Name* name)
+{
+    return name ? name->text.getUnownedSlice() : UnownedStringSlice();
+}
+
 Name* NamePool::getName(String const& text)
 {
     RefPtr<Name> name;
diff --git a/source/slang/name.h b/source/slang/name.h
index e492c4507..e1a055e60 100644
--- a/source/slang/name.h
+++ b/source/slang/name.h
@@ -36,6 +36,9 @@ public:
 // (e.g., so that it can be printed).
 String getText(Name* name);
 
+/// Get the text as unowned string slice 
+UnownedStringSlice getUnownedStringSliceText(Name* name);
+
 // A `RootNamePool` is used to store and look up names.
 // If two systems need to work together with names, and be sure that they
 // get equivalent names for a string like `"Foo"`, then they need to use
diff --git a/source/slang/parameter-binding.cpp b/source/slang/parameter-binding.cpp
index 622474116..e48b1d5bd 100644
--- a/source/slang/parameter-binding.cpp
+++ b/source/slang/parameter-binding.cpp
@@ -425,7 +425,7 @@ static bool isDigit(char c)
 /// Given a string that specifies a name and index (e.g., `COLOR0`),
 /// split it into slices for the name part and the index part.
 static void splitNameAndIndex(
-    String const&       text,
+    UnownedStringSlice const&       text,
     UnownedStringSlice& outName,
     UnownedStringSlice& outDigits)
 {
@@ -482,8 +482,8 @@ LayoutSemanticInfo ExtractLayoutSemanticInfo(
     info.index = 0;
     info.kind = LayoutResourceKind::None;
 
-    String registerName = semantic->registerName.Content;
-    if (registerName.Length() == 0)
+    UnownedStringSlice registerName = semantic->registerName.Content;
+    if (registerName.size() == 0)
         return info;
 
     // The register name is expected to be in the form:
@@ -526,7 +526,7 @@ LayoutSemanticInfo ExtractLayoutSemanticInfo(
     if( auto registerSemantic = dynamic_cast<HLSLRegisterSemantic*>(semantic) )
     {
         auto const& spaceName = registerSemantic->spaceName.Content;
-        if(spaceName.Length() != 0)
+        if(spaceName.size() != 0)
         {
             UnownedStringSlice spaceSpelling;
             UnownedStringSlice spaceDigits;
@@ -556,7 +556,7 @@ LayoutSemanticInfo ExtractLayoutSemanticInfo(
     }
 
     // TODO: handle component mask part of things...
-    if( semantic->componentMask.Content.Length() != 0 )
+    if( semantic->componentMask.Content.size() != 0 )
     {
         getSink(context)->diagnose(semantic->componentMask, Diagnostics::componentMaskNotSupported);
     }
@@ -999,7 +999,7 @@ static bool findLayoutArg(
     {
         if( modifier )
         {
-            *outVal = (UInt) strtoull(modifier->valToken.Content.Buffer(), nullptr, 10);
+            *outVal = (UInt) strtoull(String(modifier->valToken.Content).Buffer(), nullptr, 10);
             return true;
         }
     }
@@ -1981,7 +1981,7 @@ SimpleSemanticInfo decomposeSimpleSemantic(
 
     // look for a trailing sequence of decimal digits
     // at the end of the composed name
-    UInt length = composedName.Length();
+    UInt length = composedName.size();
     UInt indexLoc = length;
     while( indexLoc > 0 )
     {
@@ -2009,8 +2009,10 @@ SimpleSemanticInfo decomposeSimpleSemantic(
     else
     {
         // The name is everything before the digits
-        info.name = composedName.SubString(0, indexLoc);
-        info.index = strtol(composedName.SubString(indexLoc, length - indexLoc).begin(), nullptr, 10);
+        String stringComposedName(composedName);
+
+        info.name = stringComposedName.SubString(0, indexLoc);
+        info.index = strtol(stringComposedName.begin() + indexLoc, nullptr, 10);
     }
     return info;
 }
diff --git a/source/slang/parser.cpp b/source/slang/parser.cpp
index 2e0ca5df9..d0e965970 100644
--- a/source/slang/parser.cpp
+++ b/source/slang/parser.cpp
@@ -690,9 +690,15 @@ namespace Slang
         }
 
         // Make a 'token'
-        const String scopedIdentifier(scopedIdentifierBuilder.ToString());
+        SourceManager* sourceManager = parser->sink->sourceManager;
+        const UnownedStringSlice scopedIdentifier(sourceManager->allocateStringSlice(scopedIdentifierBuilder.getUnownedSlice()));   
         Token token(TokenType::Identifier, scopedIdentifier, scopedIdSourceLoc);
-        token.ptrValue = getName(parser, token.Content);
+
+        // Get the name pool
+        auto namePool = parser->translationUnit->compileRequest->getNamePool();
+
+        // Since it's an Identifier have to set the name.
+        token.ptrValue = namePool->getName(token.Content);
 
         return token;
     }
@@ -990,7 +996,8 @@ namespace Slang
             case TokenType::QuestionMark:
                 if (AdvanceIf(parser, TokenType::Colon))
                 {
-                    nameToken.Content = nameToken.Content + ":";
+                    // Concat : onto ?
+                    nameToken.Content = UnownedStringSlice::fromLiteral("?:"); 
                     break;
                 }
                 ;       // fall-thru
@@ -2210,8 +2217,8 @@ namespace Slang
         addModifier(bufferVarDecl, reflectionNameModifier);
 
         // Both the buffer variable and its type need to have names generated
-        bufferVarDecl->nameAndLoc.name = generateName(parser, "parameterGroup_" + reflectionNameToken.Content);
-        bufferDataTypeDecl->nameAndLoc.name = generateName(parser, "ParameterGroup_" + reflectionNameToken.Content);
+        bufferVarDecl->nameAndLoc.name = generateName(parser, "parameterGroup_" + String(reflectionNameToken.Content));
+        bufferDataTypeDecl->nameAndLoc.name = generateName(parser, "ParameterGroup_" + String(reflectionNameToken.Content));
 
         addModifier(bufferDataTypeDecl, new ImplicitParameterGroupElementTypeModifier());
         addModifier(bufferVarDecl, new ImplicitParameterGroupVariableModifier());
@@ -2379,7 +2386,7 @@ namespace Slang
         parser->FillPosition(blockVarDecl.Ptr());
 
         // Generate a unique name for the data type
-        blockDataTypeDecl->nameAndLoc.name = generateName(parser, "ParameterGroup_" + reflectionNameToken.Content);
+        blockDataTypeDecl->nameAndLoc.name = generateName(parser, "ParameterGroup_" + String(reflectionNameToken.Content));
 
         // TODO(tfoley): We end up constructing unchecked syntax here that
         // is expected to type check into the right form, but it might be
@@ -2424,7 +2431,7 @@ namespace Slang
         else
         {
             // synthesize a dummy name
-            blockVarDecl->nameAndLoc.name = generateName(parser, "parameterGroup_" + reflectionNameToken.Content);
+            blockVarDecl->nameAndLoc.name = generateName(parser, "parameterGroup_" + String(reflectionNameToken.Content));
 
             // Otherwise we have a transparent declaration, similar
             // to an HLSL `cbuffer`
@@ -3718,7 +3725,7 @@ namespace Slang
         {
         case TokenType::QuestionMark:
             opToken = parser->ReadToken();
-            opToken.Content = "?:";
+            opToken.Content = UnownedStringSlice::fromLiteral("?:");
             break;
 
         default:
@@ -3999,19 +4006,20 @@ namespace Slang
                 auto token = parser->tokenReader.AdvanceToken();
                 constExpr->token = token;
 
-                String suffix;
+                UnownedStringSlice suffix;
                 IntegerLiteralValue value = getIntegerLiteralValue(token, &suffix);
 
                 // Look at any suffix on the value
                 char const* suffixCursor = suffix.begin();
+                const char*const suffixEnd = suffix.end();
 
                 RefPtr<Type> suffixType = nullptr;
-                if( suffixCursor && *suffixCursor )
+                if( suffixCursor < suffixEnd )
                 {
                     int lCount = 0;
                     int uCount = 0;
                     int unknownCount = 0;
-                    while(*suffixCursor)
+                    while(suffixCursor < suffixEnd)
                     {
                         switch( *suffixCursor++ )
                         {
@@ -4077,20 +4085,21 @@ namespace Slang
                 auto token = parser->tokenReader.AdvanceToken();
                 constExpr->token = token;
 
-                String suffix;
+                UnownedStringSlice suffix;
                 FloatingPointLiteralValue value = getFloatingPointLiteralValue(token, &suffix);
 
                 // Look at any suffix on the value
                 char const* suffixCursor = suffix.begin();
+                const char*const suffixEnd = suffix.end();
 
                 RefPtr<Type> suffixType = nullptr;
-                if( suffixCursor && *suffixCursor )
+                if( suffixCursor < suffixEnd )
                 {
                     int fCount = 0;
                     int lCount = 0;
                     int hCount = 0;
                     int unknownCount = 0;
-                    while(*suffixCursor)
+                    while(suffixCursor < suffixEnd)
                     {
                         switch( *suffixCursor++ )
                         {
@@ -4112,9 +4121,9 @@ namespace Slang
                         }
                     }
 
-                    if(unknownCount)
+                    if (unknownCount)
                     {
-                        parser->sink->diagnose(token, Diagnostics::invalidFloatingPOintLiteralSuffix, suffix);
+                        parser->sink->diagnose(token, Diagnostics::invalidFloatingPointLiteralSuffix, suffix);
                         suffixType = parser->getSession()->getErrorType();
                     }
                     // `f` suffix -> `float`
@@ -4135,7 +4144,7 @@ namespace Slang
                     // TODO: are there other suffixes we need to handle?
                     else
                     {
-                        parser->sink->diagnose(token, Diagnostics::invalidFloatingPOintLiteralSuffix, suffix);
+                        parser->sink->diagnose(token, Diagnostics::invalidFloatingPointLiteralSuffix, suffix);
                         suffixType = parser->getSession()->getErrorType();
                     }
                 }
@@ -4424,7 +4433,7 @@ namespace Slang
             {
                 modifier->opToken = parser->ReadToken(TokenType::Identifier);
 
-                modifier->op = findIROp(modifier->opToken.Content.Buffer());
+                modifier->op = findIROp(modifier->opToken.Content);
 
                 if (modifier->op == kIROp_Invalid)
                 {
diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp
index 00784ae92..21942c8e5 100644
--- a/source/slang/preprocessor.cpp
+++ b/source/slang/preprocessor.cpp
@@ -267,11 +267,13 @@ static PreprocessorInputStream* CreateInputStreamForSource(
     Preprocessor*   preprocessor,
     SourceView*     sourceView)
 {
+    MemoryArena* memoryArena = sourceView->getSourceManager()->getMemoryArena();
+
     PrimaryInputStream* inputStream = new PrimaryInputStream();
     initializePrimaryInputStream(preprocessor, inputStream);
 
     // initialize the embedded lexer so that it can generate a token stream
-    inputStream->lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor));
+    inputStream->lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor), memoryArena);
     inputStream->token = inputStream->lexer.lexToken();
 
     return inputStream;
@@ -845,7 +847,7 @@ top:
         SourceView* sourceView = sourceManager->createSourceView(sourceFile);
 
         Lexer lexer;
-        lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor));
+        lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor), sourceManager->getMemoryArena());
 
         SimpleTokenInputStream* inputStream = new SimpleTokenInputStream();
         initializeInputStream(preprocessor, inputStream);
@@ -925,7 +927,7 @@ inline Token const& GetDirective(PreprocessorDirectiveContext* context)
 }
 
 // Get the name of the directive being parsed.
-inline String const& GetDirectiveName(PreprocessorDirectiveContext* context)
+inline UnownedStringSlice const& GetDirectiveName(PreprocessorDirectiveContext* context)
 {
     return context->directiveToken.Content;
 }
@@ -2274,7 +2276,7 @@ static void DefineMacro(
 
     // Use existing `Lexer` to generate a token stream.
     Lexer lexer;
-    lexer.initialize(valueView, GetSink(preprocessor), getNamePool(preprocessor));
+    lexer.initialize(valueView, GetSink(preprocessor), getNamePool(preprocessor), sourceManager->getMemoryArena());
     macro->tokens = lexer.lexAllTokens();
 
     Name* keyName = preprocessor->translationUnit->compileRequest->getNamePool()->getName(key);
diff --git a/source/slang/source-loc.cpp b/source/slang/source-loc.cpp
index 62926aa8c..6d6d63ae1 100644
--- a/source/slang/source-loc.cpp
+++ b/source/slang/source-loc.cpp
@@ -292,6 +292,16 @@ void SourceManager::initialize(
     m_nextLoc = m_startLoc;
 }
 
+UnownedStringSlice SourceManager::allocateStringSlice(const UnownedStringSlice& slice)
+{
+    const UInt numChars = slice.size();
+
+    char* dst = (char*)m_memoryArena.allocate(numChars);
+    ::memcpy(dst, slice.begin(), numChars);
+
+    return UnownedStringSlice(dst, numChars);
+}
+
 SourceRange SourceManager::allocateSourceRange(UInt size)
 {
     // TODO: consider using atomics here
diff --git a/source/slang/source-loc.h b/source/slang/source-loc.h
index aba9b7bd9..6949d659a 100644
--- a/source/slang/source-loc.h
+++ b/source/slang/source-loc.h
@@ -1,4 +1,4 @@
-// source-loc.h
+// source-loc.h
 #ifndef SLANG_SOURCE_LOC_H_INCLUDED
 #define SLANG_SOURCE_LOC_H_INCLUDED
 
@@ -303,6 +303,16 @@ struct SourceManager
         /// Get the parent manager to this manager. Returns nullptr if there isn't any.
     SourceManager* getParent() const { return m_parent; }
 
+        /// A memory arena to hold allocations that are in scope for the same time as SourceManager
+    MemoryArena* getMemoryArena() { return &m_memoryArena;  }
+
+        /// Allocate a string slice
+    UnownedStringSlice allocateStringSlice(const UnownedStringSlice& slice);
+    
+    SourceManager() :
+        m_memoryArena(2048)
+    {}
+
     protected:
 
     // The first location available to this source manager
@@ -320,6 +330,10 @@ struct SourceManager
     List<RefPtr<SourceView> > m_sourceViews;                
     StringSlicePool m_slicePool;
 
+    // Memory arena that can be used for holding data to held in scope as long as the Source is
+    // Can be used for storing the decoded contents of Token.Content for exampel
+    MemoryArena m_memoryArena;
+
     // Maps canonical paths to source files
     Dictionary<String, RefPtr<SourceFile> > m_sourceFiles;  
 };
diff --git a/source/slang/token.h b/source/slang/token.h
index e3b6f48ff..d7d45882d 100644
--- a/source/slang/token.h
+++ b/source/slang/token.h
@@ -36,13 +36,13 @@ public:
     SourceLoc   loc;
     void*       ptrValue;
 
-    String Content;
+    UnownedStringSlice Content;
 
     Token() = default;
 
     Token(
         TokenType typeIn,
-        const String & contentIn,
+        const UnownedStringSlice & contentIn,
         SourceLoc locIn,
         TokenFlags flagsIn = 0)
         : flags(flagsIn)