Feature/lex memory reduction (#762)

* Only do scrubbing if needed. When allocating content try to limit size (with scrubbing each token takes up 1k), now it's 16 bytes min size.

* Don't allocate for every call to write on the CallbackWriter - use the m_appendBuffer.

* Don't allocate memory for CallbackWriter use m_appendBuffer.

* Use UnownedStringSlice for suffix output for parsing float/int literals.
Fix typo in invalidFloatingPointLiteralSuffix

* Using memory arena to hold tokens that are not in SourceManager.

* Improve comment on lexing.

* Make UnownedStringSlice allocation simpler on SourceManager.

* Fix error on gcc around UnownedStringSlice - because VC converted string + UnownedStringSlice automatically into a String.

* Fix generateName needing concat string for gcc.

* When constructing a Token in parseAttributeName - because it's a Identifier, we have to set the Name.

* Remove translation through String on getIntrinsicOp

* Make func-cbuffer-param disablable with -exclude compatibility-issue

* Move memory leak in render-test.

* From review - can just use "?:" instead of performing a concat.

1 files changed, 6 insertions, 4 deletions

diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp
index 00784ae92..21942c8e5 100644
--- a/source/slang/preprocessor.cpp
+++ b/source/slang/preprocessor.cpp
@@ -267,11 +267,13 @@ static PreprocessorInputStream* CreateInputStreamForSource(
     Preprocessor*   preprocessor,
     SourceView*     sourceView)
 {
+    MemoryArena* memoryArena = sourceView->getSourceManager()->getMemoryArena();
+
     PrimaryInputStream* inputStream = new PrimaryInputStream();
     initializePrimaryInputStream(preprocessor, inputStream);
 
     // initialize the embedded lexer so that it can generate a token stream
-    inputStream->lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor));
+    inputStream->lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor), memoryArena);
     inputStream->token = inputStream->lexer.lexToken();
 
     return inputStream;
@@ -845,7 +847,7 @@ top:
         SourceView* sourceView = sourceManager->createSourceView(sourceFile);
 
         Lexer lexer;
-        lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor));
+        lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor), sourceManager->getMemoryArena());
 
         SimpleTokenInputStream* inputStream = new SimpleTokenInputStream();
         initializeInputStream(preprocessor, inputStream);
@@ -925,7 +927,7 @@ inline Token const& GetDirective(PreprocessorDirectiveContext* context)
 }
 
 // Get the name of the directive being parsed.
-inline String const& GetDirectiveName(PreprocessorDirectiveContext* context)
+inline UnownedStringSlice const& GetDirectiveName(PreprocessorDirectiveContext* context)
 {
     return context->directiveToken.Content;
 }
@@ -2274,7 +2276,7 @@ static void DefineMacro(
 
     // Use existing `Lexer` to generate a token stream.
     Lexer lexer;
-    lexer.initialize(valueView, GetSink(preprocessor), getNamePool(preprocessor));
+    lexer.initialize(valueView, GetSink(preprocessor), getNamePool(preprocessor), sourceManager->getMemoryArena());
     macro->tokens = lexer.lexAllTokens();
 
     Name* keyName = preprocessor->translationUnit->compileRequest->getNamePool()->getName(key);