From 02e44bade6370309c0292e84178095c2bae299be Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Thu, 20 Dec 2018 13:23:58 -0500 Subject: Feature/lex memory reduction (#762) * Only do scrubbing if needed. When allocating content try to limit size (with scrubbing each token takes up 1k), now it's 16 bytes min size. * Don't allocate for every call to write on the CallbackWriter - use the m_appendBuffer. * Don't allocate memory for CallbackWriter use m_appendBuffer. * Use UnownedStringSlice for suffix output for parsing float/int literals. Fix typo in invalidFloatingPointLiteralSuffix * Using memory arena to hold tokens that are not in SourceManager. * Improve comment on lexing. * Make UnownedStringSlice allocation simpler on SourceManager. * Fix error on gcc around UnownedStringSlice - because VC converted string + UnownedStringSlice automatically into a String. * Fix generateName needing concat string for gcc. * When constructing a Token in parseAttributeName - because it's a Identifier, we have to set the Name. * Remove translation through String on getIntrinsicOp * Make func-cbuffer-param disablable with -exclude compatibility-issue * Move memory leak in render-test. * From review - can just use "?:" instead of performing a concat. --- source/slang/preprocessor.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'source/slang/preprocessor.cpp') diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp index 00784ae92..21942c8e5 100644 --- a/source/slang/preprocessor.cpp +++ b/source/slang/preprocessor.cpp @@ -267,11 +267,13 @@ static PreprocessorInputStream* CreateInputStreamForSource( Preprocessor* preprocessor, SourceView* sourceView) { + MemoryArena* memoryArena = sourceView->getSourceManager()->getMemoryArena(); + PrimaryInputStream* inputStream = new PrimaryInputStream(); initializePrimaryInputStream(preprocessor, inputStream); // initialize the embedded lexer so that it can generate a token stream - inputStream->lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor)); + inputStream->lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor), memoryArena); inputStream->token = inputStream->lexer.lexToken(); return inputStream; @@ -845,7 +847,7 @@ top: SourceView* sourceView = sourceManager->createSourceView(sourceFile); Lexer lexer; - lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor)); + lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor), sourceManager->getMemoryArena()); SimpleTokenInputStream* inputStream = new SimpleTokenInputStream(); initializeInputStream(preprocessor, inputStream); @@ -925,7 +927,7 @@ inline Token const& GetDirective(PreprocessorDirectiveContext* context) } // Get the name of the directive being parsed. -inline String const& GetDirectiveName(PreprocessorDirectiveContext* context) +inline UnownedStringSlice const& GetDirectiveName(PreprocessorDirectiveContext* context) { return context->directiveToken.Content; } @@ -2274,7 +2276,7 @@ static void DefineMacro( // Use existing `Lexer` to generate a token stream. Lexer lexer; - lexer.initialize(valueView, GetSink(preprocessor), getNamePool(preprocessor)); + lexer.initialize(valueView, GetSink(preprocessor), getNamePool(preprocessor), sourceManager->getMemoryArena()); macro->tokens = lexer.lexAllTokens(); Name* keyName = preprocessor->translationUnit->compileRequest->getNamePool()->getName(key); -- cgit v1.2.3