From 02e44bade6370309c0292e84178095c2bae299be Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Thu, 20 Dec 2018 13:23:58 -0500 Subject: Feature/lex memory reduction (#762) * Only do scrubbing if needed. When allocating content try to limit size (with scrubbing each token takes up 1k), now it's 16 bytes min size. * Don't allocate for every call to write on the CallbackWriter - use the m_appendBuffer. * Don't allocate memory for CallbackWriter use m_appendBuffer. * Use UnownedStringSlice for suffix output for parsing float/int literals. Fix typo in invalidFloatingPointLiteralSuffix * Using memory arena to hold tokens that are not in SourceManager. * Improve comment on lexing. * Make UnownedStringSlice allocation simpler on SourceManager. * Fix error on gcc around UnownedStringSlice - because VC converted string + UnownedStringSlice automatically into a String. * Fix generateName needing concat string for gcc. * When constructing a Token in parseAttributeName - because it's a Identifier, we have to set the Name. * Remove translation through String on getIntrinsicOp * Make func-cbuffer-param disablable with -exclude compatibility-issue * Move memory leak in render-test. * From review - can just use "?:" instead of performing a concat. --- source/core/slang-string.h | 4 +-- source/core/slang-writer.cpp | 30 +++++++++++++++---- source/core/slang-writer.h | 1 + source/slang/diagnostic-defs.h | 2 +- source/slang/ir.cpp | 4 +-- source/slang/ir.h | 2 +- source/slang/lexer.cpp | 61 ++++++++++++++++++++++---------------- source/slang/lexer.h | 9 ++++-- source/slang/lower-to-ir.cpp | 10 +++---- source/slang/name.cpp | 5 ++++ source/slang/name.h | 3 ++ source/slang/parameter-binding.cpp | 20 +++++++------ source/slang/parser.cpp | 45 +++++++++++++++++----------- source/slang/preprocessor.cpp | 10 ++++--- source/slang/source-loc.cpp | 10 +++++++ source/slang/source-loc.h | 16 +++++++++- source/slang/token.h | 4 +-- 17 files changed, 158 insertions(+), 78 deletions(-) (limited to 'source') diff --git a/source/core/slang-string.h b/source/core/slang-string.h index 5a20d8b75..4c87ce81e 100644 --- a/source/core/slang-string.h +++ b/source/core/slang-string.h @@ -111,9 +111,9 @@ namespace Slang return -1; } - const char& operator[](int i) const + const char& operator[](UInt i) const { - assert(i >= 0 && i < int(endData - beginData)); + assert(i < UInt(endData - beginData)); return beginData[i]; } diff --git a/source/core/slang-writer.cpp b/source/core/slang-writer.cpp index 21c8c6209..433cc6992 100644 --- a/source/core/slang-writer.cpp +++ b/source/core/slang-writer.cpp @@ -80,15 +80,35 @@ SLANG_NO_THROW SlangResult SLANG_MCALL AppendBufferWriter::endAppendBuffer(char* /* !!!!!!!!!!!!!!!!!!!!!!!!! CallbackWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/ +SLANG_NO_THROW char* SLANG_MCALL CallbackWriter::beginAppendBuffer(size_t maxNumChars) +{ + // Add one so there is always space for end termination, we need for the callback. + m_appendBuffer.SetSize(maxNumChars + 1); + return m_appendBuffer.Buffer(); +} + SlangResult CallbackWriter::write(const char* chars, size_t numChars) { if (numChars > 0) { - // Make sure zero terminated - StringBuilder builder; - builder.Append(chars, numChars); - - m_callback(builder.Buffer(), (void*)m_data); + char* appendBuffer = m_appendBuffer.Buffer(); + // See if it's from an append buffer + if (chars >= appendBuffer && (chars + numChars) < (appendBuffer + m_appendBuffer.Count())) + { + // Set terminating 0 + appendBuffer[(chars + numChars) - appendBuffer] = 0; + + m_callback(chars, (void*)m_data); + } + else + { + // Use the append buffer to add the terminating 0 + m_appendBuffer.SetSize(numChars + 1); + ::memcpy(m_appendBuffer.Buffer(), chars, numChars); + m_appendBuffer[numChars] = 0; + + m_callback(m_appendBuffer.Buffer(), (void*)m_data); + } } return SLANG_OK; diff --git a/source/core/slang-writer.h b/source/core/slang-writer.h index 0e67684ec..b18af373f 100644 --- a/source/core/slang-writer.h +++ b/source/core/slang-writer.h @@ -85,6 +85,7 @@ class CallbackWriter : public AppendBufferWriter public: typedef AppendBufferWriter Parent; // ISlangWriter + SLANG_NO_THROW char* SLANG_MCALL beginAppendBuffer(size_t maxNumChars) SLANG_OVERRIDE; SLANG_NO_THROW virtual SlangResult SLANG_MCALL write(const char* chars, size_t numChars) SLANG_OVERRIDE; CallbackWriter(SlangDiagnosticCallback callback, const void* data, WriterFlags flags) : diff --git a/source/slang/diagnostic-defs.h b/source/slang/diagnostic-defs.h index 2a08dfb6a..a967fca98 100644 --- a/source/slang/diagnostic-defs.h +++ b/source/slang/diagnostic-defs.h @@ -318,7 +318,7 @@ DIAGNOSTIC(39999, Error, notEnoughArguments, "not enough arguments to call (got DIAGNOSTIC(39999, Error, tooManyArguments, "too many arguments to call (got $0, expected $1)") DIAGNOSTIC(39999, Error, invalidIntegerLiteralSuffix, "invalid suffix '$0' on integer literal") -DIAGNOSTIC(39999, Error, invalidFloatingPOintLiteralSuffix, "invalid suffix '$0' on floating-point literal") +DIAGNOSTIC(39999, Error, invalidFloatingPointLiteralSuffix, "invalid suffix '$0' on floating-point literal") diff --git a/source/slang/ir.cpp b/source/slang/ir.cpp index ca424b4a4..02a6e9f6a 100644 --- a/source/slang/ir.cpp +++ b/source/slang/ir.cpp @@ -75,11 +75,11 @@ namespace Slang return kIROps[kIROpCount].info; } - IROp findIROp(char const* name) + IROp findIROp(const UnownedStringSlice& name) { for (auto ee : kIROps) { - if (strcmp(name, ee.info.name) == 0) + if (name == ee.info.name) return ee.op; } diff --git a/source/slang/ir.h b/source/slang/ir.h index 86d1d1eb4..de644a709 100644 --- a/source/slang/ir.h +++ b/source/slang/ir.h @@ -92,7 +92,7 @@ enum IROpMeta // True if op is pseudo (or invalid which is 'pseudo-like' at least in as so far as current behavior) SLANG_FORCE_INLINE bool isPseudoOp(IROp op) { return (op & kIROpMeta_IsPseudoOp) != 0; } -IROp findIROp(char const* name); +IROp findIROp(const UnownedStringSlice& name); // A logical operation/opcode in the IR struct IROpInfo diff --git a/source/slang/lexer.cpp b/source/slang/lexer.cpp index 918668296..3d87dd7ea 100644 --- a/source/slang/lexer.cpp +++ b/source/slang/lexer.cpp @@ -14,7 +14,7 @@ namespace Slang { static Token GetEndOfFileToken() { - return Token(TokenType::EndOfFile, "", SourceLoc()); + return Token(TokenType::EndOfFile, UnownedStringSlice::fromLiteral(""), SourceLoc()); } Token* TokenList::begin() const @@ -86,11 +86,13 @@ namespace Slang void Lexer::initialize( SourceView* inSourceView, DiagnosticSink* inSink, - NamePool* inNamePool) + NamePool* inNamePool, + MemoryArena* inMemoryArena) { sourceView = inSourceView; sink = inSink; namePool = inNamePool; + memoryArena = inMemoryArena; auto content = inSourceView->getContent(); @@ -548,7 +550,7 @@ namespace Slang - IntegerLiteralValue getIntegerLiteralValue(Token const& token, String* outSuffix) + IntegerLiteralValue getIntegerLiteralValue(Token const& token, UnownedStringSlice* outSuffix) { IntegerLiteralValue value = 0; @@ -568,13 +570,13 @@ namespace Slang if(outSuffix) { - *outSuffix = String(cursor, end); + *outSuffix = UnownedStringSlice(cursor, end); } return value; } - FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, String* outSuffix) + FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, UnownedStringSlice* outSuffix) { FloatingPointLiteralValue value = 0; @@ -661,7 +663,7 @@ namespace Slang if(outSuffix) { - *outSuffix = String(cursor, end); + *outSuffix = UnownedStringSlice(cursor, end); } return value; @@ -784,7 +786,7 @@ namespace Slang return valueBuilder.ProduceString(); } - // Charcters that don't being escape sequences are easy; + // Characters that don't being escape sequences are easy; // just append them to the buffer and move on. if(c != '\\') { @@ -888,11 +890,11 @@ namespace Slang { // A file name usually doesn't process escape sequences // (this is import on Windows, where `\\` is a valid - // path separator cahracter). + // path separator character). // Just trim off the first and last characters to remove the quotes // (whether they were `""` or `<>`. - return token.Content.SubString(1, token.Content.Length()-2); + return String(token.Content.begin() + 1, token.Content.end() - 1); } @@ -1268,40 +1270,49 @@ namespace Slang // Note(tfoley): `StringBuilder::Append()` seems to crash when appending zero bytes if(textEnd != textBegin) { - // HACK(tfoley): "scrubbing" token value here to remove escaped newlines... + // "scrubbing" token value here to remove escaped newlines... // - // TODO: Only perform this work if we encountered an escaped newline + // Only perform this work if we encountered an escaped newline // while lexing this token (e.g., keep a flag on the lexer), or // do it on-demand when the actual value of the token is needed. - - StringBuilder valueBuilder; - auto tt = textBegin; - while(tt != textEnd) + if (tokenFlags & TokenFlag::ScrubbingNeeded) { - char c = *tt++; - if(c == '\\') + // Allocate space that will always be more than enough for stripped contents + char* startDst = (char*)memoryArena->allocateUnaligned(textEnd - textBegin); + char* dst = startDst; + + auto tt = textBegin; + while (tt != textEnd) { - char d = *tt; - switch(d) + char c = *tt++; + if (c == '\\') { - case '\r': case '\n': + char d = *tt; + switch (d) + { + case '\r': case '\n': { tt++; char e = *tt; - if((d ^ e) == ('\r' ^ '\n')) + if ((d ^ e) == ('\r' ^ '\n')) { tt++; } } continue; - default: - break; + default: + break; + } } + *dst++ = c; } - valueBuilder.Append(c); + token.Content = UnownedStringSlice(startDst, dst); + } + else + { + token.Content = UnownedStringSlice(textBegin, textEnd); } - token.Content = valueBuilder.ProduceString(); } token.flags = flags; diff --git a/source/slang/lexer.h b/source/slang/lexer.h index 85aca77dd..593e07c22 100644 --- a/source/slang/lexer.h +++ b/source/slang/lexer.h @@ -75,7 +75,8 @@ namespace Slang void initialize( SourceView* sourceView, DiagnosticSink* sink, - NamePool* namePool); + NamePool* namePool, + MemoryArena* memoryArena); ~Lexer(); @@ -97,6 +98,8 @@ namespace Slang TokenFlags tokenFlags; LexerFlags lexerFlags; + + MemoryArena* memoryArena; }; // Helper routines for extracting values from tokens @@ -106,8 +109,8 @@ namespace Slang typedef int64_t IntegerLiteralValue; typedef double FloatingPointLiteralValue; - IntegerLiteralValue getIntegerLiteralValue(Token const& token, String* outSuffix = 0); - FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, String* outSuffix = 0); + IntegerLiteralValue getIntegerLiteralValue(Token const& token, UnownedStringSlice* outSuffix = 0); + FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, UnownedStringSlice* outSuffix = 0); } #endif diff --git a/source/slang/lower-to-ir.cpp b/source/slang/lower-to-ir.cpp index 0d39664c8..2242619d0 100644 --- a/source/slang/lower-to-ir.cpp +++ b/source/slang/lower-to-ir.cpp @@ -458,9 +458,9 @@ IROp getIntrinsicOp( // based on the name of the declaration... auto name = decl->getName(); - auto nameText = getText(name); + auto nameText = getUnownedStringSliceText(name); - IROp op = findIROp(nameText.Buffer()); + IROp op = findIROp(nameText); SLANG_ASSERT(op != kIROp_Invalid); return op; } @@ -4909,7 +4909,7 @@ struct DeclLoweringVisitor : DeclVisitor definition = definitionToken.Content; } - builder->addTargetIntrinsicDecoration(irInst, targetMod->targetToken.Content.getUnownedSlice(), definition.getUnownedSlice()); + builder->addTargetIntrinsicDecoration(irInst, targetMod->targetToken.Content, definition.getUnownedSlice()); } } @@ -5240,7 +5240,7 @@ struct DeclLoweringVisitor : DeclVisitor // a specialized definition of the particular function for the given // target, and we need to reflect that at the IR level. - getBuilder()->addTargetDecoration(irFunc, targetMod->targetToken.Content.getUnownedSlice()); + getBuilder()->addTargetDecoration(irFunc, targetMod->targetToken.Content); } // If this declaration was marked as having a target-specific lowering @@ -5255,7 +5255,7 @@ struct DeclLoweringVisitor : DeclVisitor // for(auto extensionMod : decl->GetModifiersOfType()) { - getBuilder()->addRequireGLSLExtensionDecoration(irFunc, extensionMod->extensionNameToken.Content.getUnownedSlice()); + getBuilder()->addRequireGLSLExtensionDecoration(irFunc, extensionMod->extensionNameToken.Content); } for(auto versionMod : decl->GetModifiersOfType()) { diff --git a/source/slang/name.cpp b/source/slang/name.cpp index 995bce19f..a5cbb6541 100644 --- a/source/slang/name.cpp +++ b/source/slang/name.cpp @@ -9,6 +9,11 @@ String getText(Name* name) return name->text; } +UnownedStringSlice getUnownedStringSliceText(Name* name) +{ + return name ? name->text.getUnownedSlice() : UnownedStringSlice(); +} + Name* NamePool::getName(String const& text) { RefPtr name; diff --git a/source/slang/name.h b/source/slang/name.h index e492c4507..e1a055e60 100644 --- a/source/slang/name.h +++ b/source/slang/name.h @@ -36,6 +36,9 @@ public: // (e.g., so that it can be printed). String getText(Name* name); +/// Get the text as unowned string slice +UnownedStringSlice getUnownedStringSliceText(Name* name); + // A `RootNamePool` is used to store and look up names. // If two systems need to work together with names, and be sure that they // get equivalent names for a string like `"Foo"`, then they need to use diff --git a/source/slang/parameter-binding.cpp b/source/slang/parameter-binding.cpp index 622474116..e48b1d5bd 100644 --- a/source/slang/parameter-binding.cpp +++ b/source/slang/parameter-binding.cpp @@ -425,7 +425,7 @@ static bool isDigit(char c) /// Given a string that specifies a name and index (e.g., `COLOR0`), /// split it into slices for the name part and the index part. static void splitNameAndIndex( - String const& text, + UnownedStringSlice const& text, UnownedStringSlice& outName, UnownedStringSlice& outDigits) { @@ -482,8 +482,8 @@ LayoutSemanticInfo ExtractLayoutSemanticInfo( info.index = 0; info.kind = LayoutResourceKind::None; - String registerName = semantic->registerName.Content; - if (registerName.Length() == 0) + UnownedStringSlice registerName = semantic->registerName.Content; + if (registerName.size() == 0) return info; // The register name is expected to be in the form: @@ -526,7 +526,7 @@ LayoutSemanticInfo ExtractLayoutSemanticInfo( if( auto registerSemantic = dynamic_cast(semantic) ) { auto const& spaceName = registerSemantic->spaceName.Content; - if(spaceName.Length() != 0) + if(spaceName.size() != 0) { UnownedStringSlice spaceSpelling; UnownedStringSlice spaceDigits; @@ -556,7 +556,7 @@ LayoutSemanticInfo ExtractLayoutSemanticInfo( } // TODO: handle component mask part of things... - if( semantic->componentMask.Content.Length() != 0 ) + if( semantic->componentMask.Content.size() != 0 ) { getSink(context)->diagnose(semantic->componentMask, Diagnostics::componentMaskNotSupported); } @@ -999,7 +999,7 @@ static bool findLayoutArg( { if( modifier ) { - *outVal = (UInt) strtoull(modifier->valToken.Content.Buffer(), nullptr, 10); + *outVal = (UInt) strtoull(String(modifier->valToken.Content).Buffer(), nullptr, 10); return true; } } @@ -1981,7 +1981,7 @@ SimpleSemanticInfo decomposeSimpleSemantic( // look for a trailing sequence of decimal digits // at the end of the composed name - UInt length = composedName.Length(); + UInt length = composedName.size(); UInt indexLoc = length; while( indexLoc > 0 ) { @@ -2009,8 +2009,10 @@ SimpleSemanticInfo decomposeSimpleSemantic( else { // The name is everything before the digits - info.name = composedName.SubString(0, indexLoc); - info.index = strtol(composedName.SubString(indexLoc, length - indexLoc).begin(), nullptr, 10); + String stringComposedName(composedName); + + info.name = stringComposedName.SubString(0, indexLoc); + info.index = strtol(stringComposedName.begin() + indexLoc, nullptr, 10); } return info; } diff --git a/source/slang/parser.cpp b/source/slang/parser.cpp index 2e0ca5df9..d0e965970 100644 --- a/source/slang/parser.cpp +++ b/source/slang/parser.cpp @@ -690,9 +690,15 @@ namespace Slang } // Make a 'token' - const String scopedIdentifier(scopedIdentifierBuilder.ToString()); + SourceManager* sourceManager = parser->sink->sourceManager; + const UnownedStringSlice scopedIdentifier(sourceManager->allocateStringSlice(scopedIdentifierBuilder.getUnownedSlice())); Token token(TokenType::Identifier, scopedIdentifier, scopedIdSourceLoc); - token.ptrValue = getName(parser, token.Content); + + // Get the name pool + auto namePool = parser->translationUnit->compileRequest->getNamePool(); + + // Since it's an Identifier have to set the name. + token.ptrValue = namePool->getName(token.Content); return token; } @@ -990,7 +996,8 @@ namespace Slang case TokenType::QuestionMark: if (AdvanceIf(parser, TokenType::Colon)) { - nameToken.Content = nameToken.Content + ":"; + // Concat : onto ? + nameToken.Content = UnownedStringSlice::fromLiteral("?:"); break; } ; // fall-thru @@ -2210,8 +2217,8 @@ namespace Slang addModifier(bufferVarDecl, reflectionNameModifier); // Both the buffer variable and its type need to have names generated - bufferVarDecl->nameAndLoc.name = generateName(parser, "parameterGroup_" + reflectionNameToken.Content); - bufferDataTypeDecl->nameAndLoc.name = generateName(parser, "ParameterGroup_" + reflectionNameToken.Content); + bufferVarDecl->nameAndLoc.name = generateName(parser, "parameterGroup_" + String(reflectionNameToken.Content)); + bufferDataTypeDecl->nameAndLoc.name = generateName(parser, "ParameterGroup_" + String(reflectionNameToken.Content)); addModifier(bufferDataTypeDecl, new ImplicitParameterGroupElementTypeModifier()); addModifier(bufferVarDecl, new ImplicitParameterGroupVariableModifier()); @@ -2379,7 +2386,7 @@ namespace Slang parser->FillPosition(blockVarDecl.Ptr()); // Generate a unique name for the data type - blockDataTypeDecl->nameAndLoc.name = generateName(parser, "ParameterGroup_" + reflectionNameToken.Content); + blockDataTypeDecl->nameAndLoc.name = generateName(parser, "ParameterGroup_" + String(reflectionNameToken.Content)); // TODO(tfoley): We end up constructing unchecked syntax here that // is expected to type check into the right form, but it might be @@ -2424,7 +2431,7 @@ namespace Slang else { // synthesize a dummy name - blockVarDecl->nameAndLoc.name = generateName(parser, "parameterGroup_" + reflectionNameToken.Content); + blockVarDecl->nameAndLoc.name = generateName(parser, "parameterGroup_" + String(reflectionNameToken.Content)); // Otherwise we have a transparent declaration, similar // to an HLSL `cbuffer` @@ -3718,7 +3725,7 @@ namespace Slang { case TokenType::QuestionMark: opToken = parser->ReadToken(); - opToken.Content = "?:"; + opToken.Content = UnownedStringSlice::fromLiteral("?:"); break; default: @@ -3999,19 +4006,20 @@ namespace Slang auto token = parser->tokenReader.AdvanceToken(); constExpr->token = token; - String suffix; + UnownedStringSlice suffix; IntegerLiteralValue value = getIntegerLiteralValue(token, &suffix); // Look at any suffix on the value char const* suffixCursor = suffix.begin(); + const char*const suffixEnd = suffix.end(); RefPtr suffixType = nullptr; - if( suffixCursor && *suffixCursor ) + if( suffixCursor < suffixEnd ) { int lCount = 0; int uCount = 0; int unknownCount = 0; - while(*suffixCursor) + while(suffixCursor < suffixEnd) { switch( *suffixCursor++ ) { @@ -4077,20 +4085,21 @@ namespace Slang auto token = parser->tokenReader.AdvanceToken(); constExpr->token = token; - String suffix; + UnownedStringSlice suffix; FloatingPointLiteralValue value = getFloatingPointLiteralValue(token, &suffix); // Look at any suffix on the value char const* suffixCursor = suffix.begin(); + const char*const suffixEnd = suffix.end(); RefPtr suffixType = nullptr; - if( suffixCursor && *suffixCursor ) + if( suffixCursor < suffixEnd ) { int fCount = 0; int lCount = 0; int hCount = 0; int unknownCount = 0; - while(*suffixCursor) + while(suffixCursor < suffixEnd) { switch( *suffixCursor++ ) { @@ -4112,9 +4121,9 @@ namespace Slang } } - if(unknownCount) + if (unknownCount) { - parser->sink->diagnose(token, Diagnostics::invalidFloatingPOintLiteralSuffix, suffix); + parser->sink->diagnose(token, Diagnostics::invalidFloatingPointLiteralSuffix, suffix); suffixType = parser->getSession()->getErrorType(); } // `f` suffix -> `float` @@ -4135,7 +4144,7 @@ namespace Slang // TODO: are there other suffixes we need to handle? else { - parser->sink->diagnose(token, Diagnostics::invalidFloatingPOintLiteralSuffix, suffix); + parser->sink->diagnose(token, Diagnostics::invalidFloatingPointLiteralSuffix, suffix); suffixType = parser->getSession()->getErrorType(); } } @@ -4424,7 +4433,7 @@ namespace Slang { modifier->opToken = parser->ReadToken(TokenType::Identifier); - modifier->op = findIROp(modifier->opToken.Content.Buffer()); + modifier->op = findIROp(modifier->opToken.Content); if (modifier->op == kIROp_Invalid) { diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp index 00784ae92..21942c8e5 100644 --- a/source/slang/preprocessor.cpp +++ b/source/slang/preprocessor.cpp @@ -267,11 +267,13 @@ static PreprocessorInputStream* CreateInputStreamForSource( Preprocessor* preprocessor, SourceView* sourceView) { + MemoryArena* memoryArena = sourceView->getSourceManager()->getMemoryArena(); + PrimaryInputStream* inputStream = new PrimaryInputStream(); initializePrimaryInputStream(preprocessor, inputStream); // initialize the embedded lexer so that it can generate a token stream - inputStream->lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor)); + inputStream->lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor), memoryArena); inputStream->token = inputStream->lexer.lexToken(); return inputStream; @@ -845,7 +847,7 @@ top: SourceView* sourceView = sourceManager->createSourceView(sourceFile); Lexer lexer; - lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor)); + lexer.initialize(sourceView, GetSink(preprocessor), getNamePool(preprocessor), sourceManager->getMemoryArena()); SimpleTokenInputStream* inputStream = new SimpleTokenInputStream(); initializeInputStream(preprocessor, inputStream); @@ -925,7 +927,7 @@ inline Token const& GetDirective(PreprocessorDirectiveContext* context) } // Get the name of the directive being parsed. -inline String const& GetDirectiveName(PreprocessorDirectiveContext* context) +inline UnownedStringSlice const& GetDirectiveName(PreprocessorDirectiveContext* context) { return context->directiveToken.Content; } @@ -2274,7 +2276,7 @@ static void DefineMacro( // Use existing `Lexer` to generate a token stream. Lexer lexer; - lexer.initialize(valueView, GetSink(preprocessor), getNamePool(preprocessor)); + lexer.initialize(valueView, GetSink(preprocessor), getNamePool(preprocessor), sourceManager->getMemoryArena()); macro->tokens = lexer.lexAllTokens(); Name* keyName = preprocessor->translationUnit->compileRequest->getNamePool()->getName(key); diff --git a/source/slang/source-loc.cpp b/source/slang/source-loc.cpp index 62926aa8c..6d6d63ae1 100644 --- a/source/slang/source-loc.cpp +++ b/source/slang/source-loc.cpp @@ -292,6 +292,16 @@ void SourceManager::initialize( m_nextLoc = m_startLoc; } +UnownedStringSlice SourceManager::allocateStringSlice(const UnownedStringSlice& slice) +{ + const UInt numChars = slice.size(); + + char* dst = (char*)m_memoryArena.allocate(numChars); + ::memcpy(dst, slice.begin(), numChars); + + return UnownedStringSlice(dst, numChars); +} + SourceRange SourceManager::allocateSourceRange(UInt size) { // TODO: consider using atomics here diff --git a/source/slang/source-loc.h b/source/slang/source-loc.h index aba9b7bd9..6949d659a 100644 --- a/source/slang/source-loc.h +++ b/source/slang/source-loc.h @@ -1,4 +1,4 @@ -// source-loc.h +// source-loc.h #ifndef SLANG_SOURCE_LOC_H_INCLUDED #define SLANG_SOURCE_LOC_H_INCLUDED @@ -303,6 +303,16 @@ struct SourceManager /// Get the parent manager to this manager. Returns nullptr if there isn't any. SourceManager* getParent() const { return m_parent; } + /// A memory arena to hold allocations that are in scope for the same time as SourceManager + MemoryArena* getMemoryArena() { return &m_memoryArena; } + + /// Allocate a string slice + UnownedStringSlice allocateStringSlice(const UnownedStringSlice& slice); + + SourceManager() : + m_memoryArena(2048) + {} + protected: // The first location available to this source manager @@ -320,6 +330,10 @@ struct SourceManager List > m_sourceViews; StringSlicePool m_slicePool; + // Memory arena that can be used for holding data to held in scope as long as the Source is + // Can be used for storing the decoded contents of Token.Content for exampel + MemoryArena m_memoryArena; + // Maps canonical paths to source files Dictionary > m_sourceFiles; }; diff --git a/source/slang/token.h b/source/slang/token.h index e3b6f48ff..d7d45882d 100644 --- a/source/slang/token.h +++ b/source/slang/token.h @@ -36,13 +36,13 @@ public: SourceLoc loc; void* ptrValue; - String Content; + UnownedStringSlice Content; Token() = default; Token( TokenType typeIn, - const String & contentIn, + const UnownedStringSlice & contentIn, SourceLoc locIn, TokenFlags flagsIn = 0) : flags(flagsIn) -- cgit v1.2.3