Reduce the size of Token (#1349)

* Token size on 64 bits is 24 bytes (from 40). On 32 bits is 16 bytes from 24.

* Added hasContent method to Token.
Some other small improvements around Token.

10 files changed, 164 insertions, 98 deletions

diff --git a/source/slang/slang-check-shader.cpp b/source/slang/slang-check-shader.cpp
index b8a2a1d5e..ad5b1332e 100644
--- a/source/slang/slang-check-shader.cpp
+++ b/source/slang/slang-check-shader.cpp
@@ -396,7 +396,7 @@ namespace Slang
                 {
                     const auto& semanticToken = semantic->name;
 
-                    String lowerName = String(semanticToken.Content).toLower();
+                    String lowerName = String(semanticToken.getContent()).toLower();
 
                     if(lowerName == "sv_dispatchthreadid")
                     {
diff --git a/source/slang/slang-diagnostics.cpp b/source/slang/slang-diagnostics.cpp
index 615312661..6e0cfba4d 100644
--- a/source/slang/slang-diagnostics.cpp
+++ b/source/slang/slang-diagnostics.cpp
@@ -70,7 +70,7 @@ void printDiagnosticArg(StringBuilder& sb, TokenType tokenType)
 
 void printDiagnosticArg(StringBuilder& sb, Token const& token)
 {
-    sb << token.Content;
+    sb << token.getContent();
 }
 
 SourceLoc const& getDiagnosticPos(Token const& token)
diff --git a/source/slang/slang-lexer.cpp b/source/slang/slang-lexer.cpp
index e091de735..fe268223d 100644
--- a/source/slang/slang-lexer.cpp
+++ b/source/slang/slang-lexer.cpp
@@ -545,8 +545,10 @@ namespace Slang
     {
         IntegerLiteralValue value = 0;
 
-        char const* cursor = token.Content.begin();
-        char const* end = token.Content.end();
+        const UnownedStringSlice content = token.getContent();
+
+        char const* cursor = content.begin();
+        char const* end = content.end();
 
         int base = _readOptionalBase(&cursor);
 
@@ -571,8 +573,10 @@ namespace Slang
     {
         FloatingPointLiteralValue value = 0;
 
-        char const* cursor = token.Content.begin();
-        char const* end = token.Content.end();
+        const UnownedStringSlice content = token.getContent();
+
+        char const* cursor = content.begin();
+        char const* end = content.end();
 
         int radix = _readOptionalBase(&cursor);
 
@@ -756,8 +760,10 @@ namespace Slang
         SLANG_ASSERT(token.type == TokenType::StringLiteral
             || token.type == TokenType::CharLiteral);
 
-        char const* cursor = token.Content.begin();
-        char const* end = token.Content.end();
+        const UnownedStringSlice content = token.getContent();
+
+        char const* cursor = content.begin();
+        char const* end = content.end();
         SLANG_UNREFERENCED_VARIABLE(end);
 
         auto quote = *cursor++;
@@ -879,13 +885,15 @@ namespace Slang
 
     String getFileNameTokenValue(Token const& token)
     {
+        const UnownedStringSlice content = token.getContent();
+
         // A file name usually doesn't process escape sequences
         // (this is import on Windows, where `\\` is a valid
         // path separator character).
 
         // Just trim off the first and last characters to remove the quotes
         // (whether they were `""` or `<>`.
-        return String(token.Content.begin() + 1, token.Content.end() - 1); 
+        return String(content.begin() + 1, content.end() - 1); 
     }
 
 
@@ -1298,11 +1306,11 @@ namespace Slang
                         }
                         *dst++ = c;
                     }
-                    token.Content = UnownedStringSlice(startDst, dst);
+                    token.setContent(UnownedStringSlice(startDst, dst));
                 }
                 else
                 {
-                    token.Content = UnownedStringSlice(textBegin, textEnd);
+                    token.setContent(UnownedStringSlice(textBegin, textEnd));
                 }
             }
 
@@ -1312,7 +1320,7 @@ namespace Slang
 
             if (tokenType == TokenType::Identifier)
             {
-                token.ptrValue = m_namePool->getName(token.Content);
+                token.setName(m_namePool->getName(token.getContent()));
             }
 
             return token;
diff --git a/source/slang/slang-lower-to-ir.cpp b/source/slang/slang-lower-to-ir.cpp
index aaa9d1a4c..46afc77b7 100644
--- a/source/slang/slang-lower-to-ir.cpp
+++ b/source/slang/slang-lower-to-ir.cpp
@@ -5625,7 +5625,7 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
             }
             else if(definitionToken.type == TokenType::Identifier)
             {
-                definition = definitionToken.Content;
+                definition = definitionToken.getContent();
             }
             else
             {
@@ -5636,7 +5636,7 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
             auto& targetToken = targetMod->targetToken;
             if( targetToken.type != TokenType::Unknown )
             {
-                targetName = targetToken.Content;
+                targetName = targetToken.getContent();
             }
 
             builder->addTargetIntrinsicDecoration(irInst, targetName, definition.getUnownedSlice());
@@ -5723,7 +5723,7 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
                 //
                 if(targetMod->targetToken.type == TokenType::Unknown)
                     return;
-                else if(targetMod->targetToken.Content.getLength() == 0)
+                else if(!targetMod->targetToken.hasContent())
                     return;
             }
         }
@@ -6156,7 +6156,7 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
             // a specialized definition of the particular function for the given
             // target, and we need to reflect that at the IR level.
 
-            getBuilder()->addTargetDecoration(irFunc, targetMod->targetToken.Content);
+            getBuilder()->addTargetDecoration(irFunc, targetMod->targetToken.getContent());
         }
 
         // If this declaration was marked as having a target-specific lowering
@@ -6173,7 +6173,7 @@ struct DeclLoweringVisitor : DeclVisitor<DeclLoweringVisitor, LoweredValInfo>
         //
         for(auto extensionMod : decl->GetModifiersOfType<RequiredGLSLExtensionModifier>())
         {
-            getBuilder()->addRequireGLSLExtensionDecoration(irFunc, extensionMod->extensionNameToken.Content);
+            getBuilder()->addRequireGLSLExtensionDecoration(irFunc, extensionMod->extensionNameToken.getContent());
         }
         for(auto versionMod : decl->GetModifiersOfType<RequiredGLSLVersionModifier>())
         {
diff --git a/source/slang/slang-parameter-binding.cpp b/source/slang/slang-parameter-binding.cpp
index 353888b69..2ce7f5f2a 100644
--- a/source/slang/slang-parameter-binding.cpp
+++ b/source/slang/slang-parameter-binding.cpp
@@ -490,7 +490,7 @@ LayoutSemanticInfo ExtractLayoutSemanticInfo(
     info.index = 0;
     info.kind = LayoutResourceKind::None;
 
-    UnownedStringSlice registerName = semantic->registerName.Content;
+    UnownedStringSlice registerName = semantic->registerName.getContent();
     if (registerName.getLength() == 0)
         return info;
 
@@ -533,7 +533,7 @@ LayoutSemanticInfo ExtractLayoutSemanticInfo(
     UInt space = 0;
     if( auto registerSemantic = as<HLSLRegisterSemantic>(semantic) )
     {
-        auto const& spaceName = registerSemantic->spaceName.Content;
+        auto const& spaceName = registerSemantic->spaceName.getContent();
         if(spaceName.getLength() != 0)
         {
             UnownedStringSlice spaceSpelling;
@@ -564,7 +564,7 @@ LayoutSemanticInfo ExtractLayoutSemanticInfo(
     }
 
     // TODO: handle component mask part of things...
-    if( semantic->componentMask.Content.getLength() != 0 )
+    if( semantic->componentMask.hasContent())
     {
         getSink(context)->diagnose(semantic->componentMask, Diagnostics::componentMaskNotSupported);
     }
@@ -589,7 +589,7 @@ static bool findLayoutArg(
     {
         if( modifier )
         {
-            *outVal = (UInt) strtoull(String(modifier->valToken.Content).getBuffer(), nullptr, 10);
+            *outVal = (UInt) strtoull(String(modifier->valToken.getContent()).getBuffer(), nullptr, 10);
             return true;
         }
     }
@@ -1336,7 +1336,7 @@ struct SimpleSemanticInfo
 SimpleSemanticInfo decomposeSimpleSemantic(
     HLSLSimpleSemantic* semantic)
 {
-    auto composedName = semantic->name.Content;
+    auto composedName = semantic->name.getContent();
 
     // look for a trailing sequence of decimal digits
     // at the end of the composed name
diff --git a/source/slang/slang-parser.cpp b/source/slang/slang-parser.cpp
index cdd573a97..8d4ba36c0 100644
--- a/source/slang/slang-parser.cpp
+++ b/source/slang/slang-parser.cpp
@@ -315,7 +315,7 @@ namespace Slang
     Token Parser::ReadToken(const char* expected)
     {
         if (tokenReader.peekTokenType() == TokenType::Identifier
-                && tokenReader.peekToken().Content == expected)
+                && tokenReader.peekToken().getContent() == expected)
         {
             isRecovering = false;
             return tokenReader.advanceToken();
@@ -334,7 +334,7 @@ namespace Slang
                 // The token we expected?
                 // Then exit recovery mode and pretend like all is well.
                 if (tokenReader.peekTokenType() == TokenType::Identifier
-                    && tokenReader.peekToken().Content == expected)
+                    && tokenReader.peekToken().getContent() == expected)
                 {
                     isRecovering = false;
                     return tokenReader.advanceToken();
@@ -530,7 +530,7 @@ namespace Slang
             r.advanceToken();
 
         return r.peekTokenType() == TokenType::Identifier
-            && r.peekToken().Content == string;
+            && r.peekToken().getContent() == string;
 }
 
     bool Parser::LookAheadToken(TokenType type, int offset)
@@ -680,7 +680,7 @@ namespace Slang
         {
             scopedIdentifierBuilder.Append('_'); 
         }
-        scopedIdentifierBuilder.Append(firstIdentifier.Content);
+        scopedIdentifierBuilder.Append(firstIdentifier.getContent());
 
         while (parser->tokenReader.peekTokenType() == TokenType::Scope)
         {
@@ -688,7 +688,7 @@ namespace Slang
             scopedIdentifierBuilder.Append('_'); 
             
             const Token nextIdentifier(parser->ReadToken(TokenType::Identifier));
-            scopedIdentifierBuilder.Append(nextIdentifier.Content);
+            scopedIdentifierBuilder.Append(nextIdentifier.getContent());
         }
 
         // Make a 'token'
@@ -700,7 +700,7 @@ namespace Slang
         auto namePool = parser->getNamePool();
 
         // Since it's an Identifier have to set the name.
-        token.ptrValue = namePool->getName(token.Content);
+        token.setName(namePool->getName(token.getContent()));
 
         return token;
     }
@@ -954,7 +954,7 @@ namespace Slang
                 while (AdvanceIf(parser, TokenType::Dot))
                 {
                     sb << "/";
-                    sb << parser->ReadToken(TokenType::Identifier).Content;
+                    sb << parser->ReadToken(TokenType::Identifier).getContent();
                 }
 
                 moduleNameAndLoc.name = getName(parser, sb.ProduceString());
@@ -1003,7 +1003,7 @@ namespace Slang
                 if (AdvanceIf(parser, TokenType::Colon))
                 {
                     // Concat : onto ?
-                    nameToken.Content = UnownedStringSlice::fromLiteral("?:"); 
+                    nameToken.setContent(UnownedStringSlice::fromLiteral("?:")); 
                     break;
                 }
                 ;       // fall-thru
@@ -1013,7 +1013,7 @@ namespace Slang
             }
 
             return NameLoc(
-                getName(parser, nameToken.Content),
+                getName(parser, nameToken.getContent()),
                 nameToken.loc);
         }
         else
@@ -2301,8 +2301,8 @@ namespace Slang
         addModifier(bufferVarDecl, reflectionNameModifier);
 
         // Both the buffer variable and its type need to have names generated
-        bufferVarDecl->nameAndLoc.name = generateName(parser, "parameterGroup_" + String(reflectionNameToken.Content));
-        bufferDataTypeDecl->nameAndLoc.name = generateName(parser, "ParameterGroup_" + String(reflectionNameToken.Content));
+        bufferVarDecl->nameAndLoc.name = generateName(parser, "parameterGroup_" + String(reflectionNameToken.getContent()));
+        bufferDataTypeDecl->nameAndLoc.name = generateName(parser, "ParameterGroup_" + String(reflectionNameToken.getContent()));
 
         addModifier(bufferDataTypeDecl, new ImplicitParameterGroupElementTypeModifier());
         addModifier(bufferVarDecl, new ImplicitParameterGroupVariableModifier());
@@ -3893,7 +3893,7 @@ namespace Slang
         {
         case TokenType::QuestionMark:
             opToken = parser->ReadToken();
-            opToken.Content = UnownedStringSlice::fromLiteral("?:");
+            opToken.setContent(UnownedStringSlice::fromLiteral("?:"));
             break;
 
         default:
@@ -3902,7 +3902,7 @@ namespace Slang
         }
 
         auto opExpr = new VarExpr();
-        opExpr->name = getName(parser, opToken.Content);
+        opExpr->name = getName(parser, opToken.getContent());
         opExpr->scope = parser->currentScope;
         opExpr->loc = opToken.loc;
 
@@ -4249,7 +4249,7 @@ namespace Slang
             if ((!(maskedValue == 0 || maskedValue == mask)) && sink && token)
             {
                 // Output a warning that number has been altered
-                sink->diagnose(*token, Diagnostics::integerLiteralTruncated, token->Content, BaseTypeInfo::asText(baseType), truncatedValue);
+                sink->diagnose(*token, Diagnostics::integerLiteralTruncated, token->getContent(), BaseTypeInfo::asText(baseType), truncatedValue);
             }
 
             value = truncatedValue;
@@ -4516,12 +4516,12 @@ namespace Slang
                     }
                     case FloatFixKind::Zeroed:
                     {
-                        parser->sink->diagnose(token, Diagnostics::floatLiteralTooSmall, BaseTypeInfo::asText(suffixBaseType), token.Content, fixedValue);
+                        parser->sink->diagnose(token, Diagnostics::floatLiteralTooSmall, BaseTypeInfo::asText(suffixBaseType), token.getContent(), fixedValue);
                         break;
                     }
                     case FloatFixKind::Unrepresentable:
                     {
-                        parser->sink->diagnose(token, Diagnostics::floatLiteralUnrepresentable, BaseTypeInfo::asText(suffixBaseType), token.Content, fixedValue);
+                        parser->sink->diagnose(token, Diagnostics::floatLiteralUnrepresentable, BaseTypeInfo::asText(suffixBaseType), token.getContent(), fixedValue);
                         break;
                     }
                 }
@@ -4893,17 +4893,17 @@ namespace Slang
         {
             if (AdvanceIf(parser, TokenType::OpSub))
             {
-                modifier->op = IROp(-StringToInt(parser->ReadToken().Content));
+                modifier->op = IROp(-StringToInt(parser->ReadToken().getContent()));
             }
             else if (parser->LookAheadToken(TokenType::IntegerLiteral))
             {
-                modifier->op = IROp(StringToInt(parser->ReadToken().Content));
+                modifier->op = IROp(StringToInt(parser->ReadToken().getContent()));
             }
             else
             {
                 modifier->opToken = parser->ReadToken(TokenType::Identifier);
 
-                modifier->op = findIROp(modifier->opToken.Content);
+                modifier->op = findIROp(modifier->opToken.getContent());
 
                 if (modifier->op == kIROp_Invalid)
                 {
@@ -4984,7 +4984,7 @@ namespace Slang
         outToken = parser->ReadToken();
         parser->ReadToken(TokenType::RParent);
 
-        UnownedStringSlice content = outToken.Content;
+        UnownedStringSlice content = outToken.getContent();
         // We allow specified as major.minor or as a string (in quotes)
         switch (outToken.type)
         {
@@ -5162,7 +5162,7 @@ namespace Slang
     {
         RefPtr<BuiltinTypeModifier> modifier = new BuiltinTypeModifier();
         parser->ReadToken(TokenType::LParent);
-        modifier->tag = BaseType(StringToInt(parser->ReadToken(TokenType::IntegerLiteral).Content));
+        modifier->tag = BaseType(StringToInt(parser->ReadToken(TokenType::IntegerLiteral).getContent()));
         parser->ReadToken(TokenType::RParent);
 
         return modifier;
@@ -5172,10 +5172,10 @@ namespace Slang
     {
         RefPtr<MagicTypeModifier> modifier = new MagicTypeModifier();
         parser->ReadToken(TokenType::LParent);
-        modifier->name = parser->ReadToken(TokenType::Identifier).Content;
+        modifier->name = parser->ReadToken(TokenType::Identifier).getContent();
         if (AdvanceIf(parser, TokenType::Comma))
         {
-            modifier->tag = uint32_t(StringToInt(parser->ReadToken(TokenType::IntegerLiteral).Content));
+            modifier->tag = uint32_t(StringToInt(parser->ReadToken(TokenType::IntegerLiteral).getContent()));
         }
         parser->ReadToken(TokenType::RParent);
 
@@ -5186,10 +5186,10 @@ namespace Slang
     {
         RefPtr<IntrinsicTypeModifier> modifier = new IntrinsicTypeModifier();
         parser->ReadToken(TokenType::LParent);
-        modifier->irOp = uint32_t(StringToInt(parser->ReadToken(TokenType::IntegerLiteral).Content));
+        modifier->irOp = uint32_t(StringToInt(parser->ReadToken(TokenType::IntegerLiteral).getContent()));
         while( AdvanceIf(parser, TokenType::Comma) )
         {
-            auto operand = uint32_t(StringToInt(parser->ReadToken(TokenType::IntegerLiteral).Content));
+            auto operand = uint32_t(StringToInt(parser->ReadToken(TokenType::IntegerLiteral).getContent()));
             modifier->irOperands.add(operand);
         }
         parser->ReadToken(TokenType::RParent);
@@ -5203,7 +5203,7 @@ namespace Slang
         ConversionCost cost = kConversionCost_Default;
         if( AdvanceIf(parser, TokenType::LParent) )
         {
-            cost = ConversionCost(StringToInt(parser->ReadToken(TokenType::IntegerLiteral).Content));
+            cost = ConversionCost(StringToInt(parser->ReadToken(TokenType::IntegerLiteral).getContent()));
             parser->ReadToken(TokenType::RParent);
         }
         modifier->cost = cost;
diff --git a/source/slang/slang-preprocessor.cpp b/source/slang/slang-preprocessor.cpp
index 2f0ee69a2..60af58df0 100644
--- a/source/slang/slang-preprocessor.cpp
+++ b/source/slang/slang-preprocessor.cpp
@@ -999,7 +999,7 @@ top:
         // We are pasting tokens, which could get messy
 
         StringBuilder sb;
-        sb << token.Content;
+        sb << token.getContent();
 
         while (PeekRawTokenType(preprocessor) == TokenType::PoundPound)
         {
@@ -1012,7 +1012,7 @@ top:
             // Read the next raw token (now that expansion has been triggered)
             Token nextToken = AdvanceRawToken(preprocessor);
 
-            sb << nextToken.Content;
+            sb << nextToken.getContent();
         }
 
         // Now re-lex the input
@@ -1106,9 +1106,9 @@ inline Token const& GetDirective(PreprocessorDirectiveContext* context)
 }
 
 // Get the name of the directive being parsed.
-inline UnownedStringSlice const& GetDirectiveName(PreprocessorDirectiveContext* context)
+inline UnownedStringSlice GetDirectiveName(PreprocessorDirectiveContext* context)
 {
-    return context->directiveToken.Content;
+    return context->directiveToken.getContent();
 }
 
 // Get the location of the directive being parsed.
@@ -1360,12 +1360,12 @@ static PreprocessorExpressionValue ParseAndEvaluateUnaryExpression(PreprocessorD
         }
 
     case TokenType::IntegerLiteral:
-        return StringToInt(AdvanceToken(context).Content);
+        return StringToInt(AdvanceToken(context).getContent());
 
     case TokenType::Identifier:
         {
             Token token = AdvanceToken(context);
-            if (token.Content == "defined")
+            if (token.getContent() == "defined")
             {
                 // handle `defined(someName)`
 
@@ -1994,7 +1994,7 @@ static void HandleWarningDirective(PreprocessorDirectiveContext* context)
     Expect(context, TokenType::DirectiveMessage, Diagnostics::expectedTokenInPreprocessorDirective, &messageToken);
 
     // Report the custom error.
-    GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::userDefinedWarning, messageToken.Content);
+    GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::userDefinedWarning, messageToken.getContent());
 }
 
 // Handle a `#error` directive
@@ -2008,7 +2008,7 @@ static void HandleErrorDirective(PreprocessorDirectiveContext* context)
     Expect(context, TokenType::DirectiveMessage, Diagnostics::expectedTokenInPreprocessorDirective, &messageToken);
 
     // Report the custom error.
-    GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::userDefinedError, messageToken.Content);
+    GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::userDefinedError, messageToken.getContent());
 }
 
 // Handle a `#line` directive
@@ -2023,14 +2023,14 @@ static void HandleLineDirective(PreprocessorDirectiveContext* context)
     // `#line <integer-literal> ...`
     if (PeekTokenType(context) == TokenType::IntegerLiteral)
     {
-        line = StringToInt(AdvanceToken(context).Content);
+        line = StringToInt(AdvanceToken(context).getContent());
     }
     // `#line`
     // `#line default`
     else if (
         PeekTokenType(context) == TokenType::EndOfDirective
         || (PeekTokenType(context) == TokenType::Identifier
-            && PeekToken(context).Content == "default"))
+            && PeekToken(context).getContent() == "default"))
     {
         AdvanceToken(context);
 
@@ -2064,7 +2064,7 @@ static void HandleLineDirective(PreprocessorDirectiveContext* context)
     {
         // Note(tfoley): GLSL allows the "source string" to be indicated by an integer
         // TODO(tfoley): Figure out a better way to handle this, if it matters
-        file = AdvanceToken(context).Content;
+        file = AdvanceToken(context).getContent();
     }
     else
     {
diff --git a/source/slang/slang-reflection.cpp b/source/slang/slang-reflection.cpp
index a3cb88892..c0e83ccaf 100644
--- a/source/slang/slang-reflection.cpp
+++ b/source/slang/slang-reflection.cpp
@@ -183,8 +183,8 @@ SLANG_API const char* spReflectionUserAttribute_GetArgumentValueString(SlangRefl
     if (auto cexpr = as<StringLiteralExpr>(userAttr->args[index]))
     {
         if (bufLen)
-            *bufLen = cexpr->token.Content.getLength();
-        return cexpr->token.Content.begin();
+            *bufLen = cexpr->token.getContentLength();
+        return cexpr->token.getContent().begin();
     }
     return nullptr;
 }
diff --git a/source/slang/slang-token.cpp b/source/slang/slang-token.cpp
index a8238eb6c..a7f6d7d62 100644
--- a/source/slang/slang-token.cpp
+++ b/source/slang/slang-token.cpp
@@ -6,22 +6,7 @@
 namespace Slang {
 
 
-Name* Token::getName() const
-{
-    return getNameOrNull();
-}
-
-Name* Token::getNameOrNull() const
-{
-    switch (type)
-    {
-    default:
-        return nullptr;
 
-    case TokenType::Identifier:
-        return (Name*) ptrValue;
-    }
-}
 
 char const* TokenTypeToString(TokenType type)
 {
diff --git a/source/slang/slang-token.h b/source/slang/slang-token.h
index 193b128fa..9697a5c2d 100644
--- a/source/slang/slang-token.h
+++ b/source/slang/slang-token.h
@@ -5,12 +5,13 @@
 #include "../core/slang-basic.h"
 
 #include "slang-source-loc.h"
+#include "slang-name.h"
 
 namespace Slang {
 
 class Name;
 
-enum class TokenType
+enum class TokenType : uint8_t
 {
 #define TOKEN(NAME, DESC) NAME,
 #include "slang-token-defs.h"
@@ -18,48 +19,120 @@ enum class TokenType
 
 char const* TokenTypeToString(TokenType type);
 
-enum TokenFlag : unsigned int
+typedef uint8_t TokenFlags;
+struct TokenFlag
 {
-    AtStartOfLine           = 1 << 0,
-    AfterWhitespace         = 1 << 1,
-    SuppressMacroExpansion  = 1 << 2,
-    ScrubbingNeeded         = 1 << 3,
+    enum Enum : TokenFlags
+    {
+        AtStartOfLine           = 1 << 0,
+        AfterWhitespace         = 1 << 1,
+        SuppressMacroExpansion  = 1 << 2,
+        ScrubbingNeeded         = 1 << 3,
+        Name                    = 1 << 4,           ///< Determines if 'name' is set or 'chars' in the charsNameUnion
+    };
 };
-typedef unsigned int TokenFlags;
 
 class Token
 {
 public:
+
     TokenType   type = TokenType::Unknown;
     TokenFlags  flags = 0;
 
     SourceLoc   loc;
-    void*       ptrValue;
+    uint32_t charsCount = 0;              ///< Amount of characters. Is set if name or not.
 
-    UnownedStringSlice Content;
+    union CharsNameUnion
+    {
+        const char* chars;
+        Name* name;
+    };
 
-    Token() = default;
+    CharsNameUnion charsNameUnion;
 
-    Token(
-        TokenType typeIn,
-        const UnownedStringSlice & contentIn,
-        SourceLoc locIn,
-        TokenFlags flagsIn = 0)
-        : flags(flagsIn)
-	{
-		type = typeIn;
-		Content = contentIn;
-        loc = locIn;
-        ptrValue = nullptr;
-	}
+    bool hasContent() const { return charsCount > 0; }
+    Index getContentLength() const { return charsCount; }
+
+    UnownedStringSlice getContent() const;
+        /// Set content
+    void setContent(const UnownedStringSlice& content);
 
     Name* getName() const;
 
     Name* getNameOrNull() const;
 
     SourceLoc getLoc() const { return loc; }
+
+        /// Set the name
+    SLANG_FORCE_INLINE void setName(Name* inName);
+
+    Token()
+    {
+        charsNameUnion.chars = nullptr;
+    }
+
+    Token(
+        TokenType inType,
+        const UnownedStringSlice& inContent,
+        SourceLoc inLoc,
+        TokenFlags inFlags = 0)
+        : flags(inFlags)
+	{
+        SLANG_ASSERT((inFlags & TokenFlag::Name) == 0); 
+		type = inType;
+        charsNameUnion.chars = inContent.begin();
+        charsCount = uint32_t(inContent.getLength());
+        loc = inLoc;
+	}
+    Token(
+        TokenType inType,
+        Name* name, 
+        SourceLoc inLoc,
+        TokenFlags inFlags = 0)
+    {
+        SLANG_ASSERT(name);
+        type = inType;
+        flags = inFlags | TokenFlag::Name;
+        charsNameUnion.name = name;
+        charsCount = uint32_t(name->text.getLength());
+        loc = inLoc;
+    }
 };
 
+// ---------------------------------------------------------------------------
+SLANG_FORCE_INLINE UnownedStringSlice Token::getContent() const
+{
+    return (flags & TokenFlag::Name) ? charsNameUnion.name->text.getUnownedSlice() : UnownedStringSlice(charsNameUnion.chars, charsCount);
+}
+
+// ---------------------------------------------------------------------------
+SLANG_FORCE_INLINE Name* Token::getName() const
+{
+    return getNameOrNull();
+}
+
+// ---------------------------------------------------------------------------
+SLANG_FORCE_INLINE Name* Token::getNameOrNull() const
+{
+    return (flags & TokenFlag::Name) ? charsNameUnion.name : nullptr;
+}
+
+// ---------------------------------------------------------------------------
+SLANG_FORCE_INLINE void Token::setContent(const UnownedStringSlice& content)
+{
+    flags &= ~TokenFlag::Name;
+    charsNameUnion.chars = content.begin();
+    charsCount = uint32_t(content.getLength());
+}
+
+// ---------------------------------------------------------------------------
+SLANG_FORCE_INLINE void Token::setName(Name* inName)
+{
+    SLANG_ASSERT(inName);
+    flags |= TokenFlag::Name;
+    charsNameUnion.name = inName;
+    charsCount = uint32_t(inName->text.getLength());
+}
 
 
 } // namespace Slang