diff options
| author | Ellie Hermaszewska <ellieh@nvidia.com> | 2024-10-29 14:49:26 +0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-10-29 14:49:26 +0800 |
| commit | f65d756bff8d4c5cbc15bd0322a2ae8e6b896a21 (patch) | |
| tree | ea1d61342cd29368e19135000ec2948813096205 /source/core/slang-token-reader.h | |
| parent | a729c15e9dce9f5116a38afc66329ab2ca4cea54 (diff) | |
format
* format
* Minor test fixes
* enable checking cpp format in ci
Diffstat (limited to 'source/core/slang-token-reader.h')
| -rw-r--r-- | source/core/slang-token-reader.h | 533 |
1 files changed, 289 insertions, 244 deletions
diff --git a/source/core/slang-token-reader.h b/source/core/slang-token-reader.h index c4b552d29..41affb1d0 100644 --- a/source/core/slang-token-reader.h +++ b/source/core/slang-token-reader.h @@ -3,306 +3,351 @@ #include "slang-basic.h" -namespace Slang { -namespace Misc { +namespace Slang +{ +namespace Misc +{ - /* NOTE! This TokenReader is NOT used by the main slang compiler !*/ +/* NOTE! This TokenReader is NOT used by the main slang compiler !*/ - enum class TokenType - { - EndOfFile = -1, - // illegal - Unknown, - // identifier - Identifier, - // constant - IntLiteral, DoubleLiteral, StringLiteral, CharLiteral, - // operators - Semicolon, Comma, Dot, LBrace, RBrace, LBracket, RBracket, LParent, RParent, - OpAssign, OpAdd, OpSub, OpMul, OpDiv, OpMod, OpNot, OpBitNot, OpLsh, OpRsh, - OpEql, OpNeq, OpGreater, OpLess, OpGeq, OpLeq, - OpAnd, OpOr, OpBitXor, OpBitAnd, OpBitOr, - OpInc, OpDec, OpAddAssign, OpSubAssign, OpMulAssign, OpDivAssign, OpModAssign, - OpShlAssign, OpShrAssign, OpOrAssign, OpAndAssign, OpXorAssign, +enum class TokenType +{ + EndOfFile = -1, + // illegal + Unknown, + // identifier + Identifier, + // constant + IntLiteral, + DoubleLiteral, + StringLiteral, + CharLiteral, + // operators + Semicolon, + Comma, + Dot, + LBrace, + RBrace, + LBracket, + RBracket, + LParent, + RParent, + OpAssign, + OpAdd, + OpSub, + OpMul, + OpDiv, + OpMod, + OpNot, + OpBitNot, + OpLsh, + OpRsh, + OpEql, + OpNeq, + OpGreater, + OpLess, + OpGeq, + OpLeq, + OpAnd, + OpOr, + OpBitXor, + OpBitAnd, + OpBitOr, + OpInc, + OpDec, + OpAddAssign, + OpSubAssign, + OpMulAssign, + OpDivAssign, + OpModAssign, + OpShlAssign, + OpShrAssign, + OpOrAssign, + OpAndAssign, + OpXorAssign, - QuestionMark, Colon, RightArrow, At, Pound, PoundPound, Scope, - }; + QuestionMark, + Colon, + RightArrow, + At, + Pound, + PoundPound, + Scope, +}; - class CodePosition +class CodePosition +{ +public: + int Line = -1, Col = -1, Pos = -1; + String FileName; + String ToString() { - public: - int Line = -1, Col = -1, Pos = -1; - String FileName; - String ToString() - { - StringBuilder sb(100); - sb << FileName; - if (Line != -1) - sb << "(" << Line << ")"; - return sb.produceString(); - } - CodePosition() = default; - CodePosition(int line, int col, int pos, String fileName) - { - Line = line; - Col = col; - Pos = pos; - this->FileName = fileName; - } - bool operator < (const CodePosition & pos) const - { - return FileName < pos.FileName || (FileName == pos.FileName && Line < pos.Line) || - (FileName == pos.FileName && Line == pos.Line && Col < pos.Col); - } - bool operator == (const CodePosition & pos) const - { - return FileName == pos.FileName && Line == pos.Line && Col == pos.Col; - } - }; - - enum TokenFlag : unsigned int + StringBuilder sb(100); + sb << FileName; + if (Line != -1) + sb << "(" << Line << ")"; + return sb.produceString(); + } + CodePosition() = default; + CodePosition(int line, int col, int pos, String fileName) + { + Line = line; + Col = col; + Pos = pos; + this->FileName = fileName; + } + bool operator<(const CodePosition& pos) const + { + return FileName < pos.FileName || (FileName == pos.FileName && Line < pos.Line) || + (FileName == pos.FileName && Line == pos.Line && Col < pos.Col); + } + bool operator==(const CodePosition& pos) const { - AtStartOfLine = 1 << 0, - AfterWhitespace = 1 << 1, - }; - typedef unsigned int TokenFlags; + return FileName == pos.FileName && Line == pos.Line && Col == pos.Col; + } +}; - class Token +enum TokenFlag : unsigned int +{ + AtStartOfLine = 1 << 0, + AfterWhitespace = 1 << 1, +}; +typedef unsigned int TokenFlags; + +class Token +{ +public: + TokenType Type = TokenType::Unknown; + String Content; + CodePosition Position; + TokenFlags flags = 0; + Token() = default; + Token( + TokenType type, + const String& content, + int line, + int col, + int pos, + String fileName, + TokenFlags flags = 0) + : flags(flags) { - public: - TokenType Type = TokenType::Unknown; - String Content; - CodePosition Position; - TokenFlags flags = 0; - Token() = default; - Token(TokenType type, const String & content, int line, int col, int pos, String fileName, TokenFlags flags = 0) - : flags(flags) - { - Type = type; - Content = content; - Position = CodePosition(line, col, pos, fileName); - } - }; + Type = type; + Content = content; + Position = CodePosition(line, col, pos, fileName); + } +}; - class TextFormatException : public Exception +class TextFormatException : public Exception +{ +public: + TextFormatException(String message) + : Exception(message) { - public: - TextFormatException(String message) - : Exception(message) - {} - }; + } +}; + +class TokenReader +{ +private: + bool legal; + List<Token> tokens; + int tokenPtr; - class TokenReader +public: + TokenReader(String text); + int ReadInt() { - private: - bool legal; - List<Token> tokens; - int tokenPtr; - public: - TokenReader(String text); - int ReadInt() + auto token = ReadToken(); + bool neg = false; + if (token.Content.getUnownedSlice().isChar('-')) { - auto token = ReadToken(); - bool neg = false; - if (token.Content.getUnownedSlice().isChar('-')) - { - neg = true; - token = ReadToken(); - } - if (token.Type == TokenType::IntLiteral) - { - if (neg) - return -stringToInt(token.Content); - else - return stringToInt(token.Content); - } - throw TextFormatException("Text parsing error: int expected."); + neg = true; + token = ReadToken(); } - unsigned int ReadUInt() + if (token.Type == TokenType::IntLiteral) { - auto token = ReadToken(); - if (token.Type == TokenType::IntLiteral) - { - return stringToUInt(token.Content); - } - throw TextFormatException("Text parsing error: int expected."); + if (neg) + return -stringToInt(token.Content); + else + return stringToInt(token.Content); } - double ReadDouble() + throw TextFormatException("Text parsing error: int expected."); + } + unsigned int ReadUInt() + { + auto token = ReadToken(); + if (token.Type == TokenType::IntLiteral) { - auto token = ReadToken(); - bool neg = false; - if (token.Content.getUnownedSlice().isChar('-')) - { - neg = true; - token = ReadToken(); - } - if (token.Type == TokenType::DoubleLiteral || token.Type == TokenType::IntLiteral) - { - if (neg) - return -stringToDouble(token.Content); - else - return stringToDouble(token.Content); - } - throw TextFormatException("Text parsing error: floating point value expected."); + return stringToUInt(token.Content); } - float ReadFloat() + throw TextFormatException("Text parsing error: int expected."); + } + double ReadDouble() + { + auto token = ReadToken(); + bool neg = false; + if (token.Content.getUnownedSlice().isChar('-')) { - return (float)ReadDouble(); + neg = true; + token = ReadToken(); } - String ReadWord() + if (token.Type == TokenType::DoubleLiteral || token.Type == TokenType::IntLiteral) { - auto token = ReadToken(); - if (token.Type == TokenType::Identifier) - { - return token.Content; - } - throw TextFormatException("Text parsing error: identifier expected."); + if (neg) + return -stringToDouble(token.Content); + else + return stringToDouble(token.Content); } - String Read(const char * expectedStr) + throw TextFormatException("Text parsing error: floating point value expected."); + } + float ReadFloat() { return (float)ReadDouble(); } + String ReadWord() + { + auto token = ReadToken(); + if (token.Type == TokenType::Identifier) { - auto token = ReadToken(); - if (token.Content == expectedStr) - { - return token.Content; - } - throw TextFormatException("Text parsing error: \'" + String(expectedStr) + "\' expected."); + return token.Content; } - String Read(String expectedStr) + throw TextFormatException("Text parsing error: identifier expected."); + } + String Read(const char* expectedStr) + { + auto token = ReadToken(); + if (token.Content == expectedStr) { - auto token = ReadToken(); - if (token.Content == expectedStr) - { - return token.Content; - } - throw TextFormatException("Text parsing error: \'" + expectedStr + "\' expected."); + return token.Content; } - bool Read(TokenType tokenType) + throw TextFormatException("Text parsing error: \'" + String(expectedStr) + "\' expected."); + } + String Read(String expectedStr) + { + auto token = ReadToken(); + if (token.Content == expectedStr) { - if( NextToken().Type == tokenType ) - { - ReadToken(); - return true; - } - throw TextFormatException("Text parsing error: unexpected '" + NextToken().Content + "'."); + return token.Content; } - - String ReadStringLiteral() + throw TextFormatException("Text parsing error: \'" + expectedStr + "\' expected."); + } + bool Read(TokenType tokenType) + { + if (NextToken().Type == tokenType) { - auto token = ReadToken(); - if (token.Type == TokenType::StringLiteral) - { - return token.Content; - } - throw TextFormatException("Text parsing error: string literal expected."); + ReadToken(); + return true; } - void Back(int count) + throw TextFormatException("Text parsing error: unexpected '" + NextToken().Content + "'."); + } + + String ReadStringLiteral() + { + auto token = ReadToken(); + if (token.Type == TokenType::StringLiteral) { - tokenPtr -= count; + return token.Content; } - Token ReadMatchingToken(TokenType type) + throw TextFormatException("Text parsing error: string literal expected."); + } + void Back(int count) { tokenPtr -= count; } + Token ReadMatchingToken(TokenType type) + { + auto token = ReadToken(); + if (token.Type != type) { - auto token = ReadToken(); - if (token.Type != type) - { - throw TextFormatException("Text parsing error: unexpected token."); - } - return token; + throw TextFormatException("Text parsing error: unexpected token."); } - Token ReadToken() + return token; + } + Token ReadToken() + { + if (tokenPtr < (int)tokens.getCount()) { - if (tokenPtr < (int)tokens.getCount()) - { - auto &rs = tokens[tokenPtr]; - tokenPtr++; - return rs; - } - throw TextFormatException("Unexpected ending."); + auto& rs = tokens[tokenPtr]; + tokenPtr++; + return rs; } - Token NextToken(int offset = 0) + throw TextFormatException("Unexpected ending."); + } + Token NextToken(int offset = 0) + { + if (tokenPtr + offset < (int)tokens.getCount()) + return tokens[tokenPtr + offset]; + else { - if (tokenPtr + offset < (int)tokens.getCount()) - return tokens[tokenPtr + offset]; - else - { - Token rs; - rs.Type = TokenType::Unknown; - return rs; - } + Token rs; + rs.Type = TokenType::Unknown; + return rs; } - bool LookAhead(String token) + } + bool LookAhead(String token) + { + if (tokenPtr < (int)tokens.getCount()) { - if (tokenPtr < (int)tokens.getCount()) - { - auto next = NextToken(); - return next.Content == token; - } - else - { - return false; - } + auto next = NextToken(); + return next.Content == token; } - bool LookAhead(TokenType tokenType) + else { - if (tokenPtr < (int)tokens.getCount()) - { - auto next = NextToken(); - return next.Type == tokenType; - } - else - { - return false; - } + return false; } - bool AdvanceIf(String token) + } + bool LookAhead(TokenType tokenType) + { + if (tokenPtr < (int)tokens.getCount()) { - if( LookAhead(token) ) - { - ReadToken(); - return true; - } - return false; + auto next = NextToken(); + return next.Type == tokenType; } - bool AdvanceIf(TokenType token) + else { - if( LookAhead(token) ) - { - ReadToken(); - return true; - } return false; } - bool IsEnd() + } + bool AdvanceIf(String token) + { + if (LookAhead(token)) { - return tokenPtr == (int)tokens.getCount(); + ReadToken(); + return true; } - public: - bool IsLegalText() + return false; + } + bool AdvanceIf(TokenType token) + { + if (LookAhead(token)) { - return legal; + ReadToken(); + return true; } - }; + return false; + } + bool IsEnd() { return tokenPtr == (int)tokens.getCount(); } + +public: + bool IsLegalText() { return legal; } +}; - inline List<String> Split(String text, char c) +inline List<String> Split(String text, char c) +{ + List<String> result; + StringBuilder sb; + for (Index i = 0; i < text.getLength(); i++) { - List<String> result; - StringBuilder sb; - for (Index i = 0; i < text.getLength(); i++) + if (text[i] == c) { - if (text[i] == c) - { - auto str = sb.toString(); - if (str.getLength() != 0) - result.add(str); - sb.clear(); - } - else - sb << text[i]; + auto str = sb.toString(); + if (str.getLength() != 0) + result.add(str); + sb.clear(); } - auto lastStr = sb.toString(); - if (lastStr.getLength()) - result.add(lastStr); - return result; + else + sb << text[i]; } + auto lastStr = sb.toString(); + if (lastStr.getLength()) + result.add(lastStr); + return result; +} } // namespace Misc } // namespace Slang |
