summaryrefslogtreecommitdiff
path: root/source/core/slang-token-reader.h
diff options
context:
space:
mode:
authorEllie Hermaszewska <ellieh@nvidia.com>2024-10-29 14:49:26 +0800
committerGitHub <noreply@github.com>2024-10-29 14:49:26 +0800
commitf65d756bff8d4c5cbc15bd0322a2ae8e6b896a21 (patch)
treeea1d61342cd29368e19135000ec2948813096205 /source/core/slang-token-reader.h
parenta729c15e9dce9f5116a38afc66329ab2ca4cea54 (diff)
format
* format * Minor test fixes * enable checking cpp format in ci
Diffstat (limited to 'source/core/slang-token-reader.h')
-rw-r--r--source/core/slang-token-reader.h533
1 files changed, 289 insertions, 244 deletions
diff --git a/source/core/slang-token-reader.h b/source/core/slang-token-reader.h
index c4b552d29..41affb1d0 100644
--- a/source/core/slang-token-reader.h
+++ b/source/core/slang-token-reader.h
@@ -3,306 +3,351 @@
#include "slang-basic.h"
-namespace Slang {
-namespace Misc {
+namespace Slang
+{
+namespace Misc
+{
- /* NOTE! This TokenReader is NOT used by the main slang compiler !*/
+/* NOTE! This TokenReader is NOT used by the main slang compiler !*/
- enum class TokenType
- {
- EndOfFile = -1,
- // illegal
- Unknown,
- // identifier
- Identifier,
- // constant
- IntLiteral, DoubleLiteral, StringLiteral, CharLiteral,
- // operators
- Semicolon, Comma, Dot, LBrace, RBrace, LBracket, RBracket, LParent, RParent,
- OpAssign, OpAdd, OpSub, OpMul, OpDiv, OpMod, OpNot, OpBitNot, OpLsh, OpRsh,
- OpEql, OpNeq, OpGreater, OpLess, OpGeq, OpLeq,
- OpAnd, OpOr, OpBitXor, OpBitAnd, OpBitOr,
- OpInc, OpDec, OpAddAssign, OpSubAssign, OpMulAssign, OpDivAssign, OpModAssign,
- OpShlAssign, OpShrAssign, OpOrAssign, OpAndAssign, OpXorAssign,
+enum class TokenType
+{
+ EndOfFile = -1,
+ // illegal
+ Unknown,
+ // identifier
+ Identifier,
+ // constant
+ IntLiteral,
+ DoubleLiteral,
+ StringLiteral,
+ CharLiteral,
+ // operators
+ Semicolon,
+ Comma,
+ Dot,
+ LBrace,
+ RBrace,
+ LBracket,
+ RBracket,
+ LParent,
+ RParent,
+ OpAssign,
+ OpAdd,
+ OpSub,
+ OpMul,
+ OpDiv,
+ OpMod,
+ OpNot,
+ OpBitNot,
+ OpLsh,
+ OpRsh,
+ OpEql,
+ OpNeq,
+ OpGreater,
+ OpLess,
+ OpGeq,
+ OpLeq,
+ OpAnd,
+ OpOr,
+ OpBitXor,
+ OpBitAnd,
+ OpBitOr,
+ OpInc,
+ OpDec,
+ OpAddAssign,
+ OpSubAssign,
+ OpMulAssign,
+ OpDivAssign,
+ OpModAssign,
+ OpShlAssign,
+ OpShrAssign,
+ OpOrAssign,
+ OpAndAssign,
+ OpXorAssign,
- QuestionMark, Colon, RightArrow, At, Pound, PoundPound, Scope,
- };
+ QuestionMark,
+ Colon,
+ RightArrow,
+ At,
+ Pound,
+ PoundPound,
+ Scope,
+};
- class CodePosition
+class CodePosition
+{
+public:
+ int Line = -1, Col = -1, Pos = -1;
+ String FileName;
+ String ToString()
{
- public:
- int Line = -1, Col = -1, Pos = -1;
- String FileName;
- String ToString()
- {
- StringBuilder sb(100);
- sb << FileName;
- if (Line != -1)
- sb << "(" << Line << ")";
- return sb.produceString();
- }
- CodePosition() = default;
- CodePosition(int line, int col, int pos, String fileName)
- {
- Line = line;
- Col = col;
- Pos = pos;
- this->FileName = fileName;
- }
- bool operator < (const CodePosition & pos) const
- {
- return FileName < pos.FileName || (FileName == pos.FileName && Line < pos.Line) ||
- (FileName == pos.FileName && Line == pos.Line && Col < pos.Col);
- }
- bool operator == (const CodePosition & pos) const
- {
- return FileName == pos.FileName && Line == pos.Line && Col == pos.Col;
- }
- };
-
- enum TokenFlag : unsigned int
+ StringBuilder sb(100);
+ sb << FileName;
+ if (Line != -1)
+ sb << "(" << Line << ")";
+ return sb.produceString();
+ }
+ CodePosition() = default;
+ CodePosition(int line, int col, int pos, String fileName)
+ {
+ Line = line;
+ Col = col;
+ Pos = pos;
+ this->FileName = fileName;
+ }
+ bool operator<(const CodePosition& pos) const
+ {
+ return FileName < pos.FileName || (FileName == pos.FileName && Line < pos.Line) ||
+ (FileName == pos.FileName && Line == pos.Line && Col < pos.Col);
+ }
+ bool operator==(const CodePosition& pos) const
{
- AtStartOfLine = 1 << 0,
- AfterWhitespace = 1 << 1,
- };
- typedef unsigned int TokenFlags;
+ return FileName == pos.FileName && Line == pos.Line && Col == pos.Col;
+ }
+};
- class Token
+enum TokenFlag : unsigned int
+{
+ AtStartOfLine = 1 << 0,
+ AfterWhitespace = 1 << 1,
+};
+typedef unsigned int TokenFlags;
+
+class Token
+{
+public:
+ TokenType Type = TokenType::Unknown;
+ String Content;
+ CodePosition Position;
+ TokenFlags flags = 0;
+ Token() = default;
+ Token(
+ TokenType type,
+ const String& content,
+ int line,
+ int col,
+ int pos,
+ String fileName,
+ TokenFlags flags = 0)
+ : flags(flags)
{
- public:
- TokenType Type = TokenType::Unknown;
- String Content;
- CodePosition Position;
- TokenFlags flags = 0;
- Token() = default;
- Token(TokenType type, const String & content, int line, int col, int pos, String fileName, TokenFlags flags = 0)
- : flags(flags)
- {
- Type = type;
- Content = content;
- Position = CodePosition(line, col, pos, fileName);
- }
- };
+ Type = type;
+ Content = content;
+ Position = CodePosition(line, col, pos, fileName);
+ }
+};
- class TextFormatException : public Exception
+class TextFormatException : public Exception
+{
+public:
+ TextFormatException(String message)
+ : Exception(message)
{
- public:
- TextFormatException(String message)
- : Exception(message)
- {}
- };
+ }
+};
+
+class TokenReader
+{
+private:
+ bool legal;
+ List<Token> tokens;
+ int tokenPtr;
- class TokenReader
+public:
+ TokenReader(String text);
+ int ReadInt()
{
- private:
- bool legal;
- List<Token> tokens;
- int tokenPtr;
- public:
- TokenReader(String text);
- int ReadInt()
+ auto token = ReadToken();
+ bool neg = false;
+ if (token.Content.getUnownedSlice().isChar('-'))
{
- auto token = ReadToken();
- bool neg = false;
- if (token.Content.getUnownedSlice().isChar('-'))
- {
- neg = true;
- token = ReadToken();
- }
- if (token.Type == TokenType::IntLiteral)
- {
- if (neg)
- return -stringToInt(token.Content);
- else
- return stringToInt(token.Content);
- }
- throw TextFormatException("Text parsing error: int expected.");
+ neg = true;
+ token = ReadToken();
}
- unsigned int ReadUInt()
+ if (token.Type == TokenType::IntLiteral)
{
- auto token = ReadToken();
- if (token.Type == TokenType::IntLiteral)
- {
- return stringToUInt(token.Content);
- }
- throw TextFormatException("Text parsing error: int expected.");
+ if (neg)
+ return -stringToInt(token.Content);
+ else
+ return stringToInt(token.Content);
}
- double ReadDouble()
+ throw TextFormatException("Text parsing error: int expected.");
+ }
+ unsigned int ReadUInt()
+ {
+ auto token = ReadToken();
+ if (token.Type == TokenType::IntLiteral)
{
- auto token = ReadToken();
- bool neg = false;
- if (token.Content.getUnownedSlice().isChar('-'))
- {
- neg = true;
- token = ReadToken();
- }
- if (token.Type == TokenType::DoubleLiteral || token.Type == TokenType::IntLiteral)
- {
- if (neg)
- return -stringToDouble(token.Content);
- else
- return stringToDouble(token.Content);
- }
- throw TextFormatException("Text parsing error: floating point value expected.");
+ return stringToUInt(token.Content);
}
- float ReadFloat()
+ throw TextFormatException("Text parsing error: int expected.");
+ }
+ double ReadDouble()
+ {
+ auto token = ReadToken();
+ bool neg = false;
+ if (token.Content.getUnownedSlice().isChar('-'))
{
- return (float)ReadDouble();
+ neg = true;
+ token = ReadToken();
}
- String ReadWord()
+ if (token.Type == TokenType::DoubleLiteral || token.Type == TokenType::IntLiteral)
{
- auto token = ReadToken();
- if (token.Type == TokenType::Identifier)
- {
- return token.Content;
- }
- throw TextFormatException("Text parsing error: identifier expected.");
+ if (neg)
+ return -stringToDouble(token.Content);
+ else
+ return stringToDouble(token.Content);
}
- String Read(const char * expectedStr)
+ throw TextFormatException("Text parsing error: floating point value expected.");
+ }
+ float ReadFloat() { return (float)ReadDouble(); }
+ String ReadWord()
+ {
+ auto token = ReadToken();
+ if (token.Type == TokenType::Identifier)
{
- auto token = ReadToken();
- if (token.Content == expectedStr)
- {
- return token.Content;
- }
- throw TextFormatException("Text parsing error: \'" + String(expectedStr) + "\' expected.");
+ return token.Content;
}
- String Read(String expectedStr)
+ throw TextFormatException("Text parsing error: identifier expected.");
+ }
+ String Read(const char* expectedStr)
+ {
+ auto token = ReadToken();
+ if (token.Content == expectedStr)
{
- auto token = ReadToken();
- if (token.Content == expectedStr)
- {
- return token.Content;
- }
- throw TextFormatException("Text parsing error: \'" + expectedStr + "\' expected.");
+ return token.Content;
}
- bool Read(TokenType tokenType)
+ throw TextFormatException("Text parsing error: \'" + String(expectedStr) + "\' expected.");
+ }
+ String Read(String expectedStr)
+ {
+ auto token = ReadToken();
+ if (token.Content == expectedStr)
{
- if( NextToken().Type == tokenType )
- {
- ReadToken();
- return true;
- }
- throw TextFormatException("Text parsing error: unexpected '" + NextToken().Content + "'.");
+ return token.Content;
}
-
- String ReadStringLiteral()
+ throw TextFormatException("Text parsing error: \'" + expectedStr + "\' expected.");
+ }
+ bool Read(TokenType tokenType)
+ {
+ if (NextToken().Type == tokenType)
{
- auto token = ReadToken();
- if (token.Type == TokenType::StringLiteral)
- {
- return token.Content;
- }
- throw TextFormatException("Text parsing error: string literal expected.");
+ ReadToken();
+ return true;
}
- void Back(int count)
+ throw TextFormatException("Text parsing error: unexpected '" + NextToken().Content + "'.");
+ }
+
+ String ReadStringLiteral()
+ {
+ auto token = ReadToken();
+ if (token.Type == TokenType::StringLiteral)
{
- tokenPtr -= count;
+ return token.Content;
}
- Token ReadMatchingToken(TokenType type)
+ throw TextFormatException("Text parsing error: string literal expected.");
+ }
+ void Back(int count) { tokenPtr -= count; }
+ Token ReadMatchingToken(TokenType type)
+ {
+ auto token = ReadToken();
+ if (token.Type != type)
{
- auto token = ReadToken();
- if (token.Type != type)
- {
- throw TextFormatException("Text parsing error: unexpected token.");
- }
- return token;
+ throw TextFormatException("Text parsing error: unexpected token.");
}
- Token ReadToken()
+ return token;
+ }
+ Token ReadToken()
+ {
+ if (tokenPtr < (int)tokens.getCount())
{
- if (tokenPtr < (int)tokens.getCount())
- {
- auto &rs = tokens[tokenPtr];
- tokenPtr++;
- return rs;
- }
- throw TextFormatException("Unexpected ending.");
+ auto& rs = tokens[tokenPtr];
+ tokenPtr++;
+ return rs;
}
- Token NextToken(int offset = 0)
+ throw TextFormatException("Unexpected ending.");
+ }
+ Token NextToken(int offset = 0)
+ {
+ if (tokenPtr + offset < (int)tokens.getCount())
+ return tokens[tokenPtr + offset];
+ else
{
- if (tokenPtr + offset < (int)tokens.getCount())
- return tokens[tokenPtr + offset];
- else
- {
- Token rs;
- rs.Type = TokenType::Unknown;
- return rs;
- }
+ Token rs;
+ rs.Type = TokenType::Unknown;
+ return rs;
}
- bool LookAhead(String token)
+ }
+ bool LookAhead(String token)
+ {
+ if (tokenPtr < (int)tokens.getCount())
{
- if (tokenPtr < (int)tokens.getCount())
- {
- auto next = NextToken();
- return next.Content == token;
- }
- else
- {
- return false;
- }
+ auto next = NextToken();
+ return next.Content == token;
}
- bool LookAhead(TokenType tokenType)
+ else
{
- if (tokenPtr < (int)tokens.getCount())
- {
- auto next = NextToken();
- return next.Type == tokenType;
- }
- else
- {
- return false;
- }
+ return false;
}
- bool AdvanceIf(String token)
+ }
+ bool LookAhead(TokenType tokenType)
+ {
+ if (tokenPtr < (int)tokens.getCount())
{
- if( LookAhead(token) )
- {
- ReadToken();
- return true;
- }
- return false;
+ auto next = NextToken();
+ return next.Type == tokenType;
}
- bool AdvanceIf(TokenType token)
+ else
{
- if( LookAhead(token) )
- {
- ReadToken();
- return true;
- }
return false;
}
- bool IsEnd()
+ }
+ bool AdvanceIf(String token)
+ {
+ if (LookAhead(token))
{
- return tokenPtr == (int)tokens.getCount();
+ ReadToken();
+ return true;
}
- public:
- bool IsLegalText()
+ return false;
+ }
+ bool AdvanceIf(TokenType token)
+ {
+ if (LookAhead(token))
{
- return legal;
+ ReadToken();
+ return true;
}
- };
+ return false;
+ }
+ bool IsEnd() { return tokenPtr == (int)tokens.getCount(); }
+
+public:
+ bool IsLegalText() { return legal; }
+};
- inline List<String> Split(String text, char c)
+inline List<String> Split(String text, char c)
+{
+ List<String> result;
+ StringBuilder sb;
+ for (Index i = 0; i < text.getLength(); i++)
{
- List<String> result;
- StringBuilder sb;
- for (Index i = 0; i < text.getLength(); i++)
+ if (text[i] == c)
{
- if (text[i] == c)
- {
- auto str = sb.toString();
- if (str.getLength() != 0)
- result.add(str);
- sb.clear();
- }
- else
- sb << text[i];
+ auto str = sb.toString();
+ if (str.getLength() != 0)
+ result.add(str);
+ sb.clear();
}
- auto lastStr = sb.toString();
- if (lastStr.getLength())
- result.add(lastStr);
- return result;
+ else
+ sb << text[i];
}
+ auto lastStr = sb.toString();
+ if (lastStr.getLength())
+ result.add(lastStr);
+ return result;
+}
} // namespace Misc
} // namespace Slang