From 0ee4d4b54732239b946bae7fde32bb21aa5a3ec3 Mon Sep 17 00:00:00 2001 From: "YONGH\\yongh" Date: Fri, 20 Oct 2017 18:24:30 -0400 Subject: in-progress work: allow render-test to generate and bind various resource inputs for running test shaders with arbitrary parameter definitions. This commit contains the parser of the resource input definition. --- source/core/core.vcxproj | 2 + source/core/text-io.h | 22 -- source/core/token-reader.cpp | 868 +++++++++++++++++++++++++++++++++++++++++++ source/core/token-reader.h | 258 +++++++++++++ 4 files changed, 1128 insertions(+), 22 deletions(-) create mode 100644 source/core/token-reader.cpp create mode 100644 source/core/token-reader.h (limited to 'source/core') diff --git a/source/core/core.vcxproj b/source/core/core.vcxproj index ba9fe3d98..350482686 100644 --- a/source/core/core.vcxproj +++ b/source/core/core.vcxproj @@ -36,6 +36,7 @@ + @@ -44,6 +45,7 @@ + diff --git a/source/core/text-io.h b/source/core/text-io.h index e4bdc6e2d..c914e340a 100644 --- a/source/core/text-io.h +++ b/source/core/text-io.h @@ -311,28 +311,6 @@ namespace Slang stream = 0; } }; - - inline List Split(String text, char c) - { - List result; - StringBuilder sb; - for (int i = 0; i < text.Length(); i++) - { - if (text[i] == c) - { - auto str = sb.ToString(); - if (str.Length() != 0) - result.Add(str); - sb.Clear(); - } - else - sb << text[i]; - } - auto lastStr = sb.ToString(); - if (lastStr.Length()) - result.Add(lastStr); - return result; - } } #endif diff --git a/source/core/token-reader.cpp b/source/core/token-reader.cpp new file mode 100644 index 000000000..5f7115112 --- /dev/null +++ b/source/core/token-reader.cpp @@ -0,0 +1,868 @@ +#include "token-reader.h" + +namespace Slang +{ + enum class TokenizeErrorType + { + InvalidCharacter, InvalidEscapeSequence + }; + + enum class State + { + Start, Identifier, Operator, Int, Hex, Fixed, Double, Char, String, MultiComment, SingleComment + }; + + enum class LexDerivative + { + None, Line, File + }; + + inline bool IsLetter(char ch) + { + return ((ch >= 'a' && ch <= 'z') || + (ch >= 'A' && ch <= 'Z') || ch == '_'); + } + + inline bool IsDigit(char ch) + { + return ch >= '0' && ch <= '9'; + } + + inline bool IsPunctuation(char ch) + { + return ch == '+' || ch == '-' || ch == '*' || ch == '/' || ch == '%' || + ch == '!' || ch == '^' || ch == '&' || ch == '(' || ch == ')' || + ch == '=' || ch == '{' || ch == '}' || ch == '[' || ch == ']' || + ch == '|' || ch == ';' || ch == ',' || ch == '.' || ch == '<' || + ch == '>' || ch == '~' || ch == '@' || ch == ':' || ch == '?' || ch == '#'; + } + + inline bool IsWhiteSpace(char ch) + { + return (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\v'); + } + + void ParseOperators(const String & str, List & tokens, TokenFlags& tokenFlags, int line, int col, int startPos, String fileName) + { + int pos = 0; + while (pos < (int)str.Length()) + { + wchar_t curChar = str[pos]; + wchar_t nextChar = (pos < (int)str.Length() - 1) ? str[pos + 1] : '\0'; + wchar_t nextNextChar = (pos < (int)str.Length() - 2) ? str[pos + 2] : '\0'; + auto InsertToken = [&](TokenType type, const String & ct) + { + tokens.Add(Token(type, ct, line, col + pos, pos + startPos, fileName, tokenFlags)); + tokenFlags = 0; + }; + switch (curChar) + { + case '+': + if (nextChar == '+') + { + InsertToken(TokenType::OpInc, "++"); + pos += 2; + } + else if (nextChar == '=') + { + InsertToken(TokenType::OpAddAssign, "+="); + pos += 2; + } + else + { + InsertToken(TokenType::OpAdd, "+"); + pos++; + } + break; + case '-': + if (nextChar == '-') + { + InsertToken(TokenType::OpDec, "--"); + pos += 2; + } + else if (nextChar == '=') + { + InsertToken(TokenType::OpSubAssign, "-="); + pos += 2; + } + else if (nextChar == '>') + { + InsertToken(TokenType::RightArrow, "->"); + pos += 2; + } + else + { + InsertToken(TokenType::OpSub, "-"); + pos++; + } + break; + case '*': + if (nextChar == '=') + { + InsertToken(TokenType::OpMulAssign, "*="); + pos += 2; + } + else + { + InsertToken(TokenType::OpMul, "*"); + pos++; + } + break; + case '/': + if (nextChar == '=') + { + InsertToken(TokenType::OpDivAssign, "/="); + pos += 2; + } + else + { + InsertToken(TokenType::OpDiv, "/"); + pos++; + } + break; + case '%': + if (nextChar == '=') + { + InsertToken(TokenType::OpModAssign, "%="); + pos += 2; + } + else + { + InsertToken(TokenType::OpMod, "%"); + pos++; + } + break; + case '|': + if (nextChar == '|') + { + InsertToken(TokenType::OpOr, "||"); + pos += 2; + } + else if (nextChar == '=') + { + InsertToken(TokenType::OpOrAssign, "|="); + pos += 2; + } + else + { + InsertToken(TokenType::OpBitOr, "|"); + pos++; + } + break; + case '&': + if (nextChar == '&') + { + InsertToken(TokenType::OpAnd, "&&"); + pos += 2; + } + else if (nextChar == '=') + { + InsertToken(TokenType::OpAndAssign, "&="); + pos += 2; + } + else + { + InsertToken(TokenType::OpBitAnd, "&"); + pos++; + } + break; + case '^': + if (nextChar == '=') + { + InsertToken(TokenType::OpXorAssign, "^="); + pos += 2; + } + else + { + InsertToken(TokenType::OpBitXor, "^"); + pos++; + } + break; + case '>': + if (nextChar == '>') + { + if (nextNextChar == '=') + { + InsertToken(TokenType::OpShrAssign, ">>="); + pos += 3; + } + else + { + InsertToken(TokenType::OpRsh, ">>"); + pos += 2; + } + } + else if (nextChar == '=') + { + InsertToken(TokenType::OpGeq, ">="); + pos += 2; + } + else + { + InsertToken(TokenType::OpGreater, ">"); + pos++; + } + break; + case '<': + if (nextChar == '<') + { + if (nextNextChar == '=') + { + InsertToken(TokenType::OpShlAssign, "<<="); + pos += 3; + } + else + { + InsertToken(TokenType::OpLsh, "<<"); + pos += 2; + } + } + else if (nextChar == '=') + { + InsertToken(TokenType::OpLeq, "<="); + pos += 2; + } + else + { + InsertToken(TokenType::OpLess, "<"); + pos++; + } + break; + case '=': + if (nextChar == '=') + { + InsertToken(TokenType::OpEql, "=="); + pos += 2; + } + else + { + InsertToken(TokenType::OpAssign, "="); + pos++; + } + break; + case '!': + if (nextChar == '=') + { + InsertToken(TokenType::OpNeq, "!="); + pos += 2; + } + else + { + InsertToken(TokenType::OpNot, "!"); + pos++; + } + break; + case '?': + InsertToken(TokenType::QuestionMark, "?"); + pos++; + break; + case '@': + InsertToken(TokenType::At, "@"); + pos++; + break; + case '#': + if (nextChar == '#') + { + InsertToken(TokenType::PoundPound, "##"); + pos += 2; + } + else + { + InsertToken(TokenType::Pound, "#"); + pos++; + } + pos++; + break; + case ':': + InsertToken(TokenType::Colon, ":"); + pos++; + break; + case '~': + InsertToken(TokenType::OpBitNot, "~"); + pos++; + break; + case ';': + InsertToken(TokenType::Semicolon, ";"); + pos++; + break; + case ',': + InsertToken(TokenType::Comma, ","); + pos++; + break; + case '.': + InsertToken(TokenType::Dot, "."); + pos++; + break; + case '{': + InsertToken(TokenType::LBrace, "{"); + pos++; + break; + case '}': + InsertToken(TokenType::RBrace, "}"); + pos++; + break; + case '[': + InsertToken(TokenType::LBracket, "["); + pos++; + break; + case ']': + InsertToken(TokenType::RBracket, "]"); + pos++; + break; + case '(': + InsertToken(TokenType::LParent, "("); + pos++; + break; + case ')': + InsertToken(TokenType::RParent, ")"); + pos++; + break; + } + } + } + + List TokenizeText(const String & fileName, const String & text) + { + int lastPos = 0, pos = 0; + int line = 1, col = 0; + String file = fileName; + State state = State::Start; + StringBuilder tokenBuilder; + int tokenLine, tokenCol; + List tokenList; + LexDerivative derivative = LexDerivative::None; + TokenFlags tokenFlags = TokenFlag::AtStartOfLine; + auto InsertToken = [&](TokenType type) + { + derivative = LexDerivative::None; + tokenList.Add(Token(type, tokenBuilder.ToString(), tokenLine, tokenCol, pos, file, tokenFlags)); + tokenFlags = 0; + tokenBuilder.Clear(); + }; + auto ProcessTransferChar = [&](char nextChar) + { + switch (nextChar) + { + case '\\': + case '\"': + case '\'': + tokenBuilder.Append(nextChar); + break; + case 't': + tokenBuilder.Append('\t'); + break; + case 's': + tokenBuilder.Append(' '); + break; + case 'n': + tokenBuilder.Append('\n'); + break; + case 'r': + tokenBuilder.Append('\r'); + break; + case 'b': + tokenBuilder.Append('\b'); + break; + } + }; + while (pos <= (int)text.Length()) + { + char curChar = (pos < (int)text.Length() ? text[pos] : ' '); + char nextChar = (pos < (int)text.Length() - 1) ? text[pos + 1] : '\0'; + if (lastPos != pos) + { + if (curChar == '\n') + { + line++; + col = 0; + } + else + col++; + lastPos = pos; + } + + switch (state) + { + case State::Start: + if (IsLetter(curChar)) + { + state = State::Identifier; + tokenLine = line; + tokenCol = col; + } + else if (IsDigit(curChar)) + { + state = State::Int; + tokenLine = line; + tokenCol = col; + } + else if (curChar == '\'') + { + state = State::Char; + pos++; + tokenLine = line; + tokenCol = col; + } + else if (curChar == '"') + { + state = State::String; + pos++; + tokenLine = line; + tokenCol = col; + } + else if (curChar == '\r' || curChar == '\n') + { + tokenFlags |= TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace; + pos++; + } + else if (curChar == ' ' || curChar == '\t' || curChar == -62 || curChar == -96) // -62/-96:non-break space + { + tokenFlags |= TokenFlag::AfterWhitespace; + pos++; + } + else if (curChar == '/' && nextChar == '/') + { + state = State::SingleComment; + pos += 2; + } + else if (curChar == '/' && nextChar == '*') + { + pos += 2; + state = State::MultiComment; + } + else if (curChar == '.' && IsDigit(nextChar)) + { + tokenBuilder.Append("0."); + state = State::Fixed; + pos++; + } + else if (IsPunctuation(curChar)) + { + state = State::Operator; + tokenLine = line; + tokenCol = col; + } + else + { + pos++; + } + break; + case State::Identifier: + if (IsLetter(curChar) || IsDigit(curChar)) + { + tokenBuilder.Append(curChar); + pos++; + } + else + { + auto tokenStr = tokenBuilder.ToString(); +#if 0 + if (tokenStr == "#line_reset#") + { + line = 0; + col = 0; + tokenBuilder.Clear(); + } + else if (tokenStr == "#line") + { + derivative = LexDerivative::Line; + tokenBuilder.Clear(); + } + else if (tokenStr == "#file") + { + derivative = LexDerivative::File; + tokenBuilder.Clear(); + line = 0; + col = 0; + } + else +#endif + InsertToken(TokenType::Identifier); + state = State::Start; + } + break; + case State::Operator: + if (IsPunctuation(curChar) && !((curChar == '/' && nextChar == '/') || (curChar == '/' && nextChar == '*'))) + { + tokenBuilder.Append(curChar); + pos++; + } + else + { + //do token analyze + ParseOperators(tokenBuilder.ToString(), tokenList, tokenFlags, tokenLine, tokenCol, pos - tokenBuilder.Length(), file); + tokenBuilder.Clear(); + state = State::Start; + } + break; + case State::Int: + if (IsDigit(curChar)) + { + tokenBuilder.Append(curChar); + pos++; + } + else if (curChar == '.') + { + state = State::Fixed; + tokenBuilder.Append(curChar); + pos++; + } + else if (curChar == 'e' || curChar == 'E') + { + state = State::Double; + tokenBuilder.Append(curChar); + if (nextChar == '-' || nextChar == '+') + { + tokenBuilder.Append(nextChar); + pos++; + } + pos++; + } + else if (curChar == 'x') + { + state = State::Hex; + tokenBuilder.Append(curChar); + pos++; + } + else if (curChar == 'u') + { + pos++; + tokenBuilder.Append(curChar); + InsertToken(TokenType::IntLiteral); + state = State::Start; + } + else + { + if (derivative == LexDerivative::Line) + { + derivative = LexDerivative::None; + line = StringToInt(tokenBuilder.ToString()) - 1; + col = 0; + tokenBuilder.Clear(); + } + else + { + InsertToken(TokenType::IntLiteral); + } + state = State::Start; + } + break; + case State::Hex: + if (IsDigit(curChar) || (curChar >= 'a' && curChar <= 'f') || (curChar >= 'A' && curChar <= 'F')) + { + tokenBuilder.Append(curChar); + pos++; + } + else + { + InsertToken(TokenType::IntLiteral); + state = State::Start; + } + break; + case State::Fixed: + if (IsDigit(curChar)) + { + tokenBuilder.Append(curChar); + pos++; + } + else if (curChar == 'e' || curChar == 'E') + { + state = State::Double; + tokenBuilder.Append(curChar); + if (nextChar == '-' || nextChar == '+') + { + tokenBuilder.Append(nextChar); + pos++; + } + pos++; + } + else + { + if (curChar == 'f') + pos++; + InsertToken(TokenType::DoubleLiteral); + state = State::Start; + } + break; + case State::Double: + if (IsDigit(curChar)) + { + tokenBuilder.Append(curChar); + pos++; + } + else + { + if (curChar == 'f') + pos++; + InsertToken(TokenType::DoubleLiteral); + state = State::Start; + } + break; + case State::String: + if (curChar != '"') + { + if (curChar == '\\') + { + ProcessTransferChar(nextChar); + pos++; + } + else + tokenBuilder.Append(curChar); + } + else + { + if (derivative == LexDerivative::File) + { + derivative = LexDerivative::None; + file = tokenBuilder.ToString(); + tokenBuilder.Clear(); + } + else + { + InsertToken(TokenType::StringLiteral); + } + state = State::Start; + } + pos++; + break; + case State::Char: + if (curChar != '\'') + { + if (curChar == '\\') + { + ProcessTransferChar(nextChar); + pos++; + } + else + tokenBuilder.Append(curChar); + } + else + { + InsertToken(TokenType::CharLiteral); + state = State::Start; + } + pos++; + break; + case State::SingleComment: + if (curChar == '\n') + { + state = State::Start; + tokenFlags |= TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace; + } + pos++; + break; + case State::MultiComment: + if (curChar == '*' && nextChar == '/') + { + state = State::Start; + tokenFlags |= TokenFlag::AfterWhitespace; + pos += 2; + } + else + pos++; + break; + } + } + return tokenList; + } + List TokenizeText(const String & text) + { + return TokenizeText("", text); + } + + String EscapeStringLiteral(String str) + { + StringBuilder sb; + sb << "\""; + for (int i = 0; i < (int)str.Length(); i++) + { + switch (str[i]) + { + case ' ': + sb << "\\s"; + break; + case '\n': + sb << "\\n"; + break; + case '\r': + sb << "\\r"; + break; + case '\t': + sb << "\\t"; + break; + case '\v': + sb << "\\v"; + break; + case '\'': + sb << "\\\'"; + break; + case '\"': + sb << "\\\""; + break; + case '\\': + sb << "\\\\"; + break; + default: + sb << str[i]; + break; + } + } + sb << "\""; + return sb.ProduceString(); + } + + String UnescapeStringLiteral(String str) + { + StringBuilder sb; + for (int i = 0; i < (int)str.Length(); i++) + { + if (str[i] == '\\' && i < (int)str.Length() - 1) + { + switch (str[i + 1]) + { + case 's': + sb << " "; + break; + case 't': + sb << '\t'; + break; + case 'n': + sb << '\n'; + break; + case 'r': + sb << '\r'; + break; + case 'v': + sb << '\v'; + break; + case '\'': + sb << '\''; + break; + case '\"': + sb << "\""; + break; + case '\\': + sb << "\\"; + break; + default: + i = i - 1; + sb << str[i]; + } + i++; + } + else + sb << str[i]; + } + return sb.ProduceString(); + } + + + String TokenTypeToString(TokenType type) + { + switch (type) + { + case TokenType::EndOfFile: + return "end of file"; + case TokenType::Unknown: + return "UnknownToken"; + case TokenType::Identifier: + return "identifier"; + case TokenType::IntLiteral: + return "integer literal"; + case TokenType::DoubleLiteral: + return "floating-point literal"; + case TokenType::StringLiteral: + return "string literal"; + case TokenType::CharLiteral: + return "character literal"; + case TokenType::QuestionMark: + return "'?'"; + case TokenType::Colon: + return "':'"; + case TokenType::Semicolon: + return "';'"; + case TokenType::Comma: + return "','"; + case TokenType::LBrace: + return "'{'"; + case TokenType::RBrace: + return "'}'"; + case TokenType::LBracket: + return "'['"; + case TokenType::RBracket: + return "']'"; + case TokenType::LParent: + return "'('"; + case TokenType::RParent: + return "')'"; + case TokenType::At: + return "'@'"; + case TokenType::OpAssign: + return "'='"; + case TokenType::OpAdd: + return "'+'"; + case TokenType::OpSub: + return "'-'"; + case TokenType::OpMul: + return "'*'"; + case TokenType::OpDiv: + return "'/'"; + case TokenType::OpMod: + return "'%'"; + case TokenType::OpNot: + return "'!'"; + case TokenType::OpLsh: + return "'<<'"; + case TokenType::OpRsh: + return "'>>'"; + case TokenType::OpAddAssign: + return "'+='"; + case TokenType::OpSubAssign: + return "'-='"; + case TokenType::OpMulAssign: + return "'*='"; + case TokenType::OpDivAssign: + return "'/='"; + case TokenType::OpModAssign: + return "'%='"; + case TokenType::OpEql: + return "'=='"; + case TokenType::OpNeq: + return "'!='"; + case TokenType::OpGreater: + return "'>'"; + case TokenType::OpLess: + return "'<'"; + case TokenType::OpGeq: + return "'>='"; + case TokenType::OpLeq: + return "'<='"; + case TokenType::OpAnd: + return "'&&'"; + case TokenType::OpOr: + return "'||'"; + case TokenType::OpBitXor: + return "'^'"; + case TokenType::OpBitAnd: + return "'&'"; + case TokenType::OpBitOr: + return "'|'"; + case TokenType::OpInc: + return "'++'"; + case TokenType::OpDec: + return "'--'"; + case TokenType::Pound: + return "'#'"; + case TokenType::PoundPound: + return "'##'"; + default: + return ""; + } + } + + TokenReader::TokenReader(String text) + { + this->tokens = TokenizeText("", text); + tokenPtr = 0; + } +} diff --git a/source/core/token-reader.h b/source/core/token-reader.h new file mode 100644 index 000000000..aaebad756 --- /dev/null +++ b/source/core/token-reader.h @@ -0,0 +1,258 @@ +#ifndef CORE_TOKEN_READER_H +#define CORE_TOKEN_READER_H + +#include "basic.h" + +namespace Slang +{ + enum class TokenType + { + EndOfFile = -1, + // illegal + Unknown, + // identifier + Identifier, + // constant + IntLiteral, DoubleLiteral, StringLiteral, CharLiteral, + // operators + Semicolon, Comma, Dot, LBrace, RBrace, LBracket, RBracket, LParent, RParent, + OpAssign, OpAdd, OpSub, OpMul, OpDiv, OpMod, OpNot, OpBitNot, OpLsh, OpRsh, + OpEql, OpNeq, OpGreater, OpLess, OpGeq, OpLeq, + OpAnd, OpOr, OpBitXor, OpBitAnd, OpBitOr, + OpInc, OpDec, OpAddAssign, OpSubAssign, OpMulAssign, OpDivAssign, OpModAssign, + OpShlAssign, OpShrAssign, OpOrAssign, OpAndAssign, OpXorAssign, + + QuestionMark, Colon, RightArrow, At, Pound, PoundPound, + }; + + class CodePosition + { + public: + int Line = -1, Col = -1, Pos = -1; + String FileName; + String ToString() + { + StringBuilder sb(100); + sb << FileName; + if (Line != -1) + sb << "(" << Line << ")"; + return sb.ProduceString(); + } + CodePosition() = default; + CodePosition(int line, int col, int pos, String fileName) + { + Line = line; + Col = col; + Pos = pos; + this->FileName = fileName; + } + bool operator < (const CodePosition & pos) const + { + return FileName < pos.FileName || (FileName == pos.FileName && Line < pos.Line) || + (FileName == pos.FileName && Line == pos.Line && Col < pos.Col); + } + bool operator == (const CodePosition & pos) const + { + return FileName == pos.FileName && Line == pos.Line && Col == pos.Col; + } + }; + + enum TokenFlag : unsigned int + { + AtStartOfLine = 1 << 0, + AfterWhitespace = 1 << 1, + }; + typedef unsigned int TokenFlags; + + class Token + { + public: + TokenType Type = TokenType::Unknown; + String Content; + CodePosition Position; + TokenFlags flags; + Token() = default; + Token(TokenType type, const String & content, int line, int col, int pos, String fileName, TokenFlags flags = 0) + : flags(flags) + { + Type = type; + Content = content; + Position = CodePosition(line, col, pos, fileName); + } + }; + + class TextFormatException : public Exception + { + public: + TextFormatException(String message) + : Exception(message) + {} + }; + + class TokenReader + { + private: + bool legal; + List tokens; + int tokenPtr; + public: + TokenReader(String text); + int ReadInt() + { + auto token = ReadToken(); + bool neg = false; + if (token.Content == '-') + { + neg = true; + token = ReadToken(); + } + if (token.Type == TokenType::IntLiteral) + { + if (neg) + return -StringToInt(token.Content); + else + return StringToInt(token.Content); + } + throw TextFormatException("Text parsing error: int expected."); + } + unsigned int ReadUInt() + { + auto token = ReadToken(); + if (token.Type == TokenType::IntLiteral) + { + return StringToUInt(token.Content); + } + throw TextFormatException("Text parsing error: int expected."); + } + double ReadDouble() + { + auto token = ReadToken(); + bool neg = false; + if (token.Content == '-') + { + neg = true; + token = ReadToken(); + } + if (token.Type == TokenType::DoubleLiteral || token.Type == TokenType::IntLiteral) + { + if (neg) + return -StringToDouble(token.Content); + else + return StringToDouble(token.Content); + } + throw TextFormatException("Text parsing error: floating point value expected."); + } + float ReadFloat() + { + return (float)ReadDouble(); + } + String ReadWord() + { + auto token = ReadToken(); + if (token.Type == TokenType::Identifier) + { + return token.Content; + } + throw TextFormatException("Text parsing error: identifier expected."); + } + String Read(const char * expectedStr) + { + auto token = ReadToken(); + if (token.Content == expectedStr) + { + return token.Content; + } + throw TextFormatException("Text parsing error: \'" + String(expectedStr) + "\' expected."); + } + String Read(String expectedStr) + { + auto token = ReadToken(); + if (token.Content == expectedStr) + { + return token.Content; + } + throw TextFormatException("Text parsing error: \'" + expectedStr + "\' expected."); + } + + String ReadStringLiteral() + { + auto token = ReadToken(); + if (token.Type == TokenType::StringLiteral) + { + return token.Content; + } + throw TextFormatException("Text parsing error: string literal expected."); + } + void Back(int count) + { + tokenPtr -= count; + } + Token ReadToken() + { + if (tokenPtr < (int)tokens.Count()) + { + auto &rs = tokens[tokenPtr]; + tokenPtr++; + return rs; + } + throw TextFormatException("Unexpected ending."); + } + Token NextToken(int offset = 0) + { + if (tokenPtr + offset < (int)tokens.Count()) + return tokens[tokenPtr + offset]; + else + { + Token rs; + rs.Type = TokenType::Unknown; + return rs; + } + } + bool LookAhead(String token) + { + if (tokenPtr < (int)tokens.Count()) + { + auto next = NextToken(); + return next.Content == token; + } + else + { + return false; + } + } + bool IsEnd() + { + return tokenPtr == (int)tokens.Count(); + } + public: + bool IsLegalText() + { + return legal; + } + }; + + inline List Split(String text, char c) + { + List result; + StringBuilder sb; + for (int i = 0; i < (int)text.Length(); i++) + { + if (text[i] == c) + { + auto str = sb.ToString(); + if (str.Length() != 0) + result.Add(str); + sb.Clear(); + } + else + sb << text[i]; + } + auto lastStr = sb.ToString(); + if (lastStr.Length()) + result.Add(lastStr); + return result; + } +} + + +#endif \ No newline at end of file -- cgit v1.2.3