1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
#ifndef RASTER_RENDERER_LEXER_H
#define RASTER_RENDERER_LEXER_H
#include "../core/basic.h"
#include "diagnostics.h"
namespace Slang
{
struct NamePool;
//
struct TokenList
{
Token* begin() const;
Token* end() const;
List<Token> mTokens;
};
struct TokenSpan
{
TokenSpan();
TokenSpan(
TokenList const& tokenList)
: mBegin(tokenList.begin())
, mEnd (tokenList.end ())
{}
Token* begin() const { return mBegin; }
Token* end () const { return mEnd ; }
int GetCount() { return (int)(mEnd - mBegin); }
Token* mBegin;
Token* mEnd;
};
struct TokenReader
{
Token nextToken;
TokenReader();
explicit TokenReader(TokenSpan const& tokens)
: mCursor(tokens.begin())
, mEnd (tokens.end ())
, nextToken(tokens.begin() ? *tokens.begin() : GetEndOfFileToken())
{}
explicit TokenReader(TokenList const& tokens)
: mCursor(tokens.begin())
, mEnd (tokens.end ())
, nextToken(tokens.begin() ? *tokens.begin() : GetEndOfFileToken())
{}
struct ParsingCursor
{
Token nextToken;
Token* tokenReaderCursor = nullptr;
};
ParsingCursor getCursor()
{
ParsingCursor rs;
rs.nextToken = nextToken;
rs.tokenReaderCursor = mCursor;
return rs;
}
void setCursor(ParsingCursor cursor)
{
mCursor = cursor.tokenReaderCursor;
nextToken = cursor.nextToken;
}
bool IsAtEnd() const { return mCursor == mEnd; }
Token& PeekToken();
TokenType PeekTokenType() const;
SourceLoc PeekLoc() const;
Token AdvanceToken();
int GetCount() { return (int)(mEnd - mCursor); }
Token* mCursor;
Token* mEnd;
static Token GetEndOfFileToken();
};
typedef unsigned int LexerFlags;
enum
{
kLexerFlag_InDirective = 1 << 0, ///< Turn end-of-line and end-of-file into end-of-directive
kLexerFlag_ExpectFileName = 1 << 1, ///< Support `<>` style strings for file paths
kLexerFlag_IgnoreInvalid = 1 << 2, ///< Suppress errors about invalid/unsupported characters
kLexerFlag_ExpectDirectiveMessage = 1 << 3, ///< Don't lexer ordinary tokens, and instead consume rest of line as a string
};
struct Lexer
{
void initialize(
SourceView* sourceView,
DiagnosticSink* sink,
NamePool* namePool,
MemoryArena* memoryArena);
~Lexer();
Token lexToken(LexerFlags extraFlags = 0);
TokenList lexAllTokens();
SourceView* sourceView;
DiagnosticSink* sink;
NamePool* namePool;
char const* cursor;
char const* begin;
char const* end;
/// The starting sourceLoc (same as first location of SourceView)
SourceLoc startLoc;
TokenFlags tokenFlags;
LexerFlags lexerFlags;
MemoryArena* memoryArena;
};
// Helper routines for extracting values from tokens
String getStringLiteralTokenValue(Token const& token);
String getFileNameTokenValue(Token const& token);
typedef int64_t IntegerLiteralValue;
typedef double FloatingPointLiteralValue;
IntegerLiteralValue getIntegerLiteralValue(Token const& token, UnownedStringSlice* outSuffix = 0);
FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, UnownedStringSlice* outSuffix = 0);
}
#endif
|
