1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
// slang-json-lexer.h
#ifndef SLANG_JSON_LEXER_H
#define SLANG_JSON_LEXER_H
#include "../core/slang-basic.h"
#include "slang-diagnostic-sink.h"
#include "slang-source-loc.h"
namespace Slang
{
enum class JSONTokenType
{
Invalid,
IntegerLiteral,
FloatLiteral,
StringLiteral,
LBracket,
RBracket,
LBrace,
RBrace,
Comma,
Colon,
True,
False,
Null,
EndOfFile,
CountOf,
};
struct JSONToken
{
JSONTokenType type; ///< The token type
SourceLoc loc; ///< Location in the source file
uint32_t length; ///< The length of the token in bytes
};
UnownedStringSlice getJSONTokenAsText(JSONTokenType type);
class JSONLexer
{
public:
/// Peek the current token
JSONToken& peekToken() { return m_token; }
/// Peek the current type
JSONTokenType peekType() { return m_token.type; }
/// Peek the current SourceLoc
SourceLoc peekLoc() { return m_token.loc; }
/// Get the lexeme of JSONToken
UnownedStringSlice getLexeme(const JSONToken& tok) const;
/// Peek the lexeme at the current position
UnownedStringSlice peekLexeme() const { return getLexeme(m_token); }
JSONTokenType advance();
/// Expects a token of type type. If found advances, if not returns an error and outputs to
/// diagnostic sink
SlangResult expect(JSONTokenType type);
/// Same as expect except out will hold the token.
SlangResult expect(JSONTokenType type, JSONToken& out);
/// Returns true and advances if current token is type
bool advanceIf(JSONTokenType type);
bool advanceIf(JSONTokenType type, JSONToken& out);
/// Must be called before use
SlangResult init(SourceView* sourceView, DiagnosticSink* sink);
/// Determines the first token from text. Useful for diagnostics on DiagnosticSink
static UnownedStringSlice calcLexemeLocation(const UnownedStringSlice& text);
protected:
struct LexResult
{
JSONTokenType type;
const char* cursor;
};
/// Get the location of the cursor
SLANG_FORCE_INLINE SourceLoc _getLoc(const char* cursor) const
{
return m_startLoc + (cursor - m_contentStart);
}
const char* _lexLineComment(const char* cursor);
const char* _lexBlockComment(const char* cursor);
const char* _lexWhitespace(const char* cursor);
const char* _lexString(const char* cursor);
LexResult _lexNumber(const char* cursor);
SLANG_FORCE_INLINE JSONTokenType _setToken(JSONTokenType type, const char* cursor)
{
SLANG_ASSERT(cursor >= m_lexemeStart);
m_token.type = type;
m_token.loc = m_startLoc + (m_lexemeStart - m_contentStart);
m_token.length = uint32_t(cursor - m_lexemeStart);
m_cursor = cursor;
return type;
}
JSONTokenType _setInvalidToken();
JSONToken m_token;
const char* m_cursor;
const char* m_lexemeStart;
const char* m_contentStart;
SourceLoc m_startLoc;
SourceView* m_sourceView;
DiagnosticSink* m_sink;
};
} // namespace Slang
#endif
|
