yum-mirror/slang

Making it easier to work with shaders

git clone https://git.yummers.dev/yum-mirror/slang

Ellie HermaszewskaMove switch statement bodies to their own lines (#5493)b118451e3

master
branchesmasterall refs →
13.5 KiB527 linesraw
1// slang-json-lexer.cpp
2#include "slang-json-lexer.h"
3
4#include "../core/slang-char-util.h"
5#include "slang-json-diagnostics.h"
6
7/*
8https://www.json.org/json-en.html
9*/
10
11namespace Slang
12{
13
14/* static */ UnownedStringSlice JSONLexer::calcLexemeLocation(const UnownedStringSlice& text)
15{
16    SourceManager sourceManager;
17    sourceManager.initialize(nullptr, nullptr);
18    DiagnosticSink sink;
19    sink.init(&sourceManager, nullptr);
20
21    String contents(text);
22    SourceFile* sourceFile =
23        sourceManager.createSourceFileWithString(PathInfo::makeUnknown(), contents);
24    SourceView* sourceView = sourceManager.createSourceView(sourceFile, nullptr, SourceLoc());
25
26    JSONLexer lexer;
27
28    lexer.init(sourceView, &sink);
29
30    if (lexer.peekType() != JSONTokenType::Invalid)
31    {
32        // Get the start offset
33        auto offset = sourceView->getRange().getOffset(lexer.peekLoc());
34
35        return text.subString(offset, lexer.peekLexeme().getLength());
36    }
37    else
38    {
39        return text.head(0);
40    }
41}
42
43SlangResult JSONLexer::init(SourceView* sourceView, DiagnosticSink* sink)
44{
45    m_sourceView = sourceView;
46    m_sink = sink;
47
48    SourceFile* sourceFile = sourceView->getSourceFile();
49
50    // Note that the content must be null terminated (because of other requirements)
51    SLANG_ASSERT(sourceFile && sourceFile->hasContent());
52
53    m_contentStart = sourceFile->getContent().begin();
54
55    m_startLoc = sourceView->getRange().begin;
56
57    m_lexemeStart = m_contentStart;
58    m_cursor = m_lexemeStart;
59
60    // We need to prime the first token
61    advance();
62
63    return SLANG_OK;
64}
65
66SLANG_FORCE_INLINE static const char* _handleEndOfLine(char c, const char* cursor)
67{
68    SLANG_ASSERT(c == '\n' || c == '\r');
69    const char d = *cursor;
70    return cursor + Index((c ^ d) == ('\n' ^ '\r'));
71}
72
73JSONTokenType JSONLexer::_setInvalidToken()
74{
75    return _setToken(JSONTokenType::Invalid, m_lexemeStart);
76}
77
78SlangResult JSONLexer::expect(JSONTokenType type)
79{
80    if (type != peekType())
81    {
82        m_sink->diagnose(
83            m_token.loc,
84            JSONDiagnostics::unexpectedTokenExpectedTokenType,
85            getJSONTokenAsText(peekType()),
86            getJSONTokenAsText(type));
87        return SLANG_FAIL;
88    }
89
90    advance();
91    return SLANG_OK;
92}
93
94SlangResult JSONLexer::expect(JSONTokenType type, JSONToken& out)
95{
96    if (type != peekType())
97    {
98        m_sink->diagnose(
99            m_token.loc,
100            JSONDiagnostics::unexpectedTokenExpectedTokenType,
101            getJSONTokenAsText(peekType()),
102            getJSONTokenAsText(type));
103        return SLANG_FAIL;
104    }
105
106    out = m_token;
107    advance();
108    return SLANG_OK;
109}
110
111bool JSONLexer::advanceIf(JSONTokenType type)
112{
113    if (type == peekType())
114    {
115        advance();
116        return true;
117    }
118    return false;
119}
120
121bool JSONLexer::advanceIf(JSONTokenType type, JSONToken& out)
122{
123    if (type == peekType())
124    {
125        out = m_token;
126        advance();
127        return true;
128    }
129    return false;
130}
131
132UnownedStringSlice JSONLexer::getLexeme(const JSONToken& tok) const
133{
134    auto offset = m_sourceView->getRange().getOffset(tok.loc);
135    return UnownedStringSlice(m_sourceView->getContent().begin() + offset, tok.length);
136}
137
138JSONTokenType JSONLexer::advance()
139{
140    const char* cursor = m_cursor;
141
142    while (true)
143    {
144        m_lexemeStart = cursor;
145
146        const char c = *cursor++;
147
148        switch (c)
149        {
150        case 0:
151            return _setToken(JSONTokenType::EndOfFile, cursor - 1);
152        case '"':
153            {
154                cursor = _lexString(cursor);
155                if (cursor == nullptr)
156                {
157                    return _setInvalidToken();
158                }
159                return _setToken(JSONTokenType::StringLiteral, cursor);
160            }
161        case '/':
162            {
163                // We allow comments
164                const char nextChar = *m_cursor;
165
166                if (nextChar == '/')
167                {
168                    // Line comment
169                    cursor = _lexLineComment(cursor);
170                }
171                else if (nextChar == '*')
172                {
173                    cursor = _lexBlockComment(cursor);
174                    // Can fail...
175                    if (cursor == nullptr)
176                    {
177                        return _setInvalidToken();
178                    }
179                }
180                else
181                {
182                    return _setInvalidToken();
183                }
184                break;
185            }
186        case ' ':
187        case '\t':
188        case '\n':
189        case '\r':
190            {
191                cursor = _lexWhitespace(cursor);
192                break;
193            }
194        case ':':
195            return _setToken(JSONTokenType::Colon, cursor);
196        case ',':
197            return _setToken(JSONTokenType::Comma, cursor);
198        case '[':
199            return _setToken(JSONTokenType::LBracket, cursor);
200        case ']':
201            return _setToken(JSONTokenType::RBracket, cursor);
202        case '{':
203            return _setToken(JSONTokenType::LBrace, cursor);
204        case '}':
205            return _setToken(JSONTokenType::RBrace, cursor);
206
207        case '-':
208        case '0':
209        case '1':
210        case '2':
211        case '3':
212        case '4':
213        case '5':
214        case '6':
215        case '7':
216        case '8':
217        case '9':
218            {
219                LexResult res = _lexNumber(cursor - 1);
220                if (res.cursor == nullptr)
221                {
222                    return _setToken(JSONTokenType::Invalid, m_lexemeStart);
223                }
224                return _setToken(res.type, res.cursor);
225            }
226        case 't':
227            {
228                if (cursor[0] == 'r' && cursor[1] == 'u' && cursor[2] == 'e')
229                {
230                    return _setToken(JSONTokenType::True, cursor + 3);
231                }
232                m_sink->diagnose(_getLoc(m_lexemeStart), JSONDiagnostics::expectingValueName);
233                return _setInvalidToken();
234            }
235        case 'f':
236            {
237                if (cursor[0] == 'a' && cursor[1] == 'l' && cursor[2] == 's' && cursor[3] == 'e')
238                {
239                    return _setToken(JSONTokenType::False, cursor + 4);
240                }
241                m_sink->diagnose(_getLoc(m_lexemeStart), JSONDiagnostics::expectingValueName);
242                return _setInvalidToken();
243            }
244        case 'n':
245            {
246                if (cursor[0] == 'u' && cursor[1] == 'l' && cursor[2] == 'l')
247                {
248                    return _setToken(JSONTokenType::Null, cursor + 3);
249                }
250                m_sink->diagnose(_getLoc(m_lexemeStart), JSONDiagnostics::expectingValueName);
251                return _setInvalidToken();
252            }
253        default:
254            {
255                StringBuilder buf;
256                if (c <= ' ' || c >= 0x7e)
257                {
258                    static const char s_hex[] = "0123456789abcdef";
259
260                    char hexBuf[5] = "0x";
261
262                    uint32_t value = c;
263                    hexBuf[2] = s_hex[((value >> 4) & 0xf)];
264                    hexBuf[3] = s_hex[(value & 0xf)];
265                    hexBuf[4] = 0;
266
267                    buf << hexBuf;
268                }
269                else
270                {
271                    buf << c;
272                }
273
274                m_sink->diagnose(_getLoc(m_lexemeStart), JSONDiagnostics::unexpectedCharacter);
275                return _setInvalidToken();
276            }
277        }
278    }
279}
280
281JSONLexer::LexResult JSONLexer::_lexNumber(const char* cursor)
282{
283    JSONTokenType tokenType = JSONTokenType::IntegerLiteral;
284
285    if (*cursor == '-')
286    {
287        cursor++;
288    }
289
290    if (*cursor == '0')
291    {
292        // Can only be followed by . exponent, or nothing
293        cursor++;
294    }
295    else if (*cursor >= '1' && *cursor <= '9')
296    {
297        cursor++;
298        while (CharUtil::isDigit(*cursor))
299        {
300            cursor++;
301        }
302    }
303
304    // Theres a fraction
305    if (*cursor == '.')
306    {
307        tokenType = JSONTokenType::FloatLiteral;
308        // Skip the dot
309        cursor++;
310        // Must have at least one digit
311        if (!CharUtil::isDigit(*cursor))
312        {
313            m_sink->diagnose(_getLoc(cursor), JSONDiagnostics::expectingADigit);
314            return LexResult{JSONTokenType::Invalid, nullptr};
315        }
316        // Skip the digit
317        cursor++;
318        // Skip any more digits
319        while (CharUtil::isDigit(*cursor))
320            cursor++;
321    }
322
323    // Theres an exponent
324    if (*cursor == 'e' || *cursor == 'E')
325    {
326        tokenType = JSONTokenType::FloatLiteral;
327
328        // Has an exponent
329        cursor++;
330
331        // Skip +/- if has one
332        if (*cursor == '+' || *cursor == '-')
333        {
334            cursor++;
335        }
336
337        // Must have one digit
338        if (!CharUtil::isDigit(*cursor))
339        {
340            m_sink->diagnose(_getLoc(cursor), JSONDiagnostics::expectingADigit);
341            return LexResult{JSONTokenType::Invalid, nullptr};
342        }
343
344        // Skip the digit
345        cursor++;
346        // Skip any more digits
347        while (CharUtil::isDigit(*cursor))
348            cursor++;
349    }
350
351    return LexResult{tokenType, cursor};
352}
353
354const char* JSONLexer::_lexString(const char* cursor)
355{
356    // We've skipped the first "
357    while (true)
358    {
359        const char c = *cursor++;
360
361        switch (c)
362        {
363        case 0:
364            {
365                m_sink->diagnose(_getLoc(cursor - 1), JSONDiagnostics::endOfFileInLiteral);
366                return nullptr;
367            }
368        case '"':
369            {
370                return cursor;
371            }
372        case '\\':
373            {
374                const char nextC = *cursor;
375                switch (nextC)
376                {
377                case '"':
378                case '\\':
379                case '/':
380                case 'b':
381                case 'f':
382                case 'n':
383                case 'r':
384                case 't':
385                    {
386                        ++cursor;
387                        break;
388                    }
389                case 'u':
390                    {
391                        cursor++;
392                        for (Index i = 0; i < 4; ++i)
393                        {
394                            if (!CharUtil::isHexDigit(cursor[i]))
395                            {
396                                m_sink->diagnose(
397                                    _getLoc(cursor),
398                                    JSONDiagnostics::expectingAHexDigit);
399                                return nullptr;
400                            }
401                        }
402                        cursor += 4;
403                        break;
404                    }
405                }
406            }
407        // Somewhat surprisingly it appears it's valid to have \r\n inside of quotes.
408        default:
409            break;
410        }
411    }
412}
413
414const char* JSONLexer::_lexLineComment(const char* cursor)
415{
416    for (;;)
417    {
418        const char c = *cursor++;
419
420        switch (c)
421        {
422        case '\n':
423        case '\r':
424            {
425                // We need to skip to the next line
426                return _handleEndOfLine(c, cursor);
427            }
428        case 0:
429            {
430                return cursor - 1;
431            }
432        }
433    }
434}
435
436const char* JSONLexer::_lexBlockComment(const char* cursor)
437{
438    for (;;)
439    {
440        const char c = *cursor++;
441        switch (c)
442        {
443        case 0:
444            {
445                m_sink->diagnose(_getLoc(cursor), JSONDiagnostics::endOfFileInComment);
446                return nullptr;
447            }
448        case '*':
449            {
450                if (*cursor == '/')
451                {
452                    return cursor + 1;
453                }
454                break;
455            }
456        default:
457            break;
458        }
459    }
460}
461
462const char* JSONLexer::_lexWhitespace(const char* cursor)
463{
464    while (true)
465    {
466        const char c = *cursor;
467
468        // Might want to use CharUtil::isWhitespace...
469
470        switch (c)
471        {
472        case ' ':
473        case '\n':
474        case '\r':
475        case '\t':
476            {
477                cursor++;
478                break;
479            }
480        default:
481            {
482                // Hit non white space
483                return cursor;
484            }
485        }
486    }
487}
488
489UnownedStringSlice getJSONTokenAsText(JSONTokenType type)
490{
491    switch (type)
492    {
493    case JSONTokenType::Invalid:
494        return UnownedStringSlice::fromLiteral("invalid");
495    case JSONTokenType::IntegerLiteral:
496        return UnownedStringSlice::fromLiteral("integer literal");
497    case JSONTokenType::FloatLiteral:
498        return UnownedStringSlice::fromLiteral("float literal");
499    case JSONTokenType::StringLiteral:
500        return UnownedStringSlice::fromLiteral("string literal");
501    case JSONTokenType::LBracket:
502        return UnownedStringSlice::fromLiteral("[");
503    case JSONTokenType::RBracket:
504        return UnownedStringSlice::fromLiteral("]");
505    case JSONTokenType::LBrace:
506        return UnownedStringSlice::fromLiteral("{");
507    case JSONTokenType::RBrace:
508        return UnownedStringSlice::fromLiteral("}");
509    case JSONTokenType::Comma:
510        return UnownedStringSlice::fromLiteral(",");
511    case JSONTokenType::Colon:
512        return UnownedStringSlice::fromLiteral(":");
513    case JSONTokenType::True:
514        return UnownedStringSlice::fromLiteral("true");
515    case JSONTokenType::False:
516        return UnownedStringSlice::fromLiteral("false");
517    case JSONTokenType::Null:
518        return UnownedStringSlice::fromLiteral("null");
519    case JSONTokenType::EndOfFile:
520        return UnownedStringSlice::fromLiteral("end of file");
521    default:
522        break;
523    }
524    SLANG_UNEXPECTED("JSONTokenType not known");
525}
526
527} // namespace Slang