yum-mirror/slang

Making it easier to work with shaders

git clone https://git.yummers.dev/yum-mirror/slang

Ellie HermaszewskaMove switch statement bodies to their own lines (#5493)b118451e3

master
branchesmasterall refs →
18.9 KiB715 linesraw
1#include "slang-token-reader.h"
2
3namespace Slang
4{
5namespace Misc
6{
7
8enum class TokenizeErrorType
9{
10    InvalidCharacter,
11    InvalidEscapeSequence
12};
13
14enum class State
15{
16    Start,
17    Identifier,
18    Operator,
19    Int,
20    Hex,
21    Fixed,
22    Double,
23    Char,
24    String,
25    MultiComment,
26    SingleComment
27};
28
29enum class LexDerivative
30{
31    None,
32    Line,
33    File
34};
35
36inline bool IsLetter(char ch)
37{
38    return ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_');
39}
40
41inline bool IsDigit(char ch)
42{
43    return ch >= '0' && ch <= '9';
44}
45
46inline bool IsPunctuation(char ch)
47{
48    return ch == '+' || ch == '-' || ch == '*' || ch == '/' || ch == '%' || ch == '!' ||
49           ch == '^' || ch == '&' || ch == '(' || ch == ')' || ch == '=' || ch == '{' ||
50           ch == '}' || ch == '[' || ch == ']' || ch == '|' || ch == ';' || ch == ',' ||
51           ch == '.' || ch == '<' || ch == '>' || ch == '~' || ch == '@' || ch == ':' ||
52           ch == '?' || ch == '#';
53}
54
55inline bool IsWhiteSpace(char ch)
56{
57    return (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\v');
58}
59
60void ParseOperators(
61    const String& str,
62    List<Token>& tokens,
63    TokenFlags& tokenFlags,
64    int line,
65    int col,
66    int startPos,
67    String fileName)
68{
69    Index pos = 0;
70    while (pos < str.getLength())
71    {
72        wchar_t curChar = str[pos];
73        wchar_t nextChar = (pos < str.getLength() - 1) ? str[pos + 1] : '\0';
74        wchar_t nextNextChar = (pos < str.getLength() - 2) ? str[pos + 2] : '\0';
75        auto InsertToken = [&](TokenType type, const String& ct)
76        {
77            tokens.add(
78                Token(type, ct, line, int(col + pos), int(pos + startPos), fileName, tokenFlags));
79            tokenFlags = 0;
80        };
81        switch (curChar)
82        {
83        case '+':
84            if (nextChar == '+')
85            {
86                InsertToken(TokenType::OpInc, "++");
87                pos += 2;
88            }
89            else if (nextChar == '=')
90            {
91                InsertToken(TokenType::OpAddAssign, "+=");
92                pos += 2;
93            }
94            else
95            {
96                InsertToken(TokenType::OpAdd, "+");
97                pos++;
98            }
99            break;
100        case '-':
101            if (nextChar == '-')
102            {
103                InsertToken(TokenType::OpDec, "--");
104                pos += 2;
105            }
106            else if (nextChar == '=')
107            {
108                InsertToken(TokenType::OpSubAssign, "-=");
109                pos += 2;
110            }
111            else if (nextChar == '>')
112            {
113                InsertToken(TokenType::RightArrow, "->");
114                pos += 2;
115            }
116            else
117            {
118                InsertToken(TokenType::OpSub, "-");
119                pos++;
120            }
121            break;
122        case '*':
123            if (nextChar == '=')
124            {
125                InsertToken(TokenType::OpMulAssign, "*=");
126                pos += 2;
127            }
128            else
129            {
130                InsertToken(TokenType::OpMul, "*");
131                pos++;
132            }
133            break;
134        case '/':
135            if (nextChar == '=')
136            {
137                InsertToken(TokenType::OpDivAssign, "/=");
138                pos += 2;
139            }
140            else
141            {
142                InsertToken(TokenType::OpDiv, "/");
143                pos++;
144            }
145            break;
146        case '%':
147            if (nextChar == '=')
148            {
149                InsertToken(TokenType::OpModAssign, "%=");
150                pos += 2;
151            }
152            else
153            {
154                InsertToken(TokenType::OpMod, "%");
155                pos++;
156            }
157            break;
158        case '|':
159            if (nextChar == '|')
160            {
161                InsertToken(TokenType::OpOr, "||");
162                pos += 2;
163            }
164            else if (nextChar == '=')
165            {
166                InsertToken(TokenType::OpOrAssign, "|=");
167                pos += 2;
168            }
169            else
170            {
171                InsertToken(TokenType::OpBitOr, "|");
172                pos++;
173            }
174            break;
175        case '&':
176            if (nextChar == '&')
177            {
178                InsertToken(TokenType::OpAnd, "&&");
179                pos += 2;
180            }
181            else if (nextChar == '=')
182            {
183                InsertToken(TokenType::OpAndAssign, "&=");
184                pos += 2;
185            }
186            else
187            {
188                InsertToken(TokenType::OpBitAnd, "&");
189                pos++;
190            }
191            break;
192        case '^':
193            if (nextChar == '=')
194            {
195                InsertToken(TokenType::OpXorAssign, "^=");
196                pos += 2;
197            }
198            else
199            {
200                InsertToken(TokenType::OpBitXor, "^");
201                pos++;
202            }
203            break;
204        case '>':
205            if (nextChar == '>')
206            {
207                if (nextNextChar == '=')
208                {
209                    InsertToken(TokenType::OpShrAssign, ">>=");
210                    pos += 3;
211                }
212                else
213                {
214                    InsertToken(TokenType::OpRsh, ">>");
215                    pos += 2;
216                }
217            }
218            else if (nextChar == '=')
219            {
220                InsertToken(TokenType::OpGeq, ">=");
221                pos += 2;
222            }
223            else
224            {
225                InsertToken(TokenType::OpGreater, ">");
226                pos++;
227            }
228            break;
229        case '<':
230            if (nextChar == '<')
231            {
232                if (nextNextChar == '=')
233                {
234                    InsertToken(TokenType::OpShlAssign, "<<=");
235                    pos += 3;
236                }
237                else
238                {
239                    InsertToken(TokenType::OpLsh, "<<");
240                    pos += 2;
241                }
242            }
243            else if (nextChar == '=')
244            {
245                InsertToken(TokenType::OpLeq, "<=");
246                pos += 2;
247            }
248            else
249            {
250                InsertToken(TokenType::OpLess, "<");
251                pos++;
252            }
253            break;
254        case '=':
255            if (nextChar == '=')
256            {
257                InsertToken(TokenType::OpEql, "==");
258                pos += 2;
259            }
260            else
261            {
262                InsertToken(TokenType::OpAssign, "=");
263                pos++;
264            }
265            break;
266        case '!':
267            if (nextChar == '=')
268            {
269                InsertToken(TokenType::OpNeq, "!=");
270                pos += 2;
271            }
272            else
273            {
274                InsertToken(TokenType::OpNot, "!");
275                pos++;
276            }
277            break;
278        case '?':
279            InsertToken(TokenType::QuestionMark, "?");
280            pos++;
281            break;
282        case '@':
283            InsertToken(TokenType::At, "@");
284            pos++;
285            break;
286        case '#':
287            if (nextChar == '#')
288            {
289                InsertToken(TokenType::PoundPound, "##");
290                pos += 2;
291            }
292            else
293            {
294                InsertToken(TokenType::Pound, "#");
295                pos++;
296            }
297            pos++;
298            break;
299        case ':':
300            InsertToken(TokenType::Colon, ":");
301            pos++;
302            break;
303        case '~':
304            InsertToken(TokenType::OpBitNot, "~");
305            pos++;
306            break;
307        case ';':
308            InsertToken(TokenType::Semicolon, ";");
309            pos++;
310            break;
311        case ',':
312            InsertToken(TokenType::Comma, ",");
313            pos++;
314            break;
315        case '.':
316            InsertToken(TokenType::Dot, ".");
317            pos++;
318            break;
319        case '{':
320            InsertToken(TokenType::LBrace, "{");
321            pos++;
322            break;
323        case '}':
324            InsertToken(TokenType::RBrace, "}");
325            pos++;
326            break;
327        case '[':
328            InsertToken(TokenType::LBracket, "[");
329            pos++;
330            break;
331        case ']':
332            InsertToken(TokenType::RBracket, "]");
333            pos++;
334            break;
335        case '(':
336            InsertToken(TokenType::LParent, "(");
337            pos++;
338            break;
339        case ')':
340            InsertToken(TokenType::RParent, ")");
341            pos++;
342            break;
343        }
344    }
345}
346
347List<Token> TokenizeText(const String& fileName, const String& text)
348{
349    Index lastPos = 0, pos = 0;
350    int line = 1, col = 0;
351    String file = fileName;
352    State state = State::Start;
353    StringBuilder tokenBuilder;
354    int tokenLine, tokenCol;
355    List<Token> tokenList;
356    LexDerivative derivative = LexDerivative::None;
357    TokenFlags tokenFlags = TokenFlag::AtStartOfLine;
358    auto InsertToken = [&](TokenType type)
359    {
360        derivative = LexDerivative::None;
361        tokenList.add(
362            Token(type, tokenBuilder.toString(), tokenLine, tokenCol, int(pos), file, tokenFlags));
363        tokenFlags = 0;
364        tokenBuilder.clear();
365    };
366    auto ProcessTransferChar = [&](char nextChar)
367    {
368        switch (nextChar)
369        {
370        case '\\':
371        case '\"':
372        case '\'':
373            tokenBuilder.append(nextChar);
374            break;
375        case 't':
376            tokenBuilder.append('\t');
377            break;
378        case 's':
379            tokenBuilder.append(' ');
380            break;
381        case 'n':
382            tokenBuilder.append('\n');
383            break;
384        case 'r':
385            tokenBuilder.append('\r');
386            break;
387        case 'b':
388            tokenBuilder.append('\b');
389            break;
390        }
391    };
392    while (pos <= text.getLength())
393    {
394        char curChar = (pos < text.getLength() ? text[pos] : ' ');
395        char nextChar = (pos < text.getLength() - 1) ? text[pos + 1] : '\0';
396        if (lastPos != pos)
397        {
398            if (curChar == '\n')
399            {
400                line++;
401                col = 0;
402            }
403            else
404                col++;
405            lastPos = pos;
406        }
407
408        switch (state)
409        {
410        case State::Start:
411            if (IsLetter(curChar))
412            {
413                state = State::Identifier;
414                tokenLine = line;
415                tokenCol = col;
416            }
417            else if (IsDigit(curChar))
418            {
419                state = State::Int;
420                tokenLine = line;
421                tokenCol = col;
422            }
423            else if (curChar == '\'')
424            {
425                state = State::Char;
426                pos++;
427                tokenLine = line;
428                tokenCol = col;
429            }
430            else if (curChar == '"')
431            {
432                state = State::String;
433                pos++;
434                tokenLine = line;
435                tokenCol = col;
436            }
437            else if (curChar == '\r' || curChar == '\n')
438            {
439                tokenFlags |= TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace;
440                pos++;
441            }
442            else if (
443                curChar == ' ' || curChar == '\t' || curChar == '\xC2' ||
444                curChar == '\xA0') // -62/-96:non-break space
445            {
446                tokenFlags |= TokenFlag::AfterWhitespace;
447                pos++;
448            }
449            else if (curChar == '/' && nextChar == '/')
450            {
451                state = State::SingleComment;
452                pos += 2;
453            }
454            else if (curChar == '/' && nextChar == '*')
455            {
456                pos += 2;
457                state = State::MultiComment;
458            }
459            else if (curChar == '.' && IsDigit(nextChar))
460            {
461                tokenBuilder.append("0.");
462                state = State::Fixed;
463                pos++;
464            }
465            else if (IsPunctuation(curChar))
466            {
467                state = State::Operator;
468                tokenLine = line;
469                tokenCol = col;
470            }
471            else
472            {
473                pos++;
474            }
475            break;
476        case State::Identifier:
477            if (IsLetter(curChar) || IsDigit(curChar))
478            {
479                tokenBuilder.append(curChar);
480                pos++;
481            }
482            else
483            {
484                auto tokenStr = tokenBuilder.toString();
485#if 0
486                    if (tokenStr == "#line_reset#")
487                    {
488                        line = 0;
489                        col = 0;
490                        tokenBuilder.clear();
491                    }
492                    else if (tokenStr == "#line")
493                    {
494                        derivative = LexDerivative::Line;
495                        tokenBuilder.clear();
496                    }
497                    else if (tokenStr == "#file")
498                    {
499                        derivative = LexDerivative::File;
500                        tokenBuilder.clear();
501                        line = 0;
502                        col = 0;
503                    }
504                    else
505#endif
506                InsertToken(TokenType::Identifier);
507                state = State::Start;
508            }
509            break;
510        case State::Operator:
511            if (IsPunctuation(curChar) &&
512                !((curChar == '/' && nextChar == '/') || (curChar == '/' && nextChar == '*')))
513            {
514                tokenBuilder.append(curChar);
515                pos++;
516            }
517            else
518            {
519                // do token analyze
520                ParseOperators(
521                    tokenBuilder.toString(),
522                    tokenList,
523                    tokenFlags,
524                    tokenLine,
525                    tokenCol,
526                    (int)(pos - tokenBuilder.getLength()),
527                    file);
528                tokenBuilder.clear();
529                state = State::Start;
530            }
531            break;
532        case State::Int:
533            if (IsDigit(curChar))
534            {
535                tokenBuilder.append(curChar);
536                pos++;
537            }
538            else if (curChar == '.')
539            {
540                state = State::Fixed;
541                tokenBuilder.append(curChar);
542                pos++;
543            }
544            else if (curChar == 'e' || curChar == 'E')
545            {
546                state = State::Double;
547                tokenBuilder.append(curChar);
548                if (nextChar == '-' || nextChar == '+')
549                {
550                    tokenBuilder.append(nextChar);
551                    pos++;
552                }
553                pos++;
554            }
555            else if (curChar == 'x')
556            {
557                state = State::Hex;
558                tokenBuilder.append(curChar);
559                pos++;
560            }
561            else if (curChar == 'u')
562            {
563                pos++;
564                tokenBuilder.append(curChar);
565                InsertToken(TokenType::IntLiteral);
566                state = State::Start;
567            }
568            else
569            {
570                if (derivative == LexDerivative::Line)
571                {
572                    derivative = LexDerivative::None;
573                    line = stringToInt(tokenBuilder.toString()) - 1;
574                    col = 0;
575                    tokenBuilder.clear();
576                }
577                else
578                {
579                    InsertToken(TokenType::IntLiteral);
580                }
581                state = State::Start;
582            }
583            break;
584        case State::Hex:
585            if (IsDigit(curChar) || (curChar >= 'a' && curChar <= 'f') ||
586                (curChar >= 'A' && curChar <= 'F'))
587            {
588                tokenBuilder.append(curChar);
589                pos++;
590            }
591            else
592            {
593                InsertToken(TokenType::IntLiteral);
594                state = State::Start;
595            }
596            break;
597        case State::Fixed:
598            if (IsDigit(curChar))
599            {
600                tokenBuilder.append(curChar);
601                pos++;
602            }
603            else if (curChar == 'e' || curChar == 'E')
604            {
605                state = State::Double;
606                tokenBuilder.append(curChar);
607                if (nextChar == '-' || nextChar == '+')
608                {
609                    tokenBuilder.append(nextChar);
610                    pos++;
611                }
612                pos++;
613            }
614            else
615            {
616                if (curChar == 'f')
617                    pos++;
618                InsertToken(TokenType::DoubleLiteral);
619                state = State::Start;
620            }
621            break;
622        case State::Double:
623            if (IsDigit(curChar))
624            {
625                tokenBuilder.append(curChar);
626                pos++;
627            }
628            else
629            {
630                if (curChar == 'f')
631                    pos++;
632                InsertToken(TokenType::DoubleLiteral);
633                state = State::Start;
634            }
635            break;
636        case State::String:
637            if (curChar != '"')
638            {
639                if (curChar == '\\')
640                {
641                    ProcessTransferChar(nextChar);
642                    pos++;
643                }
644                else
645                    tokenBuilder.append(curChar);
646            }
647            else
648            {
649                if (derivative == LexDerivative::File)
650                {
651                    derivative = LexDerivative::None;
652                    file = tokenBuilder.toString();
653                    tokenBuilder.clear();
654                }
655                else
656                {
657                    InsertToken(TokenType::StringLiteral);
658                }
659                state = State::Start;
660            }
661            pos++;
662            break;
663        case State::Char:
664            if (curChar != '\'')
665            {
666                if (curChar == '\\')
667                {
668                    ProcessTransferChar(nextChar);
669                    pos++;
670                }
671                else
672                    tokenBuilder.append(curChar);
673            }
674            else
675            {
676                InsertToken(TokenType::CharLiteral);
677                state = State::Start;
678            }
679            pos++;
680            break;
681        case State::SingleComment:
682            if (curChar == '\n')
683            {
684                state = State::Start;
685                tokenFlags |= TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace;
686            }
687            pos++;
688            break;
689        case State::MultiComment:
690            if (curChar == '*' && nextChar == '/')
691            {
692                state = State::Start;
693                tokenFlags |= TokenFlag::AfterWhitespace;
694                pos += 2;
695            }
696            else
697                pos++;
698            break;
699        }
700    }
701    return tokenList;
702}
703List<Token> TokenizeText(const String& text)
704{
705    return TokenizeText("", text);
706}
707
708TokenReader::TokenReader(String text)
709{
710    this->tokens = TokenizeText("", text);
711    tokenPtr = 0;
712}
713
714} // namespace Misc
715} // namespace Slang