yum-mirror/slang

Making it easier to work with shaders

git clone https://git.yummers.dev/yum-mirror/slang

CopilotMerge NamePool and RootNamePool into a single type (#7797)3485710e9

master
branchesmasterall refs →
29.8 KiB982 linesraw
1// slang-doc-extractor.cpp
2#include "slang-doc-extractor.h"
3
4#include "../core/slang-string-util.h"
5
6namespace Slang
7{
8
9/* TODO(JS):
10
11* If Decls hand SourceRange, then we could use the range to simplify getting the Post markup, as
12will be trivial to get to the 'end'
13* Need to handle preceeding * in some markup styles
14* If we want to be able to disable markup we need a mechanism to do this. Probably define source
15ranges.
16
17* Need a way to take the extracted markup and produce suitable markdown
18** This will need to display the decoration appropriately
19*/
20
21/* static */ UnownedStringSlice DocMarkupExtractor::removeStart(
22    MarkupType type,
23    const UnownedStringSlice& comment)
24{
25    switch (type)
26    {
27    case MarkupType::BlockBefore:
28        {
29            if (comment.startsWith(UnownedStringSlice::fromLiteral("/**")) ||
30                comment.startsWith(UnownedStringSlice::fromLiteral("/*!")))
31            {
32                /// /**  */ or /*!  */.
33                return comment.tail(3);
34            }
35            return comment;
36        }
37    case MarkupType::BlockAfter:
38        {
39
40            if (comment.startsWith(UnownedStringSlice::fromLiteral("/**<")) ||
41                comment.startsWith(UnownedStringSlice::fromLiteral("/*!<")))
42            {
43                /// /*!< */ or /**< */
44                return comment.tail(4);
45            }
46            return comment;
47        }
48    case MarkupType::OrdinaryBlockBefore:
49        {
50            if (comment.startsWith(UnownedStringSlice::fromLiteral("/*")))
51            {
52                /// ordinary /* */ block.
53                return comment.tail(2);
54            }
55            return comment;
56        }
57    case MarkupType::LineBangBefore:
58        {
59            return comment.startsWith(UnownedStringSlice::fromLiteral("//!")) ? comment.tail(3)
60                                                                              : comment;
61        }
62    case MarkupType::LineSlashBefore:
63        {
64            return comment.startsWith(UnownedStringSlice::fromLiteral("///")) ? comment.tail(3)
65                                                                              : comment;
66        }
67    case MarkupType::OrdinaryLineBefore:
68    case MarkupType::OrdinaryLineAfter:
69        {
70            return comment.startsWith(UnownedStringSlice::fromLiteral("//")) ? comment.tail(2)
71                                                                             : comment;
72        }
73    case MarkupType::LineBangAfter:
74        {
75            /// //!< Can be multiple lines
76            return comment.startsWith(UnownedStringSlice::fromLiteral("//!<")) ? comment.tail(4)
77                                                                               : comment;
78        }
79    case MarkupType::LineSlashAfter:
80        {
81            return comment.startsWith(UnownedStringSlice::fromLiteral("///<")) ? comment.tail(4)
82                                                                               : comment;
83        }
84    default:
85        break;
86    }
87    return comment;
88}
89
90static Index _findTokenIndex(SourceLoc loc, const Token* toks, Index numToks)
91{
92    // Use a binary search to find the token
93    Index lo = 0;
94    Index hi = numToks;
95
96    while (lo + 1 < hi)
97    {
98        const Index mid = (hi + lo) >> 1;
99        const Token& midToken = toks[mid];
100
101        if (midToken.loc == loc)
102        {
103            return mid;
104        }
105
106        if (midToken.loc.getRaw() <= loc.getRaw())
107        {
108            lo = mid;
109        }
110        else
111        {
112            hi = mid;
113        }
114    }
115
116    // Not found
117    return -1;
118}
119
120/* static */ DocMarkupExtractor::MarkupFlags DocMarkupExtractor::getFlags(MarkupType type)
121{
122    switch (type)
123    {
124    default:
125    case MarkupType::None:
126        return 0;
127    case MarkupType::BlockBefore:
128        return MarkupFlag::Before | MarkupFlag::IsBlock;
129    case MarkupType::BlockAfter:
130        return MarkupFlag::After | MarkupFlag::IsBlock;
131    case MarkupType::OrdinaryBlockBefore:
132        return MarkupFlag::Before | MarkupFlag::IsBlock;
133
134    case MarkupType::LineBangBefore:
135        return MarkupFlag::Before | MarkupFlag::IsMultiToken;
136    case MarkupType::LineSlashBefore:
137        return MarkupFlag::Before | MarkupFlag::IsMultiToken;
138    case MarkupType::OrdinaryLineBefore:
139        return MarkupFlag::Before | MarkupFlag::IsMultiToken;
140
141    case MarkupType::LineBangAfter:
142        return MarkupFlag::After | MarkupFlag::IsMultiToken;
143    case MarkupType::LineSlashAfter:
144        return MarkupFlag::After | MarkupFlag::IsMultiToken;
145    case MarkupType::OrdinaryLineAfter:
146        return MarkupFlag::After | MarkupFlag::IsMultiToken;
147    }
148}
149
150/* static */ DocMarkupExtractor::MarkupType DocMarkupExtractor::findMarkupType(const Token& tok)
151{
152    switch (tok.type)
153    {
154    case TokenType::BlockComment:
155        {
156            UnownedStringSlice slice = tok.getContent();
157            if (slice.getLength() >= 3 && (slice[2] == '!' || slice[2] == '*'))
158            {
159                return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::BlockAfter
160                                                                   : MarkupType::BlockBefore;
161            }
162            else
163            {
164                return MarkupType::OrdinaryBlockBefore;
165            }
166            break;
167        }
168    case TokenType::LineComment:
169        {
170            UnownedStringSlice slice = tok.getContent();
171            if (slice.getLength() >= 3)
172            {
173                if (slice[2] == '!')
174                {
175                    return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::LineBangAfter
176                                                                       : MarkupType::LineBangBefore;
177                }
178                else if (slice[2] == '/')
179                {
180                    return (slice.getLength() >= 4 && slice[3] == '<')
181                               ? MarkupType::LineSlashAfter
182                               : MarkupType::LineSlashBefore;
183                }
184            }
185            return (tok.flags & TokenFlag::AtStartOfLine) != 0 ? MarkupType::OrdinaryLineBefore
186                                                               : MarkupType::OrdinaryLineAfter;
187            break;
188        }
189    default:
190        break;
191    }
192    return MarkupType::None;
193}
194
195static Index _calcWhitespaceIndent(const UnownedStringSlice& line)
196{
197    // TODO(JS): For now we ignore tabs and just work out indentation based on spaces/assume ASCII
198    Index indent = 0;
199    const Index count = line.getLength();
200    for (; indent < count && line[indent] == ' '; indent++)
201        ;
202    return indent;
203}
204
205static Index _calcIndent(const UnownedStringSlice& line)
206{
207    // TODO(JS): For now we just assume no tabs, and that every char is ASCII
208    return line.getLength();
209}
210
211static void _appendUnindenttedLine(
212    const UnownedStringSlice& line,
213    Index maxIndent,
214    StringBuilder& out)
215{
216    Index indent = _calcWhitespaceIndent(line);
217
218    // We want to remove indenting remove no more than maxIndent
219    if (maxIndent >= 0)
220    {
221        indent = (indent > maxIndent) ? maxIndent : indent;
222    }
223
224    // Remove the indenting, and append to out
225    out.append(line.tail(indent));
226}
227
228SlangResult DocMarkupExtractor::_extractMarkup(
229    const FindInfo& info,
230    const FoundMarkup& foundMarkup,
231    StringBuilder& out)
232{
233    SourceView* sourceView = info.sourceView;
234    SourceFile* sourceFile = sourceView->getSourceFile();
235
236    // Here we want to produce the text that is implied by the markup tokens.
237    // We want to removing surrounding markup, and to also keep appropriate indentation
238
239    switch (foundMarkup.type)
240    {
241    case MarkupType::BlockBefore:
242    case MarkupType::BlockAfter:
243    case MarkupType::OrdinaryBlockBefore:
244        {
245            // We should only have a single line
246            SLANG_ASSERT(foundMarkup.range.getCount() == 1);
247
248            const auto& tok = info.tokenList->m_tokens[foundMarkup.range.start];
249            uint32_t offset = sourceView->getRange().getOffset(tok.loc);
250
251            const UnownedStringSlice startLine = sourceFile->getLineContainingOffset(offset);
252
253            UnownedStringSlice content = tok.getContent();
254
255            // Split into lines
256            List<UnownedStringSlice> lines;
257
258            StringUtil::calcLines(content, lines);
259
260            Index maxIndent = -1;
261
262            StringBuilder unindentedLine;
263
264            const Index linesCount = lines.getCount();
265            for (Index i = 0; i < linesCount; ++i)
266            {
267                UnownedStringSlice line = lines[i];
268                unindentedLine.clear();
269
270                if (i == 0)
271                {
272                    if (startLine.isMemoryContained(line.begin()))
273                    {
274                        // For now we'll ignore tabs, and that the indent amount is, the amount of
275                        // *byte* NOTE! This is only appropriate for ASCII without tabs.
276                        maxIndent =
277                            _calcIndent(UnownedStringSlice(startLine.begin(), line.begin()));
278
279                        // Let's strip the start stuff
280                        line = removeStart(foundMarkup.type, line);
281                    }
282                }
283
284                if (i == linesCount - 1)
285                {
286                    SLANG_ASSERT(
287                        line.tail(line.getLength() - 2) == UnownedStringSlice::fromLiteral("*/"));
288                    // Remove the */ at the end of the line
289                    line = line.head(line.getLength() - 2);
290                }
291
292                if (i > 0)
293                {
294                    _appendUnindenttedLine(line, maxIndent, unindentedLine);
295                }
296                else
297                {
298                    unindentedLine.append(line);
299                }
300
301                // If the first or last line are all white space, just ignore them
302                if ((i == linesCount - 1 || i == 0) &&
303                    unindentedLine.getUnownedSlice().trim().getLength() == 0)
304                {
305                    continue;
306                }
307
308                out.append(unindentedLine);
309                out.appendChar('\n');
310            }
311
312            break;
313        }
314    case MarkupType::OrdinaryLineBefore:
315    case MarkupType::OrdinaryLineAfter:
316    case MarkupType::LineBangBefore:
317    case MarkupType::LineSlashBefore:
318    case MarkupType::LineBangAfter:
319    case MarkupType::LineSlashAfter:
320        {
321            // Holds the lines extracted, they may have some white space indenting (like the space
322            // at the start of //)
323            List<UnownedStringSlice> lines;
324
325            const auto& range = foundMarkup.range;
326            for (Index i = range.start; i < range.end; ++i)
327            {
328                const auto& tok = info.tokenList->m_tokens[i];
329                UnownedStringSlice line = tok.getContent();
330                line = removeStart(foundMarkup.type, line);
331
332                // If the first or last line are all white space, just ignore them
333                if ((i == range.start || i == range.end - 1) && line.trim().getLength() == 0)
334                {
335                    continue;
336                }
337                lines.add(line);
338            }
339
340            if (lines.getCount() == 0)
341            {
342                // If there are no lines, theres no content
343                return SLANG_OK;
344            }
345
346            Index minIndent = 0x7fffffff;
347            for (const auto& line : lines)
348            {
349                const Index indent = _calcWhitespaceIndent(line);
350                minIndent = (indent < minIndent) ? indent : minIndent;
351            }
352
353            for (const auto& line : lines)
354            {
355                _appendUnindenttedLine(line, minIndent, out);
356                out.appendChar('\n');
357            }
358
359            break;
360        }
361    default:
362        return SLANG_FAIL;
363    }
364
365    return SLANG_OK;
366}
367
368Index DocMarkupExtractor::_findStartIndex(const FindInfo& info, Location location)
369{
370    Index openCount = 0;
371
372    const TokenList& toks = *info.tokenList;
373    const Index tokIndex = info.tokenIndex;
374
375    Index direction = isBefore(location) ? -1 : 1;
376
377    const Index count = toks.m_tokens.getCount();
378    for (Index i = tokIndex; i >= 0 && i < count; i += direction)
379    {
380        const Token& tok = toks.m_tokens[i];
381
382        switch (tok.type)
383        {
384        case TokenType::LBrace:
385        case TokenType::LBracket:
386        case TokenType::LParent:
387        case TokenType::OpLess:
388            {
389                openCount += direction;
390                if (openCount < 0)
391                    return -1;
392                break;
393            }
394        case TokenType::RBracket:
395            {
396                openCount -= direction;
397                if (openCount < 0)
398                    return -1;
399                break;
400            }
401        case TokenType::OpGreater:
402            {
403                if (location == Location::AfterGenericParam && openCount == 0)
404                {
405                    return i + 1;
406                }
407
408                openCount -= direction;
409                if (openCount < 0)
410                    return -1;
411
412                break;
413            }
414        case TokenType::RParent:
415            {
416                if (openCount == 0 && location == Location::AfterParam)
417                {
418                    return i + 1;
419                }
420
421                openCount -= direction;
422                if (openCount < 0)
423                    return -1;
424                break;
425            }
426        case TokenType::RBrace:
427            {
428                // If we haven't hit a candidate yet before hitting } it's not going to work
429                if (location == Location::Before || location == Location::AfterEnumCase)
430                {
431                    return -1;
432                }
433                break;
434            }
435        case TokenType::BlockComment:
436        case TokenType::LineComment:
437            {
438                if (openCount == 0)
439                {
440                    // Determine the markup type
441                    const MarkupType markupType = findMarkupType(tok);
442                    if (!m_searchInOrindaryComments &&
443                        (markupType == MarkupType::OrdinaryBlockBefore ||
444                         markupType == MarkupType::OrdinaryLineBefore))
445                        break;
446                    // If the location wanted is before and the markup is, we'll assume this is it
447                    if (isBefore(location) && isBefore(markupType))
448                    {
449                        return i;
450                    }
451                    // If we are looking for enum cases, and the markup is after, we'll assume this
452                    // is it
453                    if (isAfter(location) && isAfter(markupType))
454                    {
455                        return i;
456                    }
457                }
458                break;
459            }
460        case TokenType::Comma:
461            {
462                if (openCount == 0)
463                {
464                    if (location == Location::AfterParam || location == Location::AfterEnumCase ||
465                        location == Location::AfterGenericParam)
466                    {
467                        return i + 1;
468                    }
469
470                    if (location == Location::Before)
471                    {
472                        return -1;
473                    }
474                }
475
476                break;
477            }
478        case TokenType::Semicolon:
479            {
480                // If we haven't hit a candidate yet it's not going to work
481                if (location == Location::Before)
482                {
483                    return -1;
484                }
485                if (openCount == 0 && location == Location::AfterSemicolon)
486                {
487                    return i + 1;
488                }
489                break;
490            }
491        default:
492            break;
493        }
494    }
495
496    return -1;
497}
498
499/* static */ bool DocMarkupExtractor::_isTokenOnLineIndex(
500    SourceView* sourceView,
501    MarkupType type,
502    const Token& tok,
503    Index lineIndex)
504{
505    SourceFile* sourceFile = sourceView->getSourceFile();
506    const int offset = sourceView->getRange().getOffset(tok.loc);
507
508    auto const flags = getFlags(type);
509
510    if (flags & MarkupFlag::IsBlock)
511    {
512        // Either the start or the end of the block have to be on the specified line
513        return sourceFile->isOffsetOnLine(offset, lineIndex) ||
514               sourceFile->isOffsetOnLine(offset + tok.charsCount, lineIndex);
515    }
516    else
517    {
518        // Has to be exactly on the specified line
519        return sourceFile->isOffsetOnLine(offset, lineIndex);
520    }
521}
522
523SlangResult DocMarkupExtractor::_findMarkup(
524    const FindInfo& info,
525    Location location,
526    FoundMarkup& out)
527{
528    out.reset();
529
530    const auto& toks = info.tokenList->m_tokens;
531
532    // The starting token index
533    Index startIndex = _findStartIndex(info, location);
534    if (startIndex <= 0)
535    {
536        return SLANG_E_NOT_FOUND;
537    }
538
539    SourceView* sourceView = info.sourceView;
540    SourceFile* sourceFile = sourceView->getSourceFile();
541
542    // Let's lookup the line index where this occurred
543    const int startOffset = sourceView->getRange().getOffset(toks[startIndex].loc);
544
545    // The line index that the markoff starts from
546    Index lineIndex = sourceFile->calcLineIndexFromOffset(startOffset);
547    if (lineIndex < 0)
548    {
549        return SLANG_E_NOT_FOUND;
550    }
551
552    const Index searchDirection = isBefore(location) ? -1 : 1;
553
554    // Get the type and flags
555    const MarkupType type = findMarkupType(toks[startIndex]);
556    const MarkupFlags flags = getFlags(type);
557
558    const MarkupFlag::Enum requiredFlag =
559        isBefore(location) ? MarkupFlag::Before : MarkupFlag::After;
560    if ((flags & requiredFlag) == 0)
561    {
562        return SLANG_E_NOT_FOUND;
563    }
564
565#if 0
566    // The token still isn't accepted, unless it's on the expected line
567    if (_isTokenOnLineIndex(info.sourceView, type, toks[startIndex], expectedLineIndex))
568    {
569        return SLANG_E_NOT_FOUND;
570    }
571#endif
572
573    Index endIndex = startIndex;
574
575    // If it's multiline, so look for the end index
576    if (flags & MarkupFlag::IsMultiToken)
577    {
578        Index expectedLineIndex = lineIndex;
579
580        // TODO(JS):
581        // We should probably do the work here to confirm  indentation - but that
582        // requires knowing something about tabs, so for now we leave.
583
584        while (true)
585        {
586            endIndex += searchDirection;
587            expectedLineIndex += searchDirection;
588            if (expectedLineIndex < 0)
589                break;
590            if (endIndex < 0 || endIndex >= toks.getCount())
591            {
592                break;
593            }
594
595            // Do we find a token of the right type?
596            if (findMarkupType(toks[endIndex]) != type)
597            {
598                break;
599            }
600
601            // Is it on the right line?
602            if (!_isTokenOnLineIndex(info.sourceView, type, toks[endIndex], expectedLineIndex))
603            {
604                break;
605            }
606        }
607
608        // Fix the end index (it's the last one that worked)
609        endIndex -= searchDirection;
610    }
611
612    // Put start < end order
613    if (endIndex < startIndex)
614    {
615        Swap(endIndex, startIndex);
616    }
617    // The range excludes end so increase
618    endIndex++;
619
620    // Okay we've found the markup
621    out.type = type;
622    out.location = location;
623    out.range = IndexRange{startIndex, endIndex};
624
625    SLANG_ASSERT(out.range.getCount() > 0);
626
627    return SLANG_OK;
628}
629
630SlangResult DocMarkupExtractor::_findFirstMarkup(
631    const FindInfo& info,
632    const Location* locs,
633    Index locCount,
634    FoundMarkup& out,
635    Index& outIndex)
636{
637    Index i = 0;
638    for (; i < locCount; ++i)
639    {
640        SlangResult res = _findMarkup(info, locs[i], out);
641        if (SLANG_SUCCEEDED(res) || (SLANG_FAILED(res) && res != SLANG_E_NOT_FOUND))
642        {
643            outIndex = i;
644            return res;
645        }
646    }
647    return SLANG_E_NOT_FOUND;
648}
649
650SlangResult DocMarkupExtractor::_findMarkup(
651    const FindInfo& info,
652    const Location* locs,
653    Index locCount,
654    FoundMarkup& out)
655{
656    Index foundIndex;
657    SLANG_RETURN_ON_FAIL(_findFirstMarkup(info, locs, locCount, out, foundIndex));
658
659    // Lets see if the remaining ones match
660    {
661        FoundMarkup otherMarkup;
662        for (Index i = foundIndex + 1; i < locCount; ++i)
663        {
664            SlangResult res = _findMarkup(info, locs[i], otherMarkup);
665            if (SLANG_SUCCEEDED(res))
666            {
667                // TODO(JS): Warning found markup in another location
668            }
669        }
670    }
671
672    return SLANG_OK;
673}
674
675
676SlangResult DocMarkupExtractor::_findMarkup(
677    const FindInfo& info,
678    SearchStyle searchStyle,
679    FoundMarkup& out)
680{
681    switch (searchStyle)
682    {
683    default:
684    case SearchStyle::None:
685        {
686            return SLANG_E_NOT_FOUND;
687        }
688    case SearchStyle::EnumCase:
689        {
690            Location locs[] = {Location::Before, Location::AfterEnumCase};
691            return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out);
692        }
693    case SearchStyle::Param:
694        {
695            Location locs[] = {Location::Before, Location::AfterParam};
696            return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out);
697        }
698    case SearchStyle::Before:
699        {
700            return _findMarkup(info, Location::Before, out);
701        }
702    case SearchStyle::Function:
703        {
704            return _findMarkup(info, Location::Before, out);
705        }
706    case SearchStyle::Attribute:
707        {
708            FindInfo newInfo = info;
709            newInfo.tokenIndex -= 2;
710            return _findMarkup(newInfo, Location::Before, out);
711        }
712    case SearchStyle::Variable:
713        {
714            Location locs[] = {Location::Before, Location::AfterSemicolon};
715            return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out);
716        }
717    case SearchStyle::GenericParam:
718        {
719            Location locs[] = {Location::Before, Location::AfterGenericParam};
720            return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out);
721        }
722    }
723}
724
725static void _calcLineVisibility(
726    SourceView* sourceView,
727    const TokenList& toks,
728    List<MarkupVisibility>& outLineVisibility)
729{
730    SourceFile* sourceFile = sourceView->getSourceFile();
731    const auto& lineOffsets = sourceFile->getLineBreakOffsets();
732
733    outLineVisibility.setCount(lineOffsets.getCount() + 1);
734
735    MarkupVisibility lastVisibility = MarkupVisibility::Public;
736    Index lastLine = 0;
737
738    for (const auto& tok : toks)
739    {
740        if (tok.type == TokenType::LineComment)
741        {
742            UnownedStringSlice contents = tok.getContent();
743
744            MarkupVisibility newVisibility = lastVisibility;
745
746            // Distinct from other markup
747            if (contents.startsWith(toSlice("//@")))
748            {
749                UnownedStringSlice access = contents.tail(3).trim();
750                if (access == "hidden:" || access == "private:")
751                {
752                    newVisibility = MarkupVisibility::Hidden;
753                }
754                else if (access == "internal:")
755                {
756                    newVisibility = MarkupVisibility::Internal;
757                }
758                else if (access == "public:")
759                {
760                    newVisibility = MarkupVisibility::Public;
761                }
762            }
763
764            if (newVisibility != lastVisibility)
765            {
766                // Work up the line it's on
767                const int offset = sourceView->getRange().getOffset(tok.loc);
768                Index line = sourceFile->calcLineIndexFromOffset(offset);
769
770                // Fill in the span
771                for (Index i = lastLine; i < line; ++i)
772                {
773                    outLineVisibility[i] = lastVisibility;
774                }
775
776                // Record the new access and where we are up to
777                lastLine = line;
778                lastVisibility = newVisibility;
779            }
780        }
781    }
782
783    // Fill in the remaining
784    for (Index i = lastLine; i < outLineVisibility.getCount(); ++i)
785    {
786        outLineVisibility[i] = lastVisibility;
787    }
788}
789
790SlangResult DocMarkupExtractor::extract(
791    const SearchItemInput* inputs,
792    Index inputCount,
793    SourceManager* sourceManager,
794    DiagnosticSink* sink,
795    List<SourceView*>& outViews,
796    List<SearchItemOutput>& out)
797{
798    struct Entry
799    {
800        Index viewIndex;                 ///< The view/file index this loc is found in
801        SourceLoc::RawValue locOrOffset; ///< Can be a loc or an offset into the file
802
803        SearchStyle searchStyle; ///< The search style when looking for an item
804        Index inputIndex;        ///< The index to this item in the input
805    };
806
807    List<Entry> entries;
808
809    {
810        entries.setCount(inputCount);
811        for (Index i = 0; i < inputCount; ++i)
812        {
813            const auto& input = inputs[i];
814            Entry& entry = entries[i];
815            entry.inputIndex = i;
816            entry.viewIndex = -1; //< We don't know what file/view it's in
817            entry.locOrOffset = input.sourceLoc.getRaw();
818            entry.searchStyle = input.searchStyle;
819        }
820    }
821
822    // Sort them into loc order
823    entries.sort(
824        [](const Entry& a, const Entry& b) -> bool { return a.locOrOffset < b.locOrOffset; });
825
826    {
827        SourceView* sourceView = nullptr;
828        Index viewIndex = -1;
829
830        for (auto& entry : entries)
831        {
832            if (entry.searchStyle == SearchStyle::None)
833            {
834                continue;
835            }
836
837            const SourceLoc loc = SourceLoc::fromRaw(entry.locOrOffset);
838
839            if (sourceView == nullptr || !sourceView->getRange().contains(loc))
840            {
841                // Find the new view
842                sourceView = sourceManager->findSourceView(loc);
843                if (!sourceView)
844                {
845                    entry.searchStyle = SearchStyle::None;
846                    continue;
847                }
848
849                // We want only one view per SourceFile
850                SourceFile* sourceFile = sourceView->getSourceFile();
851
852                // NOTE! The view found might be different than sourceView.
853                viewIndex = outViews.findFirstIndex(
854                    [&](SourceView* currentView) -> bool
855                    { return currentView->getSourceFile() == sourceFile; });
856
857                if (viewIndex < 0)
858                {
859                    viewIndex = outViews.getCount();
860                    outViews.add(sourceView);
861                }
862            }
863
864            SLANG_ASSERT(viewIndex >= 0);
865            SLANG_ASSERT(sourceView && sourceView->getRange().contains(loc));
866
867            // Set the file index
868            entry.viewIndex = viewIndex;
869            // Set as the offset within the file
870            entry.locOrOffset = sourceView->getRange().getOffset(loc);
871        }
872
873        // Sort into view/file and then offset order
874        entries.sort(
875            [](const Entry& a, const Entry& b) -> bool
876            {
877                return (a.viewIndex < b.viewIndex) ||
878                       ((a.viewIndex == b.viewIndex) && a.locOrOffset < b.locOrOffset);
879            });
880    }
881
882    {
883        TokenList tokens;
884        List<MarkupVisibility> lineVisibility;
885
886        MemoryArena memoryArena(4096);
887
888        NamePool namePool;
889
890        Index viewIndex = -1;
891        SourceView* sourceView = nullptr;
892
893        const Int entryCount = entries.getCount();
894
895        out.setCount(entryCount);
896
897        for (Index i = 0; i < entryCount; ++i)
898        {
899            const auto& entry = entries[i];
900            auto& dst = out[i];
901
902            dst.viewIndex = -1;
903            dst.inputIndex = entry.inputIndex;
904            dst.visibilty = MarkupVisibility::Public;
905
906            // If there isn't a mechanism to search with, just move on
907            if (entry.searchStyle == SearchStyle::None)
908            {
909                continue;
910            }
911
912            if (viewIndex != entry.viewIndex)
913            {
914                viewIndex = entry.viewIndex;
915                sourceView = outViews[viewIndex];
916
917                // Make all memory free again
918                memoryArena.reset();
919
920                // Run the lexer
921                Lexer lexer;
922                lexer.initialize(sourceView, sink, &namePool, &memoryArena);
923
924                // Lex everything
925                tokens = lexer.lexAllMarkupTokens();
926
927                // Let's work out the access
928
929                _calcLineVisibility(sourceView, tokens, lineVisibility);
930            }
931
932            dst.viewIndex = viewIndex;
933
934            // Get the offset within the source file
935            const uint32_t offset = entry.locOrOffset;
936
937            // We need to get the loc in the source views space, so we look up appropriately in the
938            // list of tokens (which uses the views loc range)
939            const SourceLoc loc = sourceView->getRange().getSourceLocFromOffset(offset);
940
941            // Work out the line number
942            SourceFile* sourceFile = sourceView->getSourceFile();
943            const Index lineIndex = sourceFile->calcLineIndexFromOffset(int(offset));
944
945            dst.visibilty = lineVisibility[lineIndex];
946
947            // Okay, lets find the token index with a binary chop
948            Index tokenIndex =
949                _findTokenIndex(loc, tokens.m_tokens.getBuffer(), tokens.m_tokens.getCount());
950            if (tokenIndex >= 0 && lineIndex >= 0)
951            {
952                FindInfo findInfo;
953                findInfo.tokenIndex = tokenIndex;
954                findInfo.lineIndex = lineIndex;
955                findInfo.tokenList = &tokens;
956                findInfo.sourceView = sourceView;
957
958                // Okay let's see if we extract some documentation then for this.
959                FoundMarkup foundMarkup;
960                SlangResult res = _findMarkup(findInfo, entry.searchStyle, foundMarkup);
961
962                if (SLANG_SUCCEEDED(res))
963                {
964                    // We need to extract
965                    StringBuilder buf;
966                    SLANG_RETURN_ON_FAIL(_extractMarkup(findInfo, foundMarkup, buf));
967
968                    // Save the extracted text in the output
969                    dst.text = buf;
970                }
971                else if (res != SLANG_E_NOT_FOUND)
972                {
973                    return res;
974                }
975            }
976        }
977    }
978
979    return SLANG_OK;
980}
981
982} // namespace Slang