Repository: llvm/llvm-project
File: clang-tools-extra/clang-tidy/utils/LexerUtils.cpp (C++)

Code before:
    if (TheLexer.LexFromRawLexer(Tok))
      break;
    if (Tok.is(tok::eof) || Tok.getLocation() == FileRange.getEnd() ||
        SM.isBeforeInTranslationUnit(FileRange.getEnd(), Tok.getLocation()))
      break;

    if (Tok.is(tok::comment)) {
      const std::pair<FileID, unsigned> CommentLoc =
          SM.getDecomposedLoc(Tok.getLocation());
      assert(CommentLoc.first == BeginLoc.first);
      Comments.emplace_back(CommentToken{
          Tok.getLocation(),
          StringRef(Buffer.begin() + CommentLoc.second, Tok.getLength()),
      });
    } else {
      // Clear comments found before the different token, e.g. comma. Callers
      // use this to retrieve only the contiguous comment block that directly
      // precedes a token of interest.
      Comments.clear();
    }
  }

  return Comments;
}

CharSourceRange findTokenInRange(CharSourceRange Range, const SourceManager &SM,
                                 const LangOptions &LangOpts,
                                 llvm::function_ref<bool(const Token &)> Pred) {
  if (Range.isInvalid())
    return {};

  // Normalize to a file-based char range so raw lexing can operate on one
  // contiguous buffer and reject unmappable (e.g. macro) ranges.
  const CharSourceRange FileRange =
      Lexer::makeFileCharRange(Range, SM, LangOpts);
  if (FileRange.isInvalid())
    return {};

  const auto [BeginFID, BeginOffset] =
      SM.getDecomposedLoc(FileRange.getBegin());
  const auto [EndFID, EndOffset] = SM.getDecomposedLoc(FileRange.getEnd());
  if (BeginFID != EndFID || BeginOffset > EndOffset)
    return {};

  bool Invalid = false;
  const StringRef Buffer = SM.getBufferData(BeginFID, &Invalid);
  if (Invalid)
    return {};

  const char *LexStart = Buffer.data() + BeginOffset;
  // Re-lex raw tokens in the bounded file buffer while preserving comments so

Review comment (suggestion):
Maybe we will need `pair<Token, CharSourceRange>` someday, like in
```
std::pair<std::optional<Token>, SourceLocation>
getPreviousTokenAndStart
```

But I think we should leave as is now, I'm fine with it.

Code after:
  // preprocessing or macro expansion effects.
  TheLexer.SetCommentRetentionState(true);

  while (true) {
    Token Tok;
    if (TheLexer.LexFromRawLexer(Tok))
      break;
    if (Tok.is(tok::eof) || Tok.getLocation() == FileRange.getEnd() ||
        SM.isBeforeInTranslationUnit(FileRange.getEnd(), Tok.getLocation()))
      break;

    if (Tok.is(tok::comment)) {
      const std::pair<FileID, unsigned> CommentLoc =
          SM.getDecomposedLoc(Tok.getLocation());
      assert(CommentLoc.first == BeginLoc.first);
      Comments.emplace_back(CommentToken{
          Tok.getLocation(),
          StringRef(Buffer.begin() + CommentLoc.second, Tok.getLength()),
      });
    } else if (Mode == CommentCollectionMode::TrailingComments) {
      // Clear comments found before the different token, e.g. comma. Callers
      // use this to retrieve only the contiguous comment block that directly
      // precedes a token of interest.
      Comments.clear();
    }
  }

  return Comments;
}

std::vector<CommentToken> getCommentsInRange(CharSourceRange Range,
                                             const SourceManager &SM,
                                             const LangOptions &LangOpts) {
  return collectCommentsInRange(Range, SM, LangOpts,
                                CommentCollectionMode::AllComments);
}

std::vector<CommentToken>
getTrailingCommentsInRange(CharSourceRange Range, const SourceManager &SM,
                           const LangOptions &LangOpts) {
  return collectCommentsInRange(Range, SM, LangOpts,
                                CommentCollectionMode::TrailingComments);
}

CharSourceRange
findTokenTextInRange(CharSourceRange Range, const SourceManager &SM,
                     const LangOptions &LangOpts,
                     llvm::function_ref<bool(const Token &)> Pred) {
  if (Range.isInvalid())
    return {};