Change:
In , we would like to tweak the structure of the tokens rustc uses. One of the components which heavily depends on the current structure is ( and module use ). I think, for this issue and in general, it makes sense to switch over to , which produces lower-level guaranteed lossless tokens, which look like a better fit for the task of syntax highlighting. I plan to do this refactor myself, but I want to check with if this is a good idea. EDIT: cc
sgtm
I'm fine with it as well. :)
BTW, is there some chat room for rustdoc team on zulip on discord? I'd like to ask some questions :)
There is one on discord (simply called "rustdoc").
It's linked at (which is I agree pretty hard to find). Also the text there is wrong and mentions , we should probably fix that.

Discussion:
pub candidate_span: Option<Span>, } <del> pub struct StringReader<'a> { </del> <ins> crate struct StringReader<'a> { </ins> sess: &'a ParseSess, /// Initial position, read-only. start_pos: BytePos,

} impl<'a> StringReader<'a> { <del> pub fn new( </del> <ins> crate fn new( </ins> sess: &'a ParseSess, source_file: Lrc<rustc_span::SourceFile>, override_span: Option<Span>,

} /// Returns the next token, including trivia like whitespace or comments. <del> pub fn next_token(&mut self) -> Token { </del> <ins> fn next_token(&mut self) -> Token { </ins> let start_src_index = self.src_index(self.pos); let text: &str = &self.src[start_src_index..self.end_src_index];

<del> use rustc_ast::token; </del> use rustc_data_structures::sync::{Lock, Lrc}; use rustc_errors::{emitter::Emitter, Applicability, Diagnostic, Handler}; <del> use rustc_parse::lexer::StringReader as Lexer; </del> <ins> use rustc_parse::parse_stream_from_source_str; </ins> use rustc_session::parse::ParseSess; use rustc_span::source_map::{FilePathMapping, SourceMap}; use rustc_span::{FileName, InnerSpan};

impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> { fn check_rust_syntax(&self, item: &clean::Item, dox: &str, code_block: RustCodeBlock) { <del> let buffered_messages = Lrc::new(Lock::new(vec![])); let emitter = BufferEmitter { messages: Lrc::clone(&buffered_messages) }; </del> <ins> let buffer = Lrc::new(Lock::new(Buffer::default())); let emitter = BufferEmitter { buffer: Lrc::clone(&buffer) }; </ins> let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); let handler = Handler::with_emitter(false, None, Box::new(emitter)); <ins> let source = dox[code_block.code].to_owned(); </ins> let sess = ParseSess::with_span_handler(handler, sm); <del> let source_file = sess.source_map().new_source_file( FileName::Custom(String::from("doctest")), dox[code_block.code].to_owned(), ); let validation_status = rustc_driver::catch_fatal_errors(|| { let mut has_syntax_errors = false; let mut only_whitespace = true; // even if there is a syntax error, we need to run the lexer over the whole file let mut lexer = Lexer::new(&sess, source_file, None); loop { match lexer.next_token().kind { token::Eof => break, token::Whitespace => (), token::Unknown(..) => has_syntax_errors = true, _ => only_whitespace = false, } } if has_syntax_errors { Some(CodeBlockInvalid::SyntaxError) } else if only_whitespace { Some(CodeBlockInvalid::Empty) } else { None } </del> <ins> let is_empty = rustc_driver::catch_fatal_errors(|| { parse_stream_from_source_str( FileName::Custom(String::from("doctest")), source, &sess, None, ) .is_empty() </ins> }) <del> .unwrap_or(Some(CodeBlockInvalid::SyntaxError)); </del> <ins> .unwrap_or(false); let buffer = buffer.borrow(); </ins> <del> if let Some(code_block_invalid) = validation_status { </del> <ins> if buffer.has_errors || is_empty { </ins> let mut diag = if let Some(sp) = super::source_span_for_markdown_range(self.cx, &dox, &code_block.range, &item.attrs) { <del> let warning_message = match code_block_invalid { CodeBlockInvalid::SyntaxError => "could not parse code block as Rust code", CodeBlockInvalid::Empty => "Rust code block is empty", </del> <ins> let warning_message = if buffer.has_errors { "could not parse code block as Rust code" } else { "Rust code block is empty" </ins> }; let mut diag = self.cx.sess().struct_span_warn(sp, warning_message);

}; // FIXME(#67563): Provide more context for these errors by displaying the spans inline. <del> for message in buffered_messages.borrow().iter() { </del> <ins> for message in buffer.messages.iter() { </ins> diag.note(&message); }

} } <ins> #[derive(Default)] struct Buffer { messages: Vec<String>, has_errors: bool, } </ins> struct BufferEmitter { <del> messages: Lrc<Lock<Vec<String>>>, </del> <ins> buffer: Lrc<Lock<Buffer>>, </ins> } impl Emitter for BufferEmitter { fn emit_diagnostic(&mut self, diag: &Diagnostic) { <del> self.messages.borrow_mut().push(format!("error from rustc: {}", diag.message[0].0)); </del> <ins> let mut buffer = self.buffer.borrow_mut(); buffer.messages.push(format!("error from rustc: {}", diag.message[0].0)); if diag.is_error() { buffer.has_errors = true; } </ins> } fn source_map(&self) -> Option<&Lrc<SourceMap>> { None } } <del> enum CodeBlockInvalid { SyntaxError, Empty, } </del>