Change:
<!-- Thank you for filing a regression report! A regression is something that changed between versions of Rust but was not supposed to. Please provide a short summary of the regression, along with any information you feel is relevant to replicate it. --We have a library that compiles fine on 1.47.0 but after uggrading to 1.48.0 the compilation hangs. This code-snippet above is reduced from the original code as much as I can. I'm having problems understanding exactly where the problem is coming from and any further modifications to reduce it haven't worked. It uses the "paste" crate, and I've tried a few different versions including the latest one but it doesn't make a difference . I've been building it with at 1.47.0 and 1.48.0 to reproduce the issue which I got just by running doesn't work on works on </p</details<!-- If you know when this regression occurred, please add a line like below, replacing with one of stable, beta, or nightly. modify labels: +regression-from-stable-to-{channel} -regression-untriaged --
Some quick notes as I find them: This reproduces on just a cargo check, so has nothing to do with codegen (unsurprising) Compilation time scales by a little more than 2x for each additional number (going from 0 to 20 took 11 seconds, 21 took 26, 22 took 1 min 5) Given that I don't think this is a hard hang per se, just a really really long running compilation Definitely something in macro expansion, compiling the post- code is instant Reproduces on current nightly. -Zself-profile shows all the time is spent in . Unfortunately I don't have a debug build of rustc handy, but I imagine slapping a profiler on one should make the issue fairly obvious. I'd recommend cutting the provided example down from 64 to 20 though.
searched nightlies: from nightly-2020-08-24 to nightly-2020-10-03 regressed nightly: nightly-2020-10-02 searched commits: from to regressed commit: <details<summarybisected with <a href=''cargo-bisect-rustc</av0.6.0</summaryHost triple: x86_64-unknown-linux-gnu Reproduce with: </detailsLooks like
If it helps, this crate was exploring some ideas related to the crate
Assigning and removing as in the prioritization working group.
This is caused by some of the extra logic I in to ensure that we don't throw out the captured unnecessarily. I'm looking into improving the performance. In the longer term, this will be solved by getting rid of the pretty-print/retokenize hack entirely.

Discussion:
use rustc_ast::token::{self, DelimToken, Nonterminal, Token, TokenKind}; use rustc_ast::tokenstream::{self, LazyTokenStream, TokenStream, TokenTree}; use rustc_ast_pretty::pprust; <ins> use rustc_data_structures::fx::FxHashSet; </ins> use rustc_data_structures::sync::Lrc; use rustc_errors::{Diagnostic, FatalError, Level, PResult}; use rustc_session::parse::ParseSess; use rustc_span::{symbol::kw, FileName, SourceFile, Span, DUMMY_SP}; use smallvec::SmallVec; <ins> use std::cell::RefCell; </ins> use std::mem; use std::path::Path; use std::str;

} }; <ins> // Caches the stringification of 'good' `TokenStreams` which passed // `tokenstream_probably_equal_for_proc_macro`. This allows us to avoid // repeatedly stringifying and comparing the same `TokenStream` for deeply // nested nonterminals. // // We cache by the strinification instead of the `TokenStream` to avoid // needing to implement `Hash` for `TokenStream`. Note that it's possible to // have two distinct `TokenStream`s that stringify to the same result // (e.g. if they differ only in hygiene information). However, any // information lost during the stringification process is also intentionally // ignored by `tokenstream_probably_equal_for_proc_macro`, so it's fine // that a single cache entry may 'map' to multiple distinct `TokenStream`s. // // This is a temporary hack to prevent compilation blowup on certain inputs. // The entire pretty-print/retokenize process will be removed soon. thread_local! { static GOOD_TOKEN_CACHE: RefCell<FxHashSet<String>> = Default::default(); } </ins> // FIXME(#43081): Avoid this pretty-print + reparse hack // Pretty-print the AST struct without inserting any parenthesis // beyond those explicitly written by the user (e.g. `ExpnKind::Paren`).

// ever used for a comparison against the capture tokenstream. let source = pprust::nonterminal_to_string_no_extra_parens(nt); let filename = FileName::macro_expansion_source_code(&source); <del> let reparsed_tokens = parse_stream_from_source_str(filename, source, sess, Some(span)); </del> <ins> let reparsed_tokens = parse_stream_from_source_str(filename, source.clone(), sess, Some(span)); </ins> // During early phases of the compiler the AST could get modified // directly (e.g., attributes added or removed) and the internal cache

// modifications, including adding/removing typically non-semantic // tokens such as extra braces and commas, don't happen. if let Some(tokens) = tokens { <ins> if GOOD_TOKEN_CACHE.with(|cache| cache.borrow().contains(&source)) { return tokens; } </ins> // Compare with a non-relaxed delim match to start. if tokenstream_probably_equal_for_proc_macro(&tokens, &reparsed_tokens, sess, false) { <ins> GOOD_TOKEN_CACHE.with(|cache| cache.borrow_mut().insert(source.clone())); </ins> return tokens; }

// token stream to match up with inserted parenthesis in the reparsed stream. let source_with_parens = pprust::nonterminal_to_string(nt); let filename_with_parens = FileName::macro_expansion_source_code(&source_with_parens); <ins> if GOOD_TOKEN_CACHE.with(|cache| cache.borrow().contains(&source_with_parens)) { return tokens; } </ins> let reparsed_tokens_with_parens = parse_stream_from_source_str( filename_with_parens, source_with_parens,

sess, true, ) { <ins> GOOD_TOKEN_CACHE.with(|cache| cache.borrow_mut().insert(source.clone())); </ins> return tokens; }

// to iterate breaking tokens mutliple times. For example: // '[BinOpEq(Shr)] => [Gt, Ge] -> [Gt, Gt, Eq]' let mut token_trees: SmallVec<[_; 2]>; <del> if let TokenTree::Token(token) = &tree { </del> <ins> if let TokenTree::Token(token) = tree { </ins> let mut out = SmallVec::<[_; 2]>::new(); <del> out.push(token.clone()); </del> <ins> out.push(token); </ins> // Iterate to fixpoint: // * We start off with 'out' containing our initial token, and `temp` empty // * If we are able to break any tokens in `out`, then `out` will have

<ins> // force-host // no-prefer-dynamic #![crate_type = "proc-macro"] extern crate proc_macro; use proc_macro::TokenStream; #[proc_macro] pub fn dummy(input: TokenStream) -> TokenStream { // Iterate to force internal conversion of nonterminals // to `proc_macro` structs for _ in input {} TokenStream::new() } </ins>

<ins> // check-pass // aux-build:issue-79242.rs // Regression test for issue #79242 // Tests that compilation time doesn't blow up for a proc-macro // invocation with deeply nested nonterminals #![allow(unused)] extern crate issue_79242; macro_rules! declare_nats { ($prev:ty) => {}; ($prev:ty, $n:literal$(, $tail:literal)*) => { issue_79242::dummy! { $prev } declare_nats!(Option<$prev>$(, $tail)*); }; (0, $($n:literal),+) => { pub struct N0; declare_nats!(N0, $($n),+); }; } declare_nats! { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28 } fn main() {} </ins>