Repository: rust-analyzer/rust-analyzer
Pull request #10838 (2021-11-22 14:59:58)

internal: Add the ability to emit token map ids in macro expansion tests


Patch:
diff --git a/crates/hir_expand/src/lib.rs b/crates/hir_expand/src/lib.rs
index fdb639f55d..409d0cc66f 100644
--- a/crates/hir_expand/src/lib.rs
+++ b/crates/hir_expand/src/lib.rs
@@ -370,8 +370,9 @@ impl MacroCallId {
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct ExpansionInfo {
     expanded: InFile<SyntaxNode>,
+    /// The argument TokenTree or item for attributes
     arg: InFile<SyntaxNode>,
-    /// The `macro_rules!` arguments or attribute input.
+    /// The `macro_rules!` or attribute input.
     attr_input_or_mac_def: Option<InFile<ast::TokenTree>>,
 
     macro_def: Arc<TokenExpander>,
diff --git a/crates/ide/src/hover.rs b/crates/ide/src/hover.rs
index 7c098b13ae..dd77dd3e4b 100644
--- a/crates/ide/src/hover.rs
+++ b/crates/ide/src/hover.rs
@@ -22,7 +22,6 @@ use crate::{
     runnables::{runnable_fn, runnable_mod},
     FileId, FilePosition, NavigationTarget, RangeInfo, Runnable, TryToNav,
 };
-
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct HoverConfig {
     pub links_in_hover: bool,
diff --git a/crates/ide_assists/src/handlers/merge_imports.rs b/crates/ide_assists/src/handlers/merge_imports.rs
index 94400be7a5..fc117bd9a8 100644
--- a/crates/ide_assists/src/handlers/merge_imports.rs
+++ b/crates/ide_assists/src/handlers/merge_imports.rs
@@ -80,7 +80,7 @@ use std::fmt$0::{Display, Debug};
 use std::fmt::{Display, Debug};
 ",
             r"
-use std::fmt::{Display, Debug};
+use std::fmt::{Debug, Display};
 ",
         )
     }
@@ -108,7 +108,7 @@ use std::fmt::Debug;
 use std::fmt$0::Display;
 ",
             r"
-use std::fmt::{Display, Debug};
+use std::fmt::{Debug, Display};
 ",
         );
     }
@@ -135,7 +135,7 @@ use std::fmt::{self, Display};
 use std::{fmt, $0fmt::Display};
 ",
             r"
-use std::{fmt::{Display, self}};
+use std::{fmt::{self, Display}};
 ",
         );
     }
@@ -247,7 +247,7 @@ use std::{fmt::{Debug, Display}};
 use std::{fmt::Debug, fmt$0::Display};
 ",
             r"
-use std::{fmt::{Display, Debug}};
+use std::{fmt::{Debug, Display}};
 ",
         );
     }
@@ -341,9 +341,7 @@ use foo::$0{
 };
 ",
             r"
-use foo::{
-    FooBar, bar::baz,
-};
+use foo::{FooBar, bar::baz};
 ",
         )
     }
diff --git a/crates/ide_assists/src/handlers/replace_qualified_name_with_use.rs b/crates/ide_assists/src/handlers/replace_qualified_name_with_use.rs
index 6b196681ca..8df8c4b726 100644
--- a/crates/ide_assists/src/handlers/replace_qualified_name_with_use.rs
+++ b/crates/ide_assists/src/handlers/replace_qualified_name_with_use.rs
@@ -299,7 +299,7 @@ fn main() {
     ",
             r"
 mod std { pub mod fmt { pub trait Display {} } }
-use std::fmt::{Display, self};
+use std::fmt::{self, Display};
 
 fn main() {
     fmt;
diff --git a/crates/ide_assists/src/handlers/split_import.rs b/crates/ide_assists/src/handlers/split_import.rs
index ac67874b31..090df25cbd 100644
--- a/crates/ide_assists/src/handlers/split_import.rs
+++ b/crates/ide_assists/src/handlers/split_import.rs
@@ -19,20 +19,14 @@ pub(crate) fn split_import(acc: &mut Assists, ctx: &AssistContext) -> Option<()>
 
     let use_tree = path.top_path().syntax().ancestors().find_map(ast::UseTree::cast)?;
 
-    let has_errors = use_tree
-        .syntax()
-        .descendants_with_tokens()
-        .any(|it| it.kind() == syntax::SyntaxKind::ERROR);
-    let last_segment = use_tree.path().and_then(|it| it.segment());
-    if has_errors || last_segment.is_none() {
+    let new_tree = use_tree.split_prefix(&path);
+    if new_tree == use_tree {
         return None;
     }
 
     let target = colon_colon.text_range();
     acc.add(AssistId("split_import", AssistKind::RefactorRewrite), "Split import", target, |edit| {
-        let use_tree = edit.make_mut(use_tree.clone());
-        let path = edit.make_mut(path);
-        use_tree.split_prefix(&path);
+        edit.replace_ast(use_tree, new_tree);
     })
 }
 
diff --git a/crates/ide_db/src/helpers/merge_imports.rs b/crates/ide_db/src/helpers/merge_imports.rs
index 8581a83436..6a7a9b8f61 100644
--- a/crates/ide_db/src/helpers/merge_imports.rs
+++ b/crates/ide_db/src/helpers/merge_imports.rs
@@ -44,10 +44,10 @@ pub fn try_merge_imports(
     }
 
     let lhs = lhs.clone_subtree().clone_for_update();
-    let rhs = rhs.clone_subtree().clone_for_update();
     let lhs_tree = lhs.use_tree()?;
     let rhs_tree = rhs.use_tree()?;
-    try_merge_trees_mut(&lhs_tree, &rhs_tree, merge_behavior)?;
+    let merged = try_merge_trees(&lhs_tree, &rhs_tree, merge_behavior)?;
+    ted::replace(lhs_tree.syntax(), merged.syntax());
     Some(lhs)
 }
 
@@ -56,49 +56,57 @@ pub fn try_merge_trees(
     rhs: &ast::UseTree,
     merge: MergeBehavior,
 ) -> Option<ast::UseTree> {
-    let lhs = lhs.clone_subtree().clone_for_update();
-    let rhs = rhs.clone_subtree().clone_for_update();
-    try_merge_trees_mut(&lhs, &rhs, merge)?;
-    Some(lhs)
-}
-
-fn try_merge_trees_mut(lhs: &ast::UseTree, rhs: &ast::UseTree, merge: MergeBehavior) -> Option<()> {
     let lhs_path = lhs.path()?;
     let rhs_path = rhs.path()?;
 
     let (lhs_prefix, rhs_prefix) = common_prefix(&lhs_path, &rhs_path)?;
-    if !(lhs.is_simple_path()
+    let (lhs, rhs) = if lhs.is_simple_path()
         && rhs.is_simple_path()
         && lhs_path == lhs_prefix
-        && rhs_path == rhs_prefix)
+        && rhs_path == rhs_prefix
     {
-        lhs.split_prefix(&lhs_prefix);
-        rhs.split_prefix(&rhs_prefix);
-    }
-    recursive_merge(&lhs, &rhs, merge)
+        (lhs.clone(), rhs.clone())
+    } else {
+        (lhs.split_prefix(&lhs_prefix), rhs.split_prefix(&rhs_prefix))
+    };
+    recursive_merge(&lhs, &rhs, merge).map(|it| it.clone_for_update())
 }
 
-/// Recursively merges rhs to lhs
-#[must_use]
-fn recursive_merge(lhs: &ast::UseTree, rhs: &ast::UseTree, merge: MergeBehavior) -> Option<()> {
-    let mut use_trees: Vec<ast::UseTree> = lhs
+/// Recursively "zips" together lhs and rhs.
+fn recursive_merge(
+    lhs: &ast::UseTree,
+    rhs: &ast::UseTree,
+    merge: MergeBehavior,
+) -> Option<ast::UseTree> {
+    let mut use_trees = lhs
         .use_tree_list()
         .into_iter()
         .flat_map(|list| list.use_trees())
         // We use Option here to early return from this function(this is not the
         // same as a `filter` op).
-        .map(|tree| merge.is_tree_allowed(&tree).then(|| tree))
-        .collect::<Option<_>>()?;
+        .map(|tree| match merge.is_tree_allowed(&tree) {
+            true => Some(tree),
+            false => None,
+        })
+        .collect::<Option<Vec<_>>>()?;
     use_trees.sort_unstable_by(|a, b| path_cmp_for_sort(a.path(), b.path()));
     for rhs_t in rhs.use_tree_list().into_iter().flat_map(|list| list.use_trees()) {
         if !merge.is_tree_allowed(&rhs_t) {
             return None;
         }
         let rhs_path = rhs_t.path();
+        match use_trees.binary_search_by(|lhs_t| {
+            let (lhs_t, rhs_t) = match lhs_t
+                .path()
+                .zip(rhs_path.clone())
+                .and_then(|(lhs, rhs)| common_prefix(&lhs, &rhs))
+            {
+                Some((lhs_p, rhs_p)) => (lhs_t.split_prefix(&lhs_p), rhs_t.split_prefix(&rhs_p)),
+                None => (lhs_t.clone(), rhs_t.clone()),
+            };
 
-        match use_trees
-            .binary_search_by(|lhs_t| path_cmp_bin_search(lhs_t.path(), rhs_path.as_ref()))
-        {
+            path_cmp_bin_search(lhs_t.path(), rhs_t.path())
+        }) {
             Ok(idx) => {
                 let lhs_t = &mut use_trees[idx];
                 let lhs_path = lhs_t.path()?;
@@ -120,7 +128,6 @@ fn recursive_merge(lhs: &ast::UseTree, rhs: &ast::UseTree, merge: MergeBehavior)
                     match (tree_contains_self(lhs_t), tree_contains_self(&rhs_t)) {
                         (true, false) => continue,
                         (false, true) => {
-                            ted::replace(lhs_t.syntax(), rhs_t.syntax());
                             *lhs_t = rhs_t;
                             continue;
                         }
@@ -135,27 +142,25 @@ fn recursive_merge(lhs: &ast::UseTree, rhs: &ast::UseTree, merge: MergeBehavior)
                     if lhs_t.star_token().is_some() || rhs_t.star_token().is_some() {
                         if tree_is_self(lhs_t) || tree_is_self(&rhs_t) {
                             cov_mark::hit!(merge_self_glob);
-                            let self_tree = make::use_tree(
+                            *lhs_t = make::use_tree(
                                 make::path_unqualified(make::path_segment_self()),
                                 None,
                                 None,
                                 false,
-                            )
-                            .clone_for_update();
-                            ted::replace(lhs_t.syntax(), self_tree.syntax());
-                            *lhs_t = self_tree;
-                            let glob = make::use_tree_glob().clone_for_update();
-                            use_trees.insert(idx, glob.clone());
-                            lhs.get_or_create_use_tree_list().add_use_tree(glob);
+                            );
+                            use_trees.insert(idx, make::use_tree_glob());
                             continue;
                         }
                     } else if lhs_t.use_tree_list().is_none() && rhs_t.use_tree_list().is_none() {
                         continue;
                     }
                 }
-                lhs_t.split_prefix(&lhs_prefix);
-                rhs_t.split_prefix(&rhs_prefix);
-                recursive_merge(&lhs_t, &rhs_t, merge)?;
+                let lhs = lhs_t.split_prefix(&lhs_prefix);
+                let rhs = rhs_t.split_prefix(&rhs_prefix);
+                match recursive_merge(&lhs, &rhs, merge) {
+                    Some(use_tree) => use_trees[idx] = use_tree,
+                    None => return None,
+                }
             }
             Err(_)
                 if merge == MergeBehavior::Module
@@ -165,12 +170,16 @@ fn recursive_merge(lhs: &ast::UseTree, rhs: &ast::UseTree, merge: MergeBehavior)
                 return None
             }
             Err(idx) => {
-                use_trees.insert(idx, rhs_t.clone());
-                lhs.get_or_create_use_tree_list().add_use_tree(rhs_t);
+                use_trees.insert(idx, rhs_t);
             }
         }
     }
-    Some(())
+
+    let lhs = lhs.clone_subtree().clone_for_update();
+    if let Some(old) = lhs.use_tree_list() {
+        ted::replace(old.syntax(), make::use_tree_list(use_trees).syntax().clone_for_update());
+    }
+    ast::UseTree::cast(lhs.syntax().clone_subtree())
 }
 
 /// Traverses both paths until they differ, returning the common prefix of both.
@@ -215,9 +224,11 @@ fn path_cmp_for_sort(a: Option<ast::Path>, b: Option<ast::Path>) -> Ordering {
 }
 
 /// Path comparison func for binary searching for merging.
-fn path_cmp_bin_search(lhs: Option<ast::Path>, rhs: Option<&ast::Path>) -> Ordering {
-    match (lhs.as_ref().and_then(ast::Path::first_segment), rhs.and_then(ast::Path::first_segment))
-    {
+fn path_cmp_bin_search(lhs: Option<ast::Path>, rhs: Option<ast::Path>) -> Ordering {
+    match (
+        lhs.as_ref().and_then(ast::Path::first_segment),
+        rhs.as_ref().and_then(ast::Path::first_segment),
+    ) {
         (None, None) => Ordering::Equal,
         (None, Some(_)) => Ordering::Less,
         (Some(_), None) => Ordering::Greater,
diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs
index 498a1b3323..833a67ad99 100644
--- a/crates/mbe/src/lib.rs
+++ b/crates/mbe/src/lib.rs
@@ -245,6 +245,10 @@ impl DeclarativeMacro {
             None => (id, Origin::Def),
         }
     }
+
+    pub fn shift(&self) -> Shift {
+        self.shift
+    }
 }
 
 impl Rule {
diff --git a/crates/syntax/src/ast/edit.rs b/crates/syntax/src/ast/edit.rs
index 15805dfc86..43a9c6756d 100644
--- a/crates/syntax/src/ast/edit.rs
+++ b/crates/syntax/src/ast/edit.rs
@@ -4,10 +4,48 @@
 use std::{fmt, iter, ops};
 
 use crate::{
+    algo,
     ast::{self, make, AstNode},
     ted, AstToken, NodeOrToken, SyntaxElement, SyntaxNode, SyntaxToken,
 };
 
+impl ast::UseTree {
+    /// Splits off the given prefix, making it the path component of the use tree, appending the rest of the path to all UseTreeList items.
+    #[must_use]
+    pub fn split_prefix(&self, prefix: &ast::Path) -> ast::UseTree {
+        let suffix = if self.path().as_ref() == Some(prefix) && self.use_tree_list().is_none() {
+            make::path_unqualified(make::path_segment_self())
+        } else {
+            match split_path_prefix(prefix) {
+                Some(it) => it,
+                None => return self.clone(),
+            }
+        };
+
+        let use_tree = make::use_tree(
+            suffix,
+            self.use_tree_list(),
+            self.rename(),
+            self.star_token().is_some(),
+        );
+        let nested = make::use_tree_list(iter::once(use_tree));
+        return make::use_tree(prefix.clone(), Some(nested), None, false);
+
+        fn split_path_prefix(prefix: &ast::Path) -> Option<ast::Path> {
+            let parent = prefix.parent_path()?;
+            let segment = parent.segment()?;
+            if algo::has_errors(segment.syntax()) {
+                return None;
+            }
+            let mut res = make::path_unqualified(segment);
+            for p in iter::successors(parent.parent_path(), |it| it.parent_path()) {
+                res = make::path_qualified(res, p.segment()?);
+            }
+            Some(res)
+        }
+    }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct IndentLevel(pub u8);
 
diff --git a/crates/syntax/src/ast/edit_in_place.rs b/crates/syntax/src/ast/edit_in_place.rs
index bf4f9d6d18..552d6fc1e3 100644
--- a/crates/syntax/src/ast/edit_in_place.rs
+++ b/crates/syntax/src/ast/edit_in_place.rs
@@ -1,12 +1,12 @@
 //! Structural editing for ast.
 
-use std::iter::{empty, successors};
+use std::iter::empty;
 
 use parser::{SyntaxKind, T};
 use rowan::SyntaxElement;
 
 use crate::{
-    algo::{self, neighbor},
+    algo::neighbor,
     ast::{self, edit::IndentLevel, make, HasGenericParams},
     ted::{self, Position},
     AstNode, AstToken, Direction,
@@ -282,78 +282,6 @@ impl ast::UseTree {
         }
         ted::remove(self.syntax());
     }
-
-    pub fn get_or_create_use_tree_list(&self) -> ast::UseTreeList {
-        match self.use_tree_list() {
-            Some(it) => it,
-            None => {
-                let position = Position::last_child_of(self.syntax());
-                let use_tree_list = make::use_tree_list(empty()).clone_for_update();
-                let mut elements = Vec::with_capacity(2);
-                if self.coloncolon_token().is_none() {
-                    elements.push(make::token(T![::]).into());
-                }
-                elements.push(use_tree_list.syntax().clone().into());
-                ted::insert_all_raw(position, elements);
-                use_tree_list
-            }
-        }
-    }
-
-    /// Splits off the given prefix, making it the path component of the use tree,
-    /// appending the rest of the path to all UseTreeList items.
-    pub fn split_prefix(&self, prefix: &ast::Path) {
-        debug_assert_eq!(self.path(), Some(prefix.top_path()));
-        let path = self.path().unwrap();
-        if &path == prefix && self.use_tree_list().is_none() {
-            let self_suffix = make::path_unqualified(make::path_segment_self()).clone_for_update();
-            ted::replace(path.syntax(), self_suffix.syntax());
-        } else if split_path_prefix(prefix).is_none() {
-            return;
-        }
-
-        let subtree = self.clone_subtree().clone_for_update();
-        ted::remove_all_iter(self.syntax().children_with_tokens());
-        ted::insert(Position::first_child_of(self.syntax()), prefix.syntax());
-        self.get_or_create_use_tree_list().add_use_tree(subtree);
-
-        fn split_path_prefix(prefix: &ast::Path) -> Option<()> {
-            let parent = prefix.parent_path()?;
-            let segment = parent.segment()?;
-            if algo::has_errors(segment.syntax()) {
-                return None;
-            }
-            for p in successors(parent.parent_path(), |it| it.parent_path()) {
-                p.segment()?;
-            }
-            prefix.parent_path().and_then(|p| p.coloncolon_token()).map(ted::remove);
-            ted::remove(prefix.syntax());
-            Some(())
-        }
-    }
-}
-
-impl ast::UseTreeList {
-    pub fn add_use_tree(&self, use_tree: ast::UseTree) {
-        let (position, elements) = match self.use_trees().last() {
-            Some(last_tree) => (
-                Position::after(last_tree.syntax()),
-                vec![
-                    make::token(T![,]).into(),
-                    make::tokens::single_space().into(),
-                    use_tree.syntax.into(),
-                ],
-            ),
-            None => {
-                let position = match self.l_curly_token() {
-                    Some(l_curly) => Position::after(l_curly),
-                    None => Position::last_child_of(self.syntax()),
-                };
-                (position, vec![use_tree.syntax.into()])
-            }
-        };
-        ted::insert_all_raw(position, elements);
-    }
 }
 
 impl ast::Use {
diff --git a/xtask/src/release.rs b/xtask/src/release.rs
index acb13282f6..d0ddf27b4d 100644
--- a/xtask/src/release.rs
+++ b/xtask/src/release.rs
@@ -27,11 +27,6 @@ impl flags::Release {
             .run()?;
 
         let website_root = project_root().join("../rust-analyzer.github.io");
-        {
-            let _dir = pushd(&website_root)?;
-            cmd!("git switch src").run()?;
-            cmd!("git pull").run()?;
-        }
         let changelog_dir = website_root.join("./thisweek/_posts");
 
         let today = date_iso()?;
@@ -81,7 +76,7 @@ impl flags::Promote {
         cmd!("git add src/tools/rust-analyzer").run()?;
         cmd!("git commit -m':arrow_up: rust-analyzer'").run()?;
         if !self.dry_run {
-            cmd!("git push -u origin {branch}").run()?;
+            cmd!("git push -u").run()?;
             cmd!("xdg-open https://github.com/matklad/rust/pull/new/{branch}?body=r%3F%20%40ghost")
                 .run()?;
         }


Test patch:
diff --git a/crates/hir_def/src/macro_expansion_tests.rs b/crates/hir_def/src/macro_expansion_tests.rs
index 036d9147a0..c3116edc88 100644
--- a/crates/hir_def/src/macro_expansion_tests.rs
+++ b/crates/hir_def/src/macro_expansion_tests.rs
@@ -16,16 +16,21 @@ mod proc_macros;
 
 use std::{iter, ops::Range};
 
+use ::mbe::TokenMap;
 use base_db::{fixture::WithFixture, SourceDatabase};
 use expect_test::Expect;
-use hir_expand::{db::AstDatabase, InFile, MacroFile};
+use hir_expand::{
+    db::{AstDatabase, TokenExpander},
+    AstId, InFile, MacroDefId, MacroDefKind, MacroFile,
+};
 use stdx::format_to;
 use syntax::{
     ast::{self, edit::IndentLevel},
-    AstNode,
-    SyntaxKind::{COMMENT, EOF, IDENT, LIFETIME_IDENT},
-    SyntaxNode, T,
+    AstNode, SyntaxElement,
+    SyntaxKind::{self, COMMENT, EOF, IDENT, LIFETIME_IDENT},
+    SyntaxNode, TextRange, T,
 };
+use tt::{Subtree, TokenId};
 
 use crate::{
     db::DefDatabase, nameres::ModuleSource, resolver::HasResolver, src::HasSource, test_db::TestDB,
@@ -61,7 +66,47 @@ fn check(ra_fixture: &str, mut expect: Expect) {
     // in the module and assume that, if impls's source is a different
     // `HirFileId`, than it came from macro expansion.
 
+    let mut text_edits = Vec::new();
     let mut expansions = Vec::new();
+
+    for macro_ in source_file.syntax().descendants().filter_map(ast::Macro::cast) {
+        let mut show_token_ids = false;
+        for comment in macro_.syntax().children_with_tokens().filter(|it| it.kind() == COMMENT) {
+            show_token_ids |= comment.to_string().contains("+tokenids");
+        }
+        if !show_token_ids {
+            continue;
+        }
+
+        let call_offset = macro_.syntax().text_range().start().into();
+        let file_ast_id = db.ast_id_map(source.file_id).ast_id(&macro_);
+        let ast_id = AstId::new(source.file_id, file_ast_id.upcast());
+        let kind = MacroDefKind::Declarative(ast_id);
+
+        let macro_def = db.macro_def(MacroDefId { krate, kind, local_inner: false }).unwrap();
+        if let TokenExpander::DeclarativeMacro { mac, def_site_token_map } = &*macro_def {
+            let tt = match &macro_ {
+                ast::Macro::MacroRules(mac) => mac.token_tree().unwrap(),
+                ast::Macro::MacroDef(_) => unimplemented!(""),
+            };
+
+            let tt_start = tt.syntax().text_range().start();
+            tt.syntax().descendants_with_tokens().filter_map(SyntaxElement::into_token).for_each(
+                |token| {
+                    let range = token.text_range().checked_sub(tt_start).unwrap();
+                    if let Some(id) = def_site_token_map.token_by_range(range) {
+                        let offset = (range.end() + tt_start).into();
+                        text_edits.push((offset..offset, format!("#{}", id.0)));
+                    }
+                },
+            );
+            text_edits.push((
+                call_offset..call_offset,
+                format!("// call ids will be shifted by {:?}\n", mac.shift()),
+            ));
+        }
+    }
+
     for macro_call in source_file.syntax().descendants().filter_map(ast::MacroCall::cast) {
         let macro_call = InFile::new(source.file_id, &macro_call);
         let mut error = None;
@@ -77,24 +122,24 @@ fn check(ra_fixture: &str, mut expect: Expect) {
         let macro_file = MacroFile { macro_call_id };
         let mut expansion_result = db.parse_macro_expansion(macro_file);
         expansion_result.err = expansion_result.err.or(error);
-        expansions.push((macro_call.value.clone(), expansion_result));
+        expansions.push((macro_call.value.clone(), expansion_result, db.macro_arg(macro_call_id)));
     }
 
-    let mut expanded_text = source_file.to_string();
-
-    for (call, exp) in expansions.into_iter().rev() {
+    for (call, exp, arg) in expansions.into_iter().rev() {
         let mut tree = false;
         let mut expect_errors = false;
+        let mut show_token_ids = false;
         for comment in call.syntax().children_with_tokens().filter(|it| it.kind() == COMMENT) {
             tree |= comment.to_string().contains("+tree");
             expect_errors |= comment.to_string().contains("+errors");
+            show_token_ids |= comment.to_string().contains("+tokenids");
         }
 
         let mut expn_text = String::new();
         if let Some(err) = exp.err {
             format_to!(expn_text, "/* error: {} */", err);
         }
-        if let Some((parse, _token_map)) = exp.value {
+        if let Some((parse, token_map)) = exp.value {
             if expect_errors {
                 assert!(!parse.errors().is_empty(), "no parse errors in expansion");
                 for e in parse.errors() {
@@ -107,7 +152,10 @@ fn check(ra_fixture: &str, mut expect: Expect) {
                     parse.errors()
                 );
             }
-            let pp = pretty_print_macro_expansion(parse.syntax_node());
+            let pp = pretty_print_macro_expansion(
+                parse.syntax_node(),
+                show_token_ids.then(|| &*token_map),
+            );
             let indent = IndentLevel::from_node(call.syntax());
             let pp = reindent(indent, pp);
             format_to!(expn_text, "{}", pp);
@@ -122,14 +170,41 @@ fn check(ra_fixture: &str, mut expect: Expect) {
         }
         let range = call.syntax().text_range();
         let range: Range<usize> = range.into();
-        expanded_text.replace_range(range, &expn_text)
+
+        if show_token_ids {
+            if let Some((tree, map)) = arg.as_deref() {
+                let tt_range = call.token_tree().unwrap().syntax().text_range();
+                let mut ranges = Vec::new();
+                extract_id_ranges(&mut ranges, &map, &tree);
+                for (range, id) in ranges {
+                    let idx = (tt_range.start() + range.end()).into();
+                    text_edits.push((idx..idx, format!("#{}", id.0)));
+                }
+            }
+            text_edits.push((range.start..range.start, "// ".into()));
+            call.to_string().match_indices('\n').for_each(|(offset, _)| {
+                let offset = offset + 1 + range.start;
+                text_edits.push((offset..offset, "// ".into()));
+            });
+            text_edits.push((range.end..range.end, "\n".into()));
+            text_edits.push((range.end..range.end, expn_text));
+        } else {
+            text_edits.push((range, expn_text));
+        }
+    }
+
+    text_edits.sort_by_key(|(range, _)| range.start);
+    text_edits.reverse();
+    let mut expanded_text = source_file.to_string();
+    for (range, text) in text_edits {
+        expanded_text.replace_range(range, &text);
     }
 
     for decl_id in def_map[local_id].scope.declarations() {
         if let ModuleDefId::AdtId(AdtId::StructId(struct_id)) = decl_id {
             let src = struct_id.lookup(&db).source(&db);
             if src.file_id.is_attr_macro(&db) || src.file_id.is_custom_derive(&db) {
-                let pp = pretty_print_macro_expansion(src.value.syntax().clone());
+                let pp = pretty_print_macro_expansion(src.value.syntax().clone(), None);
                 format_to!(expanded_text, "\n{}", pp)
             }
         }
@@ -138,7 +213,7 @@ fn check(ra_fixture: &str, mut expect: Expect) {
     for impl_id in def_map[local_id].scope.impls() {
         let src = impl_id.lookup(&db).source(&db);
         if src.file_id.is_builtin_derive(&db).is_some() {
-            let pp = pretty_print_macro_expansion(src.value.syntax().clone());
+            let pp = pretty_print_macro_expansion(src.value.syntax().clone(), None);
             format_to!(expanded_text, "\n{}", pp)
         }
     }
@@ -147,6 +222,20 @@ fn check(ra_fixture: &str, mut expect: Expect) {
     expect.assert_eq(&expanded_text);
 }
 
+fn extract_id_ranges(ranges: &mut Vec<(TextRange, TokenId)>, map: &TokenMap, tree: &Subtree) {
+    tree.token_trees.iter().for_each(|tree| match tree {
+        tt::TokenTree::Leaf(leaf) => {
+            let id = match leaf {
+                tt::Leaf::Literal(it) => it.id,
+                tt::Leaf::Punct(it) => it.id,
+                tt::Leaf::Ident(it) => it.id,
+            };
+            ranges.extend(map.ranges_by_token(id, SyntaxKind::ERROR).map(|range| (range, id)));
+        }
+        tt::TokenTree::Subtree(tree) => extract_id_ranges(ranges, map, tree),
+    });
+}
+
 fn reindent(indent: IndentLevel, pp: String) -> String {
     if !pp.contains('\n') {
         return pp;
@@ -163,7 +252,7 @@ fn reindent(indent: IndentLevel, pp: String) -> String {
     res
 }
 
-fn pretty_print_macro_expansion(expn: SyntaxNode) -> String {
+fn pretty_print_macro_expansion(expn: SyntaxNode, map: Option<&TokenMap>) -> String {
     let mut res = String::new();
     let mut prev_kind = EOF;
     let mut indent_level = 0;
@@ -206,7 +295,12 @@ fn pretty_print_macro_expansion(expn: SyntaxNode) -> String {
             res.push_str(&"    ".repeat(level));
         }
         prev_kind = curr_kind;
-        format_to!(res, "{}", token)
+        format_to!(res, "{}", token);
+        if let Some(map) = map {
+            if let Some(id) = map.token_by_range(token.text_range()) {
+                format_to!(res, "#{}", id.0);
+            }
+        }
     }
     res
 }
diff --git a/crates/hir_def/src/macro_expansion_tests/mbe.rs b/crates/hir_def/src/macro_expansion_tests/mbe.rs
index 953750d360..962e1e7459 100644
--- a/crates/hir_def/src/macro_expansion_tests/mbe.rs
+++ b/crates/hir_def/src/macro_expansion_tests/mbe.rs
@@ -10,6 +10,42 @@ use expect_test::expect;
 
 use crate::macro_expansion_tests::check;
 
+#[test]
+fn token_mapping_smoke_test() {
+    check(
+        r#"
+// +tokenids
+macro_rules! f {
+    ( struct $ident:ident ) => {
+        struct $ident {
+            map: ::std::collections::HashSet<()>,
+        }
+    };
+}
+
+// +tokenids
+f!(struct MyTraitMap2);
+"#,
+        expect![[r##"
+// call ids will be shifted by Shift(27)
+// +tokenids
+macro_rules! f {#0
+    (#1 struct#2 $#3ident#4:#5ident#6 )#1 =#7>#8 {#9
+        struct#10 $#11ident#12 {#13
+            map#14:#15 :#16:#17std#18:#19:#20collections#21:#22:#23HashSet#24<#25(#26)#26>#27,#28
+        }#13
+    }#9;#29
+}#0
+
+// // +tokenids
+// f!(struct#1 MyTraitMap2#2);
+struct#10 MyTraitMap2#29 {#13
+    map#14:#15 ::std#18::collections#21::HashSet#24<#25(#26)#26>#27,#28
+}#13
+"##]],
+    );
+}
+
 #[test]
 fn mbe_smoke_test() {
     check(
diff --git a/crates/ide_db/src/helpers/insert_use/tests.rs b/crates/ide_db/src/helpers/insert_use/tests.rs
index a5763eb41c..34a6900e26 100644
--- a/crates/ide_db/src/helpers/insert_use/tests.rs
+++ b/crates/ide_db/src/helpers/insert_use/tests.rs
@@ -463,7 +463,7 @@ fn merge_groups_full() {
 
 #[test]
 fn merge_groups_long_full() {
-    check_crate("std::foo::bar::Baz", r"use std::foo::bar::Qux;", r"use std::foo::bar::{Qux, Baz};")
+    check_crate("std::foo::bar::Baz", r"use std::foo::bar::Qux;", r"use std::foo::bar::{Baz, Qux};")
 }
 
 #[test]
@@ -471,7 +471,7 @@ fn merge_groups_long_last() {
     check_module(
         "std::foo::bar::Baz",
         r"use std::foo::bar::Qux;",
-        r"use std::foo::bar::{Qux, Baz};",
+        r"use std::foo::bar::{Baz, Qux};",
     )
 }
 
@@ -480,7 +480,7 @@ fn merge_groups_long_full_list() {
     check_crate(
         "std::foo::bar::Baz",
         r"use std::foo::bar::{Qux, Quux};",
-        r"use std::foo::bar::{Qux, Quux, Baz};",
+        r"use std::foo::bar::{Baz, Quux, Qux};",
     )
 }
 
@@ -489,7 +489,7 @@ fn merge_groups_long_last_list() {
     check_module(
         "std::foo::bar::Baz",
         r"use std::foo::bar::{Qux, Quux};",
-        r"use std::foo::bar::{Qux, Quux, Baz};",
+        r"use std::foo::bar::{Baz, Quux, Qux};",
     )
 }
 
@@ -498,7 +498,7 @@ fn merge_groups_long_full_nested() {
     check_crate(
         "std::foo::bar::Baz",
         r"use std::foo::bar::{Qux, quux::{Fez, Fizz}};",
-        r"use std::foo::bar::{Qux, quux::{Fez, Fizz}, Baz};",
+        r"use std::foo::bar::{Baz, Qux, quux::{Fez, Fizz}};",
     )
 }
 
@@ -517,7 +517,7 @@ fn merge_groups_full_nested_deep() {
     check_crate(
         "std::foo::bar::quux::Baz",
         r"use std::foo::bar::{Qux, quux::{Fez, Fizz}};",
-        r"use std::foo::bar::{Qux, quux::{Fez, Fizz, Baz}};",
+        r"use std::foo::bar::{Qux, quux::{Baz, Fez, Fizz}};",
     )
 }
 
@@ -526,7 +526,7 @@ fn merge_groups_full_nested_long() {
     check_crate(
         "std::foo::bar::Baz",
         r"use std::{foo::bar::Qux};",
-        r"use std::{foo::bar::{Qux, Baz}};",
+        r"use std::{foo::bar::{Baz, Qux}};",
     );
 }
 
@@ -535,7 +535,7 @@ fn merge_groups_last_nested_long() {
     check_crate(
         "std::foo::bar::Baz",
         r"use std::{foo::bar::Qux};",
-        r"use std::{foo::bar::{Qux, Baz}};",
+        r"use std::{foo::bar::{Baz, Qux}};",
     );
 }
 
@@ -600,7 +600,7 @@ fn merge_mod_into_glob() {
     check_with_config(
         "token::TokenKind",
         r"use token::TokenKind::*;",
-        r"use token::TokenKind::{self, *};",
+        r"use token::TokenKind::{*, self};",
         &InsertUseConfig {
             granularity: ImportGranularity::Crate,
             enforce_granularity: true,
@@ -618,7 +618,7 @@ fn merge_self_glob() {
     check_with_config(
         "self",
         r"use self::*;",
-        r"use self::{self, *};",
+        r"use self::{*, self};",
         &InsertUseConfig {
             granularity: ImportGranularity::Crate,
             enforce_granularity: true,
@@ -637,7 +637,7 @@ fn merge_glob() {
         r"
 use syntax::{SyntaxKind::*};",
         r"
-use syntax::{SyntaxKind::{self, *}};",
+use syntax::{SyntaxKind::{*, self}};",
     )
 }