Repository: rust-analyzer/rust-analyzer (rust)

Issue:
MBE failed to parse single composited token
```rust
macro_rules! m {
    ( -> ) => { "->" };
    ( $first:tt $($tail:tt)+ ) => {
        m!($first)
    }
}

fn main() {
    let a = m!(-> a b c);  // a is unknown 
}
```

We should make `$var:tt` meta variable respect to composited token. 

Pull request description:
Fix mbe composited token bug
Fix #3436

Patch:
diff --git a/Cargo.lock b/Cargo.lock
index 91edb460c3..316dae053f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -46,9 +46,9 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
 
 [[package]]
 name = "backtrace"
-version = "0.3.45"
+version = "0.3.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad235dabf00f36301792cfe82499880ba54c6486be094d1047b02bacb67c14e8"
+checksum = "e4036b9bf40f3cf16aba72a3d65e8a520fc4bafcdc7079aea8f848c58c5b5536"
 dependencies = [
  "backtrace-sys",
  "cfg-if",
@@ -58,9 +58,9 @@ dependencies = [
 
 [[package]]
 name = "backtrace-sys"
-version = "0.1.33"
+version = "0.1.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e17b52e737c40a7d75abca20b29a19a0eb7ba9fc72c5a72dd282a0a3c2c0dc35"
+checksum = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491"
 dependencies = [
  "cc",
  "libc",
@@ -1259,9 +1259,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.16"
+version = "0.6.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1132f845907680735a84409c3bebc64d1364a5683ffbce899550cd09d5eaefc1"
+checksum = "7246cd0a0a6ec2239a5405b2b16e3f404fa0dcc6d28f5f5b877bf80e33e0f294"
 
 [[package]]
 name = "relative-path"
diff --git a/crates/ra_assists/src/handlers/inline_local_variable.rs b/crates/ra_assists/src/handlers/inline_local_variable.rs
index 3bfcba8ff3..eb5112343c 100644
--- a/crates/ra_assists/src/handlers/inline_local_variable.rs
+++ b/crates/ra_assists/src/handlers/inline_local_variable.rs
@@ -1,4 +1,3 @@
-use ra_ide_db::defs::Definition;
 use ra_syntax::{
     ast::{self, AstNode, AstToken},
     TextRange,
@@ -38,15 +37,6 @@ pub(crate) fn inline_local_variable(ctx: AssistCtx) -> Option<Assist> {
         return None;
     }
     let initializer_expr = let_stmt.initializer()?;
-
-    let def = ctx.sema.to_def(&bind_pat)?;
-    let def = Definition::Local(def);
-    let refs = def.find_usages(ctx.db, None);
-    if refs.is_empty() {
-        tested_by!(test_not_applicable_if_variable_unused);
-        return None;
-    };
-
     let delete_range = if let Some(whitespace) = let_stmt
         .syntax()
         .next_sibling_or_token()
@@ -59,14 +49,16 @@ pub(crate) fn inline_local_variable(ctx: AssistCtx) -> Option<Assist> {
     } else {
         let_stmt.syntax().text_range()
     };
+    let refs = ctx.sema.find_all_refs(&bind_pat);
+    if refs.is_empty() {
+        return None;
+    };
 
     let mut wrap_in_parens = vec![true; refs.len()];
 
     for (i, desc) in refs.iter().enumerate() {
-        let usage_node = ctx
-            .covering_node_for_range(desc.file_range.range)
-            .ancestors()
-            .find_map(ast::PathExpr::cast)?;
+        let usage_node =
+            ctx.covering_node_for_range(desc.range).ancestors().find_map(ast::PathExpr::cast)?;
         let usage_parent_option = usage_node.syntax().parent().and_then(ast::Expr::cast);
         let usage_parent = match usage_parent_option {
             Some(u) => u,
@@ -111,9 +103,11 @@ pub(crate) fn inline_local_variable(ctx: AssistCtx) -> Option<Assist> {
         move |edit: &mut ActionBuilder| {
             edit.delete(delete_range);
             for (desc, should_wrap) in refs.iter().zip(wrap_in_parens) {
-                let replacement =
-                    if should_wrap { init_in_paren.clone() } else { init_str.clone() };
-                edit.replace(desc.file_range.range, replacement)
+                if should_wrap {
+                    edit.replace(desc.range, init_in_paren.clone())
+                } else {
+                    edit.replace(desc.range, init_str.clone())
+                }
             }
             edit.set_cursor(delete_range.start())
         },
@@ -663,7 +657,6 @@ fn foo() {
 
     #[test]
     fn test_not_applicable_if_variable_unused() {
-        covers!(test_not_applicable_if_variable_unused);
         check_assist_not_applicable(
             inline_local_variable,
             r"
diff --git a/crates/ra_assists/src/marks.rs b/crates/ra_assists/src/marks.rs
index 22404ee80a..cef3df4e5a 100644
--- a/crates/ra_assists/src/marks.rs
+++ b/crates/ra_assists/src/marks.rs
@@ -1,10 +1,9 @@
 //! See test_utils/src/marks.rs
 
-test_utils::marks![
+test_utils::marks!(
     introduce_var_in_comment_is_not_applicable
     test_introduce_var_expr_stmt
     test_introduce_var_last_expr
     not_applicable_outside_of_bind_pat
     test_not_inline_mut_variable
-    test_not_applicable_if_variable_unused
-];
+);
diff --git a/crates/ra_hir/src/semantics.rs b/crates/ra_hir/src/semantics.rs
index 7ce7857913..afc7f7ee77 100644
--- a/crates/ra_hir/src/semantics.rs
+++ b/crates/ra_hir/src/semantics.rs
@@ -19,7 +19,7 @@ use rustc_hash::{FxHashMap, FxHashSet};
 use crate::{
     db::HirDatabase,
     semantics::source_to_def::{ChildContainer, SourceToDefCache, SourceToDefCtx},
-    source_analyzer::{resolve_hir_path, SourceAnalyzer},
+    source_analyzer::{resolve_hir_path, ReferenceDescriptor, SourceAnalyzer},
     Function, HirFileId, InFile, Local, MacroDef, Module, ModuleDef, Name, Origin, Path,
     PathResolution, ScopeDef, StructField, Trait, Type, TypeParam, VariantDef,
 };
@@ -171,6 +171,12 @@ impl<'db, DB: HirDatabase> Semantics<'db, DB> {
         SemanticsScope { db: self.db, resolver }
     }
 
+    // FIXME: we only use this in `inline_local_variable` assist, ideally, we
+    // should switch to general reference search infra there.
+    pub fn find_all_refs(&self, pat: &ast::BindPat) -> Vec<ReferenceDescriptor> {
+        self.analyze(pat.syntax()).find_all_refs(pat)
+    }
+
     fn analyze(&self, node: &SyntaxNode) -> SourceAnalyzer {
         let src = self.find_file(node.clone());
         self.analyze2(src.as_ref(), None)
diff --git a/crates/ra_hir/src/source_analyzer.rs b/crates/ra_hir/src/source_analyzer.rs
index 33c566025e..015389fb06 100644
--- a/crates/ra_hir/src/source_analyzer.rs
+++ b/crates/ra_hir/src/source_analyzer.rs
@@ -7,6 +7,7 @@
 //! purely for "IDE needs".
 use std::{iter::once, sync::Arc};
 
+use either::Either;
 use hir_def::{
     body::{
         scope::{ExprScopes, ScopeId},
@@ -20,7 +21,7 @@ use hir_expand::{hygiene::Hygiene, name::AsName, HirFileId, InFile};
 use hir_ty::{InEnvironment, InferenceResult, TraitEnvironment};
 use ra_syntax::{
     ast::{self, AstNode},
-    SyntaxNode, SyntaxNodePtr, TextRange, TextUnit,
+    AstPtr, SyntaxNode, SyntaxNodePtr, TextRange, TextUnit,
 };
 
 use crate::{
@@ -110,16 +111,20 @@ impl SourceAnalyzer {
     fn expand_expr(
         &self,
         db: &impl HirDatabase,
-        expr: InFile<ast::MacroCall>,
+        expr: InFile<&ast::Expr>,
     ) -> Option<InFile<ast::Expr>> {
-        let macro_file = self.body_source_map.as_ref()?.node_macro_file(expr.as_ref())?;
+        let macro_call = ast::MacroCall::cast(expr.value.syntax().clone())?;
+        let macro_file =
+            self.body_source_map.as_ref()?.node_macro_file(expr.with_value(&macro_call))?;
         let expanded = db.parse_or_expand(macro_file)?;
+        let kind = expanded.kind();
+        let expr = InFile::new(macro_file, ast::Expr::cast(expanded)?);
 
-        let res = match ast::MacroCall::cast(expanded.clone()) {
-            Some(call) => self.expand_expr(db, InFile::new(macro_file, call))?,
-            _ => InFile::new(macro_file, ast::Expr::cast(expanded)?),
-        };
-        Some(res)
+        if ast::MacroCall::can_cast(kind) {
+            self.expand_expr(db, expr.as_ref())
+        } else {
+            Some(expr)
+        }
     }
 
     fn trait_env(&self, db: &impl HirDatabase) -> Arc<TraitEnvironment> {
@@ -127,13 +132,11 @@ impl SourceAnalyzer {
     }
 
     pub(crate) fn type_of(&self, db: &impl HirDatabase, expr: &ast::Expr) -> Option<Type> {
-        let expr_id = match expr {
-            ast::Expr::MacroCall(call) => {
-                let expr = self.expand_expr(db, InFile::new(self.file_id, call.clone()))?;
-                self.body_source_map.as_ref()?.node_expr(expr.as_ref())
-            }
-            _ => self.expr_id(expr),
-        }?;
+        let expr_id = if let Some(expr) = self.expand_expr(db, InFile::new(self.file_id, expr)) {
+            self.body_source_map.as_ref()?.node_expr(expr.as_ref())?
+        } else {
+            self.expr_id(expr)?
+        };
 
         let ty = self.infer.as_ref()?[expr_id].clone();
         let environment = self.trait_env(db);
@@ -248,6 +251,38 @@ impl SourceAnalyzer {
         resolve_hir_path(db, &self.resolver, &hir_path)
     }
 
+    fn resolve_local_name(
+        &self,
+        name_ref: &ast::NameRef,
+    ) -> Option<Either<AstPtr<ast::Pat>, AstPtr<ast::SelfParam>>> {
+        let name = name_ref.as_name();
+        let source_map = self.body_source_map.as_ref()?;
+        let scopes = self.scopes.as_ref()?;
+        let scope = scope_for(scopes, source_map, InFile::new(self.file_id, name_ref.syntax()))?;
+        let entry = scopes.resolve_name_in_scope(scope, &name)?;
+        Some(source_map.pat_syntax(entry.pat())?.value)
+    }
+
+    // FIXME: we only use this in `inline_local_variable` assist, ideally, we
+    // should switch to general reference search infra there.
+    pub(crate) fn find_all_refs(&self, pat: &ast::BindPat) -> Vec<ReferenceDescriptor> {
+        let fn_def = pat.syntax().ancestors().find_map(ast::FnDef::cast).unwrap();
+        let ptr = Either::Left(AstPtr::new(&ast::Pat::from(pat.clone())));
+        fn_def
+            .syntax()
+            .descendants()
+            .filter_map(ast::NameRef::cast)
+            .filter(|name_ref| match self.resolve_local_name(&name_ref) {
+                None => false,
+                Some(d_ptr) => d_ptr == ptr,
+            })
+            .map(|name_ref| ReferenceDescriptor {
+                name: name_ref.text().to_string(),
+                range: name_ref.syntax().text_range(),
+            })
+            .collect()
+    }
+
     pub(crate) fn expand(
         &self,
         db: &impl HirDatabase,
diff --git a/crates/ra_hir_expand/src/builtin_derive.rs b/crates/ra_hir_expand/src/builtin_derive.rs
index 87224481ce..a5b50a832b 100644
--- a/crates/ra_hir_expand/src/builtin_derive.rs
+++ b/crates/ra_hir_expand/src/builtin_derive.rs
@@ -266,7 +266,7 @@ mod tests {
             BuiltinDeriveExpander::Copy,
         );
 
-        assert_eq!(expanded, "impl <>std::marker::CopyforFoo <>{}");
+        assert_eq!(expanded, "impl< >std::marker::CopyforFoo< >{}");
     }
 
     #[test]
diff --git a/crates/ra_hir_expand/src/builtin_macro.rs b/crates/ra_hir_expand/src/builtin_macro.rs
index 1f380b5716..b2c8a911f0 100644
--- a/crates/ra_hir_expand/src/builtin_macro.rs
+++ b/crates/ra_hir_expand/src/builtin_macro.rs
@@ -367,7 +367,7 @@ mod tests {
             "#,
         );
 
-        assert_eq!(expanded, "std::option::Option::None:: <&str>");
+        assert_eq!(expanded, "std::option::Option::None:: < &str>");
     }
 
     #[test]
@@ -414,7 +414,7 @@ mod tests {
 
         assert_eq!(
             expanded,
-            r#"std::fmt::Arguments::new_v1(&[] ,&[std::fmt::ArgumentV1::new(&(arg1(a,b,c)),std::fmt::Display::fmt),std::fmt::ArgumentV1::new(&(arg2),std::fmt::Display::fmt),])"#
+            r#"std::fmt::Arguments::new_v1(&[], &[std::fmt::ArgumentV1::new(&(arg1(a,b,c)),std::fmt::Display::fmt),std::fmt::ArgumentV1::new(&(arg2),std::fmt::Display::fmt),])"#
         );
     }
 }
diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs
index 32e0d5ced3..f3a84cacc3 100644
--- a/crates/ra_hir_expand/src/db.rs
+++ b/crates/ra_hir_expand/src/db.rs
@@ -176,10 +176,20 @@ pub(crate) fn parse_macro(
                 MacroCallId::LazyMacro(id) => {
                     let loc: MacroCallLoc = db.lookup_intern_macro(id);
                     let node = loc.kind.node(db);
+
+                    // collect parent information for warning log
+                    let parents = std::iter::successors(loc.kind.file_id().call_node(db), |it| {
+                        it.file_id.call_node(db)
+                    })
+                    .map(|n| format!("{:#}", n.value))
+                    .collect::<Vec<_>>()
+                    .join("\n");
+
                     log::warn!(
-                        "fail on macro_parse: (reason: {} macro_call: {:#})",
+                        "fail on macro_parse: (reason: {} macro_call: {:#}) parents: {}",
                         err,
-                        node.value
+                        node.value,
+                        parents
                     );
                 }
                 _ => {
diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs
index 2bdea11e14..ffba03898a 100644
--- a/crates/ra_mbe/src/mbe_expander/matcher.rs
+++ b/crates/ra_mbe/src/mbe_expander/matcher.rs
@@ -155,6 +155,60 @@ impl<'a> TtIter<'a> {
         ok
     }
 
+    pub(crate) fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
+        let tt = self.next().ok_or_else(|| ())?.clone();
+        let punct = match tt {
+            tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => {
+                punct
+            }
+            _ => return Ok(tt),
+        };
+
+        let (second, third) = match (self.peek_n(0), self.peek_n(1)) {
+            (
+                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))),
+                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))),
+            ) if p2.spacing == tt::Spacing::Joint => (p2.char, Some(p3.char)),
+            (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2.char, None),
+            _ => return Ok(tt),
+        };
+
+        match (punct.char, second, third) {
+            ('.', '.', Some('.'))
+            | ('.', '.', Some('='))
+            | ('<', '<', Some('='))
+            | ('>', '>', Some('=')) => {
+                let tt2 = self.next().unwrap().clone();
+                let tt3 = self.next().unwrap().clone();
+                Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2, tt3] }.into())
+            }
+            ('-', '=', None)
+            | ('-', '>', None)
+            | (':', ':', None)
+            | ('!', '=', None)
+            | ('.', '.', None)
+            | ('*', '=', None)
+            | ('/', '=', None)
+            | ('&', '&', None)
+            | ('&', '=', None)
+            | ('%', '=', None)
+            | ('^', '=', None)
+            | ('+', '=', None)
+            | ('<', '<', None)
+            | ('<', '=', None)
+            | ('=', '=', None)
+            | ('=', '>', None)
+            | ('>', '=', None)
+            | ('>', '>', None)
+            | ('|', '=', None)
+            | ('|', '|', None) => {
+                let tt2 = self.next().unwrap().clone();
+                Ok(tt::Subtree { delimiter: None, token_trees: vec![tt.clone(), tt2] }.into())
+            }
+            _ => Ok(tt),
+        }
+    }
+
     pub(crate) fn expect_lifetime(&mut self) -> Result<&tt::Ident, ()> {
         let ident = self.expect_ident()?;
         // check if it start from "`"
@@ -302,7 +356,7 @@ fn match_meta_var(kind: &str, input: &mut TtIter) -> Result<Option<Fragment>, Ex
                     let ident = input.expect_ident().map_err(|()| err!("expected ident"))?.clone();
                     tt::Leaf::from(ident).into()
                 }
-                "tt" => input.next().ok_or_else(|| err!())?.clone(),
+                "tt" => input.expect_tt().map_err(|()| err!())?.clone(),
                 "lifetime" => {
                     let ident = input.expect_lifetime().map_err(|()| err!())?;
                     tt::Leaf::Ident(ident.clone()).into()
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 2aaf0215fa..d8ee74faaa 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -388,11 +388,12 @@ impl<'a> TreeSink for TtTreeSink<'a> {
             return;
         }
 
+        let mut last = self.cursor;
         for _ in 0..n_tokens {
             if self.cursor.eof() {
                 break;
             }
-
+            last = self.cursor;
             let text: SmolStr = match self.cursor.token_tree() {
                 Some(tt::TokenTree::Leaf(leaf)) => {
                     // Mark the range if needed
@@ -441,11 +442,11 @@ impl<'a> TreeSink for TtTreeSink<'a> {
         self.inner.token(kind, text);
 
         // Add whitespace between adjoint puncts
-        let next = self.cursor.bump();
+        let next = last.bump();
         if let (
             Some(tt::TokenTree::Leaf(tt::Leaf::Punct(curr))),
             Some(tt::TokenTree::Leaf(tt::Leaf::Punct(_))),
-        ) = (self.cursor.token_tree(), next.token_tree())
+        ) = (last.token_tree(), next.token_tree())
         {
             if curr.spacing == tt::Spacing::Alone {
                 self.inner.token(WHITESPACE, " ".into());
diff --git a/crates/ra_mbe/src/tt_iter.rs b/crates/ra_mbe/src/tt_iter.rs
index 319f1ee65f..100184e66a 100644
--- a/crates/ra_mbe/src/tt_iter.rs
+++ b/crates/ra_mbe/src/tt_iter.rs
@@ -53,6 +53,10 @@ impl<'a> TtIter<'a> {
             _ => Err(()),
         }
     }
+
+    pub(crate) fn peek_n(&self, n: usize) -> Option<&tt::TokenTree> {
+        self.inner.as_slice().get(n)
+    }
 }
 
 impl<'a> Iterator for TtIter<'a> {


Test patch:
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index 1dba829154..304867881d 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -825,6 +825,50 @@ fn test_tt_group() {
     )
     .assert_expand_items(r#"foo! { fn foo() {} }"#, r#"fn foo () {}"#);
 }
+
+#[test]
+fn test_tt_composite() {
+    parse_macro(
+        r#"
+            macro_rules! foo {
+                 ($i:tt) => { 0 }
+            }
+    "#,
+    )
+    .assert_expand_items(r#"foo! { => }"#, r#"0"#);
+}
+
+#[test]
+fn test_tt_composite2() {
+    let node = parse_macro(
+        r#"
+            macro_rules! foo {
+                ($($tt:tt)*) => { abs!(=> $($tt)*) }
+            }
+    "#,
+    )
+    .expand_items(r#"foo!{#}"#);
+
+    let res = format!("{:#?}", &node);
+    assert_eq_text!(
+        res.trim(),
+        r###"MACRO_ITEMS@[0; 10)
+  MACRO_CALL@[0; 10)
+    PATH@[0; 3)
+      PATH_SEGMENT@[0; 3)
+        NAME_REF@[0; 3)
+          IDENT@[0; 3) "abs"
+    EXCL@[3; 4) "!"
+    TOKEN_TREE@[4; 10)
+      L_PAREN@[4; 5) "("
+      EQ@[5; 6) "="
+      R_ANGLE@[6; 7) ">"
+      WHITESPACE@[7; 8) " "
+      POUND@[8; 9) "#"
+      R_PAREN@[9; 10) ")""###
+    );
+}
+
 #[test]
 fn test_lifetime() {
     parse_macro(