Repository: rust-lang/rust-analyzer (rust)

Issue:
False Positive macro error when using `guard` macro with path in pattern
<!--
Troubleshooting guide: https://rust-analyzer.github.io/manual.html#troubleshooting
Forum for questions: https://users.rust-lang.org/c/ide/14

Before submitting, please make sure that you're not running into one of these known issues:

 1. extension doesn't load in VSCodium: #11080
 2. VS Code Marketplace serves old stable version: #11098
 3. on-the-fly diagnostics are mostly unimplemented (`cargo check` diagnostics will be shown when saving a file): #3107

Otherwise please try to provide information which will help us to fix the issue faster. Minimal reproducible examples with few dependencies are especially lovely <3.
-->

With this code:

```rust
use guard::guard; // guard = "0.5.1"

pub fn demo() {
    guard!(let Some(_) = Some(1) else { panic!() });
    guard!(let Option::Some(_) = Some(1) else { panic!() });
}
```

Rustc compilles this code fine, but Rust-Analyzer thinks their is an error, but only for the secound line

![image](https://user-images.githubusercontent.com/28781354/154723390-510764be-b6d1-4a3b-b21d-8b1c8aed9215.png)

<details>
<summary> version with `guard` definitions inlined (~200 loc), to make debugging easier </summary>

```rust
pub enum LetElseBodyMustDiverge {}

#[macro_export(local_inner_macros)]
macro_rules! __guard_output {
    ((($($imms:ident)*) ($($muts:ident)*)), [($($guard:tt)*) ($($pattern:tt)*) ($rhs:expr) ($diverge:expr)]) => {
        __guard_impl!(@as_stmt
               let ($($imms,)* $(mut $muts,)*) = { #[allow(unused_mut)]
                                                 match $rhs {
                                                   $($pattern)* => {
                                                       if $($guard)* { // move the guard inside to avoid "cannot bind by-move"
                                                           ($($imms,)* $($muts,)*)
                                                       } else {
                                                           let _: $crate::LetElseBodyMustDiverge = $diverge;
                                                       }
                                                   },

                                                   _ => {
                                                       let _: $crate::LetElseBodyMustDiverge = $diverge;
                                                   },
               } }
              )
    };
}

#[doc(hidden)]
#[macro_export(local_inner_macros)]
macro_rules! __guard_impl {
    // 0. cast a series of token trees to a statement
    (@as_stmt $s:stmt) => { $s };

    // 1. output stage
    (@collect () -> $($rest:tt)*) => {
        __guard_output!($($rest)*)
    };


    // 2. identifier collection stage
    //      The pattern is scanned destructively. Anything that looks like a capture (including
    //      false positives, like un-namespaced/empty structs or enum variants) is copied into the
    //      appropriate identifier list. Irrelevant symbols are discarded. The scanning descends
    //      recursively into bracketed structures.

    // unwrap brackets and prepend their contents to the pattern remainder, in case there are captures inside
    (@collect (($($inside:tt)*) $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($inside)* $($tail)*) -> $idents, $thru)
    };
    (@collect ({$($inside:tt)*} $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($inside)* $($tail)*) -> $idents, $thru)
    };
    (@collect ([$($inside:tt)*] $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($inside)* $($tail)*) -> $idents, $thru)
    };

    // discard irrelevant symbols
    (@collect (, $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> $idents, $thru)
    };
    (@collect (.. $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> $idents, $thru)
    };
    (@collect (@ $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> $idents, $thru)
    };
    (@collect (_ $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> $idents, $thru)
    };
    (@collect (& $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> $idents, $thru)
    };

    // ignore generic parameters
    (@collect (:: <$($generic:tt),*> $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> $idents, $thru)
    };
    // a path can't be a capture, and a path can't end with ::, so the ident after :: is never a capture
    (@collect (:: $pathend:ident $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> $idents, $thru)
    };

    // alternative patterns may be given with | as long as the same captures (including type) appear on each side
    // due to this property, if we see a | we've already parsed all the captures and can simply stop
    (@collect (| $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect () -> $idents, $thru) // discard the rest of the pattern, proceed to output stage
    };

    // an explicitly provided pattern guard replaces the default, if there was one
    (@collect (if $($tail:tt)*) -> $idents:tt, [$guard:tt $($rest:tt)*]) => {
        __guard_impl!(@collect () -> $idents, [($($tail)*) $($rest)*])
    };

    // throw away some identifiers that do not represent captures

    // box destructuring
    (@collect (box $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> $idents, $thru)
    };

    // an ident followed by a colon is the name of a structure member
    (@collect ($id:ident: $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> $idents, $thru)
    };
    // paths do not represent captures
    (@collect ($pathcomp:ident :: $pathend:ident $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> $idents, $thru)
    };
    // an ident followed by parentheses is the name of a tuple-like struct or enum variant
    // (unwrap the parens to parse the contents)
    (@collect ($id:ident ($($inside:tt)*) $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($inside)* $($tail)*) -> $idents, $thru)
    };
    // an ident followed by curly braces is the name of a struct or struct-like enum variant
    // (unwrap the braces to parse the contents)
    (@collect ($id:ident {$($inside:tt)*} $($tail:tt)*) -> $idents:tt, $thru:tt) => {
        __guard_impl!(@collect ($($inside)* $($tail)*) -> $idents, $thru)
    };

    // actually identifier collection happens here!

    // capture by mutable reference!
    (@collect (ref mut $id:ident $($tail:tt)*) -> (($($imms:ident)*) $muts:tt), $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> (($($imms)* $id) $muts), $thru)
    };
    // capture by immutable reference!
    (@collect (ref $id:ident $($tail:tt)*) -> (($($imms:ident)*) $muts:tt), $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> (($($imms)* $id) $muts), $thru)
    };
    // capture by move into mutable binding!
    (@collect (mut $id:ident $($tail:tt)*) -> ($imms:tt ($($muts:ident)*)), $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> ($imms ($($muts)* $id)), $thru)
    };
    // capture by move into an immutable binding!
    (@collect ($id:ident $($tail:tt)*) -> (($($imms:ident)*) $muts:tt), $thru:tt) => {
        __guard_impl!(@collect ($($tail)*) -> (($($imms)* $id) $muts), $thru)
    };

    // 3. splitting (for new syntax)

    // done with pattern (and it's LPED=X)
    (@split (else { $($diverge:tt)* } = $($tail:tt)*) -> ($pat:tt $guard:tt)) => {
        __guard_impl!(@collect $pat -> (() ()), [$guard $pat ($($tail)*) ({ $($diverge)* })])
    };

    // done with pattern (and it's LP=XED)
    (@split (= $($tail:tt)*) -> ($pat:tt $guard:tt)) => {
        __guard_impl!(@split expr ($($tail)*) -> ($pat $guard ()))
    };

    // found a guard in the pattern
    (@split (if $($tail:tt)*) -> ($pat:tt $guard:tt)) => {
        __guard_impl!(@split guard ($($tail)*) -> ($pat ()))
    };

    // done with guard (and it's LP=XED)
    (@split guard (= $($tail:tt)*) -> ($pat:tt $guard:tt)) => {
        __guard_impl!(@split expr ($($tail)*) -> ($pat $guard ()))
    };

    // done with guard (and it's LPED=X)
    (@split guard (else { $($diverge:tt)* } = $($tail:tt)*) -> ($pat:tt $guard:tt)) => {
        __guard_impl!(@collect $pat -> (() ()), [$guard $pat ($($tail)*) ({ $($diverge)* })])
    };

    // found a token in the guard
    (@split guard ($head:tt $($tail:tt)*) -> ($pat:tt ($($guard:tt)*))) => {
        __guard_impl!(@split guard ($($tail)*) -> ($pat ($($guard)* $head)))
    };

    // found a token in the pattern
    (@split ($head:tt $($tail:tt)*) -> (($($pat:tt)*) $guard:tt)) => {
        __guard_impl!(@split ($($tail)*) -> (($($pat)* $head) $guard))
    };

    // found an "else DIVERGE" in the expr
    (@split expr (else { $($tail:tt)* }) -> ($pat:tt $guard:tt $expr:tt)) => {
        __guard_impl!(@collect $pat -> (() ()), [$guard $pat $expr ({ $($tail)* })])
    };

    // found an else in the expr with more stuff after it
    (@split expr (else { $($body:tt)* } $($tail:tt)*) -> ($pat:tt $guard:tt ($($expr:tt)*))) => {
        __guard_impl!(@split expr ($($tail)*) -> ($pat $guard ($($expr)* else { $($body)* })))
    };

    // found another token in the expr
    (@split expr ($head:tt $($tail:tt)*) -> ($pat:tt $guard:tt ($($expr:tt)*))) => {
        __guard_impl!(@split expr ($($tail)*) -> ($pat $guard ($($expr)* $head)))
    };

    // 4. entry points

    // old syntax
    ({ $($diverge:tt)* } unless $rhs:expr => $($pattern:tt)*) => {
        __guard_impl!(@collect ($($pattern)*) -> (() ()), [(true) ($($pattern)*) ($rhs) ({$($diverge)*})])
        //            |        |                 ||  |    ||      |              |      |
        //            |        |                 ||  |    ||      |              |      ^ diverging expression
        //            |        |                 ||  |    ||      |              ^ expression
        //            |        |                 ||  |    ||      ^ saved copy of pattern
        //            |        |                 ||  |    | ^ pattern guard
        //            |        |                 ||  |    ^ parameters that will be carried through to output stage
        //            |        |                 ||  ^ identifiers bound to mutable captures
        //            |        |                 |^ identifiers bound to immutable captures
        //            |        |                 ^ identifiers found by the scan
        //            |        ^ pattern to be destructively scanned for identifiers
        //            ^ proceed to identifier collection stage

        // FIXME once #14252 is fixed, put "if true" in as the default guard to defeat E0008
    };

    // new syntax
    (let $($tail:tt)*) => {
        __guard_impl!(@split ($($tail)*) -> (() (true)))
        //            |      |               |  |
        //            |      |               |  ^ guard
        //            |      |               ^ pattern
        //            |      ^ tail to be split into "PAT = EXPR else DIVERGE"
        //            ^ first pass will do the parsing
    };
}

macro_rules! guard {
    ($($input:tt)*) => {
        __guard_impl!($($input)*)
    };
}

pub fn demo() {
    guard!(let Some(_) = Some(1) else { panic!() });
    guard!(let Option::Some(_) = Some(1) else { panic!() });
}

```


</details>

**rust-analyzer version**: rust-analyzer version: 02904e99a 2022-02-14 nightly

**rustc version**: rustc 1.58.1 (db9d1b20b 2022-01-20)


Pull request description:
Support multi-character punct tokens in MBE
Fixes #11497

In the context of MBE, consecutive puncts are parsed as multi-character punct tokens whenever possible. For example, `:::` is parsed as ``[Punct(`::`), Punct(`:`)]`` and shouldn't get matched to patterns like `: : :` or `: ::`.

We have implemented this behavior only for when we match puncts against `tt` fragments, but not when we match puncts literally. This PR extracts the multi-character punct handling procedure into a separate method and extends its support for literal matching.

For good measure, this PR adds support for `<-` token, which is still [considered as one token in rustc](https://github.com/rust-lang/rust/blob/e3961864075eaa9e855e5eec6b4f148029684539/compiler/rustc_ast/src/token.rs#L249) despite the placement syntax having been removed.

Patch:
diff --git a/crates/hir-ty/src/lib.rs b/crates/hir-ty/src/lib.rs
index cbe6873c7d..d3b445c017 100644
--- a/crates/hir-ty/src/lib.rs
+++ b/crates/hir-ty/src/lib.rs
@@ -40,7 +40,7 @@ use std::sync::Arc;
 use chalk_ir::{
     fold::{Shift, TypeFoldable},
     interner::HasInterner,
-    NoSolution,
+    NoSolution, UniverseIndex,
 };
 use hir_def::{expr::ExprId, type_ref::Rawness, TypeOrConstParamId};
 use hir_expand::name;
@@ -48,9 +48,7 @@ use itertools::Either;
 use traits::FnTrait;
 use utils::Generics;
 
-use crate::{
-    consteval::unknown_const, db::HirDatabase, infer::unify::InferenceTable, utils::generics,
-};
+use crate::{consteval::unknown_const, db::HirDatabase, utils::generics};
 
 pub use autoderef::autoderef;
 pub use builder::{ParamKind, TyBuilder};
@@ -535,31 +533,53 @@ pub fn callable_sig_from_fnonce(
     let fn_once_trait = FnTrait::FnOnce.get_id(db, krate)?;
     let output_assoc_type = db.trait_data(fn_once_trait).associated_type_by_name(&name![Output])?;
 
-    let mut table = InferenceTable::new(db, env.clone());
     let b = TyBuilder::trait_ref(db, fn_once_trait);
     if b.remaining() != 2 {
         return None;
     }
+    let fn_once = b.push(self_ty.clone()).fill_with_bound_vars(DebruijnIndex::INNERMOST, 0).build();
+    let kinds = fn_once
+        .substitution
+        .iter(Interner)
+        .skip(1)
+        .map(|x| {
+            let vk = match x.data(Interner) {
+                chalk_ir::GenericArgData::Ty(_) => {
+                    chalk_ir::VariableKind::Ty(chalk_ir::TyVariableKind::General)
+                }
+                chalk_ir::GenericArgData::Lifetime(_) => chalk_ir::VariableKind::Lifetime,
+                chalk_ir::GenericArgData::Const(c) => {
+                    chalk_ir::VariableKind::Const(c.data(Interner).ty.clone())
+                }
+            };
+            chalk_ir::WithKind::new(vk, UniverseIndex::ROOT)
+        })
+        .collect::<Vec<_>>();
+
+    // FIXME: chalk refuses to solve `<Self as FnOnce<^0.0>>::Output == ^0.1`, so we first solve
+    // `<Self as FnOnce<^0.0>>` and then replace `^0.0` with the concrete argument tuple.
+    let trait_env = env.env.clone();
+    let obligation = InEnvironment { goal: fn_once.cast(Interner), environment: trait_env };
+    let canonical =
+        Canonical { binders: CanonicalVarKinds::from_iter(Interner, kinds), value: obligation };
+    let subst = match db.trait_solve(krate, canonical) {
+        Some(Solution::Unique(vars)) => vars.value.subst,
+        _ => return None,
+    };
+    let args = subst.at(Interner, 0).ty(Interner)?;
+    let params = match args.kind(Interner) {
+        chalk_ir::TyKind::Tuple(_, subst) => {
+            subst.iter(Interner).filter_map(|arg| arg.ty(Interner).cloned()).collect::<Vec<_>>()
+        }
+        _ => return None,
+    };
 
-    // Register two obligations:
-    // - Self: FnOnce<?args_ty>
-    // - <Self as FnOnce<?args_ty>>::Output == ?ret_ty
-    let args_ty = table.new_type_var();
-    let trait_ref = b.push(self_ty.clone()).push(args_ty.clone()).build();
-    let projection = TyBuilder::assoc_type_projection(
-        db,
-        output_assoc_type,
-        Some(trait_ref.substitution.clone()),
-    )
-    .build();
-    table.register_obligation(trait_ref.cast(Interner));
-    let ret_ty = table.normalize_projection_ty(projection);
-
-    let ret_ty = table.resolve_completely(ret_ty);
-    let args_ty = table.resolve_completely(args_ty);
+    let fn_once =
+        TyBuilder::trait_ref(db, fn_once_trait).push(self_ty.clone()).push(args.clone()).build();
+    let projection =
+        TyBuilder::assoc_type_projection(db, output_assoc_type, Some(fn_once.substitution)).build();
 
-    let params =
-        args_ty.as_tuple()?.iter(Interner).map(|it| it.assert_ty_ref(Interner)).cloned().collect();
+    let ret_ty = db.normalize_projection(projection, env);
 
     Some(CallableSig::from_params_and_return(params, ret_ty, false, Safety::Safe))
 }
diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs
index 1915c0b661..4b75002501 100644
--- a/crates/mbe/src/benchmark.rs
+++ b/crates/mbe/src/benchmark.rs
@@ -141,7 +141,13 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
                 None => (),
                 Some(kind) => panic!("Unhandled kind {kind:?}"),
             },
-            Op::Leaf(leaf) => parent.token_trees.push(leaf.clone().into()),
+            Op::Literal(it) => parent.token_trees.push(tt::Leaf::from(it.clone()).into()),
+            Op::Ident(it) => parent.token_trees.push(tt::Leaf::from(it.clone()).into()),
+            Op::Punct(puncts) => {
+                for punct in puncts {
+                    parent.token_trees.push(tt::Leaf::from(punct.clone()).into());
+                }
+            }
             Op::Repeat { tokens, kind, separator } => {
                 let max = 10;
                 let cnt = match kind {
diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs
index b6c5c1026e..88eae136f7 100644
--- a/crates/mbe/src/expander/matcher.rs
+++ b/crates/mbe/src/expander/matcher.rs
@@ -68,7 +68,7 @@ use crate::{
     expander::{Binding, Bindings, ExpandResult, Fragment},
     parser::{MetaVarKind, Op, RepeatKind, Separator},
     tt_iter::TtIter,
-    ExpandError, MetaTemplate,
+    ExpandError, MetaTemplate, ValueResult,
 };
 
 impl Bindings {
@@ -321,8 +321,8 @@ struct MatchState<'t> {
     /// The KleeneOp of this sequence if we are in a repetition.
     sep_kind: Option<RepeatKind>,
 
-    /// Number of tokens of separator parsed
-    sep_parsed: Option<usize>,
+    /// Whether we already matched separator token.
+    sep_matched: bool,
 
     /// Matched meta variables bindings
     bindings: BindingsIdx,
@@ -387,7 +387,7 @@ fn match_loop_inner<'t>(
             None => {
                 // We are at or past the end of the matcher of `item`.
                 if let Some(up) = &item.up {
-                    if item.sep_parsed.is_none() {
+                    if !item.sep_matched {
                         // Get the `up` matcher
                         let mut new_pos = (**up).clone();
                         new_pos.bindings = bindings_builder.copy(&new_pos.bindings);
@@ -401,14 +401,17 @@ fn match_loop_inner<'t>(
                     }
 
                     // Check if we need a separator.
-                    // We check the separator one by one
-                    let sep_idx = item.sep_parsed.unwrap_or(0);
-                    let sep_len = item.sep.as_ref().map_or(0, Separator::tt_count);
-                    if item.sep.is_some() && sep_idx != sep_len {
+                    if item.sep.is_some() && !item.sep_matched {
                         let sep = item.sep.as_ref().unwrap();
-                        if src.clone().expect_separator(sep, sep_idx) {
+                        let mut fork = src.clone();
+                        if fork.expect_separator(sep) {
+                            // HACK: here we use `meta_result` to pass `TtIter` back to caller because
+                            // it might have been advanced multiple times. `ValueResult` is
+                            // insignificant.
+                            item.meta_result = Some((fork, ValueResult::ok(None)));
                             item.dot.next();
-                            item.sep_parsed = Some(sep_idx + 1);
+                            // item.sep_parsed = Some(sep_len);
+                            item.sep_matched = true;
                             try_push!(next_items, item);
                         }
                     }
@@ -416,7 +419,7 @@ fn match_loop_inner<'t>(
                     // and try to match again UNLESS we are only allowed to have _one_ repetition.
                     else if item.sep_kind != Some(RepeatKind::ZeroOrOne) {
                         item.dot = item.dot.reset();
-                        item.sep_parsed = None;
+                        item.sep_matched = false;
                         bindings_builder.push_default(&mut item.bindings);
                         cur_items.push(item);
                     }
@@ -451,7 +454,7 @@ fn match_loop_inner<'t>(
                     up: Some(Box::new(item)),
                     sep: separator.clone(),
                     sep_kind: Some(*kind),
-                    sep_parsed: None,
+                    sep_matched: false,
                     bindings: bindings_builder.alloc(),
                     meta_result: None,
                     is_error: false,
@@ -500,18 +503,69 @@ fn match_loop_inner<'t>(
                     }
                 }
             }
-            OpDelimited::Op(Op::Leaf(leaf)) => {
-                if let Err(err) = match_leaf(leaf, &mut src.clone()) {
-                    res.add_err(err);
+            OpDelimited::Op(Op::Literal(lhs)) => {
+                if let Ok(rhs) = src.clone().expect_leaf() {
+                    if matches!(rhs, tt::Leaf::Literal(it) if it.text == lhs.text) {
+                        item.dot.next();
+                    } else {
+                        res.add_err(ExpandError::UnexpectedToken);
+                        item.is_error = true;
+                    }
+                } else {
+                    res.add_err(ExpandError::binding_error(format!("expected literal: `{lhs}`")));
                     item.is_error = true;
+                }
+                try_push!(next_items, item);
+            }
+            OpDelimited::Op(Op::Ident(lhs)) => {
+                if let Ok(rhs) = src.clone().expect_leaf() {
+                    if matches!(rhs, tt::Leaf::Ident(it) if it.text == lhs.text) {
+                        item.dot.next();
+                    } else {
+                        res.add_err(ExpandError::UnexpectedToken);
+                        item.is_error = true;
+                    }
                 } else {
-                    item.dot.next();
+                    res.add_err(ExpandError::binding_error(format!("expected ident: `{lhs}`")));
+                    item.is_error = true;
                 }
                 try_push!(next_items, item);
             }
+            OpDelimited::Op(Op::Punct(lhs)) => {
+                let mut fork = src.clone();
+                let error = if let Ok(rhs) = fork.expect_glued_punct() {
+                    let first_is_single_quote = rhs[0].char == '\'';
+                    let lhs = lhs.iter().map(|it| it.char);
+                    let rhs = rhs.iter().map(|it| it.char);
+                    if lhs.clone().eq(rhs) {
+                        // HACK: here we use `meta_result` to pass `TtIter` back to caller because
+                        // it might have been advanced multiple times. `ValueResult` is
+                        // insignificant.
+                        item.meta_result = Some((fork, ValueResult::ok(None)));
+                        item.dot.next();
+                        next_items.push(item);
+                        continue;
+                    }
+
+                    if first_is_single_quote {
+                        // If the first punct token is a single quote, that's a part of a lifetime
+                        // ident, not a punct.
+                        ExpandError::UnexpectedToken
+                    } else {
+                        let lhs: SmolStr = lhs.collect();
+                        ExpandError::binding_error(format!("expected punct: `{lhs}`"))
+                    }
+                } else {
+                    ExpandError::UnexpectedToken
+                };
+
+                res.add_err(error);
+                item.is_error = true;
+                error_items.push(item);
+            }
             OpDelimited::Op(Op::Ignore { .. } | Op::Index { .. }) => {}
             OpDelimited::Open => {
-                if matches!(src.clone().next(), Some(tt::TokenTree::Subtree(..))) {
+                if matches!(src.peek_n(0), Some(tt::TokenTree::Subtree(..))) {
                     item.dot.next();
                     try_push!(next_items, item);
                 }
@@ -541,7 +595,7 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree) -> Match {
         up: None,
         sep: None,
         sep_kind: None,
-        sep_parsed: None,
+        sep_matched: false,
         bindings: bindings_builder.alloc(),
         is_error: false,
         meta_result: None,
@@ -616,21 +670,33 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree) -> Match {
         }
         // Dump all possible `next_items` into `cur_items` for the next iteration.
         else if !next_items.is_empty() {
-            // Now process the next token
-            cur_items.extend(next_items.drain(..));
-
-            match src.next() {
-                Some(tt::TokenTree::Subtree(subtree)) => {
-                    stack.push(src.clone());
-                    src = TtIter::new(subtree);
+            if let Some((iter, _)) = next_items[0].meta_result.take() {
+                // We've matched a possibly "glued" punct. The matched punct (hence
+                // `meta_result` also) must be the same for all items.
+                // FIXME: If there are multiple items, it's definitely redundant (and it's hacky!
+                // `meta_result` isn't supposed to be used this way).
+
+                // We already bumped, so no need to call `.next()` like in the other branch.
+                src = iter;
+                for item in next_items.iter_mut() {
+                    item.meta_result = None;
                 }
-                None => {
-                    if let Some(iter) = stack.pop() {
-                        src = iter;
+            } else {
+                match src.next() {
+                    Some(tt::TokenTree::Subtree(subtree)) => {
+                        stack.push(src.clone());
+                        src = TtIter::new(subtree);
                     }
+                    None => {
+                        if let Some(iter) = stack.pop() {
+                            src = iter;
+                        }
+                    }
+                    _ => (),
                 }
-                _ => (),
             }
+            // Now process the next token
+            cur_items.extend(next_items.drain(..));
         }
         // Finally, we have the case where we need to call the black-box parser to get some
         // nonterminal.
@@ -663,27 +729,6 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree) -> Match {
     }
 }
 
-fn match_leaf(lhs: &tt::Leaf, src: &mut TtIter<'_>) -> Result<(), ExpandError> {
-    let rhs = src
-        .expect_leaf()
-        .map_err(|()| ExpandError::binding_error(format!("expected leaf: `{lhs}`")))?;
-    match (lhs, rhs) {
-        (
-            tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
-            tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
-        ) if lhs == rhs => Ok(()),
-        (
-            tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
-            tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
-        ) if lhs == rhs => Ok(()),
-        (
-            tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
-            tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
-        ) if lhs == rhs => Ok(()),
-        _ => Err(ExpandError::UnexpectedToken),
-    }
-}
-
 fn match_meta_var(kind: MetaVarKind, input: &mut TtIter<'_>) -> ExpandResult<Option<Fragment>> {
     let fragment = match kind {
         MetaVarKind::Path => parser::PrefixEntryPoint::Path,
@@ -756,10 +801,10 @@ fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate)
     for op in pattern.iter() {
         match op {
             Op::Var { name, .. } => collector_fun(name.clone()),
-            Op::Leaf(_) => (),
             Op::Subtree { tokens, .. } => collect_vars(collector_fun, tokens),
             Op::Repeat { tokens, .. } => collect_vars(collector_fun, tokens),
-            Op::Ignore { .. } | Op::Index { .. } => {}
+            Op::Ignore { .. } | Op::Index { .. } | Op::Literal(_) | Op::Ident(_) | Op::Punct(_) => {
+            }
         }
     }
 }
@@ -822,14 +867,14 @@ impl<'a> Iterator for OpDelimitedIter<'a> {
 }
 
 impl<'a> TtIter<'a> {
-    fn expect_separator(&mut self, separator: &Separator, idx: usize) -> bool {
+    fn expect_separator(&mut self, separator: &Separator) -> bool {
         let mut fork = self.clone();
         let ok = match separator {
-            Separator::Ident(lhs) if idx == 0 => match fork.expect_ident_or_underscore() {
+            Separator::Ident(lhs) => match fork.expect_ident_or_underscore() {
                 Ok(rhs) => rhs.text == lhs.text,
                 Err(_) => false,
             },
-            Separator::Literal(lhs) if idx == 0 => match fork.expect_literal() {
+            Separator::Literal(lhs) => match fork.expect_literal() {
                 Ok(rhs) => match rhs {
                     tt::Leaf::Literal(rhs) => rhs.text == lhs.text,
                     tt::Leaf::Ident(rhs) => rhs.text == lhs.text,
@@ -837,11 +882,14 @@ impl<'a> TtIter<'a> {
                 },
                 Err(_) => false,
             },
-            Separator::Puncts(lhss) if idx < lhss.len() => match fork.expect_punct() {
-                Ok(rhs) => rhs.char == lhss[idx].char,
+            Separator::Puncts(lhs) => match fork.expect_glued_punct() {
+                Ok(rhs) => {
+                    let lhs = lhs.iter().map(|it| it.char);
+                    let rhs = rhs.iter().map(|it| it.char);
+                    lhs.eq(rhs)
+                }
                 Err(_) => false,
             },
-            _ => false,
         };
         if ok {
             *self = fork;
@@ -850,52 +898,21 @@ impl<'a> TtIter<'a> {
     }
 
     fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
-        match self.peek_n(0) {
-            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => {
-                return self.expect_lifetime();
-            }
-            _ => (),
-        }
-
-        let tt = self.next().ok_or(())?.clone();
-        let punct = match tt {
-            tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => {
-                punct
-            }
-            _ => return Ok(tt),
-        };
-
-        let (second, third) = match (self.peek_n(0), self.peek_n(1)) {
-            (
-                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))),
-                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))),
-            ) if p2.spacing == tt::Spacing::Joint => (p2.char, Some(p3.char)),
-            (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2.char, None),
-            _ => return Ok(tt),
-        };
-
-        match (punct.char, second, third) {
-            ('.', '.', Some('.' | '=')) | ('<', '<', Some('=')) | ('>', '>', Some('=')) => {
-                let tt2 = self.next().unwrap().clone();
-                let tt3 = self.next().unwrap().clone();
-                Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2, tt3] }.into())
-            }
-            ('-' | '!' | '*' | '/' | '&' | '%' | '^' | '+' | '<' | '=' | '>' | '|', '=', _)
-            | ('-' | '=' | '>', '>', _)
-            | (':', ':', _)
-            | ('.', '.', _)
-            | ('&', '&', _)
-            | ('<', '<', _)
-            | ('|', '|', _) => {
-                let tt2 = self.next().unwrap().clone();
-                Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2] }.into())
+        if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = self.peek_n(0) {
+            if punct.char == '\'' {
+                self.expect_lifetime()
+            } else {
+                let puncts = self.expect_glued_punct()?;
+                let token_trees = puncts.into_iter().map(|p| tt::Leaf::Punct(p).into()).collect();
+                Ok(tt::TokenTree::Subtree(tt::Subtree { delimiter: None, token_trees }))
             }
-            _ => Ok(tt),
+        } else {
+            self.next().ok_or(()).cloned()
         }
     }
 
     fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
-        let punct = self.expect_punct()?;
+        let punct = self.expect_single_punct()?;
         if punct.char != '\'' {
             return Err(());
         }
diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs
index cbb59ab8e6..db0d327bf4 100644
--- a/crates/mbe/src/expander/transcriber.rs
+++ b/crates/mbe/src/expander/transcriber.rs
@@ -134,7 +134,13 @@ fn expand_subtree(
     let mut err = None;
     for op in template.iter() {
         match op {
-            Op::Leaf(tt) => arena.push(tt.clone().into()),
+            Op::Literal(it) => arena.push(tt::Leaf::from(it.clone()).into()),
+            Op::Ident(it) => arena.push(tt::Leaf::from(it.clone()).into()),
+            Op::Punct(puncts) => {
+                for punct in puncts {
+                    arena.push(tt::Leaf::from(punct.clone()).into());
+                }
+            }
             Op::Subtree { tokens, delimiter } => {
                 let ExpandResult { value: tt, err: e } =
                     expand_subtree(ctx, tokens, *delimiter, arena);
diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs
index 351c359b73..3d9a61dbc8 100644
--- a/crates/mbe/src/parser.rs
+++ b/crates/mbe/src/parser.rs
@@ -1,7 +1,7 @@
 //! Parser recognizes special macro syntax, `$var` and `$(repeat)*`, in token
 //! trees.
 
-use smallvec::SmallVec;
+use smallvec::{smallvec, SmallVec};
 use syntax::SmolStr;
 
 use crate::{tt_iter::TtIter, ParseError};
@@ -39,7 +39,7 @@ impl MetaTemplate {
         let mut src = TtIter::new(tt);
 
         let mut res = Vec::new();
-        while let Some(first) = src.next() {
+        while let Some(first) = src.peek_n(0) {
             let op = next_op(first, &mut src, mode)?;
             res.push(op);
         }
@@ -54,8 +54,10 @@ pub(crate) enum Op {
     Ignore { name: SmolStr, id: tt::TokenId },
     Index { depth: u32 },
     Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
-    Leaf(tt::Leaf),
     Subtree { tokens: MetaTemplate, delimiter: Option<tt::Delimiter> },
+    Literal(tt::Literal),
+    Punct(SmallVec<[tt::Punct; 3]>),
+    Ident(tt::Ident),
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -108,28 +110,23 @@ impl PartialEq for Separator {
     }
 }
 
-impl Separator {
-    pub(crate) fn tt_count(&self) -> usize {
-        match self {
-            Separator::Literal(_) => 1,
-            Separator::Ident(_) => 1,
-            Separator::Puncts(it) => it.len(),
-        }
-    }
-}
-
 #[derive(Clone, Copy)]
 enum Mode {
     Pattern,
     Template,
 }
 
-fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Result<Op, ParseError> {
-    let res = match first {
-        tt::TokenTree::Leaf(leaf @ tt::Leaf::Punct(tt::Punct { char: '$', .. })) => {
+fn next_op<'a>(
+    first_peeked: &tt::TokenTree,
+    src: &mut TtIter<'a>,
+    mode: Mode,
+) -> Result<Op, ParseError> {
+    let res = match first_peeked {
+        tt::TokenTree::Leaf(tt::Leaf::Punct(p @ tt::Punct { char: '$', .. })) => {
+            src.next().expect("first token already peeked");
             // Note that the '$' itself is a valid token inside macro_rules.
             let second = match src.next() {
-                None => return Ok(Op::Leaf(leaf.clone())),
+                None => return Ok(Op::Punct(smallvec![p.clone()])),
                 Some(it) => it,
             };
             match second {
@@ -160,7 +157,7 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
                 tt::TokenTree::Leaf(leaf) => match leaf {
                     tt::Leaf::Ident(ident) if ident.text == "crate" => {
                         // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path.
-                        Op::Leaf(tt::Leaf::from(tt::Ident { text: "$crate".into(), id: ident.id }))
+                        Op::Ident(tt::Ident { text: "$crate".into(), id: ident.id })
                     }
                     tt::Leaf::Ident(ident) => {
                         let kind = eat_fragment_kind(src, mode)?;
@@ -180,7 +177,7 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
                                 "`$$` is not allowed on the pattern side",
                             ))
                         }
-                        Mode::Template => Op::Leaf(tt::Leaf::Punct(*punct)),
+                        Mode::Template => Op::Punct(smallvec![*punct]),
                     },
                     tt::Leaf::Punct(_) | tt::Leaf::Literal(_) => {
                         return Err(ParseError::expected("expected ident"))
@@ -188,8 +185,25 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
                 },
             }
         }
-        tt::TokenTree::Leaf(tt) => Op::Leaf(tt.clone()),
+
+        tt::TokenTree::Leaf(tt::Leaf::Literal(it)) => {
+            src.next().expect("first token already peeked");
+            Op::Literal(it.clone())
+        }
+
+        tt::TokenTree::Leaf(tt::Leaf::Ident(it)) => {
+            src.next().expect("first token already peeked");
+            Op::Ident(it.clone())
+        }
+
+        tt::TokenTree::Leaf(tt::Leaf::Punct(_)) => {
+            // There's at least one punct so this shouldn't fail.
+            let puncts = src.expect_glued_punct().unwrap();
+            Op::Punct(puncts)
+        }
+
         tt::TokenTree::Subtree(subtree) => {
+            src.next().expect("first token already peeked");
             let tokens = MetaTemplate::parse(subtree, mode)?;
             Op::Subtree { tokens, delimiter: subtree.delimiter }
         }
diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs
index 7aceb676c7..bee7b5de6a 100644
--- a/crates/mbe/src/tt_iter.rs
+++ b/crates/mbe/src/tt_iter.rs
@@ -1,6 +1,7 @@
 //! A "Parser" structure for token trees. We use this when parsing a declarative
 //! macro definition into a list of patterns and templates.
 
+use smallvec::{smallvec, SmallVec};
 use syntax::SyntaxKind;
 use tt::buffer::TokenBuffer;
 
@@ -80,13 +81,56 @@ impl<'a> TtIter<'a> {
         }
     }
 
-    pub(crate) fn expect_punct(&mut self) -> Result<&'a tt::Punct, ()> {
+    pub(crate) fn expect_single_punct(&mut self) -> Result<&'a tt::Punct, ()> {
         match self.expect_leaf()? {
             tt::Leaf::Punct(it) => Ok(it),
             _ => Err(()),
         }
     }
 
+    /// Returns consecutive `Punct`s that can be glued together.
+    ///
+    /// This method currently may return a single quotation, which is part of lifetime ident and
+    /// conceptually not a punct in the context of mbe. Callers should handle this.
+    pub(crate) fn expect_glued_punct(&mut self) -> Result<SmallVec<[tt::Punct; 3]>, ()> {
+        let tt::TokenTree::Leaf(tt::Leaf::Punct(first)) = self.next().ok_or(())?.clone() else {
+            return Err(());
+        };
+
+        if first.spacing == tt::Spacing::Alone {
+            return Ok(smallvec![first]);
+        }
+
+        let (second, third) = match (self.peek_n(0), self.peek_n(1)) {
+            (
+                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))),
+                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))),
+            ) if p2.spacing == tt::Spacing::Joint => (p2, Some(p3)),
+            (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2, None),
+            _ => return Ok(smallvec![first]),
+        };
+
+        match (first.char, second.char, third.map(|it| it.char)) {
+            ('.', '.', Some('.' | '=')) | ('<', '<', Some('=')) | ('>', '>', Some('=')) => {
+                let _ = self.next().unwrap();
+                let _ = self.next().unwrap();
+                Ok(smallvec![first, second.clone(), third.unwrap().clone()])
+            }
+            ('-' | '!' | '*' | '/' | '&' | '%' | '^' | '+' | '<' | '=' | '>' | '|', '=', _)
+            | ('-' | '=' | '>', '>', _)
+            | ('<', '-', _)
+            | (':', ':', _)
+            | ('.', '.', _)
+            | ('&', '&', _)
+            | ('<', '<', _)
+            | ('|', '|', _) => {
+                let _ = self.next().unwrap();
+                Ok(smallvec![first, second.clone()])
+            }
+            _ => Ok(smallvec![first]),
+        }
+    }
+
     pub(crate) fn expect_fragment(
         &mut self,
         entry_point: parser::PrefixEntryPoint,
@@ -141,7 +185,7 @@ impl<'a> TtIter<'a> {
         ExpandResult { value: res, err }
     }
 
-    pub(crate) fn peek_n(&self, n: usize) -> Option<&tt::TokenTree> {
+    pub(crate) fn peek_n(&self, n: usize) -> Option<&'a tt::TokenTree> {
         self.inner.as_slice().get(n)
     }
 }


Test patch:
diff --git a/crates/hir-def/src/macro_expansion_tests/mbe.rs b/crates/hir-def/src/macro_expansion_tests/mbe.rs
index 457e43925c..2d5f2a692e 100644
--- a/crates/hir-def/src/macro_expansion_tests/mbe.rs
+++ b/crates/hir-def/src/macro_expansion_tests/mbe.rs
@@ -1630,3 +1630,48 @@ const _: i32 = -0--1--2;
     "#]],
     );
 }
+
+#[test]
+fn test_punct_without_space() {
+    // Puncts are "glued" greedily.
+    check(
+        r#"
+macro_rules! foo {
+    (: : :) => { "1 1 1" };
+    (: ::) => { "1 2" };
+    (:: :) => { "2 1" };
+
+    (: : : :) => { "1 1 1 1" };
+    (:: : :) => { "2 1 1" };
+    (: :: :) => { "1 2 1" };
+    (: : ::) => { "1 1 2" };
+    (:: ::) => { "2 2" };
+}
+
+fn test() {
+    foo!(:::);
+    foo!(: :::);
+    foo!(::::);
+}
+"#,
+        expect![[r#"
+macro_rules! foo {
+    (: : :) => { "1 1 1" };
+    (: ::) => { "1 2" };
+    (:: :) => { "2 1" };
+
+    (: : : :) => { "1 1 1 1" };
+    (:: : :) => { "2 1 1" };
+    (: :: :) => { "1 2 1" };
+    (: : ::) => { "1 1 2" };
+    (:: ::) => { "2 2" };
+}
+
+fn test() {
+    "2 1";
+    "1 2 1";
+    "2 2";
+}
+"#]],
+    );
+}
diff --git a/crates/ide/src/syntax_highlighting/tests.rs b/crates/ide/src/syntax_highlighting/tests.rs
index 2f870d769c..46cc667fc4 100644
--- a/crates/ide/src/syntax_highlighting/tests.rs
+++ b/crates/ide/src/syntax_highlighting/tests.rs
@@ -1028,26 +1028,6 @@ macro_rules! test {}
     let _ = analysis.highlight(HL_CONFIG, file_id).unwrap();
 }
 
-#[test]
-fn highlight_callable_no_crash() {
-    // regression test for #13838.
-    let (analysis, file_id) = fixture::file(
-        r#"
-//- minicore: fn, sized
-impl<A, F: ?Sized> FnOnce<A> for &F
-where
-    F: Fn<A>,
-{
-    type Output = F::Output;
-}
-
-trait Trait {}
-fn foo(x: &fn(&dyn Trait)) {}
-"#,
-    );
-    let _ = analysis.highlight(HL_CONFIG, file_id).unwrap();
-}
-
 /// Highlights the code given by the `ra_fixture` argument, renders the
 /// result as HTML, and compares it with the HTML file given as `snapshot`.
 /// Note that the `snapshot` file is overwritten by the rendered HTML.