Repository: rust-analyzer/rust-analyzer
Pull request #11141 (2021-12-29 18:10:40)

internal: add tests for extra parser entry points
bors r+
🤖


Patch:
diff --git a/crates/hir_expand/src/ast_id_map.rs b/crates/hir_expand/src/ast_id_map.rs
index c1ddef03ba..43c8a3db5c 100644
--- a/crates/hir_expand/src/ast_id_map.rs
+++ b/crates/hir_expand/src/ast_id_map.rs
@@ -137,7 +137,7 @@ impl AstIdMap {
     }
 
     pub fn get<N: AstNode>(&self, id: FileAstId<N>) -> AstPtr<N> {
-        AstPtr::try_from_raw(self.arena[id.raw].clone()).unwrap()
+        self.arena[id.raw].clone().cast::<N>().unwrap()
     }
 
     fn alloc(&mut self, item: &SyntaxNode) -> ErasedFileAstId {
diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs
index 42426a1df2..e1a265d817 100644
--- a/crates/parser/src/grammar.rs
+++ b/crates/parser/src/grammar.rs
@@ -59,7 +59,7 @@ pub(crate) mod entry {
         }
 
         pub(crate) fn stmt(p: &mut Parser) {
-            expressions::stmt(p, expressions::StmtWithSemi::No, true);
+            expressions::stmt(p, expressions::Semicolon::Forbidden);
         }
 
         pub(crate) fn pat(p: &mut Parser) {
@@ -98,12 +98,7 @@ pub(crate) mod entry {
             let m = p.start();
 
             while !p.at(EOF) {
-                if p.at(T![;]) {
-                    p.bump(T![;]);
-                    continue;
-                }
-
-                expressions::stmt(p, expressions::StmtWithSemi::Optional, true);
+                expressions::stmt(p, expressions::Semicolon::Optional);
             }
 
             m.complete(p, MACRO_STMTS);
diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs
index 64057a4a67..9dbba89c56 100644
--- a/crates/parser/src/grammar/expressions.rs
+++ b/crates/parser/src/grammar/expressions.rs
@@ -5,10 +5,11 @@ use super::*;
 pub(crate) use self::atom::{block_expr, match_arm_list};
 pub(super) use self::atom::{literal, LITERAL_FIRST};
 
-pub(super) enum StmtWithSemi {
-    Yes,
-    No,
+#[derive(PartialEq, Eq)]
+pub(super) enum Semicolon {
+    Required,
     Optional,
+    Forbidden,
 }
 
 const EXPR_FIRST: TokenSet = LHS_FIRST;
@@ -28,7 +29,11 @@ fn expr_no_struct(p: &mut Parser) {
     expr_bp(p, None, r, 1);
 }
 
-pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi, prefer_expr: bool) {
+pub(super) fn stmt(p: &mut Parser, semicolon: Semicolon) {
+    if p.eat(T![;]) {
+        return;
+    }
+
     let m = p.start();
     // test attr_on_expr_stmt
     // fn foo() {
@@ -40,7 +45,7 @@ pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi, prefer_expr: bool) {
     attributes::outer_attrs(p);
 
     if p.at(T![let]) {
-        let_stmt(p, m, with_semi);
+        let_stmt(p, m, semicolon);
         return;
     }
 
@@ -52,7 +57,7 @@ pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi, prefer_expr: bool) {
     };
 
     if let Some((cm, blocklike)) = expr_stmt(p, Some(m)) {
-        if !(p.at(T!['}']) || (prefer_expr && p.at(EOF))) {
+        if !(p.at(T!['}']) || (semicolon != Semicolon::Required && p.at(EOF))) {
             // test no_semi_after_block
             // fn foo() {
             //     if true {}
@@ -68,27 +73,26 @@ pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi, prefer_expr: bool) {
             //     test!{}
             // }
             let m = cm.precede(p);
-            match with_semi {
-                StmtWithSemi::No => (),
-                StmtWithSemi::Optional => {
-                    p.eat(T![;]);
-                }
-                StmtWithSemi::Yes => {
+            match semicolon {
+                Semicolon::Required => {
                     if blocklike.is_block() {
                         p.eat(T![;]);
                     } else {
                         p.expect(T![;]);
                     }
                 }
+                Semicolon::Optional => {
+                    p.eat(T![;]);
+                }
+                Semicolon::Forbidden => (),
             }
-
             m.complete(p, EXPR_STMT);
         }
     }
 
     // test let_stmt
     // fn f() { let x: i32 = 92; }
-    fn let_stmt(p: &mut Parser, m: Marker, with_semi: StmtWithSemi) {
+    fn let_stmt(p: &mut Parser, m: Marker, with_semi: Semicolon) {
         p.bump(T![let]);
         patterns::pattern(p);
         if p.at(T![:]) {
@@ -113,11 +117,11 @@ pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi, prefer_expr: bool) {
         }
 
         match with_semi {
-            StmtWithSemi::No => (),
-            StmtWithSemi::Optional => {
+            Semicolon::Forbidden => (),
+            Semicolon::Optional => {
                 p.eat(T![;]);
             }
-            StmtWithSemi::Yes => {
+            Semicolon::Required => {
                 p.expect(T![;]);
             }
         }
@@ -143,13 +147,7 @@ pub(super) fn expr_block_contents(p: &mut Parser) {
         //     fn f() {};
         //     struct S {};
         // }
-
-        if p.at(T![;]) {
-            p.bump(T![;]);
-            continue;
-        }
-
-        stmt(p, StmtWithSemi::Yes, false);
+        stmt(p, Semicolon::Required);
     }
 }
 
diff --git a/crates/parser/src/output.rs b/crates/parser/src/output.rs
index b613df029f..e9ec9822d6 100644
--- a/crates/parser/src/output.rs
+++ b/crates/parser/src/output.rs
@@ -22,6 +22,7 @@ pub struct Output {
     error: Vec<String>,
 }
 
+#[derive(Debug)]
 pub enum Step<'a> {
     Token { kind: SyntaxKind, n_input_tokens: u8 },
     Enter { kind: SyntaxKind },
diff --git a/crates/parser/src/shortcuts.rs b/crates/parser/src/shortcuts.rs
index 3d28f814c9..b038d44fe0 100644
--- a/crates/parser/src/shortcuts.rs
+++ b/crates/parser/src/shortcuts.rs
@@ -16,6 +16,7 @@ use crate::{
     SyntaxKind::{self, *},
 };
 
+#[derive(Debug)]
 pub enum StrStep<'a> {
     Token { kind: SyntaxKind, text: &'a str },
     Enter { kind: SyntaxKind },
@@ -49,6 +50,7 @@ impl<'a> LexedStr<'a> {
         res
     }
 
+    /// NB: only valid to call with Output from Reparser/TopLevelEntry.
     pub fn intersperse_trivia(
         &self,
         output: &crate::Output,
diff --git a/crates/syntax/src/ptr.rs b/crates/syntax/src/ptr.rs
index 4a54bed0d4..57ae64cafe 100644
--- a/crates/syntax/src/ptr.rs
+++ b/crates/syntax/src/ptr.rs
@@ -11,15 +11,54 @@
 
 use std::{
     hash::{Hash, Hasher},
+    iter::successors,
     marker::PhantomData,
 };
 
-use crate::{syntax_node::RustLanguage, AstNode, SyntaxNode};
+use crate::{AstNode, SyntaxKind, SyntaxNode, TextRange};
 
-/// A "pointer" to a [`SyntaxNode`], via location in the source code.
-pub type SyntaxNodePtr = rowan::ast::SyntaxNodePtr<RustLanguage>;
+/// A pointer to a syntax node inside a file. It can be used to remember a
+/// specific node across reparses of the same file.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct SyntaxNodePtr {
+    // Don't expose this field further. At some point, we might want to replace
+    // range with node id.
+    pub(crate) range: TextRange,
+    kind: SyntaxKind,
+}
+
+impl SyntaxNodePtr {
+    pub fn new(node: &SyntaxNode) -> SyntaxNodePtr {
+        SyntaxNodePtr { range: node.text_range(), kind: node.kind() }
+    }
+
+    /// "Dereference" the pointer to get the node it points to.
+    ///
+    /// Panics if node is not found, so make sure that `root` syntax tree is
+    /// equivalent (is build from the same text) to the tree which was
+    /// originally used to get this [`SyntaxNodePtr`].
+    ///
+    /// The complexity is linear in the depth of the tree and logarithmic in
+    /// tree width. As most trees are shallow, thinking about this as
+    /// `O(log(N))` in the size of the tree is not too wrong!
+    pub fn to_node(&self, root: &SyntaxNode) -> SyntaxNode {
+        assert!(root.parent().is_none());
+        successors(Some(root.clone()), |node| {
+            node.child_or_token_at_range(self.range).and_then(|it| it.into_node())
+        })
+        .find(|it| it.text_range() == self.range && it.kind() == self.kind)
+        .unwrap_or_else(|| panic!("can't resolve local ptr to SyntaxNode: {:?}", self))
+    }
+
+    pub fn cast<N: AstNode>(self) -> Option<AstPtr<N>> {
+        if !N::can_cast(self.kind) {
+            return None;
+        }
+        Some(AstPtr { raw: self, _ty: PhantomData })
+    }
+}
 
-/// Like `SyntaxNodePtr`, but remembers the type of node.
+/// Like `SyntaxNodePtr`, but remembers the type of node
 #[derive(Debug)]
 pub struct AstPtr<N: AstNode> {
     raw: SyntaxNodePtr,
@@ -61,16 +100,11 @@ impl<N: AstNode> AstPtr<N> {
     }
 
     pub fn cast<U: AstNode>(self) -> Option<AstPtr<U>> {
-        if !U::can_cast(self.raw.kind()) {
+        if !U::can_cast(self.raw.kind) {
             return None;
         }
         Some(AstPtr { raw: self.raw, _ty: PhantomData })
     }
-
-    /// Like `SyntaxNodePtr::cast` but the trait bounds work out.
-    pub fn try_from_raw(raw: SyntaxNodePtr) -> Option<AstPtr<N>> {
-        N::can_cast(raw.kind()).then(|| AstPtr { raw, _ty: PhantomData })
-    }
 }
 
 impl<N: AstNode> From<AstPtr<N>> for SyntaxNodePtr {


Test patch:
diff --git a/crates/hir_def/src/macro_expansion_tests/mbe/matching.rs b/crates/hir_def/src/macro_expansion_tests/mbe/matching.rs
index 9fb6d96b72..b93072d446 100644
--- a/crates/hir_def/src/macro_expansion_tests/mbe/matching.rs
+++ b/crates/hir_def/src/macro_expansion_tests/mbe/matching.rs
@@ -50,3 +50,56 @@ macro_rules! m{ ($fmt:expr) => (); }
 "#]],
     );
 }
+
+#[test]
+fn asi() {
+    // Thanks, Christopher!
+    //
+    // https://internals.rust-lang.org/t/understanding-decisions-behind-semicolons/15181/29
+    check(
+        r#"
+macro_rules! asi { ($($stmt:stmt)*) => ($($stmt)*); }
+
+fn main() {
+    asi! {
+        let a = 2
+        let b = 5
+        drop(b-a)
+        println!("{}", a+b)
+    }
+}
+"#,
+        expect![[r#"
+macro_rules! asi { ($($stmt:stmt)*) => ($($stmt)*); }
+
+fn main() {
+    let a = 2let b = 5drop(b-a)println!("{}", a+b)
+}
+"#]],
+    )
+}
+
+#[test]
+fn stmt_boundaries() {
+    // FIXME: this actually works OK under rustc.
+    check(
+        r#"
+macro_rules! m {
+    ($($s:stmt)*) => (stringify!($($s |)*);)
+}
+m!(;;92;let x = 92; loop {};);
+"#,
+        expect![[r#"
+macro_rules! m {
+    ($($s:stmt)*) => (stringify!($($s |)*);)
+}
+stringify!(;
+|;
+|92|;
+|let x = 92|;
+|loop {}
+|;
+|);
+"#]],
+    );
+}
diff --git a/crates/ide_db/src/test_data/test_symbol_index_collection.txt b/crates/ide_db/src/test_data/test_symbol_index_collection.txt
index 176d7e1579..fe6eebc4f0 100644
--- a/crates/ide_db/src/test_data/test_symbol_index_collection.txt
+++ b/crates/ide_db/src/test_data/test_symbol_index_collection.txt
@@ -21,12 +21,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: STRUCT,
                         range: 0..22,
+                        kind: STRUCT,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 6..21,
+                        kind: NAME,
                     },
                 },
                 kind: Struct,
@@ -43,12 +43,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: STRUCT,
                         range: 170..184,
+                        kind: STRUCT,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 177..183,
+                        kind: NAME,
                     },
                 },
                 kind: Struct,
@@ -65,12 +65,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: ENUM,
                         range: 185..207,
+                        kind: ENUM,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 190..194,
+                        kind: NAME,
                     },
                 },
                 kind: Enum,
@@ -87,12 +87,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: UNION,
                         range: 208..222,
+                        kind: UNION,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 214..219,
+                        kind: NAME,
                     },
                 },
                 kind: Union,
@@ -109,12 +109,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: TRAIT,
                         range: 261..300,
+                        kind: TRAIT,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 267..272,
+                        kind: NAME,
                     },
                 },
                 kind: Trait,
@@ -131,12 +131,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: FN,
                         range: 279..298,
+                        kind: FN,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 282..290,
+                        kind: NAME,
                     },
                 },
                 kind: Function,
@@ -155,12 +155,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: FN,
                         range: 302..338,
+                        kind: FN,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 305..309,
+                        kind: NAME,
                     },
                 },
                 kind: Function,
@@ -177,12 +177,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: CONST,
                         range: 340..361,
+                        kind: CONST,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 346..351,
+                        kind: NAME,
                     },
                 },
                 kind: Const,
@@ -199,12 +199,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: STATIC,
                         range: 362..396,
+                        kind: STATIC,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 369..375,
+                        kind: NAME,
                     },
                 },
                 kind: Static,
@@ -221,12 +221,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: TYPE_ALIAS,
                         range: 397..417,
+                        kind: TYPE_ALIAS,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 402..407,
+                        kind: NAME,
                     },
                 },
                 kind: TypeAlias,
@@ -243,12 +243,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: MODULE,
                         range: 419..457,
+                        kind: MODULE,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 423..428,
+                        kind: NAME,
                     },
                 },
                 kind: Module,
@@ -265,12 +265,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: CONST,
                         range: 520..592,
+                        kind: CONST,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 526..542,
+                        kind: NAME,
                     },
                 },
                 kind: Const,
@@ -287,12 +287,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: MODULE,
                         range: 594..604,
+                        kind: MODULE,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 598..603,
+                        kind: NAME,
                     },
                 },
                 kind: Module,
@@ -309,12 +309,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: FN,
                         range: 242..257,
+                        kind: FN,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 245..252,
+                        kind: NAME,
                     },
                 },
                 kind: Function,
@@ -331,12 +331,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: MACRO_RULES,
                         range: 1..48,
+                        kind: MACRO_RULES,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 14..31,
+                        kind: NAME,
                     },
                 },
                 kind: Macro,
@@ -353,12 +353,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: MACRO_RULES,
                         range: 51..131,
+                        kind: MACRO_RULES,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 64..77,
+                        kind: NAME,
                     },
                 },
                 kind: Macro,
@@ -375,12 +375,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: MACRO_DEF,
                         range: 153..168,
+                        kind: MACRO_DEF,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 159..164,
+                        kind: NAME,
                     },
                 },
                 kind: Macro,
@@ -397,12 +397,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: STRUCT,
                         range: 479..507,
+                        kind: STRUCT,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 486..506,
+                        kind: NAME,
                     },
                 },
                 kind: Struct,
@@ -419,12 +419,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: STRUCT,
                         range: 555..581,
+                        kind: STRUCT,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 562..580,
+                        kind: NAME,
                     },
                 },
                 kind: Struct,
@@ -443,12 +443,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: STRUCT,
                         range: 318..336,
+                        kind: STRUCT,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 325..335,
+                        kind: NAME,
                     },
                 },
                 kind: Struct,
@@ -478,12 +478,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: STRUCT,
                         range: 435..455,
+                        kind: STRUCT,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 442..454,
+                        kind: NAME,
                     },
                 },
                 kind: Struct,
@@ -511,12 +511,12 @@
                         ),
                     ),
                     ptr: SyntaxNodePtr {
-                        kind: STRUCT,
                         range: 0..20,
+                        kind: STRUCT,
                     },
                     name_ptr: SyntaxNodePtr {
-                        kind: NAME,
                         range: 7..19,
+                        kind: NAME,
                     },
                 },
                 kind: Struct,
diff --git a/crates/parser/src/tests.rs b/crates/parser/src/tests.rs
index 512f7ddb95..fb4885e98d 100644
--- a/crates/parser/src/tests.rs
+++ b/crates/parser/src/tests.rs
@@ -1,4 +1,5 @@
 mod sourcegen_inline_tests;
+mod entries;
 
 use std::{
     fmt::Write,
diff --git a/crates/parser/src/tests/entries.rs b/crates/parser/src/tests/entries.rs
new file mode 100644
index 0000000000..947922d8b3
--- /dev/null
+++ b/crates/parser/src/tests/entries.rs
@@ -0,0 +1,54 @@
+use crate::{LexedStr, PrefixEntryPoint, Step};
+
+#[test]
+fn vis() {
+    check_prefix(PrefixEntryPoint::Vis, "pub(crate) fn foo() {}", "pub(crate)");
+    check_prefix(PrefixEntryPoint::Vis, "fn foo() {}", "");
+    check_prefix(PrefixEntryPoint::Vis, "pub(fn foo() {}", "pub");
+    check_prefix(PrefixEntryPoint::Vis, "pub(crate fn foo() {}", "pub(crate");
+    check_prefix(PrefixEntryPoint::Vis, "crate fn foo() {}", "crate");
+}
+
+#[test]
+fn block() {
+    check_prefix(PrefixEntryPoint::Block, "{}, 92", "{}");
+    check_prefix(PrefixEntryPoint::Block, "{, 92)", "{, 92)");
+    check_prefix(PrefixEntryPoint::Block, "()", "");
+}
+
+#[test]
+fn stmt() {
+    check_prefix(PrefixEntryPoint::Stmt, "92; fn", "92");
+    check_prefix(PrefixEntryPoint::Stmt, "let _ = 92; 1", "let _ = 92");
+    check_prefix(PrefixEntryPoint::Stmt, "pub fn f() {} = 92", "pub fn f() {}");
+    check_prefix(PrefixEntryPoint::Stmt, ";;;", ";");
+    check_prefix(PrefixEntryPoint::Stmt, "+", "+");
+    check_prefix(PrefixEntryPoint::Stmt, "@", "@");
+    check_prefix(PrefixEntryPoint::Stmt, "loop {} - 1", "loop {}");
+}
+
+fn check_prefix(entry: PrefixEntryPoint, input: &str, prefix: &str) {
+    let lexed = LexedStr::new(input);
+    let input = lexed.to_input();
+
+    let mut n_tokens = 0;
+    for step in entry.parse(&input).iter() {
+        match step {
+            Step::Token { n_input_tokens, .. } => n_tokens += n_input_tokens as usize,
+            Step::Enter { .. } | Step::Exit | Step::Error { .. } => (),
+        }
+    }
+
+    let mut i = 0;
+    loop {
+        if n_tokens == 0 {
+            break;
+        }
+        if !lexed.kind(i).is_trivia() {
+            n_tokens -= 1;
+        }
+        i += 1;
+    }
+    let buf = &lexed.as_str()[..lexed.text_start(i)];
+    assert_eq!(buf, prefix);
+}