Repository: rust-analyzer/rust-analyzer
Pull request #3230 (2020-02-19 07:57:10)

ra_mbe: convert_literal now works with malformed tokens
Fixes: #3226

Veetaha: Sorry, didn't have time to provide a unit test and didn't get to study ra_mbe yet...
SomeoneToIgnore: Does this PR fixes the same issue as the https://github.com/rust-analyzer/rust-analyzer/pull/3229 ?

The other one has tests ;)
VitaliiKryvenkoTR: I don't think so, tokenizer doesn't report errors for non-ascii symbols diagnostics in comments... It reports only unterminated [block comments](https://github.com/rust-lang/rust/blob/3a8108d8e52e13f3f1a151a93addfcf4a86b97f5/src/librustc_lexer/src/lib.rs#L41-L47)
Veetaha: I don't think so, tokenizer doesn't report errors for non-ascii symbols diagnostics in comments... It reports only unterminated [block comments](https://github.com/rust-lang/rust/blob/3a8108d8e52e13f3f1a151a93addfcf4a86b97f5/src/librustc_lexer/src/lib.rs#L41-L47)
SomeoneToIgnore: Ok, never mind me then.
edwin0cheng: You could add a test in:

https://github.com/rust-analyzer/rust-analyzer/blob/20252efb32bfdfe7392934a95a6c6d6b583d10e7/crates/ra_mbe/src/tests.rs#L1524

Something like:

```rust
#[test]
fn test_expand_bad_literal() {
    parse_macro(
        r#"
        macro_rules! foo { ($i:literal) => {}; }
    "#,
    )
    .assert_expand_items("foo!(&k");", "");
}
Veetaha: Sure, thanks for the pointer, I'll add it tonight!
Veetaha: @edwin0cheng , see my last commit. It turns out macro expansion just returns `NoMatchingRule` error in case of malformed token...
edwin0cheng: LGTM

Thanks

bors r+
Veetaha: I guess is bors is stuck
bors r+
bors[bot]: :lock: Permission denied

Existing reviewers: [click here to make Veetaha a reviewer](https://app.bors.tech/repositories/4610/add-reviewer/Veetaha)

edwin0cheng: Maybe you need to rebase it to make Mr bors happy first

Patch:
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index dacca82794..91e324db90 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -1,7 +1,7 @@
 //! FIXME: write short doc here
 
 use ra_parser::{Token, TokenSource};
-use ra_syntax::{lex_single_valid_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
+use ra_syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
 use std::cell::{Cell, Ref, RefCell};
 use tt::buffer::{Cursor, TokenBuffer};
 
@@ -129,7 +129,8 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
 }
 
 fn convert_literal(l: &tt::Literal) -> TtToken {
-    let kind = lex_single_valid_syntax_kind(&l.text)
+    let kind = lex_single_syntax_kind(&l.text)
+        .map(|(kind, _error)| kind)
         .filter(|kind| kind.is_literal())
         .unwrap_or_else(|| match l.text.as_ref() {
             "true" => T![true],


Test patch:
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index e0d689704b..cb228702fb 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -1374,14 +1374,22 @@ pub(crate) struct MacroFixture {
 
 impl MacroFixture {
     pub(crate) fn expand_tt(&self, invocation: &str) -> tt::Subtree {
-        let source_file = ast::SourceFile::parse(invocation).ok().unwrap();
+        self.try_expand_tt(invocation).unwrap()
+    }
+
+    fn try_expand_tt(&self, invocation: &str) -> Result<tt::Subtree, ExpandError> {
+        let source_file = ast::SourceFile::parse(invocation).tree();
         let macro_invocation =
             source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
 
         let (invocation_tt, _) =
             ast_to_token_tree(&macro_invocation.token_tree().unwrap()).unwrap();
 
-        self.rules.expand(&invocation_tt).unwrap()
+        self.rules.expand(&invocation_tt)
+    }
+
+    fn assert_expand_err(&self, invocation: &str, err: &ExpandError) {
+        assert_eq!(self.try_expand_tt(invocation).as_ref(), Err(err));
     }
 
     fn expand_items(&self, invocation: &str) -> SyntaxNode {
@@ -1539,3 +1547,13 @@ fn test_repeat_bad_var() {
     )
     .assert_expand_items("foo!(b0 b1);", "b0 b1");
 }
+
+#[test]
+fn test_expand_bad_literal() {
+    parse_macro(
+        r#"
+        macro_rules! foo { ($i:literal) => {}; }
+    "#,
+    )
+    .assert_expand_err(r#"foo!(&k");"#, &ExpandError::NoMatchingRule);
+}