Repository: rust-analyzer/rust-analyzer
Pull request #10995 (2021-12-12 16:14:25)

internal: switch from trait-based TokenSource to simple struct of arrays
cc #10765 

The idea here is to try to simplify the interface as best as we can. The original trait-based approach is a bit over-engineered and hard to debug. Here, we replace callback with just data. The next PR in series will replace the output `TreeSink` trait with data as well. 


The biggest drawback here is that we now require to materialize all parser's input up-front. This is a bad fit for macro by example: when you parse `$e:expr`, you might consume only part of the input. However, today's trait-based solution doesn't really help -- we were already materializing the whole thing! So, let's keep it simple!


Patch:
diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs
index b58b86b38d..1a56878fdb 100644
--- a/crates/mbe/src/lib.rs
+++ b/crates/mbe/src/lib.rs
@@ -10,7 +10,7 @@ mod parser;
 mod expander;
 mod syntax_bridge;
 mod tt_iter;
-mod subtree_source;
+mod to_parser_tokens;
 
 #[cfg(test)]
 mod benchmark;
diff --git a/crates/mbe/src/subtree_source.rs b/crates/mbe/src/subtree_source.rs
deleted file mode 100644
index 6bdd787e30..0000000000
--- a/crates/mbe/src/subtree_source.rs
+++ /dev/null
@@ -1,174 +0,0 @@
-//! Our parser is generic over the source of tokens it parses.
-//!
-//! This module defines tokens sourced from declarative macros.
-
-use parser::{Token, TokenSource};
-use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
-use tt::buffer::TokenBuffer;
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-struct TtToken {
-    tt: Token,
-    text: SmolStr,
-}
-
-pub(crate) struct SubtreeTokenSource {
-    cached: Vec<TtToken>,
-    curr: (Token, usize),
-}
-
-impl<'a> SubtreeTokenSource {
-    pub(crate) fn new(buffer: &TokenBuffer) -> SubtreeTokenSource {
-        let mut current = buffer.begin();
-        let mut cached = Vec::with_capacity(100);
-
-        while !current.eof() {
-            let cursor = current;
-            let tt = cursor.token_tree();
-
-            // Check if it is lifetime
-            if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tt {
-                if punct.char == '\'' {
-                    let next = cursor.bump();
-                    if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(ident), _)) =
-                        next.token_tree()
-                    {
-                        let text = SmolStr::new("'".to_string() + &ident.text);
-                        cached.push(TtToken {
-                            tt: Token { kind: LIFETIME_IDENT, is_jointed_to_next: false },
-                            text,
-                        });
-                        current = next.bump();
-                        continue;
-                    } else {
-                        panic!("Next token must be ident : {:#?}", next.token_tree());
-                    }
-                }
-            }
-
-            current = match tt {
-                Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
-                    cached.push(convert_leaf(leaf));
-                    cursor.bump()
-                }
-                Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
-                    if let Some(d) = subtree.delimiter_kind() {
-                        cached.push(convert_delim(d, false));
-                    }
-                    cursor.subtree().unwrap()
-                }
-                None => match cursor.end() {
-                    Some(subtree) => {
-                        if let Some(d) = subtree.delimiter_kind() {
-                            cached.push(convert_delim(d, true));
-                        }
-                        cursor.bump()
-                    }
-                    None => continue,
-                },
-            };
-        }
-
-        let mut res = SubtreeTokenSource {
-            curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
-            cached,
-        };
-        res.curr = (res.token(0), 0);
-        res
-    }
-
-    fn token(&self, pos: usize) -> Token {
-        match self.cached.get(pos) {
-            Some(it) => it.tt,
-            None => Token { kind: EOF, is_jointed_to_next: false },
-        }
-    }
-}
-
-impl<'a> TokenSource for SubtreeTokenSource {
-    fn current(&self) -> Token {
-        self.curr.0
-    }
-
-    /// Lookahead n token
-    fn lookahead_nth(&self, n: usize) -> Token {
-        self.token(self.curr.1 + n)
-    }
-
-    /// bump cursor to next token
-    fn bump(&mut self) {
-        if self.current().kind == EOF {
-            return;
-        }
-        self.curr = (self.token(self.curr.1 + 1), self.curr.1 + 1);
-    }
-
-    /// Is the current token a specified keyword?
-    fn is_keyword(&self, kw: &str) -> bool {
-        match self.cached.get(self.curr.1) {
-            Some(t) => t.text == *kw,
-            None => false,
-        }
-    }
-}
-
-fn convert_delim(d: tt::DelimiterKind, closing: bool) -> TtToken {
-    let (kinds, texts) = match d {
-        tt::DelimiterKind::Parenthesis => ([T!['('], T![')']], "()"),
-        tt::DelimiterKind::Brace => ([T!['{'], T!['}']], "{}"),
-        tt::DelimiterKind::Bracket => ([T!['['], T![']']], "[]"),
-    };
-
-    let idx = closing as usize;
-    let kind = kinds[idx];
-    let text = &texts[idx..texts.len() - (1 - idx)];
-    TtToken { tt: Token { kind, is_jointed_to_next: false }, text: SmolStr::new(text) }
-}
-
-fn convert_literal(l: &tt::Literal) -> TtToken {
-    let is_negated = l.text.starts_with('-');
-    let inner_text = &l.text[if is_negated { 1 } else { 0 }..];
-
-    let kind = lex_single_syntax_kind(inner_text)
-        .map(|(kind, _error)| kind)
-        .filter(|kind| {
-            kind.is_literal() && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER))
-        })
-        .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
-
-    TtToken { tt: Token { kind, is_jointed_to_next: false }, text: l.text.clone() }
-}
-
-fn convert_ident(ident: &tt::Ident) -> TtToken {
-    let kind = match ident.text.as_ref() {
-        "true" => T![true],
-        "false" => T![false],
-        "_" => UNDERSCORE,
-        i if i.starts_with('\'') => LIFETIME_IDENT,
-        _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT),
-    };
-
-    TtToken { tt: Token { kind, is_jointed_to_next: false }, text: ident.text.clone() }
-}
-
-fn convert_punct(p: tt::Punct) -> TtToken {
-    let kind = match SyntaxKind::from_char(p.char) {
-        None => panic!("{:#?} is not a valid punct", p),
-        Some(kind) => kind,
-    };
-
-    let text = {
-        let mut buf = [0u8; 4];
-        let s: &str = p.char.encode_utf8(&mut buf);
-        SmolStr::new(s)
-    };
-    TtToken { tt: Token { kind, is_jointed_to_next: p.spacing == tt::Spacing::Joint }, text }
-}
-
-fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
-    match leaf {
-        tt::Leaf::Literal(l) => convert_literal(l),
-        tt::Leaf::Ident(ident) => convert_ident(ident),
-        tt::Leaf::Punct(punct) => convert_punct(*punct),
-    }
-}
diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs
index 0b65fa171f..28a23f6be2 100644
--- a/crates/mbe/src/syntax_bridge.rs
+++ b/crates/mbe/src/syntax_bridge.rs
@@ -12,7 +12,7 @@ use syntax::{
 use tt::buffer::{Cursor, TokenBuffer};
 
 use crate::{
-    subtree_source::SubtreeTokenSource, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,
+    to_parser_tokens::to_parser_tokens, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,
 };
 
 /// Convert the syntax node to a `TokenTree` (what macro
@@ -56,9 +56,9 @@ pub fn token_tree_to_syntax_node(
         }
         _ => TokenBuffer::from_subtree(tt),
     };
-    let mut token_source = SubtreeTokenSource::new(&buffer);
+    let parser_tokens = to_parser_tokens(&buffer);
     let mut tree_sink = TtTreeSink::new(buffer.begin());
-    parser::parse(&mut token_source, &mut tree_sink, entry_point);
+    parser::parse(&parser_tokens, &mut tree_sink, entry_point);
     if tree_sink.roots.len() != 1 {
         return Err(ExpandError::ConversionError);
     }
diff --git a/crates/mbe/src/to_parser_tokens.rs b/crates/mbe/src/to_parser_tokens.rs
new file mode 100644
index 0000000000..644689f432
--- /dev/null
+++ b/crates/mbe/src/to_parser_tokens.rs
@@ -0,0 +1,99 @@
+//! Convert macro-by-example tokens which are specific to macro expansion into a
+//! format that works for our parser.
+
+use syntax::{lex_single_syntax_kind, SyntaxKind, SyntaxKind::*, T};
+use tt::buffer::TokenBuffer;
+
+pub(crate) fn to_parser_tokens(buffer: &TokenBuffer) -> parser::Tokens {
+    let mut res = parser::Tokens::default();
+
+    let mut current = buffer.begin();
+
+    while !current.eof() {
+        let cursor = current;
+        let tt = cursor.token_tree();
+
+        // Check if it is lifetime
+        if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tt {
+            if punct.char == '\'' {
+                let next = cursor.bump();
+                match next.token_tree() {
+                    Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(_ident), _)) => {
+                        res.push(LIFETIME_IDENT);
+                        current = next.bump();
+                        continue;
+                    }
+                    _ => panic!("Next token must be ident : {:#?}", next.token_tree()),
+                }
+            }
+        }
+
+        current = match tt {
+            Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
+                match leaf {
+                    tt::Leaf::Literal(lit) => {
+                        let is_negated = lit.text.starts_with('-');
+                        let inner_text = &lit.text[if is_negated { 1 } else { 0 }..];
+
+                        let kind = lex_single_syntax_kind(inner_text)
+                            .map(|(kind, _error)| kind)
+                            .filter(|kind| {
+                                kind.is_literal()
+                                    && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER))
+                            })
+                            .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit));
+
+                        res.push(kind);
+                    }
+                    tt::Leaf::Ident(ident) => match ident.text.as_ref() {
+                        "_" => res.push(T![_]),
+                        i if i.starts_with('\'') => res.push(LIFETIME_IDENT),
+                        _ => match SyntaxKind::from_keyword(&ident.text) {
+                            Some(kind) => res.push(kind),
+                            None => {
+                                let contextual_keyword =
+                                    SyntaxKind::from_contextual_keyword(&ident.text)
+                                        .unwrap_or(SyntaxKind::IDENT);
+                                res.push_ident(contextual_keyword);
+                            }
+                        },
+                    },
+                    tt::Leaf::Punct(punct) => {
+                        let kind = SyntaxKind::from_char(punct.char)
+                            .unwrap_or_else(|| panic!("{:#?} is not a valid punct", punct));
+                        res.push(kind);
+                        if punct.spacing == tt::Spacing::Joint {
+                            res.was_joint();
+                        }
+                    }
+                }
+                cursor.bump()
+            }
+            Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
+                if let Some(d) = subtree.delimiter_kind() {
+                    res.push(match d {
+                        tt::DelimiterKind::Parenthesis => T!['('],
+                        tt::DelimiterKind::Brace => T!['{'],
+                        tt::DelimiterKind::Bracket => T!['['],
+                    });
+                }
+                cursor.subtree().unwrap()
+            }
+            None => match cursor.end() {
+                Some(subtree) => {
+                    if let Some(d) = subtree.delimiter_kind() {
+                        res.push(match d {
+                            tt::DelimiterKind::Parenthesis => T![')'],
+                            tt::DelimiterKind::Brace => T!['}'],
+                            tt::DelimiterKind::Bracket => T![']'],
+                        })
+                    }
+                    cursor.bump()
+                }
+                None => continue,
+            },
+        };
+    }
+
+    res
+}
diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs
index ff0272808b..d05e84b0f0 100644
--- a/crates/mbe/src/tt_iter.rs
+++ b/crates/mbe/src/tt_iter.rs
@@ -1,7 +1,7 @@
 //! A "Parser" structure for token trees. We use this when parsing a declarative
 //! macro definition into a list of patterns and templates.
 
-use crate::{subtree_source::SubtreeTokenSource, ExpandError, ExpandResult, ParserEntryPoint};
+use crate::{to_parser_tokens::to_parser_tokens, ExpandError, ExpandResult, ParserEntryPoint};
 
 use parser::TreeSink;
 use syntax::SyntaxKind;
@@ -116,10 +116,10 @@ impl<'a> TtIter<'a> {
         }
 
         let buffer = TokenBuffer::from_tokens(self.inner.as_slice());
-        let mut src = SubtreeTokenSource::new(&buffer);
+        let parser_tokens = to_parser_tokens(&buffer);
         let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
 
-        parser::parse(&mut src, &mut sink, entry_point);
+        parser::parse(&parser_tokens, &mut sink, entry_point);
 
         let mut err = if !sink.cursor.is_root() || sink.error {
             Some(err!("expected {:?}", entry_point))
diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs
index 54eb96d84e..4b9c579a05 100644
--- a/crates/parser/src/grammar/expressions.rs
+++ b/crates/parser/src/grammar/expressions.rs
@@ -296,10 +296,7 @@ fn lhs(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)>
         T![&] => {
             m = p.start();
             p.bump(T![&]);
-            if p.at(IDENT)
-                && p.at_contextual_kw("raw")
-                && (p.nth_at(1, T![mut]) || p.nth_at(1, T![const]))
-            {
+            if p.at_contextual_kw(T![raw]) && (p.nth_at(1, T![mut]) || p.nth_at(1, T![const])) {
                 p.bump_remap(T![raw]);
                 p.bump_any();
             } else {
diff --git a/crates/parser/src/grammar/items.rs b/crates/parser/src/grammar/items.rs
index 39be0e1a19..896efaf375 100644
--- a/crates/parser/src/grammar/items.rs
+++ b/crates/parser/src/grammar/items.rs
@@ -122,14 +122,14 @@ pub(super) fn opt_item(p: &mut Parser, m: Marker) -> Result<(), Marker> {
         has_mods = true;
         abi(p);
     }
-    if p.at(IDENT) && p.at_contextual_kw("auto") && p.nth(1) == T![trait] {
+    if p.at_contextual_kw(T![auto]) && p.nth(1) == T![trait] {
         p.bump_remap(T![auto]);
         has_mods = true;
     }
 
     // test default_item
     // default impl T for Foo {}
-    if p.at(IDENT) && p.at_contextual_kw("default") {
+    if p.at_contextual_kw(T![default]) {
         match p.nth(1) {
             T![fn] | T![type] | T![const] | T![impl] => {
                 p.bump_remap(T![default]);
@@ -176,7 +176,7 @@ pub(super) fn opt_item(p: &mut Parser, m: Marker) -> Result<(), Marker> {
 
     // test existential_type
     // existential type Foo: Fn() -> usize;
-    if p.at(IDENT) && p.at_contextual_kw("existential") && p.nth(1) == T![type] {
+    if p.at_contextual_kw(T![existential]) && p.nth(1) == T![type] {
         p.bump_remap(T![existential]);
         has_mods = true;
     }
@@ -224,10 +224,10 @@ fn opt_item_without_modifiers(p: &mut Parser, m: Marker) -> Result<(), Marker> {
         T![type] => type_alias(p, m),
         T![struct] => adt::strukt(p, m),
         T![enum] => adt::enum_(p, m),
-        IDENT if p.at_contextual_kw("union") && p.nth(1) == IDENT => adt::union(p, m),
+        IDENT if p.at_contextual_kw(T![union]) && p.nth(1) == IDENT => adt::union(p, m),
 
         T![macro] => macro_def(p, m),
-        IDENT if p.at_contextual_kw("macro_rules") && p.nth(1) == BANG => macro_rules(p, m),
+        IDENT if p.at_contextual_kw(T![macro_rules]) && p.nth(1) == BANG => macro_rules(p, m),
 
         T![const] if (la == IDENT || la == T![_] || la == T![mut]) => consts::konst(p, m),
         T![static] => consts::static_(p, m),
@@ -319,7 +319,7 @@ pub(crate) fn extern_item_list(p: &mut Parser) {
 }
 
 fn macro_rules(p: &mut Parser, m: Marker) {
-    assert!(p.at_contextual_kw("macro_rules"));
+    assert!(p.at_contextual_kw(T![macro_rules]));
     p.bump_remap(T![macro_rules]);
     p.expect(T![!]);
 
diff --git a/crates/parser/src/grammar/items/adt.rs b/crates/parser/src/grammar/items/adt.rs
index c5bd5b14ba..83b7ff0578 100644
--- a/crates/parser/src/grammar/items/adt.rs
+++ b/crates/parser/src/grammar/items/adt.rs
@@ -10,7 +10,7 @@ pub(super) fn strukt(p: &mut Parser, m: Marker) {
 // test union_item
 // struct U { i: i32, f: f32 }
 pub(super) fn union(p: &mut Parser, m: Marker) {
-    assert!(p.at_contextual_kw("union"));
+    assert!(p.at_contextual_kw(T![union]));
     p.bump_remap(T![union]);
     struct_or_union(p, m, false);
 }
diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs
index 51ee969204..2e2d96d027 100644
--- a/crates/parser/src/lib.rs
+++ b/crates/parser/src/lib.rs
@@ -1,8 +1,11 @@
 //! The Rust parser.
 //!
+//! NOTE: The crate is undergoing refactors, don't believe everything the docs
+//! say :-)
+//!
 //! The parser doesn't know about concrete representation of tokens and syntax
-//! trees. Abstract [`TokenSource`] and [`TreeSink`] traits are used instead.
-//! As a consequence, this crate does not contain a lexer.
+//! trees. Abstract [`TokenSource`] and [`TreeSink`] traits are used instead. As
+//! a consequence, this crate does not contain a lexer.
 //!
 //! The [`Parser`] struct from the [`parser`] module is a cursor into the
 //! sequence of tokens.  Parsing routines use [`Parser`] to inspect current
@@ -20,40 +23,15 @@ mod syntax_kind;
 mod event;
 mod parser;
 mod grammar;
+mod tokens;
 
 pub(crate) use token_set::TokenSet;
 
-pub use syntax_kind::SyntaxKind;
+pub use crate::{syntax_kind::SyntaxKind, tokens::Tokens};
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct ParseError(pub Box<String>);
 
-/// `TokenSource` abstracts the source of the tokens parser operates on.
-///
-/// Hopefully this will allow us to treat text and token trees in the same way!
-pub trait TokenSource {
-    fn current(&self) -> Token;
-
-    /// Lookahead n token
-    fn lookahead_nth(&self, n: usize) -> Token;
-
-    /// bump cursor to next token
-    fn bump(&mut self);
-
-    /// Is the current token a specified keyword?
-    fn is_keyword(&self, kw: &str) -> bool;
-}
-
-/// `Token` abstracts the cursor of `TokenSource` operates on.
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub struct Token {
-    /// What is the current token?
-    pub kind: SyntaxKind,
-
-    /// Is the current token joined to the next one (`> >` vs `>>`).
-    pub is_jointed_to_next: bool,
-}
-
 /// `TreeSink` abstracts details of a particular syntax tree implementation.
 pub trait TreeSink {
     /// Adds new token to the current branch.
@@ -92,15 +70,11 @@ pub enum ParserEntryPoint {
 }
 
 /// Parse given tokens into the given sink as a rust file.
-pub fn parse_source_file(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
-    parse(token_source, tree_sink, ParserEntryPoint::SourceFile);
+pub fn parse_source_file(tokens: &Tokens, tree_sink: &mut dyn TreeSink) {
+    parse(tokens, tree_sink, ParserEntryPoint::SourceFile);
 }
 
-pub fn parse(
-    token_source: &mut dyn TokenSource,
-    tree_sink: &mut dyn TreeSink,
-    entry_point: ParserEntryPoint,
-) {
+pub fn parse(tokens: &Tokens, tree_sink: &mut dyn TreeSink, entry_point: ParserEntryPoint) {
     let entry_point: fn(&'_ mut parser::Parser) = match entry_point {
         ParserEntryPoint::SourceFile => grammar::entry_points::source_file,
         ParserEntryPoint::Path => grammar::entry_points::path,
@@ -118,7 +92,7 @@ pub fn parse(
         ParserEntryPoint::Attr => grammar::entry_points::attr,
     };
 
-    let mut p = parser::Parser::new(token_source);
+    let mut p = parser::Parser::new(tokens);
     entry_point(&mut p);
     let events = p.finish();
     event::process(tree_sink, events);
@@ -141,9 +115,9 @@ impl Reparser {
     ///
     /// Tokens must start with `{`, end with `}` and form a valid brace
     /// sequence.
-    pub fn parse(self, token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+    pub fn parse(self, tokens: &Tokens, tree_sink: &mut dyn TreeSink) {
         let Reparser(r) = self;
-        let mut p = parser::Parser::new(token_source);
+        let mut p = parser::Parser::new(tokens);
         r(&mut p);
         let events = p.finish();
         event::process(tree_sink, events);
diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs
index 44c5f8e12f..4c891108a6 100644
--- a/crates/parser/src/parser.rs
+++ b/crates/parser/src/parser.rs
@@ -7,9 +7,10 @@ use limit::Limit;
 
 use crate::{
     event::Event,
+    tokens::Tokens,
     ParseError,
     SyntaxKind::{self, EOF, ERROR, TOMBSTONE},
-    TokenSet, TokenSource, T,
+    TokenSet, T,
 };
 
 /// `Parser` struct provides the low-level API for
@@ -22,7 +23,8 @@ use crate::{
 /// "start expression, consume number literal,
 /// finish expression". See `Event` docs for more.
 pub(crate) struct Parser<'t> {
-    token_source: &'t mut dyn TokenSource,
+    tokens: &'t Tokens,
+    pos: usize,
     events: Vec<Event>,
     steps: Cell<u32>,
 }
@@ -30,8 +32,8 @@ pub(crate) struct Parser<'t> {
 static PARSER_STEP_LIMIT: Limit = Limit::new(15_000_000);
 
 impl<'t> Parser<'t> {
-    pub(super) fn new(token_source: &'t mut dyn TokenSource) -> Parser<'t> {
-        Parser { token_source, events: Vec::new(), steps: Cell::new(0) }
+    pub(super) fn new(tokens: &'t Tokens) -> Parser<'t> {
+        Parser { tokens, pos: 0, events: Vec::new(), steps: Cell::new(0) }
     }
 
     pub(crate) fn finish(self) -> Vec<Event> {
@@ -54,7 +56,7 @@ impl<'t> Parser<'t> {
         assert!(PARSER_STEP_LIMIT.check(steps as usize).is_ok(), "the parser seems stuck");
         self.steps.set(steps + 1);
 
-        self.token_source.lookahead_nth(n).kind
+        self.tokens.kind(self.pos + n)
     }
 
     /// Checks if the current token is `kind`.
@@ -90,7 +92,7 @@ impl<'t> Parser<'t> {
             T![<<=] => self.at_composite3(n, T![<], T![<], T![=]),
             T![>>=] => self.at_composite3(n, T![>], T![>], T![=]),
 
-            _ => self.token_source.lookahead_nth(n).kind == kind,
+            _ => self.tokens.kind(self.pos + n) == kind,
         }
     }
 
@@ -129,25 +131,17 @@ impl<'t> Parser<'t> {
     }
 
     fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool {
-        let t1 = self.token_source.lookahead_nth(n);
-        if t1.kind != k1 || !t1.is_jointed_to_next {
-            return false;
-        }
-        let t2 = self.token_source.lookahead_nth(n + 1);
-        t2.kind == k2
+        self.tokens.kind(self.pos + n) == k1
+            && self.tokens.kind(self.pos + n + 1) == k2
+            && self.tokens.is_joint(self.pos + n)
     }
 
     fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool {
-        let t1 = self.token_source.lookahead_nth(n);
-        if t1.kind != k1 || !t1.is_jointed_to_next {
-            return false;
-        }
-        let t2 = self.token_source.lookahead_nth(n + 1);
-        if t2.kind != k2 || !t2.is_jointed_to_next {
-            return false;
-        }
-        let t3 = self.token_source.lookahead_nth(n + 2);
-        t3.kind == k3
+        self.tokens.kind(self.pos + n) == k1
+            && self.tokens.kind(self.pos + n + 1) == k2
+            && self.tokens.kind(self.pos + n + 2) == k3
+            && self.tokens.is_joint(self.pos + n)
+            && self.tokens.is_joint(self.pos + n + 1)
     }
 
     /// Checks if the current token is in `kinds`.
@@ -156,8 +150,8 @@ impl<'t> Parser<'t> {
     }
 
     /// Checks if the current token is contextual keyword with text `t`.
-    pub(crate) fn at_contextual_kw(&self, kw: &str) -> bool {
-        self.token_source.is_keyword(kw)
+    pub(crate) fn at_contextual_kw(&self, kw: SyntaxKind) -> bool {
+        self.tokens.contextual_kind(self.pos) == kw
     }
 
     /// Starts a new node in the syntax tree. All nodes and tokens
@@ -243,10 +237,7 @@ impl<'t> Parser<'t> {
     }
 
     fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) {
-        for _ in 0..n_raw_tokens {
-            self.token_source.bump();
-        }
-
+        self.pos += n_raw_tokens as usize;
         self.push_event(Event::Token { kind, n_raw_tokens });
     }
 
diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs
index 99e7651906..601a5792af 100644
--- a/crates/parser/src/syntax_kind/generated.rs
+++ b/crates/parser/src/syntax_kind/generated.rs
@@ -334,6 +334,18 @@ impl SyntaxKind {
         };
         Some(kw)
     }
+    pub fn from_contextual_keyword(ident: &str) -> Option<SyntaxKind> {
+        let kw = match ident {
+            "auto" => AUTO_KW,
+            "default" => DEFAULT_KW,
+            "existential" => EXISTENTIAL_KW,
+            "union" => UNION_KW,
+            "raw" => RAW_KW,
+            "macro_rules" => MACRO_RULES_KW,
+            _ => return None,
+        };
+        Some(kw)
+    }
     pub fn from_char(c: char) -> Option<SyntaxKind> {
         let tok = match c {
             ';' => SEMICOLON,
diff --git a/crates/parser/src/tokens.rs b/crates/parser/src/tokens.rs
new file mode 100644
index 0000000000..1c0672492d
--- /dev/null
+++ b/crates/parser/src/tokens.rs
@@ -0,0 +1,87 @@
+//! Input for the parser -- a sequence of tokens.
+//!
+//! As of now, parser doesn't have access to the *text* of the tokens, and makes
+//! decisions based solely on their classification.
+
+use crate::SyntaxKind;
+
+#[allow(non_camel_case_types)]
+type bits = u64;
+
+/// Main input to the parser.
+///
+/// A sequence of tokens represented internally as a struct of arrays.
+#[derive(Default)]
+pub struct Tokens {
+    kind: Vec<SyntaxKind>,
+    joint: Vec<bits>,
+    contextual_kind: Vec<SyntaxKind>,
+}
+
+/// `pub` impl used by callers to create `Tokens`.
+impl Tokens {
+    #[inline]
+    pub fn push(&mut self, kind: SyntaxKind) {
+        self.push_impl(kind, SyntaxKind::EOF)
+    }
+    #[inline]
+    pub fn push_ident(&mut self, contextual_kind: SyntaxKind) {
+        self.push_impl(SyntaxKind::IDENT, contextual_kind)
+    }
+    /// Sets jointness for the last token we've pushed.
+    ///
+    /// This is a separate API rather than an argument to the `push` to make it
+    /// convenient both for textual and mbe tokens. With text, you know whether
+    /// the *previous* token was joint, with mbe, you know whether the *current*
+    /// one is joint. This API allows for styles of usage:
+    ///
+    /// ```
+    /// // In text:
+    /// tokens.was_joint(prev_joint);
+    /// tokens.push(curr);
+    ///
+    /// // In MBE:
+    /// token.push(curr);
+    /// tokens.push(curr_joint)
+    /// ```
+    #[inline]
+    pub fn was_joint(&mut self) {
+        let n = self.len() - 1;
+        let (idx, b_idx) = self.bit_index(n);
+        self.joint[idx] |= 1 << b_idx;
+    }
+    #[inline]
+    fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind) {
+        let idx = self.len();
+        if idx % (bits::BITS as usize) == 0 {
+            self.joint.push(0);
+        }
+        self.kind.push(kind);
+        self.contextual_kind.push(contextual_kind);
+    }
+}
+
+/// pub(crate) impl used by the parser to consume `Tokens`.
+impl Tokens {
+    pub(crate) fn kind(&self, idx: usize) -> SyntaxKind {
+        self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
+    }
+    pub(crate) fn contextual_kind(&self, idx: usize) -> SyntaxKind {
+        self.contextual_kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
+    }
+    pub(crate) fn is_joint(&self, n: usize) -> bool {
+        let (idx, b_idx) = self.bit_index(n);
+        self.joint[idx] & 1 << b_idx != 0
+    }
+}
+
+impl Tokens {
+    fn bit_index(&self, n: usize) -> (usize, usize) {
+        let idx = n / (bits::BITS as usize);
+        let b_idx = n % (bits::BITS as usize);
+        (idx, b_idx)
+    }
+    fn len(&self) -> usize {
+        self.kind.len()
+    }
+}
diff --git a/crates/syntax/src/parsing.rs b/crates/syntax/src/parsing.rs
index a45f262877..865e146482 100644
--- a/crates/syntax/src/parsing.rs
+++ b/crates/syntax/src/parsing.rs
@@ -2,12 +2,10 @@
 //! incremental reparsing.
 
 pub(crate) mod lexer;
-mod text_token_source;
 mod text_tree_sink;
 mod reparsing;
 
 use parser::SyntaxKind;
-use text_token_source::TextTokenSource;
 use text_tree_sink::TextTreeSink;
 
 use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode};
@@ -15,12 +13,12 @@ use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode};
 pub(crate) use crate::parsing::{lexer::*, reparsing::incremental_reparse};
 
 pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
-    let (tokens, lexer_errors) = tokenize(text);
+    let (lexer_tokens, lexer_errors) = tokenize(text);
+    let parser_tokens = to_parser_tokens(text, &lexer_tokens);
 
-    let mut token_source = TextTokenSource::new(text, &tokens);
-    let mut tree_sink = TextTreeSink::new(text, &tokens);
+    let mut tree_sink = TextTreeSink::new(text, &lexer_tokens);
 
-    parser::parse_source_file(&mut token_source, &mut tree_sink);
+    parser::parse_source_file(&parser_tokens, &mut tree_sink);
 
     let (tree, mut parser_errors) = tree_sink.finish();
     parser_errors.extend(lexer_errors);
@@ -33,26 +31,52 @@ pub(crate) fn parse_text_as<T: AstNode>(
     text: &str,
     entry_point: parser::ParserEntryPoint,
 ) -> Result<T, ()> {
-    let (tokens, lexer_errors) = tokenize(text);
+    let (lexer_tokens, lexer_errors) = tokenize(text);
     if !lexer_errors.is_empty() {
         return Err(());
     }
 
-    let mut token_source = TextTokenSource::new(text, &tokens);
-    let mut tree_sink = TextTreeSink::new(text, &tokens);
+    let parser_tokens = to_parser_tokens(text, &lexer_tokens);
+
+    let mut tree_sink = TextTreeSink::new(text, &lexer_tokens);
 
     // TextTreeSink assumes that there's at least some root node to which it can attach errors and
     // tokens. We arbitrarily give it a SourceFile.
     use parser::TreeSink;
     tree_sink.start_node(SyntaxKind::SOURCE_FILE);
-    parser::parse(&mut token_source, &mut tree_sink, entry_point);
+    parser::parse(&parser_tokens, &mut tree_sink, entry_point);
     tree_sink.finish_node();
 
-    let (tree, parser_errors) = tree_sink.finish();
-    use parser::TokenSource;
-    if !parser_errors.is_empty() || token_source.current().kind != SyntaxKind::EOF {
+    let (tree, parser_errors, eof) = tree_sink.finish_eof();
+    if !parser_errors.is_empty() || !eof {
         return Err(());
     }
 
     SyntaxNode::new_root(tree).first_child().and_then(T::cast).ok_or(())
 }
+
+pub(crate) fn to_parser_tokens(text: &str, lexer_tokens: &[lexer::Token]) -> ::parser::Tokens {
+    let mut off = 0;
+    let mut res = parser::Tokens::default();
+    let mut was_joint = false;
+    for t in lexer_tokens {
+        if t.kind.is_trivia() {
+            was_joint = false;
+        } else {
+            if t.kind == SyntaxKind::IDENT {
+                let token_text = &text[off..][..usize::from(t.len)];
+                let contextual_kw =
+                    SyntaxKind::from_contextual_keyword(token_text).unwrap_or(SyntaxKind::IDENT);
+                res.push_ident(contextual_kw);
+            } else {
+                if was_joint {
+                    res.was_joint();
+                }
+                res.push(t.kind);
+            }
+            was_joint = true;
+        }
+        off += usize::from(t.len);
+    }
+    res
+}
diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs
index 186cc9e74c..62f39a9347 100644
--- a/crates/syntax/src/parsing/reparsing.rs
+++ b/crates/syntax/src/parsing/reparsing.rs
@@ -12,8 +12,8 @@ use text_edit::Indel;
 use crate::{
     parsing::{
         lexer::{lex_single_syntax_kind, tokenize, Token},
-        text_token_source::TextTokenSource,
         text_tree_sink::TextTreeSink,
+        to_parser_tokens,
     },
     syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode},
     SyntaxError,
@@ -91,14 +91,14 @@ fn reparse_block(
     let (node, reparser) = find_reparsable_node(root, edit.delete)?;
     let text = get_text_after_edit(node.clone().into(), edit);
 
-    let (tokens, new_lexer_errors) = tokenize(&text);
-    if !is_balanced(&tokens) {
+    let (lexer_tokens, new_lexer_errors) = tokenize(&text);
+    if !is_balanced(&lexer_tokens) {
         return None;
     }
+    let parser_tokens = to_parser_tokens(&text, &lexer_tokens);
 
-    let mut token_source = TextTokenSource::new(&text, &tokens);
-    let mut tree_sink = TextTreeSink::new(&text, &tokens);
-    reparser.parse(&mut token_source, &mut tree_sink);
+    let mut tree_sink = TextTreeSink::new(&text, &lexer_tokens);
+    reparser.parse(&parser_tokens, &mut tree_sink);
 
     let (green, mut new_parser_errors) = tree_sink.finish();
     new_parser_errors.extend(new_lexer_errors);
diff --git a/crates/syntax/src/parsing/text_token_source.rs b/crates/syntax/src/parsing/text_token_source.rs
deleted file mode 100644
index 11dfc63a65..0000000000
--- a/crates/syntax/src/parsing/text_token_source.rs
+++ /dev/null
@@ -1,82 +0,0 @@
-//! See `TextTokenSource` docs.
-
-use parser::TokenSource;
-
-use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextSize};
-
-/// Implementation of `parser::TokenSource` that takes tokens from source code text.
-pub(crate) struct TextTokenSource<'t> {
-    text: &'t str,
-    /// token and its start position (non-whitespace/comment tokens)
-    /// ```non-rust
-    ///  struct Foo;
-    ///  ^------^--^-
-    ///  |      |    \________
-    ///  |      \____         \
-    ///  |           \         |
-    ///  (struct, 0) (Foo, 7) (;, 10)
-    /// ```
-    /// `[(struct, 0), (Foo, 7), (;, 10)]`
-    token_offset_pairs: Vec<(Token, TextSize)>,
-
-    /// Current token and position
-    curr: (parser::Token, usize),
-}
-
-impl<'t> TokenSource for TextTokenSource<'t> {
-    fn current(&self) -> parser::Token {
-        self.curr.0
-    }
-
-    fn lookahead_nth(&self, n: usize) -> parser::Token {
-        mk_token(self.curr.1 + n, &self.token_offset_pairs)
-    }
-
-    fn bump(&mut self) {
-        if self.curr.0.kind == EOF {
-            return;
-        }
-
-        let pos = self.curr.1 + 1;
-        self.curr = (mk_token(pos, &self.token_offset_pairs), pos);
-    }
-
-    fn is_keyword(&self, kw: &str) -> bool {
-        self.token_offset_pairs
-            .get(self.curr.1)
-            .map_or(false, |(token, offset)| &self.text[TextRange::at(*offset, token.len)] == kw)
-    }
-}
-
-fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> parser::Token {
-    let (kind, is_jointed_to_next) = match token_offset_pairs.get(pos) {
-        Some((token, offset)) => (
-            token.kind,
-            token_offset_pairs
-                .get(pos + 1)
-                .map_or(false, |(_, next_offset)| offset + token.len == *next_offset),
-        ),
-        None => (EOF, false),
-    };
-    parser::Token { kind, is_jointed_to_next }
-}
-
-impl<'t> TextTokenSource<'t> {
-    /// Generate input from tokens(expect comment and whitespace).
-    pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> {
-        let token_offset_pairs: Vec<_> = raw_tokens
-            .iter()
-            .filter_map({
-                let mut len = 0.into();
-                move |token| {
-                    let pair = if token.kind.is_trivia() { None } else { Some((*token, len)) };
-                    len += token.len;
-                    pair
-                }
-            })
-            .collect();
-
-        let first = mk_token(0, &token_offset_pairs);
-        TextTokenSource { text, token_offset_pairs, curr: (first, 0) }
-    }
-}
diff --git a/crates/syntax/src/parsing/text_tree_sink.rs b/crates/syntax/src/parsing/text_tree_sink.rs
index 8c1de92048..c1792199fd 100644
--- a/crates/syntax/src/parsing/text_tree_sink.rs
+++ b/crates/syntax/src/parsing/text_tree_sink.rs
@@ -104,7 +104,7 @@ impl<'a> TextTreeSink<'a> {
         }
     }
 
-    pub(super) fn finish(mut self) -> (GreenNode, Vec<SyntaxError>) {
+    pub(super) fn finish_eof(mut self) -> (GreenNode, Vec<SyntaxError>, bool) {
         match mem::replace(&mut self.state, State::Normal) {
             State::PendingFinish => {
                 self.eat_trivias();
@@ -113,7 +113,15 @@ impl<'a> TextTreeSink<'a> {
             State::PendingStart | State::Normal => unreachable!(),
         }
 
-        self.inner.finish_raw()
+        let (node, errors) = self.inner.finish_raw();
+        let is_eof = self.token_pos == self.tokens.len();
+
+        (node, errors, is_eof)
+    }
+
+    pub(super) fn finish(self) -> (GreenNode, Vec<SyntaxError>) {
+        let (node, errors, _eof) = self.finish_eof();
+        (node, errors)
     }
 
     fn eat_trivias(&mut self) {


Test patch:
diff --git a/crates/syntax/src/tests/sourcegen_ast.rs b/crates/syntax/src/tests/sourcegen_ast.rs
index dcd813bbe0..c66edadc3c 100644
--- a/crates/syntax/src/tests/sourcegen_ast.rs
+++ b/crates/syntax/src/tests/sourcegen_ast.rs
@@ -359,6 +359,10 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String {
     let full_keywords =
         full_keywords_values.iter().map(|kw| format_ident!("{}_KW", to_upper_snake_case(kw)));
 
+    let contextual_keywords_values = &grammar.contextual_keywords;
+    let contextual_keywords =
+        contextual_keywords_values.iter().map(|kw| format_ident!("{}_KW", to_upper_snake_case(kw)));
+
     let all_keywords_values =
         grammar.keywords.iter().chain(grammar.contextual_keywords.iter()).collect::<Vec<_>>();
     let all_keywords_idents = all_keywords_values.iter().map(|kw| format_ident!("{}", kw));
@@ -428,6 +432,14 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String {
                 Some(kw)
             }
 
+            pub fn from_contextual_keyword(ident: &str) -> Option<SyntaxKind> {
+                let kw = match ident {
+                    #(#contextual_keywords_values => #contextual_keywords,)*
+                    _ => return None,
+                };
+                Some(kw)
+            }
+
             pub fn from_char(c: char) -> Option<SyntaxKind> {
                 let tok = match c {
                     #(#single_byte_tokens_values => #single_byte_tokens,)*