Repository: rust-analyzer/rust-analyzer
Pull request #3518 (2020-03-08 16:13:38)

Add parse_to_token_tree
This PR introduce a function for parsing `&str` to `tt::TokenTree`:

```rust
// Convert a string to a `TokenTree`
pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> {
````


Patch:
diff --git a/crates/ra_hir_expand/src/builtin_macro.rs b/crates/ra_hir_expand/src/builtin_macro.rs
index 9bc33cfa80..3f60b1cca2 100644
--- a/crates/ra_hir_expand/src/builtin_macro.rs
+++ b/crates/ra_hir_expand/src/builtin_macro.rs
@@ -7,6 +7,7 @@ use crate::{
 
 use crate::{quote, EagerMacroId, LazyMacroId, MacroCallId};
 use either::Either;
+use mbe::parse_to_token_tree;
 use ra_db::{FileId, RelativePath};
 use ra_parser::FragmentKind;
 
@@ -306,10 +307,9 @@ fn include_expand(
 
     // FIXME:
     // Handle include as expression
-    let node =
-        db.parse_or_expand(file_id.into()).ok_or_else(|| mbe::ExpandError::ConversionError)?;
-    let res =
-        mbe::syntax_node_to_token_tree(&node).ok_or_else(|| mbe::ExpandError::ConversionError)?.0;
+    let res = parse_to_token_tree(&db.file_text(file_id.into()))
+        .ok_or_else(|| mbe::ExpandError::ConversionError)?
+        .0;
 
     Ok((res, FragmentKind::Items))
 }
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs
index 2c6ae56589..43afe24ccb 100644
--- a/crates/ra_mbe/src/lib.rs
+++ b/crates/ra_mbe/src/lib.rs
@@ -31,7 +31,8 @@ pub enum ExpandError {
 }
 
 pub use crate::syntax_bridge::{
-    ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, TokenMap,
+    ast_to_token_tree, parse_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node,
+    TokenMap,
 };
 
 /// This struct contains AST for a single `macro_rules` definition. What might
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index fb9fa53149..fcb73fbc7c 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -2,8 +2,10 @@
 
 use ra_parser::{FragmentKind, ParseError, TreeSink};
 use ra_syntax::{
-    ast, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode,
-    SyntaxTreeBuilder, TextRange, TextUnit, T,
+    ast::{self, make::tokens::doc_comment},
+    tokenize, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind,
+    SyntaxKind::*,
+    SyntaxNode, SyntaxTreeBuilder, TextRange, TextUnit, Token, T,
 };
 use rustc_hash::FxHashMap;
 use std::iter::successors;
@@ -48,9 +50,11 @@ pub fn ast_to_token_tree(ast: &impl ast::AstNode) -> Option<(tt::Subtree, TokenM
 /// will consume).
 pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> {
     let global_offset = node.text_range().start();
-    let mut c = Convertor { map: TokenMap::default(), global_offset, next_id: 0 };
+    let mut c = Convertor {
+        id_alloc: { TokenIdAlloc { map: TokenMap::default(), global_offset, next_id: 0 } },
+    };
     let subtree = c.go(node)?;
-    Some((subtree, c.map))
+    Some((subtree, c.id_alloc.map))
 }
 
 // The following items are what `rustc` macro can be parsed into :
@@ -89,6 +93,28 @@ pub fn token_tree_to_syntax_node(
     Ok((parse, range_map))
 }
 
+/// Convert a string to a `TokenTree`
+pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> {
+    let (tokens, errors) = tokenize(text);
+    if !errors.is_empty() {
+        return None;
+    }
+
+    let mut conv = RawConvertor {
+        text,
+        offset: TextUnit::default(),
+        inner: tokens.iter(),
+        id_alloc: TokenIdAlloc {
+            map: Default::default(),
+            global_offset: TextUnit::default(),
+            next_id: 0,
+        },
+    };
+
+    let subtree = conv.go()?;
+    Some((subtree, conv.id_alloc.map))
+}
+
 impl TokenMap {
     pub fn token_by_range(&self, relative_range: TextRange) -> Option<tt::TokenId> {
         let &(token_id, _) = self.entries.iter().find(|(_, range)| match range {
@@ -118,6 +144,14 @@ impl TokenMap {
         self.entries
             .push((token_id, TokenTextRange::Delimiter(open_relative_range, close_relative_range)));
     }
+
+    fn update_close_delim(&mut self, token_id: tt::TokenId, close_relative_range: TextRange) {
+        if let Some(entry) = self.entries.iter_mut().find(|(tid, _)| *tid == token_id) {
+            if let TokenTextRange::Delimiter(dim, _) = entry.1 {
+                entry.1 = TokenTextRange::Delimiter(dim, close_relative_range);
+            }
+        }
+    }
 }
 
 /// Returns the textual content of a doc comment block as a quoted string
@@ -188,12 +222,161 @@ fn convert_doc_comment(token: &ra_syntax::SyntaxToken) -> Option<Vec<tt::TokenTr
     }
 }
 
-struct Convertor {
+struct TokenIdAlloc {
     map: TokenMap,
     global_offset: TextUnit,
     next_id: u32,
 }
 
+impl TokenIdAlloc {
+    fn alloc(&mut self, absolute_range: TextRange) -> tt::TokenId {
+        let relative_range = absolute_range - self.global_offset;
+        let token_id = tt::TokenId(self.next_id);
+        self.next_id += 1;
+        self.map.insert(token_id, relative_range);
+        token_id
+    }
+
+    fn delim(&mut self, open_abs_range: TextRange, close_abs_range: TextRange) -> tt::TokenId {
+        let open_relative_range = open_abs_range - self.global_offset;
+        let close_relative_range = close_abs_range - self.global_offset;
+        let token_id = tt::TokenId(self.next_id);
+        self.next_id += 1;
+
+        self.map.insert_delim(token_id, open_relative_range, close_relative_range);
+        token_id
+    }
+
+    fn open_delim(&mut self, open_abs_range: TextRange) -> tt::TokenId {
+        let token_id = tt::TokenId(self.next_id);
+        self.next_id += 1;
+        self.map.insert_delim(token_id, open_abs_range, open_abs_range);
+        token_id
+    }
+
+    fn close_delim(&mut self, id: tt::TokenId, close_abs_range: TextRange) {
+        self.map.update_close_delim(id, close_abs_range);
+    }
+}
+
+/// A Raw Token (straightly from lexer) convertor
+struct RawConvertor<'a> {
+    text: &'a str,
+    offset: TextUnit,
+    id_alloc: TokenIdAlloc,
+    inner: std::slice::Iter<'a, Token>,
+}
+
+impl RawConvertor<'_> {
+    fn go(&mut self) -> Option<tt::Subtree> {
+        let mut subtree = tt::Subtree::default();
+        subtree.delimiter = None;
+        while self.peek().is_some() {
+            self.collect_leaf(&mut subtree.token_trees);
+        }
+        if subtree.token_trees.is_empty() {
+            return None;
+        }
+        if subtree.token_trees.len() == 1 {
+            if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] {
+                return Some(first.clone());
+            }
+        }
+        Some(subtree)
+    }
+
+    fn bump(&mut self) -> Option<(Token, TextRange)> {
+        let token = self.inner.next()?;
+        let range = TextRange::offset_len(self.offset, token.len);
+        self.offset += token.len;
+        Some((*token, range))
+    }
+
+    fn peek(&self) -> Option<Token> {
+        self.inner.as_slice().get(0).cloned()
+    }
+
+    fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) {
+        let (token, range) = match self.bump() {
+            None => return,
+            Some(it) => it,
+        };
+
+        let k: SyntaxKind = token.kind;
+        if k == COMMENT {
+            let node = doc_comment(&self.text[range]);
+            if let Some(tokens) = convert_doc_comment(&node) {
+                result.extend(tokens);
+            }
+            return;
+        }
+
+        result.push(if k.is_punct() {
+            let delim = match k {
+                T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])),
+                T!['{'] => Some((tt::DelimiterKind::Brace, T!['}'])),
+                T!['['] => Some((tt::DelimiterKind::Bracket, T![']'])),
+                _ => None,
+            };
+
+            if let Some((kind, closed)) = delim {
+                let mut subtree = tt::Subtree::default();
+                let id = self.id_alloc.open_delim(range);
+                subtree.delimiter = Some(tt::Delimiter { kind, id });
+
+                while self.peek().map(|it| it.kind != closed).unwrap_or(false) {
+                    self.collect_leaf(&mut subtree.token_trees);
+                }
+                let last_range = match self.bump() {
+                    None => return,
+                    Some(it) => it.1,
+                };
+                self.id_alloc.close_delim(id, last_range);
+                subtree.into()
+            } else {
+                let spacing = match self.peek() {
+                    Some(next)
+                        if next.kind.is_trivia()
+                            || next.kind == T!['[']
+                            || next.kind == T!['{']
+                            || next.kind == T!['('] =>
+                    {
+                        tt::Spacing::Alone
+                    }
+                    Some(next) if next.kind.is_punct() => tt::Spacing::Joint,
+                    _ => tt::Spacing::Alone,
+                };
+                let char =
+                    self.text[range].chars().next().expect("Token from lexer must be single char");
+
+                tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc.alloc(range) }).into()
+            }
+        } else {
+            macro_rules! make_leaf {
+                ($i:ident) => {
+                    tt::$i { id: self.id_alloc.alloc(range), text: self.text[range].into() }.into()
+                };
+            }
+            let leaf: tt::Leaf = match k {
+                T![true] | T![false] => make_leaf!(Literal),
+                IDENT | LIFETIME => make_leaf!(Ident),
+                k if k.is_keyword() => make_leaf!(Ident),
+                k if k.is_literal() => make_leaf!(Literal),
+                _ => return,
+            };
+
+            leaf.into()
+        });
+    }
+}
+
+// FIXME: There are some duplicate logic between RawConvertor and Convertor
+// It would be nice to refactor to converting SyntaxNode to ra_parser::Token and thus
+// use RawConvertor directly. But performance-wise it may not be a good idea ?
+struct Convertor {
+    id_alloc: TokenIdAlloc,
+}
+
 impl Convertor {
     fn go(&mut self, tt: &SyntaxNode) -> Option<tt::Subtree> {
         // This tree is empty
@@ -236,7 +419,7 @@ impl Convertor {
         };
         let delimiter = delimiter_kind.map(|kind| tt::Delimiter {
             kind,
-            id: self.alloc_delim(first_child.text_range(), last_child.text_range()),
+            id: self.id_alloc.delim(first_child.text_range(), last_child.text_range()),
         });
 
         let mut token_trees = Vec::new();
@@ -273,7 +456,7 @@ impl Convertor {
                                 tt::Leaf::from(tt::Punct {
                                     char,
                                     spacing,
-                                    id: self.alloc(token.text_range()),
+                                    id: self.id_alloc.alloc(token.text_range()),
                                 })
                                 .into(),
                             );
@@ -282,7 +465,7 @@ impl Convertor {
                         macro_rules! make_leaf {
                             ($i:ident) => {
                                 tt::$i {
-                                    id: self.alloc(token.text_range()),
+                                    id: self.id_alloc.alloc(token.text_range()),
                                     text: token.text().clone(),
                                 }
                                 .into()
@@ -313,28 +496,6 @@ impl Convertor {
         let res = tt::Subtree { delimiter, token_trees };
         Some(res)
     }
-
-    fn alloc(&mut self, absolute_range: TextRange) -> tt::TokenId {
-        let relative_range = absolute_range - self.global_offset;
-        let token_id = tt::TokenId(self.next_id);
-        self.next_id += 1;
-        self.map.insert(token_id, relative_range);
-        token_id
-    }
-
-    fn alloc_delim(
-        &mut self,
-        open_abs_range: TextRange,
-        close_abs_range: TextRange,
-    ) -> tt::TokenId {
-        let open_relative_range = open_abs_range - self.global_offset;
-        let close_relative_range = close_abs_range - self.global_offset;
-        let token_id = tt::TokenId(self.next_id);
-        self.next_id += 1;
-
-        self.map.insert_delim(token_id, open_relative_range, close_relative_range);
-        token_id
-    }
 }
 
 struct TtTreeSink<'a> {
diff --git a/crates/ra_syntax/src/ast/make.rs b/crates/ra_syntax/src/ast/make.rs
index 53d6fa5620..ae8829807c 100644
--- a/crates/ra_syntax/src/ast/make.rs
+++ b/crates/ra_syntax/src/ast/make.rs
@@ -267,6 +267,12 @@ pub mod tokens {
         sf.syntax().first_child_or_token().unwrap().into_token().unwrap()
     }
 
+    pub fn doc_comment(text: &str) -> SyntaxToken {
+        assert!(!text.trim().is_empty());
+        let sf = SourceFile::parse(text).ok().unwrap();
+        sf.syntax().first_child_or_token().unwrap().into_token().unwrap()
+    }
+
     pub fn literal(text: &str) -> SyntaxToken {
         assert_eq!(text.trim(), text);
         let lit: ast::Literal = super::ast_from_text(&format!("fn f() {{ let _ = {}; }}", text));
diff --git a/editors/code/src/ctx.ts b/editors/code/src/ctx.ts
index 25ef38aed0..b4e983a0cd 100644
--- a/editors/code/src/ctx.ts
+++ b/editors/code/src/ctx.ts
@@ -3,7 +3,7 @@ import * as lc from 'vscode-languageclient';
 
 import { Config } from './config';
 import { createClient } from './client';
-import { isRustEditor, RustEditor } from './util';
+import { isRustDocument } from './util';
 
 export class Ctx {
     private constructor(
@@ -22,15 +22,17 @@ export class Ctx {
         return res;
     }
 
-    get activeRustEditor(): RustEditor | undefined {
+    get activeRustEditor(): vscode.TextEditor | undefined {
         const editor = vscode.window.activeTextEditor;
-        return editor && isRustEditor(editor)
+        return editor && isRustDocument(editor.document)
             ? editor
             : undefined;
     }
 
-    get visibleRustEditors(): RustEditor[] {
-        return vscode.window.visibleTextEditors.filter(isRustEditor);
+    get visibleRustEditors(): vscode.TextEditor[] {
+        return vscode.window.visibleTextEditors.filter(
+            editor => isRustDocument(editor.document),
+        );
     }
 
     registerCommand(name: string, factory: (ctx: Ctx) => Cmd) {
diff --git a/editors/code/src/inlay_hints.ts b/editors/code/src/inlay_hints.ts
index e1a82e03e8..08d3a64a77 100644
--- a/editors/code/src/inlay_hints.ts
+++ b/editors/code/src/inlay_hints.ts
@@ -1,214 +1,156 @@
-import * as lc from "vscode-languageclient";
 import * as vscode from 'vscode';
 import * as ra from './rust-analyzer-api';
 
-import { Ctx, Disposable } from './ctx';
-import { sendRequestWithRetry, isRustDocument, RustDocument, RustEditor } from './util';
-
+import { Ctx } from './ctx';
+import { log, sendRequestWithRetry, isRustDocument } from './util';
 
 export function activateInlayHints(ctx: Ctx) {
-    const maybeUpdater = {
-        updater: null as null | HintsUpdater,
-        onConfigChange() {
-            if (!ctx.config.displayInlayHints) {
-                return this.dispose();
-            }
-            if (!this.updater) this.updater = new HintsUpdater(ctx);
-        },
-        dispose() {
-            this.updater?.dispose();
-            this.updater = null;
-        }
-    };
+    const hintsUpdater = new HintsUpdater(ctx);
+    vscode.window.onDidChangeVisibleTextEditors(
+        async _ => hintsUpdater.refresh(),
+        null,
+        ctx.subscriptions
+    );
 
-    ctx.pushCleanup(maybeUpdater);
+    vscode.workspace.onDidChangeTextDocument(
+        async event => {
+            if (event.contentChanges.length === 0) return;
+            if (!isRustDocument(event.document)) return;
+            await hintsUpdater.refresh();
+        },
+        null,
+        ctx.subscriptions
+    );
 
     vscode.workspace.onDidChangeConfiguration(
-        maybeUpdater.onConfigChange, maybeUpdater, ctx.subscriptions
+        async _ => hintsUpdater.setEnabled(ctx.config.displayInlayHints),
+        null,
+        ctx.subscriptions
     );
 
-    maybeUpdater.onConfigChange();
-}
-
-
-const typeHints = {
-    decorationType: vscode.window.createTextEditorDecorationType({
-        after: {
-            color: new vscode.ThemeColor('rust_analyzer.inlayHint'),
-            fontStyle: "normal",
+    ctx.pushCleanup({
+        dispose() {
+            hintsUpdater.clear();
         }
-    }),
+    });
 
-    toDecoration(hint: ra.InlayHint.TypeHint, conv: lc.Protocol2CodeConverter): vscode.DecorationOptions {
-        return {
-            range: conv.asRange(hint.range),
-            renderOptions: { after: { contentText: `: ${hint.label}` } }
-        };
-    }
-};
-
-const paramHints = {
-    decorationType: vscode.window.createTextEditorDecorationType({
-        before: {
-            color: new vscode.ThemeColor('rust_analyzer.inlayHint'),
-            fontStyle: "normal",
-        }
-    }),
+    // XXX: we don't await this, thus Promise rejections won't be handled, but
+    // this should never throw in fact...
+    void hintsUpdater.setEnabled(ctx.config.displayInlayHints);
+}
 
-    toDecoration(hint: ra.InlayHint.ParamHint, conv: lc.Protocol2CodeConverter): vscode.DecorationOptions {
-        return {
-            range: conv.asRange(hint.range),
-            renderOptions: { before: { contentText: `${hint.label}: ` } }
-        };
+const typeHintDecorationType = vscode.window.createTextEditorDecorationType({
+    after: {
+        color: new vscode.ThemeColor('rust_analyzer.inlayHint'),
+        fontStyle: "normal",
+    },
+});
+
+const parameterHintDecorationType = vscode.window.createTextEditorDecorationType({
+    before: {
+        color: new vscode.ThemeColor('rust_analyzer.inlayHint'),
+        fontStyle: "normal",
+    },
+});
+
+class HintsUpdater {
+    private pending = new Map<string, vscode.CancellationTokenSource>();
+    private ctx: Ctx;
+    private enabled: boolean;
+
+    constructor(ctx: Ctx) {
+        this.ctx = ctx;
+        this.enabled = false;
     }
-};
 
-class HintsUpdater implements Disposable {
-    private sourceFiles = new Map<string, RustSourceFile>(); // map Uri -> RustSourceFile
-    private readonly disposables: Disposable[] = [];
+    async setEnabled(enabled: boolean): Promise<void> {
+        log.debug({ enabled, prev: this.enabled });
 
-    constructor(private readonly ctx: Ctx) {
-        vscode.window.onDidChangeVisibleTextEditors(
-            this.onDidChangeVisibleTextEditors,
-            this,
-            this.disposables
-        );
+        if (this.enabled === enabled) return;
+        this.enabled = enabled;
 
-        vscode.workspace.onDidChangeTextDocument(
-            this.onDidChangeTextDocument,
-            this,
-            this.disposables
-        );
-
-        // Set up initial cache shape
-        ctx.visibleRustEditors.forEach(editor => this.sourceFiles.set(
-            editor.document.uri.toString(),
-            {
-                document: editor.document,
-                inlaysRequest: null,
-                cachedDecorations: null
-            }
-        ));
-
-        this.syncCacheAndRenderHints();
-    }
-
-    dispose() {
-        this.sourceFiles.forEach(file => file.inlaysRequest?.cancel());
-        this.ctx.visibleRustEditors.forEach(editor => this.renderDecorations(editor, { param: [], type: [] }));
-        this.disposables.forEach(d => d.dispose());
+        if (this.enabled) {
+            return await this.refresh();
+        } else {
+            return this.clear();
+        }
     }
 
-    onDidChangeTextDocument({ contentChanges, document }: vscode.TextDocumentChangeEvent) {
-        if (contentChanges.length === 0 || !isRustDocument(document)) return;
-        this.syncCacheAndRenderHints();
+    clear() {
+        this.ctx.visibleRustEditors.forEach(it => {
+            this.setTypeDecorations(it, []);
+            this.setParameterDecorations(it, []);
+        });
     }
 
-    private syncCacheAndRenderHints() {
-        // FIXME: make inlayHints request pass an array of files?
-        this.sourceFiles.forEach((file, uri) => this.fetchHints(file).then(hints => {
-            if (!hints) return;
-
-            file.cachedDecorations = this.hintsToDecorations(hints);
-
-            for (const editor of this.ctx.visibleRustEditors) {
-                if (editor.document.uri.toString() === uri) {
-                    this.renderDecorations(editor, file.cachedDecorations);
-                }
-            }
-        }));
+    async refresh() {
+        if (!this.enabled) return;
+        await Promise.all(this.ctx.visibleRustEditors.map(it => this.refreshEditor(it)));
     }
 
-    onDidChangeVisibleTextEditors() {
-        const newSourceFiles = new Map<string, RustSourceFile>();
-
-        // Rerendering all, even up-to-date editors for simplicity
-        this.ctx.visibleRustEditors.forEach(async editor => {
-            const uri = editor.document.uri.toString();
-            const file = this.sourceFiles.get(uri) ?? {
-                document: editor.document,
-                inlaysRequest: null,
-                cachedDecorations: null
-            };
-            newSourceFiles.set(uri, file);
-
-            // No text documents changed, so we may try to use the cache
-            if (!file.cachedDecorations) {
-                file.inlaysRequest?.cancel();
-
-                const hints = await this.fetchHints(file);
-                if (!hints) return;
-
-                file.cachedDecorations = this.hintsToDecorations(hints);
-            }
-
-            this.renderDecorations(editor, file.cachedDecorations);
-        });
-
-        // Cancel requests for no longer visible (disposed) source files
-        this.sourceFiles.forEach((file, uri) => {
-            if (!newSourceFiles.has(uri)) file.inlaysRequest?.cancel();
-        });
-
-        this.sourceFiles = newSourceFiles;
+    private async refreshEditor(editor: vscode.TextEditor): Promise<void> {
+        const newHints = await this.queryHints(editor.document.uri.toString());
+        if (newHints == null) return;
+
+        const newTypeDecorations = newHints
+            .filter(hint => hint.kind === ra.InlayKind.TypeHint)
+            .map(hint => ({
+                range: this.ctx.client.protocol2CodeConverter.asRange(hint.range),
+                renderOptions: {
+                    after: {
+                        contentText: `: ${hint.label}`,
+                    },
+                },
+            }));
+        this.setTypeDecorations(editor, newTypeDecorations);
+
+        const newParameterDecorations = newHints
+            .filter(hint => hint.kind === ra.InlayKind.ParameterHint)
+            .map(hint => ({
+                range: this.ctx.client.protocol2CodeConverter.asRange(hint.range),
+                renderOptions: {
+                    before: {
+                        contentText: `${hint.label}: `,
+                    },
+                },
+            }));
+        this.setParameterDecorations(editor, newParameterDecorations);
     }
 
-    private renderDecorations(editor: RustEditor, decorations: InlaysDecorations) {
-        editor.setDecorations(typeHints.decorationType, decorations.type);
-        editor.setDecorations(paramHints.decorationType, decorations.param);
+    private setTypeDecorations(
+        editor: vscode.TextEditor,
+        decorations: vscode.DecorationOptions[],
+    ) {
+        editor.setDecorations(
+            typeHintDecorationType,
+            this.enabled ? decorations : [],
+        );
     }
 
-    private hintsToDecorations(hints: ra.InlayHint[]): InlaysDecorations {
-        const decorations: InlaysDecorations = { type: [], param: [] };
-        const conv = this.ctx.client.protocol2CodeConverter;
-
-        for (const hint of hints) {
-            switch (hint.kind) {
-                case ra.InlayHint.Kind.TypeHint: {
-                    decorations.type.push(typeHints.toDecoration(hint, conv));
-                    continue;
-                }
-                case ra.InlayHint.Kind.ParamHint: {
-                    decorations.param.push(paramHints.toDecoration(hint, conv));
-                    continue;
-                }
-            }
-        }
-        return decorations;
+    private setParameterDecorations(
+        editor: vscode.TextEditor,
+        decorations: vscode.DecorationOptions[],
+    ) {
+        editor.setDecorations(
+            parameterHintDecorationType,
+            this.enabled ? decorations : [],
+        );
     }
 
-    private async fetchHints(file: RustSourceFile): Promise<null | ra.InlayHint[]> {
-        file.inlaysRequest?.cancel();
+    private async queryHints(documentUri: string): Promise<ra.InlayHint[] | null> {
+        this.pending.get(documentUri)?.cancel();
 
         const tokenSource = new vscode.CancellationTokenSource();
-        file.inlaysRequest = tokenSource;
+        this.pending.set(documentUri, tokenSource);
 
-        const request = { textDocument: { uri: file.document.uri.toString() } };
+        const request = { textDocument: { uri: documentUri } };
 
         return sendRequestWithRetry(this.ctx.client, ra.inlayHints, request, tokenSource.token)
             .catch(_ => null)
             .finally(() => {
-                if (file.inlaysRequest === tokenSource) {
-                    file.inlaysRequest = null;
+                if (!tokenSource.token.isCancellationRequested) {
+                    this.pending.delete(documentUri);
                 }
             });
     }
 }
-
-interface InlaysDecorations {
-    type: vscode.DecorationOptions[];
-    param: vscode.DecorationOptions[];
-}
-
-interface RustSourceFile {
-    /*
-    * Source of the token to cancel in-flight inlay hints request if any.
-    */
-    inlaysRequest: null | vscode.CancellationTokenSource;
-    /**
-    * Last applied decorations.
-    */
-    cachedDecorations: null | InlaysDecorations;
-
-    document: RustDocument;
-}
diff --git a/editors/code/src/rust-analyzer-api.ts b/editors/code/src/rust-analyzer-api.ts
index bd6e3ada08..c5a010e947 100644
--- a/editors/code/src/rust-analyzer-api.ts
+++ b/editors/code/src/rust-analyzer-api.ts
@@ -86,20 +86,14 @@ export interface Runnable {
 export const runnables = request<RunnablesParams, Vec<Runnable>>("runnables");
 
 
-
-export type InlayHint = InlayHint.TypeHint | InlayHint.ParamHint;
-
-export namespace InlayHint {
-    export const enum Kind {
-        TypeHint = "TypeHint",
-        ParamHint = "ParameterHint",
-    }
-    interface Common {
-        range: lc.Range;
-        label: string;
-    }
-    export type TypeHint = Common & { kind: Kind.TypeHint };
-    export type ParamHint = Common & { kind: Kind.ParamHint };
+export const enum InlayKind {
+    TypeHint = "TypeHint",
+    ParameterHint = "ParameterHint",
+}
+export interface InlayHint {
+    range: lc.Range;
+    kind: InlayKind;
+    label: string;
 }
 export interface InlayHintsParams {
     textDocument: lc.TextDocumentIdentifier;
diff --git a/editors/code/src/util.ts b/editors/code/src/util.ts
index 95a5f1227c..7c95769bb8 100644
--- a/editors/code/src/util.ts
+++ b/editors/code/src/util.ts
@@ -1,6 +1,7 @@
 import * as lc from "vscode-languageclient";
 import * as vscode from "vscode";
 import { strict as nativeAssert } from "assert";
+import { TextDocument } from "vscode";
 
 export function assert(condition: boolean, explanation: string): asserts condition {
     try {
@@ -66,16 +67,9 @@ function sleep(ms: number) {
     return new Promise(resolve => setTimeout(resolve, ms));
 }
 
-export type RustDocument = vscode.TextDocument & { languageId: "rust" };
-export type RustEditor = vscode.TextEditor & { document: RustDocument; id: string };
-
-export function isRustDocument(document: vscode.TextDocument): document is RustDocument {
+export function isRustDocument(document: TextDocument) {
     return document.languageId === 'rust'
         // SCM diff views have the same URI as the on-disk document but not the same content
         && document.uri.scheme !== 'git'
         && document.uri.scheme !== 'svn';
-}
-
-export function isRustEditor(editor: vscode.TextEditor): editor is RustEditor {
-    return isRustDocument(editor.document);
-}
+}
\ No newline at end of file


Test patch:
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index 066ce150b5..6d5d1e9e60 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -1499,12 +1499,20 @@ impl MacroFixture {
     }
 }
 
-pub(crate) fn parse_macro(macro_definition: &str) -> MacroFixture {
-    let source_file = ast::SourceFile::parse(macro_definition).ok().unwrap();
+pub(crate) fn parse_macro(ra_fixture: &str) -> MacroFixture {
+    let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap();
     let macro_definition =
         source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
 
     let (definition_tt, _) = ast_to_token_tree(&macro_definition.token_tree().unwrap()).unwrap();
+
+    let parsed = parse_to_token_tree(
+        &ra_fixture[macro_definition.token_tree().unwrap().syntax().text_range()],
+    )
+    .unwrap()
+    .0;
+    assert_eq!(definition_tt, parsed);
+
     let rules = MacroRules::parse(&definition_tt).unwrap();
     MacroFixture { rules }
 }