Repository: intellij-rust/intellij-rust (language: rust)

Issue:
`rust-analyzer-proc-macro-srv` API breakage
I feel like we should inform you about this here as well, some months ago we started adding a version check to the proc-macro server so that we can add more functionality without breaking support for older servers as long as the client supports it (as our proc-macro server protocol is rather rigid). See the corresponding PR here https://github.com/rust-lang/rust-analyzer/pull/14070

The toolchain distributed `rust-analyzer-proc-macro-srv` component has this rolled out now on recent nightlies now (~1-2 week old as of posting). I think if a "client" (intellij in this case) does not know of this versioning things might start breaking for you as the server itself only supports the latest version. The only thing that really changed right now for version 2 is that the protocol now encodes the open and close span for groups separately, instead of assigning one to both.

(If you want we can keep this open after you've implemented this as a tracking issue for me to post whenever a new version gets added)
serde_with macro expand failed
<!--
Hello and thank you for the issue!
If you would like to report a bug, we have added some points below that you can fill out.
Consider using "Help | Create New Issue" action that fills out all necessary information automatically.
Feel free to remove all the irrelevant text to request a new feature.
-->

## Environment

* **IntelliJ Rust plugin version:** 0.4.197.5408-231-nightly
* **Rust toolchain version:** rustc 1.72.0-nightly (37998ab50 2023-06-11)
* **IDE name and version:** 2023.1.3
* **Operating system:** Archlinux

## Problem description
![image](https://github.com/intellij-rust/intellij-rust/assets/10096425/e4ecb08e-6aaa-408d-8b9c-a8ee5444ae92)

it seems intellij-rust expand the `serde_as` attr macro failed, that make the IDE report `use std::net::Ipv4Addr` `DisplayFromStr` and `Deserialize` is unused

## Steps to reproduce
```rust
use std::net::Ipv4Addr;
use serde_with::{serde_as, DisplayFromStr};
use serde::Deserialize;

#[serde_as]
#[derive(Debug, Deserialize)]
pub struct Config {
    pub tun_name: String,

    #[serde_as(as = "DisplayFromStr")]
    pub local_ipv4: Ipv4Addr,
}
```

try this codes with `serde` and `serde_with` 3.0.0 version




Discussion / hints:
Hi! Thank you so much for the information! I'll take a look soon


Fix patch:
diff --git a/src/main/kotlin/org/rust/lang/core/macros/proc/ProMacroExpanderVersion.kt b/src/main/kotlin/org/rust/lang/core/macros/proc/ProMacroExpanderVersion.kt
index 2204b6e4b5c..c732b6a6c3e 100644
--- a/src/main/kotlin/org/rust/lang/core/macros/proc/ProMacroExpanderVersion.kt
+++ b/src/main/kotlin/org/rust/lang/core/macros/proc/ProMacroExpanderVersion.kt
@@ -8,7 +8,7 @@ package org.rust.lang.core.macros.proc
 enum class ProMacroExpanderVersion {
     NO_VERSION_CHECK_VERSION,
     VERSION_CHECK_VERSION,
-//    ENCODE_CLOSE_SPAN_VERSION,
+    ENCODE_CLOSE_SPAN_VERSION,
     ;
 
     companion object {
diff --git a/src/main/kotlin/org/rust/lang/core/macros/proc/ProcMacroExpander.kt b/src/main/kotlin/org/rust/lang/core/macros/proc/ProcMacroExpander.kt
index bc904acdc78..77517fe3701 100644
--- a/src/main/kotlin/org/rust/lang/core/macros/proc/ProcMacroExpander.kt
+++ b/src/main/kotlin/org/rust/lang/core/macros/proc/ProcMacroExpander.kt
@@ -116,9 +116,9 @@ class ProcMacroExpander private constructor(
         val envMapped = env.mapValues { (_, v) -> toolchain?.toRemotePath(v) ?: v }
         val version = server.requestExpanderVersion().unwrapOrElse { return Err(it.toProcMacroExpansionError()) }
         val request = Request.ExpandMacro(
-            FlatTree.fromSubtree(macroCallBody),
+            FlatTree.fromSubtree(macroCallBody, version),
             macroName,
-            attributes?.let { FlatTree.fromSubtree(it) },
+            attributes?.let { FlatTree.fromSubtree(it, version) },
             remoteLib,
             envMapped.map { listOf(it.key, it.value) },
             envMapped["CARGO_MANIFEST_DIR"],
@@ -126,7 +126,7 @@ class ProcMacroExpander private constructor(
         val response = server.send(request, timeout).unwrapOrElse { return Err(it.toProcMacroExpansionError()) }
         check(response is Response.ExpandMacro)
         return response.expansion.map {
-            it.toTokenTree()
+            it.toTokenTree(version)
         }.mapErr {
             ProcMacroExpansionError.ServerSideError(it.message)
         }
diff --git a/src/main/kotlin/org/rust/lang/core/macros/tt/FlatTree.kt b/src/main/kotlin/org/rust/lang/core/macros/tt/FlatTree.kt
index b9132913f47..8ed9fb8f2b1 100644
--- a/src/main/kotlin/org/rust/lang/core/macros/tt/FlatTree.kt
+++ b/src/main/kotlin/org/rust/lang/core/macros/tt/FlatTree.kt
@@ -6,6 +6,7 @@
 package org.rust.lang.core.macros.tt
 
 import it.unimi.dsi.fastutil.ints.IntArrayList
+import org.rust.lang.core.macros.proc.ProMacroExpanderVersion
 import org.rust.lang.core.psi.MacroBraces
 import org.rust.stdext.dequeOf
 import java.util.*
@@ -21,15 +22,19 @@ class FlatTree(
     val tokenTree: IntArrayList,
     val text: List<String>,
 ) {
-    fun toTokenTree(): TokenTree.Subtree {
+    fun toTokenTree(version: ProMacroExpanderVersion): TokenTree.Subtree {
+        val encodeCloseSpan = version >= ProMacroExpanderVersion.ENCODE_CLOSE_SPAN_VERSION
+        val offset = if (encodeCloseSpan) 1 else 0
+        val stepSize = if (encodeCloseSpan) 5 else 4
+
         val res: MutableList<TokenTree.Subtree?> = ArrayList(subtree.size)
         repeat(subtree.size) { res.add(null) }
 
-        for (i in (0 until subtree.size).step(4).reversed()) {
+        for (i in (0 until subtree.size).step(stepSize).reversed()) {
             val delimiterId = subtree.getInt(i)
-            val kind = subtree.getInt(i + 1)
-            val lo = subtree.getInt(i + 2)
-            val len = subtree.getInt(i + 3)
+            val kind = subtree.getInt(i + offset + 1)
+            val lo = subtree.getInt(i + offset + 2)
+            val len = subtree.getInt(i + offset + 3)
 
             val rawTokenTrees = tokenTree
             val tokenTrees = ArrayList<TokenTree>(len - lo)
@@ -74,7 +79,7 @@ class FlatTree(
                 else -> error("Unknown kind $kind")
             }
 
-            res[i / 4] = TokenTree.Subtree(
+            res[i / stepSize] = TokenTree.Subtree(
                 delimiterKind?.let { Delimiter(delimiterId, delimiterKind) },
                 tokenTrees,
             )
@@ -84,12 +89,12 @@ class FlatTree(
     }
 
     companion object {
-        fun fromSubtree(root: TokenTree.Subtree): FlatTree =
-            FlatTreeBuilder().apply { write(root) }.toFlatTree()
+        fun fromSubtree(root: TokenTree.Subtree, version: ProMacroExpanderVersion): FlatTree =
+            FlatTreeBuilder(version >= ProMacroExpanderVersion.ENCODE_CLOSE_SPAN_VERSION).apply { write(root) }.toFlatTree()
     }
 }
 
-private class FlatTreeBuilder {
+private class FlatTreeBuilder(private val encodeCloseSpan: Boolean) {
     private val work: Deque<Pair<Int, TokenTree.Subtree>> = dequeOf()
     private val stringTable: HashMap<String, Int> = hashMapOf()
 
@@ -118,8 +123,11 @@ private class FlatTreeBuilder {
             tokenTree.add(-1)
         }
 
-        this.subtree[subtreeId * 4 + 2] = firstTt
-        this.subtree[subtreeId * 4 + 3] = firstTt + nTt
+        val offset = if (encodeCloseSpan) 1 else 0
+        val stepSize = if (encodeCloseSpan) 5 else 4
+
+        this.subtree[subtreeId * stepSize + offset + 2] = firstTt
+        this.subtree[subtreeId * stepSize + offset + 3] = firstTt + nTt
 
         for (child in subtree.tokenTrees) {
             val idxTag = when (child) {
@@ -158,11 +166,15 @@ private class FlatTreeBuilder {
     }
 
     private fun enqueue(subtree: TokenTree.Subtree): Int {
-        val idx = this.subtree.size / 4
+        val stepSize = if (encodeCloseSpan) 5 else 4
+        val idx = this.subtree.size / stepSize
         val delimiterId = subtree.delimiter?.id ?: -1
         val delimiterKind = subtree.delimiter?.kind
         this.subtree.apply {
             add(delimiterId)
+            if (encodeCloseSpan) {
+                add(-1) // closeId
+            }
             add(when (delimiterKind) {
                 null -> 0
                 MacroBraces.PARENS -> 1


Test patch:
diff --git a/src/test/kotlin/org/rust/lang/core/macros/tt/TokenTreeJsonTest.kt b/src/test/kotlin/org/rust/lang/core/macros/tt/TokenTreeJsonTest.kt
index 045b2dc674a..2808864e27a 100644
--- a/src/test/kotlin/org/rust/lang/core/macros/tt/TokenTreeJsonTest.kt
+++ b/src/test/kotlin/org/rust/lang/core/macros/tt/TokenTreeJsonTest.kt
@@ -9,13 +9,14 @@ import com.fasterxml.jackson.core.util.DefaultIndenter
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter
 import org.intellij.lang.annotations.Language
 import org.rust.RsTestBase
+import org.rust.lang.core.macros.proc.ProMacroExpanderVersion
 import org.rust.lang.core.macros.proc.ProcMacroJsonParser
 import org.rust.lang.core.parser.createRustPsiBuilder
 
 class TokenTreeJsonTest : RsTestBase() {
     fun `test 1`() = doTest(".", """
         {
-          "subtree": [ 4294967295, 0, 0, 1 ],
+          "subtree": [ 4294967295, 4294967295, 0, 0, 1 ],
           "literal": [ ],
           "punct": [ 0, 46, 0 ],
           "ident": [ ],
@@ -26,7 +27,7 @@ class TokenTreeJsonTest : RsTestBase() {
 
     fun `test 2`() = doTest("..", """
         {
-          "subtree": [ 4294967295, 0, 0, 2 ],
+          "subtree": [ 4294967295, 4294967295, 0, 0, 2 ],
           "literal": [ ],
           "punct": [ 0, 46, 1, 1, 46, 0 ],
           "ident": [ ],
@@ -37,7 +38,7 @@ class TokenTreeJsonTest : RsTestBase() {
 
     fun `test 3`() = doTest(".foo", """
         {
-          "subtree": [ 4294967295, 0, 0, 2 ],
+          "subtree": [ 4294967295, 4294967295, 0, 0, 2 ],
           "literal": [ ],
           "punct": [ 0, 46, 0 ],
           "ident": [ 1, 0 ],
@@ -48,7 +49,7 @@ class TokenTreeJsonTest : RsTestBase() {
 
     fun `test 4`() = doTest(":::", """
         {
-          "subtree": [ 4294967295, 0, 0, 3 ],
+          "subtree": [ 4294967295, 4294967295, 0, 0, 3 ],
           "literal": [ ],
           "punct": [ 0, 58, 1, 1, 58, 1, 2, 58, 0 ],
           "ident": [ ],
@@ -59,7 +60,7 @@ class TokenTreeJsonTest : RsTestBase() {
 
     fun `test 5`() = doTest(". asd .. \"asd\" ...", """
         {
-          "subtree": [ 4294967295, 0, 0, 8 ],
+          "subtree": [ 4294967295, 4294967295, 0, 0, 8 ],
           "literal": [ 4, 1 ],
           "punct": [ 0, 46, 0, 2, 46, 1, 3, 46, 0, 5, 46, 1, 6, 46, 1, 7, 46, 0 ],
           "ident": [ 1, 0 ],
@@ -70,7 +71,7 @@ class TokenTreeJsonTest : RsTestBase() {
 
     fun `test 6`() = doTest("{}", """
         {
-          "subtree": [ 0, 2, 0, 0 ],
+          "subtree": [ 0, 4294967295, 2, 0, 0 ],
           "literal": [ ],
           "punct": [ ],
           "ident": [ ],
@@ -81,7 +82,7 @@ class TokenTreeJsonTest : RsTestBase() {
 
     fun `test 7`() = doTest("[]", """
         {
-          "subtree": [ 0, 3, 0, 0 ],
+          "subtree": [ 0, 4294967295, 3, 0, 0 ],
           "literal": [ ],
           "punct": [ ],
           "ident": [ ],
@@ -92,7 +93,7 @@ class TokenTreeJsonTest : RsTestBase() {
 
     fun `test 8`() = doTest("()", """
         {
-          "subtree": [ 0, 1, 0, 0 ],
+          "subtree": [ 0, 4294967295, 1, 0, 0 ],
           "literal": [ ],
           "punct": [ ],
           "ident": [ ],
@@ -103,7 +104,7 @@ class TokenTreeJsonTest : RsTestBase() {
 
     fun `test 9`() = doTest("(..)", """
         {
-          "subtree": [ 0, 1, 0, 2 ],
+          "subtree": [ 0, 4294967295, 1, 0, 2 ],
           "literal": [ ],
           "punct": [ 1, 46, 1, 2, 46, 1 ],
           "ident": [ ],
@@ -114,7 +115,7 @@ class TokenTreeJsonTest : RsTestBase() {
 
     fun `test 10`() = doTest("([{.}])", """
         {
-          "subtree": [ 0, 1, 0, 1, 1, 3, 1, 2, 2, 2, 2, 3 ],
+          "subtree": [ 0, 4294967295, 1, 0, 1, 1, 4294967295, 3, 1, 2, 2, 4294967295, 2, 2, 3 ],
           "literal": [ ],
           "punct": [ 3, 46, 1 ],
           "ident": [ ],
@@ -126,14 +127,16 @@ class TokenTreeJsonTest : RsTestBase() {
     fun doTest(code: String, @Language("Json") expectedJson: String) {
         val subtree = project.createRustPsiBuilder(code).parseSubtree().subtree
 
+        val version = ProMacroExpanderVersion.ENCODE_CLOSE_SPAN_VERSION
+
         val jackson = ProcMacroJsonParser.JSON_MAPPER
         val actualJson = jackson
             .writer()
             .with(DefaultPrettyPrinter(TestPrettyPrinter()))
-            .writeValueAsString(FlatTree.fromSubtree(subtree))
+            .writeValueAsString(FlatTree.fromSubtree(subtree, version))
 
         assertEquals(expectedJson.trimIndent(), actualJson)
-        assertEquals(jackson.readValue(actualJson, FlatTree::class.java).toTokenTree(), subtree)
+        assertEquals(jackson.readValue(actualJson, FlatTree::class.java).toTokenTree(version), subtree)
     }
 }
 
diff --git a/src/test/kotlin/org/rust/lang/core/macros/tt/TokenTreeParserTest.kt b/src/test/kotlin/org/rust/lang/core/macros/tt/TokenTreeParserTest.kt
index 919315ff80f..e694d3e5a40 100644
--- a/src/test/kotlin/org/rust/lang/core/macros/tt/TokenTreeParserTest.kt
+++ b/src/test/kotlin/org/rust/lang/core/macros/tt/TokenTreeParserTest.kt
@@ -6,6 +6,7 @@
 package org.rust.lang.core.macros.tt
 
 import org.rust.RsTestBase
+import org.rust.lang.core.macros.proc.ProMacroExpanderVersion
 import org.rust.lang.core.parser.createRustPsiBuilder
 
 class TokenTreeParserTest : RsTestBase() {
@@ -246,7 +247,8 @@ class TokenTreeParserTest : RsTestBase() {
 
     fun doTest(code: String, expected: String) {
         val subtree = project.createRustPsiBuilder(code).parseSubtree().subtree
-        assertEquals(subtree, FlatTree.fromSubtree(subtree).toTokenTree())
+        val version = ProMacroExpanderVersion.ENCODE_CLOSE_SPAN_VERSION
+        assertEquals(subtree, FlatTree.fromSubtree(subtree, version).toTokenTree(version))
         assertEquals(expected.trimIndent(), subtree.toDebugString())
     }
 }