Repository: toml-rs/toml (rust)

Issue:
Spanned values don't play well with dotted notation: getting "invalid type: string [..], expected a borrowed string"
I'm using `Spanned` to deserialize TOML to Nickel (a configuration language) while preserving spans as much as possible, as Nickel adds validation capabilities and we'd like to link back validation errors to the precise piece of TOML data that failed.

To do so, we define a bespoke datastructure that is more or less like a TOML value but with `Spanned` appropriately sprinkled, and write a custom deserializer using `serde_untagged`. You can find the type definition and the deserializer here: https://github.com/tweag/nickel/blob/927ee23993747b7851e51bcfe3eb3e685ba4ebb1/core/src/serialize.rs#L491-L582

However, when deserializing the following file:

```
[foo.bar]
baz = "qux"
```

This gives the following surprising error:

```
error: toml parse error: TOML parse error at line 1, column 1
  |
1 | [foo.bar]
  | ^
invalid type: string "bar", expected a borrowed string



in `foo`
```

It's surprising because we never try to deserialize a borrowed string: all strings, both as terminal values and keys, are owned in `SpannedValue` (`NickelString` is a simple wrapper around `String`). Also, any TOML file without dotted notation is parsed fine. After some experimentation, it seems that this happens when trying to deserialize the value (and not the key) of the outer map, that is the value associated to `foo`.

I suspect that there are some shenanigans around getting the location of the nested map `{bar = {baz = "qux"}}`. It _seems_ that the spanned deserializer of `toml-rs` tries to deserialize markers as borrowed string (https://github.com/toml-rs/toml/blob/b05e8c489be8ebfc0acacc1ec3556d95cd8d2198/crates/serde_spanned/src/spanned.rs#L161) but it also expects a very precise structure, so I'm not entirely sure what's going on here.

The issue is that I don't see any easy work-around: once we've tried to deserialize the content of a map as spanned (which is entirely legit for files that don't have the dotted notation), there doesn't seem to be anyway to retry the same deserialization at a different type.

Pull request:
fix(edit): Correctly deserialize Spanned implicit tables
Fixes #798

Patch:
diff --git a/Cargo.lock b/Cargo.lock
index 1de58f9e..57678d3d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -927,6 +927,7 @@ dependencies = [
  "indexmap 2.3.0",
  "itertools",
  "serde",
+ "serde-untagged",
  "serde_json",
  "serde_spanned",
  "snapbox",
@@ -1006,6 +1007,7 @@ dependencies = [
  "kstring",
  "proptest",
  "serde",
+ "serde-untagged",
  "serde_json",
  "serde_spanned",
  "snapbox",
diff --git a/crates/toml/Cargo.toml b/crates/toml/Cargo.toml
index 8d16a8af..1adee871 100644
--- a/crates/toml/Cargo.toml
+++ b/crates/toml/Cargo.toml
@@ -66,6 +66,7 @@ toml-test-data = "2.3.0"
 snapbox = "0.6.0"
 walkdir = "2.5.0"
 itertools = "0.14.0"
+serde-untagged = "0.1.7"
 
 [[test]]
 name = "decoder_compliance"
diff --git a/crates/toml_edit/Cargo.toml b/crates/toml_edit/Cargo.toml
index 3e47e1aa..68d7fe83 100644
--- a/crates/toml_edit/Cargo.toml
+++ b/crates/toml_edit/Cargo.toml
@@ -62,6 +62,7 @@ proptest = "1.5.0"
 walkdir = "2.5.0"
 serde_spanned = { path = "../serde_spanned" }
 toml_types = { path = "../toml", package = "toml", default-features = false, features = ["preserve_order"] }
+serde-untagged = "0.1.7"
 
 [[test]]
 name = "testsuite"
diff --git a/crates/toml_edit/src/parser/document.rs b/crates/toml_edit/src/parser/document.rs
index 3e1e2606..b123d57f 100644
--- a/crates/toml_edit/src/parser/document.rs
+++ b/crates/toml_edit/src/parser/document.rs
@@ -458,6 +458,7 @@ fn descend_path<'t>(
         table = match table.entry_format(key) {
             crate::Entry::Vacant(entry) => {
                 let mut new_table = Table::new();
+                new_table.span = key.span();
                 new_table.set_implicit(true);
                 new_table.set_dotted(dotted);
 
diff --git a/crates/toml_edit/src/parser/inline_table.rs b/crates/toml_edit/src/parser/inline_table.rs
index c0133d47..104e1612 100644
--- a/crates/toml_edit/src/parser/inline_table.rs
+++ b/crates/toml_edit/src/parser/inline_table.rs
@@ -232,6 +232,7 @@ fn descend_path<'a>(
         table = match table.entry_format(key) {
             crate::InlineEntry::Vacant(entry) => {
                 let mut new_table = InlineTable::new();
+                new_table.span = key.span();
                 new_table.set_implicit(true);
                 new_table.set_dotted(dotted);
                 entry


Test patch:
diff --git a/crates/toml/tests/serde/spanned.rs b/crates/toml/tests/serde/spanned.rs
index ca422039..0082db82 100644
--- a/crates/toml/tests/serde/spanned.rs
+++ b/crates/toml/tests/serde/spanned.rs
@@ -4,7 +4,9 @@
 use std::collections::HashMap;
 use std::fmt::Debug;
 
+use serde::de::{Deserializer, MapAccess};
 use serde::Deserialize;
+use serde_untagged::UntaggedEnumVisitor;
 use snapbox::assert_data_eq;
 use snapbox::prelude::*;
 use snapbox::str;
@@ -302,3 +304,111 @@ unknown field `fake`, expected `real`
         .raw()
     );
 }
+
+#[test]
+fn implicit_tables() {
+    #[derive(Debug)]
+    #[allow(dead_code)]
+    enum SpannedValue {
+        String(String),
+        Map(Vec<(String, Spanned<SpannedValue>)>),
+    }
+
+    impl<'de> Deserialize<'de> for SpannedValue {
+        fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+        where
+            D: Deserializer<'de>,
+        {
+            let data = UntaggedEnumVisitor::new()
+                .string(|str| Ok(SpannedValue::String(str.into())))
+                .map(|mut map| {
+                    let mut result = Vec::new();
+
+                    while let Some((k, v)) = map.next_entry()? {
+                        result.push((k, v));
+                    }
+
+                    Ok(SpannedValue::Map(result))
+                })
+                .deserialize(deserializer)?;
+
+            Ok(data)
+        }
+    }
+
+    const INPUT: &str = r#"
+[foo.bar]
+alice.bob = { one.two = "qux" }
+"#;
+
+    let result = crate::from_str::<SpannedValue>(INPUT);
+    assert_data_eq!(
+        result.unwrap().to_debug(),
+        str![[r#"
+Map(
+    [
+        (
+            "foo",
+            Spanned {
+                span: 2..5,
+                value: Map(
+                    [
+                        (
+                            "bar",
+                            Spanned {
+                                span: 1..10,
+                                value: Map(
+                                    [
+                                        (
+                                            "alice",
+                                            Spanned {
+                                                span: 11..16,
+                                                value: Map(
+                                                    [
+                                                        (
+                                                            "bob",
+                                                            Spanned {
+                                                                span: 23..42,
+                                                                value: Map(
+                                                                    [
+                                                                        (
+                                                                            "one",
+                                                                            Spanned {
+                                                                                span: 25..28,
+                                                                                value: Map(
+                                                                                    [
+                                                                                        (
+                                                                                            "two",
+                                                                                            Spanned {
+                                                                                                span: 35..40,
+                                                                                                value: String(
+                                                                                                    "qux",
+                                                                                                ),
+                                                                                            },
+                                                                                        ),
+                                                                                    ],
+                                                                                ),
+                                                                            },
+                                                                        ),
+                                                                    ],
+                                                                ),
+                                                            },
+                                                        ),
+                                                    ],
+                                                ),
+                                            },
+                                        ),
+                                    ],
+                                ),
+                            },
+                        ),
+                    ],
+                ),
+            },
+        ),
+    ],
+)
+
+"#]]
+    );
+}
diff --git a/crates/toml_edit/tests/serde/spanned.rs b/crates/toml_edit/tests/serde/spanned.rs
index ca422039..0082db82 100644
--- a/crates/toml_edit/tests/serde/spanned.rs
+++ b/crates/toml_edit/tests/serde/spanned.rs
@@ -4,7 +4,9 @@
 use std::collections::HashMap;
 use std::fmt::Debug;
 
+use serde::de::{Deserializer, MapAccess};
 use serde::Deserialize;
+use serde_untagged::UntaggedEnumVisitor;
 use snapbox::assert_data_eq;
 use snapbox::prelude::*;
 use snapbox::str;
@@ -302,3 +304,111 @@ unknown field `fake`, expected `real`
         .raw()
     );
 }
+
+#[test]
+fn implicit_tables() {
+    #[derive(Debug)]
+    #[allow(dead_code)]
+    enum SpannedValue {
+        String(String),
+        Map(Vec<(String, Spanned<SpannedValue>)>),
+    }
+
+    impl<'de> Deserialize<'de> for SpannedValue {
+        fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+        where
+            D: Deserializer<'de>,
+        {
+            let data = UntaggedEnumVisitor::new()
+                .string(|str| Ok(SpannedValue::String(str.into())))
+                .map(|mut map| {
+                    let mut result = Vec::new();
+
+                    while let Some((k, v)) = map.next_entry()? {
+                        result.push((k, v));
+                    }
+
+                    Ok(SpannedValue::Map(result))
+                })
+                .deserialize(deserializer)?;
+
+            Ok(data)
+        }
+    }
+
+    const INPUT: &str = r#"
+[foo.bar]
+alice.bob = { one.two = "qux" }
+"#;
+
+    let result = crate::from_str::<SpannedValue>(INPUT);
+    assert_data_eq!(
+        result.unwrap().to_debug(),
+        str![[r#"
+Map(
+    [
+        (
+            "foo",
+            Spanned {
+                span: 2..5,
+                value: Map(
+                    [
+                        (
+                            "bar",
+                            Spanned {
+                                span: 1..10,
+                                value: Map(
+                                    [
+                                        (
+                                            "alice",
+                                            Spanned {
+                                                span: 11..16,
+                                                value: Map(
+                                                    [
+                                                        (
+                                                            "bob",
+                                                            Spanned {
+                                                                span: 23..42,
+                                                                value: Map(
+                                                                    [
+                                                                        (
+                                                                            "one",
+                                                                            Spanned {
+                                                                                span: 25..28,
+                                                                                value: Map(
+                                                                                    [
+                                                                                        (
+                                                                                            "two",
+                                                                                            Spanned {
+                                                                                                span: 35..40,
+                                                                                                value: String(
+                                                                                                    "qux",
+                                                                                                ),
+                                                                                            },
+                                                                                        ),
+                                                                                    ],
+                                                                                ),
+                                                                            },
+                                                                        ),
+                                                                    ],
+                                                                ),
+                                                            },
+                                                        ),
+                                                    ],
+                                                ),
+                                            },
+                                        ),
+                                    ],
+                                ),
+                            },
+                        ),
+                    ],
+                ),
+            },
+        ),
+    ],
+)
+
+"#]]
+    );
+}