Repository: nushell/nushell (language: rust)

Issue:
Insert/Upsert does not work as expected on nested lists
### Describe the bug

When inserting or upserting, records are created, at arbitrary depth, to fill in gaps:
```nu 
$ {} | insert a value
╭───┬───────╮
│ a │ value │
╰───┴───────╯
$ {} | upsert a value
╭───┬───────╮
│ a │ value │
╰───┴───────╯
$ {} | insert a.b.c.d.e.f.g.h.i value
╭───┬───────────────────────────────────────────────────────────────────────╮
│   │ ╭───┬───────────────────────────────────────────────────────────────╮ │
│ a │ │   │ ╭───┬───────────────────────────────────────────────────────╮ │ │
│   │ │ b │ │   │ ╭───┬───────────────────────────────────────────────╮ │ │ │
│   │ │   │ │ c │ │   │ ╭───┬───────────────────────────────────────╮ │ │ │ │
│   │ │   │ │   │ │ d │ │   │ ╭───┬───────────────────────────────╮ │ │ │ │ │
│   │ │   │ │   │ │   │ │ e │ │   │ ╭───┬───────────────────────╮ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │ f │ │   │ ╭───┬───────────────╮ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │ g │ │   │ ╭───┬───────╮ │ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │   │ │ h │ │ i │ value │ │ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │   │ │   │ ╰───┴───────╯ │ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │   │ ╰───┴───────────────╯ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ ╰───┴───────────────────────╯ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ ╰───┴───────────────────────────────╯ │ │ │ │ │
│   │ │   │ │   │ │   │ ╰───┴───────────────────────────────────────╯ │ │ │ │
│   │ │   │ │   │ ╰───┴───────────────────────────────────────────────╯ │ │ │
│   │ │   │ ╰───┴───────────────────────────────────────────────────────╯ │ │
│   │ ╰───┴───────────────────────────────────────────────────────────────╯ │
╰───┴───────────────────────────────────────────────────────────────────────╯
$ {} | upsert a.b.c.d.e.f.g.h.i value
╭───┬───────────────────────────────────────────────────────────────────────╮
│   │ ╭───┬───────────────────────────────────────────────────────────────╮ │
│ a │ │   │ ╭───┬───────────────────────────────────────────────────────╮ │ │
│   │ │ b │ │   │ ╭───┬───────────────────────────────────────────────╮ │ │ │
│   │ │   │ │ c │ │   │ ╭───┬───────────────────────────────────────╮ │ │ │ │
│   │ │   │ │   │ │ d │ │   │ ╭───┬───────────────────────────────╮ │ │ │ │ │
│   │ │   │ │   │ │   │ │ e │ │   │ ╭───┬───────────────────────╮ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │ f │ │   │ ╭───┬───────────────╮ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │ g │ │   │ ╭───┬───────╮ │ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │   │ │ h │ │ i │ value │ │ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │   │ │   │ ╰───┴───────╯ │ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │   │ ╰───┴───────────────╯ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ ╰───┴───────────────────────╯ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ ╰───┴───────────────────────────────╯ │ │ │ │ │
│   │ │   │ │   │ │   │ ╰───┴───────────────────────────────────────╯ │ │ │ │
│   │ │   │ │   │ ╰───┴───────────────────────────────────────────────╯ │ │ │
│   │ │   │ ╰───┴───────────────────────────────────────────────────────╯ │ │
│   │ ╰───┴───────────────────────────────────────────────────────────────╯ │
╰───┴───────────────────────────────────────────────────────────────────────╯
```

This behavior should be expected of lists as well, and works if the lists encountered along the path already exist:
```nu
{a: {b: {c: {d: {e: {f: {g: {h: {i: []}}}}}}}}}  | insert a.b.c.d.e.f.g.h.i.0 value
╭───┬───────────────────────────────────────────────────────────────────────────────╮
│   │ ╭───┬───────────────────────────────────────────────────────────────────────╮ │
│ a │ │   │ ╭───┬───────────────────────────────────────────────────────────────╮ │ │
│   │ │ b │ │   │ ╭───┬───────────────────────────────────────────────────────╮ │ │ │
│   │ │   │ │ c │ │   │ ╭───┬───────────────────────────────────────────────╮ │ │ │ │
│   │ │   │ │   │ │ d │ │   │ ╭───┬───────────────────────────────────────╮ │ │ │ │ │
│   │ │   │ │   │ │   │ │ e │ │   │ ╭───┬───────────────────────────────╮ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │ f │ │   │ ╭───┬───────────────────────╮ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │ g │ │   │ ╭───┬───────────────╮ │ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │   │ │ h │ │   │ ╭───┬───────╮ │ │ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │   │ │   │ │ i │ │ 0 │ value │ │ │ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │   │ │   │ │   │ ╰───┴───────╯ │ │ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │   │ │   │ ╰───┴───────────────╯ │ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ │   │ ╰───┴───────────────────────╯ │ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ │   │ ╰───┴───────────────────────────────╯ │ │ │ │ │ │
│   │ │   │ │   │ │   │ │   │ ╰───┴───────────────────────────────────────╯ │ │ │ │ │
│   │ │   │ │   │ │   │ ╰───┴───────────────────────────────────────────────╯ │ │ │ │
│   │ │   │ │   │ ╰───┴───────────────────────────────────────────────────────╯ │ │ │
│   │ │   │ ╰───┴───────────────────────────────────────────────────────────────╯ │ │
│   │ ╰───┴───────────────────────────────────────────────────────────────────────╯ │
╰───┴───────────────────────────────────────────────────────────────────────────────╯
```
But returns an error when it does not (e.g when the list at `i` is removed above):
```nu
{a: {b: {c: {d: {e: {f: {g: {h: {}}}}}}}}}  | insert a.b.c.d.e.f.g.h.i.0 value
Error: nu::shell::not_a_list

  × Not a list value
   ╭─[entry #24:1:72]
 1 │ {a: {b: {c: {d: {e: {f: {g: {h: {}}}}}}}}}  | insert a.b.c.d.e.f.g.h.i.0 value
   ·                                                                        ┬ ──┬──
   ·                                                                        │   ╰── value originates here
   ·                                                                        ╰── value not a list
   ╰────
```
Similarly if any of its ancestors are missing:
```nu
$ {} | insert a.0 value
Error: nu::shell::not_a_list

  × Not a list value
   ╭─[entry #25:1:15]
 1 │ {} | insert a.0 value
   ·               ┬   ┬
   ·               │   ╰── value originates here
   ·               ╰── value not a list
   ╰────
```
A similar, though distinct, error is found when inserting descendants:
```nu
$ {a:[]} | upsert a.0.b value
Error: nu::shell::access_beyond_end

  × Row number too large (max: 18446744073709551615).
   ╭─[entry #29:1:19]
 1 │ {a:[]} | upsert a.0.b value
   ·                   ┬
   ·                   ╰── index too large (max: 18446744073709551615)
   ╰────
```

### How to reproduce

1. Insert value into non-existent list:
    ```nu
    {}  | insert a.0 value
    Error: nu::shell::not_a_list
    
      × Not a list value
       ╭─[entry #35:1:16]
     1 │ {}  | insert a.0 value
       ·                ┬ ──┬──
       ·                │   ╰── value originates here
       ·                ╰── value not a list
       ╰────
    ```
1. Insert value into descendents in non-existent list:
    ```nu
    {}  | insert a.0.b value # {a:[{ b:"value"}]}
    Error: nu::shell::not_a_list
    
      × Not a list value
       ╭─[entry #38:1:16]
     1 │ {}  | insert a.0.b value
       ·                ┬   ──┬──
       ·                │     ╰── value originates here
       ·                ╰── value not a list
       ╰────
    ```

### Expected behavior

I expected `nu` to: 
1. Insert value into non-existent list:    
    ```nu
    {}  | insert a.0 value # {a:["value"]}
    ```
    Should output
    ```
    ╭───┬───────────────╮
    │   │ ╭───┬───────╮ │
    │ a │ │ 0 │ value │ │
    │   │ ╰───┴───────╯ │
    ╰───┴───────────────╯
    ```
1. Insert value into descendents in non-existent list:
    ```nu
    {}  | insert a.0.b value # {a:[{ b:"value"}]}
    ```
    Should output
    ```
    ╭───┬───────────────╮
    │   │ ╭───┬───────╮ │
    │ a │ │ # │   b   │ │
    │   │ ├───┼───────┤ │
    │   │ │ 0 │ value │ │
    │   │ ╰───┴───────╯ │
    ╰───┴───────────────╯
    ```

### Configuration

| key                | value                                          |
| ------------------ | ---------------------------------------------- |
| version            | 0.102.0                                        |
| major              | 0                                              |
| minor              | 102                                            |
| patch              | 0                                              |
| branch             |                                                |
| commit_hash        |                                                |
| build_os           | macos-aarch64                                  |
| build_target       | aarch64-apple-darwin                           |
| rust_version       | rustc 1.84.1 (e71f9a9a9 2025-01-27) (Homebrew) |
| cargo_version      | cargo 1.84.1                                   |
| build_time         | 2025-02-04 15:49:35 +00:00                     |
| build_rust_channel | release                                        |
| allocator          | mimalloc                                       |
| features           | default, sqlite, trash                         |
| installed_plugins  | diff 0.1.0                                     |


Discussion / hints:
1. Currently upsert/insert assumes the value that's missing is a record rather than a list [here](https://github.com/nushell/nushell/blob/main/crates/nu-protocol/src/value/mod.rs#L1367)
2. The decendants issue seems to be introduced in [this change](https://github.com/nushell/nushell/commit/fa5d7babb9c1ff94cf2262a0db16b59df9e02936#diff-654f056f1c72a4868170c32aa44d6dd497a83cb1c8b919e24a7dc50e8f81e624R1650-R1654), where this condition was added, restricting any created lists to be inserted at the end of the path in inserts and upserts.
    ```rust
    } else if !path.is_empty() {
        return Err(ShellError::AccessBeyondEnd {
            max_idx: vals.len() - 1,
            span: *span,
        });
    }
    ```
    I think removing this will be enough to address this

I can try to get a PR up for this today or tomorrow

Fix patch:
diff --git a/crates/nu-command/src/filters/insert.rs b/crates/nu-command/src/filters/insert.rs
index 0d3a234f58944..97a087ec6ee0b 100644
--- a/crates/nu-command/src/filters/insert.rs
+++ b/crates/nu-command/src/filters/insert.rs
@@ -114,6 +114,22 @@ When inserting into a specific index, the closure will instead get the current v
                     Value::test_int(4),
                 ])),
             },
+            Example {
+                description: "Insert into a nested path, creating new values as needed",
+                example: "[{} {a: [{}]}] | insert a.0.b \"value\"",
+                result: Some(Value::test_list(vec![
+                    Value::test_record(record!(
+                        "a" => Value::test_list(vec![Value::test_record(record!(
+                            "b" => Value::test_string("value"),
+                        ))]),
+                    )),
+                    Value::test_record(record!(
+                        "a" => Value::test_list(vec![Value::test_record(record!(
+                            "b" => Value::test_string("value"),
+                        ))]),
+                    )),
+                ])),
+            },
         ]
     }
 }
diff --git a/crates/nu-command/src/filters/upsert.rs b/crates/nu-command/src/filters/upsert.rs
index a85e98f63949d..5bcd7b9a8ad47 100644
--- a/crates/nu-command/src/filters/upsert.rs
+++ b/crates/nu-command/src/filters/upsert.rs
@@ -144,6 +144,22 @@ If the command is inserting at the end of a list or table, then both of these va
                     Value::test_int(4),
                 ])),
             },
+            Example {
+                description: "Upsert into a nested path, creating new values as needed",
+                example: "[{} {a: [{}]}] | upsert a.0.b \"value\"",
+                result: Some(Value::test_list(vec![
+                    Value::test_record(record!(
+                        "a" => Value::test_list(vec![Value::test_record(record!(
+                            "b" => Value::test_string("value"),
+                        ))]),
+                    )),
+                    Value::test_record(record!(
+                        "a" => Value::test_list(vec![Value::test_record(record!(
+                            "b" => Value::test_string("value"),
+                        ))]),
+                    )),
+                ])),
+            },
         ]
     }
 }
diff --git a/crates/nu-protocol/src/value/mod.rs b/crates/nu-protocol/src/value/mod.rs
index a03b1ddb4e5ba..74c04327f1519 100644
--- a/crates/nu-protocol/src/value/mod.rs
+++ b/crates/nu-protocol/src/value/mod.rs
@@ -1333,15 +1333,8 @@ impl Value {
                                     if let Some(val) = record.get_mut(col_name) {
                                         val.upsert_data_at_cell_path(path, new_val.clone())?;
                                     } else {
-                                        let new_col = if path.is_empty() {
-                                            new_val.clone()
-                                        } else {
-                                            let mut new_col =
-                                                Value::record(Record::new(), new_val.span());
-                                            new_col
-                                                .upsert_data_at_cell_path(path, new_val.clone())?;
-                                            new_col
-                                        };
+                                        let new_col =
+                                            Value::with_data_at_cell_path(path, new_val.clone())?;
                                         record.push(col_name, new_col);
                                     }
                                 }
@@ -1361,13 +1354,7 @@ impl Value {
                         if let Some(val) = record.get_mut(col_name) {
                             val.upsert_data_at_cell_path(path, new_val)?;
                         } else {
-                            let new_col = if path.is_empty() {
-                                new_val
-                            } else {
-                                let mut new_col = Value::record(Record::new(), new_val.span());
-                                new_col.upsert_data_at_cell_path(path, new_val)?;
-                                new_col
-                            };
+                            let new_col = Value::with_data_at_cell_path(path, new_val.clone())?;
                             record.push(col_name, new_col);
                         }
                     }
@@ -1391,14 +1378,9 @@ impl Value {
                                 available_idx: vals.len(),
                                 span: *span,
                             });
-                        } else if !path.is_empty() {
-                            return Err(ShellError::AccessBeyondEnd {
-                                max_idx: vals.len() - 1,
-                                span: *span,
-                            });
                         } else {
                             // If the upsert is at 1 + the end of the list, it's OK.
-                            vals.push(new_val);
+                            vals.push(Value::with_data_at_cell_path(path, new_val)?);
                         }
                     }
                     Value::Error { error, .. } => return Err(*error.clone()),
@@ -1679,7 +1661,6 @@ impl Value {
             }
         }
     }
-
     pub fn insert_data_at_cell_path(
         &mut self,
         cell_path: &[PathMember],
@@ -1715,18 +1696,8 @@ impl Value {
                                             )?;
                                         }
                                     } else {
-                                        let new_col = if path.is_empty() {
-                                            new_val.clone()
-                                        } else {
-                                            let mut new_col =
-                                                Value::record(Record::new(), new_val.span());
-                                            new_col.insert_data_at_cell_path(
-                                                path,
-                                                new_val.clone(),
-                                                head_span,
-                                            )?;
-                                            new_col
-                                        };
+                                        let new_col =
+                                            Value::with_data_at_cell_path(path, new_val.clone())?;
                                         record.push(col_name, new_col);
                                     }
                                 }
@@ -1755,13 +1726,7 @@ impl Value {
                                 val.insert_data_at_cell_path(path, new_val, head_span)?;
                             }
                         } else {
-                            let new_col = if path.is_empty() {
-                                new_val
-                            } else {
-                                let mut new_col = Value::record(Record::new(), new_val.span());
-                                new_col.insert_data_at_cell_path(path, new_val, head_span)?;
-                                new_col
-                            };
+                            let new_col = Value::with_data_at_cell_path(path, new_val)?;
                             record.push(col_name, new_col);
                         }
                     }
@@ -1789,14 +1754,9 @@ impl Value {
                                 available_idx: vals.len(),
                                 span: *span,
                             });
-                        } else if !path.is_empty() {
-                            return Err(ShellError::AccessBeyondEnd {
-                                max_idx: vals.len() - 1,
-                                span: *span,
-                            });
                         } else {
                             // If the insert is at 1 + the end of the list, it's OK.
-                            vals.push(new_val);
+                            vals.push(Value::with_data_at_cell_path(path, new_val)?);
                         }
                     }
                     _ => {
@@ -1813,6 +1773,36 @@ impl Value {
         Ok(())
     }
 
+    /// Creates a new [Value] with the specified member at the specified path.
+    /// This is used by [Value::insert_data_at_cell_path] and [Value::upsert_data_at_cell_path] whenever they have the need to insert a non-existent element
+    fn with_data_at_cell_path(cell_path: &[PathMember], value: Value) -> Result<Value, ShellError> {
+        if let Some((member, path)) = cell_path.split_first() {
+            let span = value.span();
+            match member {
+                PathMember::String { val, .. } => Ok(Value::record(
+                    std::iter::once((val.clone(), Value::with_data_at_cell_path(path, value)?))
+                        .collect(),
+                    span,
+                )),
+                PathMember::Int { val, .. } => {
+                    if *val == 0usize {
+                        Ok(Value::list(
+                            vec![Value::with_data_at_cell_path(path, value)?],
+                            span,
+                        ))
+                    } else {
+                        Err(ShellError::InsertAfterNextFreeIndex {
+                            available_idx: 0,
+                            span,
+                        })
+                    }
+                }
+            }
+        } else {
+            Ok(value)
+        }
+    }
+
     /// Visits all values contained within the value (including this value) with a mutable reference
     /// given to the closure.
     ///
@@ -4027,6 +4017,161 @@ mod tests {
     use super::{Record, Value};
     use crate::record;
 
+    mod at_cell_path {
+        use crate::{IntoValue, Span};
+
+        use super::super::PathMember;
+        use super::*;
+
+        #[test]
+        fn test_record_with_data_at_cell_path() {
+            let value_to_insert = Value::test_string("value");
+            let span = Span::test_data();
+            assert_eq!(
+                Value::with_data_at_cell_path(
+                    &[
+                        PathMember::test_string("a".to_string(), false),
+                        PathMember::test_string("b".to_string(), false),
+                        PathMember::test_string("c".to_string(), false),
+                        PathMember::test_string("d".to_string(), false),
+                    ],
+                    value_to_insert,
+                ),
+                // {a:{b:c{d:"value"}}}
+                Ok(record!(
+                    "a" => record!(
+                        "b" => record!(
+                            "c" => record!(
+                                "d" => Value::test_string("value")
+                            ).into_value(span)
+                        ).into_value(span)
+                    ).into_value(span)
+                )
+                .into_value(span))
+            );
+        }
+
+        #[test]
+        fn test_lists_with_data_at_cell_path() {
+            let value_to_insert = Value::test_string("value");
+            assert_eq!(
+                Value::with_data_at_cell_path(
+                    &[
+                        PathMember::test_int(0, false),
+                        PathMember::test_int(0, false),
+                        PathMember::test_int(0, false),
+                        PathMember::test_int(0, false),
+                    ],
+                    value_to_insert.clone(),
+                ),
+                // [[[[["value"]]]]]
+                Ok(Value::test_list(vec![Value::test_list(vec![
+                    Value::test_list(vec![Value::test_list(vec![value_to_insert])])
+                ])]))
+            );
+        }
+        #[test]
+        fn test_mixed_with_data_at_cell_path() {
+            let value_to_insert = Value::test_string("value");
+            let span = Span::test_data();
+            assert_eq!(
+                Value::with_data_at_cell_path(
+                    &[
+                        PathMember::test_string("a".to_string(), false),
+                        PathMember::test_int(0, false),
+                        PathMember::test_string("b".to_string(), false),
+                        PathMember::test_int(0, false),
+                        PathMember::test_string("c".to_string(), false),
+                        PathMember::test_int(0, false),
+                        PathMember::test_string("d".to_string(), false),
+                        PathMember::test_int(0, false),
+                    ],
+                    value_to_insert.clone(),
+                ),
+                // [{a:[{b:[{c:[{d:["value"]}]}]}]]}
+                Ok(record!(
+                    "a" => Value::test_list(vec![record!(
+                        "b" => Value::test_list(vec![record!(
+                            "c" => Value::test_list(vec![record!(
+                                "d" => Value::test_list(vec![value_to_insert])
+                            ).into_value(span)])
+                        ).into_value(span)])
+                    ).into_value(span)])
+                )
+                .into_value(span))
+            );
+        }
+
+        #[test]
+        fn test_nested_upsert_data_at_cell_path() {
+            let span = Span::test_data();
+            let mut base_value = record!(
+                "a" => Value::test_list(vec![])
+            )
+            .into_value(span);
+
+            let value_to_insert = Value::test_string("value");
+            let res = base_value.upsert_data_at_cell_path(
+                &[
+                    PathMember::test_string("a".to_string(), false),
+                    PathMember::test_int(0, false),
+                    PathMember::test_string("b".to_string(), false),
+                    PathMember::test_int(0, false),
+                ],
+                value_to_insert.clone(),
+            );
+            assert_eq!(res, Ok(()));
+            assert_eq!(
+                base_value,
+                // {a:[{b:["value"]}]}
+                record!(
+                    "a" => Value::test_list(vec![
+                        record!(
+                            "b" => Value::test_list(vec![value_to_insert])
+                        )
+                        .into_value(span)
+                    ])
+                )
+                .into_value(span)
+            );
+        }
+
+        #[test]
+        fn test_nested_insert_data_at_cell_path() {
+            let span = Span::test_data();
+            let mut base_value = record!(
+                "a" => Value::test_list(vec![])
+            )
+            .into_value(span);
+
+            let value_to_insert = Value::test_string("value");
+            let res = base_value.insert_data_at_cell_path(
+                &[
+                    PathMember::test_string("a".to_string(), false),
+                    PathMember::test_int(0, false),
+                    PathMember::test_string("b".to_string(), false),
+                    PathMember::test_int(0, false),
+                ],
+                value_to_insert.clone(),
+                span,
+            );
+            assert_eq!(res, Ok(()));
+            assert_eq!(
+                base_value,
+                // {a:[{b:["value"]}]}
+                record!(
+                    "a" => Value::test_list(vec![
+                        record!(
+                            "b" => Value::test_list(vec![value_to_insert])
+                        )
+                        .into_value(span)
+                    ])
+                )
+                .into_value(span)
+            );
+        }
+    }
+
     mod is_empty {
         use super::*;
 


Test patch: