Repository: omni-us/jsonargparse

Issue:
Parsing to a list of dataclasses with nested dataclass fails
<!-- If you like this project, please ⭐ star it https://github.com/omni-us/jsonargparse/ -->

## 🐛 Bug report

Parsing a configuration to a list of dataclasses which contain nested dataclass fails.

### To reproduce

Run following code: 

```python
import jsonargparse
from dataclasses import dataclass

@dataclass
class B:
    # THIS WORKS
    # an_int: int = 7
    # THIS CAUSES AN ERROR
    an_int: int

@dataclass
class A:
    b: B

def fun(a: list[A]):
    print(a)

jsonargparse.CLI(fun)
```

Like so:
```
python python-jsonargparse-list-of-dataclasses.py '[{b: {an_int: 3}}]'
```

I'm seeing an error:
```
argparse.ArgumentError: Validation failed: Key "b.an_int" is required but not included in config object or its value is None.
```
Attaching full traceback.
[traceback.txt](https://github.com/user-attachments/files/17618428/traceback.txt)


### Expected behavior

We should see following printed to the stdout:
```
[A(b=B(an_int=3))]
```

### Environment

- jsonargparse version: 4.33.2
- Python version: 3.10.12
- How jsonargparse was installed: `pip install --user --update jsonargparse`
- OS: Ubuntu 22.04.4 LTS

Discussion:
@mauvilsa: Thank you for reporting! I will look at it later.

Solution patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 739f9b04..fee10a5b 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -12,6 +12,17 @@ The semantic versioning only considers the public API as described in
 paths are considered internals and can change in minor and patch releases.
 
 
+v4.34.1 (2024-11-??)
+--------------------
+
+Fixed
+^^^^^
+- List of dataclass with nested dataclass attribute fails to parse (`#625
+  <https://github.com/omni-us/jsonargparse/pull/625>`__).
+- ``NotRequired`` incorrectly having ``inspect._empty`` as default (`#625
+  <https://github.com/omni-us/jsonargparse/pull/625>`__).
+
+
 v4.34.0 (2024-11-08)
 --------------------
 
diff --git a/jsonargparse/_core.py b/jsonargparse/_core.py
index a9e253ab..548e5b66 100644
--- a/jsonargparse/_core.py
+++ b/jsonargparse/_core.py
@@ -61,7 +61,6 @@
     load_value,
     loaders,
     set_omegaconf_loader,
-    yaml_load,
 )
 from ._namespace import (
     Namespace,
@@ -223,7 +222,7 @@ def __init__(
             print_config: Add this as argument to print config, set None to disable.
             parser_mode: Mode for parsing config files: ``'yaml'``, ``'jsonnet'`` or ones added via :func:`.set_loader`.
             dump_header: Header to include as comment when dumping a config object.
-            default_config_files: Default config file locations, e.g. :code:`['~/.config/myapp/*.yaml']`.
+            default_config_files: Default config file locations, e.g. ``['~/.config/myapp/*.yaml']``.
             default_env: Set the default value on whether to parse environment variables.
             default_meta: Set the default value on whether to include metadata in config objects.
         """
@@ -442,6 +441,7 @@ def parse_object(
             if cfg_base:
                 cfg = self.merge_config(cfg_base, cfg)
 
+            cfg = self._apply_actions(cfg)
             cfg_apply = self._apply_actions(cfg_obj, prev_cfg=cfg)
             cfg = self.merge_config(cfg_apply, cfg)
 
@@ -1341,7 +1341,6 @@ def merge_config(self, cfg_from: Namespace, cfg_to: Namespace) -> Namespace:
         cfg_to = cfg_to.clone()
         with parser_context(parent_parser=self):
             ActionTypeHint.discard_init_args_on_class_path_change(self, cfg_to, cfg_from)
-        ActionTypeHint.delete_not_required_args(cfg_from, cfg_to)
         cfg_to.update(cfg_from)
         ActionTypeHint.apply_appends(self, cfg_to)
         return cfg_to
@@ -1396,7 +1395,7 @@ def default_config_files(self) -> List[str]:
         """Default config file locations.
 
         :getter: Returns the current default config file locations.
-        :setter: Sets new default config file locations, e.g. :code:`['~/.config/myapp/*.yaml']`.
+        :setter: Sets new default config file locations, e.g. ``['~/.config/myapp/*.yaml']``.
 
         Raises:
             ValueError: If an invalid value is given.
@@ -1441,8 +1440,9 @@ def default_env(self) -> bool:
 
     @default_env.setter
     def default_env(self, default_env: bool):
-        if os.environ.get("JSONARGPARSE_DEFAULT_ENV", "").lower() in {"true", "false"}:
-            self._default_env = yaml_load(os.environ.get("JSONARGPARSE_DEFAULT_ENV"))
+        os_default_env = os.getenv("JSONARGPARSE_DEFAULT_ENV", "").lower()
+        if os_default_env in {"true", "false"}:
+            self._default_env = os_default_env == "true"
         elif isinstance(default_env, bool):
             self._default_env = default_env
         else:
diff --git a/jsonargparse/_formatters.py b/jsonargparse/_formatters.py
index 92975433..5ded5239 100644
--- a/jsonargparse/_formatters.py
+++ b/jsonargparse/_formatters.py
@@ -54,7 +54,7 @@ class DefaultHelpFormatter(HelpFormatter):
     This class is an extension of `argparse.HelpFormatter
     <https://docs.python.org/3/library/argparse.html#argparse.HelpFormatter>`_.
     Default values are always included. Furthermore, if the parser is configured
-    with :code:`default_env=True` command line options are preceded by 'ARG:' and
+    with ``default_env=True`` command line options are preceded by 'ARG:' and
     the respective environment variable name is included preceded by 'ENV:'.
     """
 
diff --git a/jsonargparse/_signatures.py b/jsonargparse/_signatures.py
index 7d75e197..efd57e21 100644
--- a/jsonargparse/_signatures.py
+++ b/jsonargparse/_signatures.py
@@ -3,7 +3,7 @@
 import dataclasses
 import inspect
 import re
-from argparse import ArgumentParser
+from argparse import SUPPRESS, ArgumentParser
 from contextlib import suppress
 from typing import Any, Callable, List, Optional, Set, Tuple, Type, Union
 
@@ -71,7 +71,7 @@ def add_class_arguments(
             as_positional: Whether to add required parameters as positional arguments.
             default: Default value used to override parameter defaults.
             skip: Names of parameters or number of positionals that should be skipped.
-            instantiate: Whether the class group should be instantiated by :code:`instantiate_classes`.
+            instantiate: Whether the class group should be instantiated by ``instantiate_classes``.
             fail_untyped: Whether to raise exception if a required parameter does not have a type.
             sub_configs: Whether subclass type hints should be loadable from inner config file.
 
@@ -242,7 +242,7 @@ def _add_signature_arguments(
             skip: Names of parameters or number of positionals that should be skipped.
             fail_untyped: Whether to raise exception if a required parameter does not have a type.
             sub_configs: Whether subclass type hints should be loadable from inner config file.
-            instantiate: Whether the class group should be instantiated by :code:`instantiate_classes`.
+            instantiate: Whether the class group should be instantiated by ``instantiate_classes``.
 
         Returns:
             The list of arguments added.
@@ -321,9 +321,12 @@ def _add_signature_parameter(
         annotation = param.annotation
         if default == inspect_empty:
             default = param.default
-            if default == inspect_empty and is_optional(annotation):
-                default = None
-        is_required = default == inspect_empty and get_typehint_origin(annotation) not in not_required_types
+            if default == inspect_empty:
+                if is_optional(annotation):
+                    default = None
+                elif get_typehint_origin(annotation) in not_required_types:
+                    default = SUPPRESS
+        is_required = default == inspect_empty
         src = get_parameter_origins(param.component, param.parent)
         skip_message = f'Skipping parameter "{name}" from "{src}" because of: '
         if not fail_untyped and annotation == inspect_empty:
diff --git a/jsonargparse/_typehints.py b/jsonargparse/_typehints.py
index c8f5afb4..bc44a2f8 100644
--- a/jsonargparse/_typehints.py
+++ b/jsonargparse/_typehints.py
@@ -431,12 +431,6 @@ def discard_init_args_on_class_path_change(parser_or_action, prev_cfg, cfg):
                     keys = keys[: num + 1] + [k for k in keys[num + 1 :] if not k.startswith(key + ".")]
             num += 1
 
-    @staticmethod
-    def delete_not_required_args(cfg_from, cfg_to):
-        for key, val in list(cfg_to.items(branches=True)):
-            if val == inspect._empty and key not in cfg_from:
-                del cfg_to[key]
-
     @staticmethod
     @contextmanager
     def subclass_arg_context(parser):


Test patch:
diff --git a/jsonargparse_tests/test_dataclass_like.py b/jsonargparse_tests/test_dataclass_like.py
index 67a820eb..ff736d0d 100644
--- a/jsonargparse_tests/test_dataclass_like.py
+++ b/jsonargparse_tests/test_dataclass_like.py
@@ -336,6 +336,25 @@ def test_instantiate_dataclass_within_classes(parser):
     assert isinstance(init["class"].data, NestedData)
 
 
+@dataclasses.dataclass
+class RequiredAttr:
+    an_int: int
+
+
+@dataclasses.dataclass
+class NestedRequiredAttr:
+    b: RequiredAttr
+
+
+def test_list_nested_dataclass_required_attr(parser):
+    parser.add_argument("--a", type=List[NestedRequiredAttr])
+    cfg = parser.parse_args(['--a=[{"b": {"an_int": 3}}]'])
+    assert cfg == Namespace(a=[Namespace(b=Namespace(an_int=3))])
+    init = parser.instantiate_classes(cfg)
+    assert isinstance(init.a[0].b, RequiredAttr)
+    assert isinstance(init.a[0], NestedRequiredAttr)
+
+
 @dataclasses.dataclass
 class WithAttrDocs:
     attr_str: str = "a"