Repository: omni-us/jsonargparse (python)

Issue:
`CLI` fails to instantiate a Pydantic object if it has `typing.Optional` annotation
<!-- If you like this project, please ⭐ star it https://github.com/omni-us/jsonargparse/ -->

## 🐛 Bug report

`CLI` fails to instantiate a Pydantic object if it has `typing.Optional` annotation.

### To reproduce

In `main.py`:

```Python
import typing as tp

import pydantic
from jsonargparse import CLI


class A(pydantic.BaseModel):
    value: int


class B:

    def __init__(self, a: tp.Optional[A] = None) -> None:
        self.a = a


def main(b: B) -> None:
    print(b)


if __name__ == "__main__":
    CLI(main)
```

In `config.yaml`:

```Yaml
b:
  class_path: __main__.B
  init_args:
    a:
      value: 1
```

In terminal:

`python -m main --config config.yaml`

Gives error:

```
error: Validation failed: Parser key "b":
  Problem with given class_path '__main__.B':
    Parser key "a":
      Does not validate against any of the Union subtypes
      Subtypes: (<class '__main__.A'>, <class 'NoneType'>)
      Errors:
        - __pydantic_private__
        - Expected a <class 'NoneType'>
      Given value type: <class 'jsonargparse._namespace.Namespace'>
      Given value: Namespace(value=1)
```

If I use this signature:

```Python
def __init__(self, a: A) -> None:
        self.a = a
```

Everything works as expected.

### Expected behavior

The above code should run without errors.

### Environment



- jsonargparse version (e.g., 4.8.0): 4.29.0
- Python version (e.g., 3.9): 3.9.19
- How jsonargparse was installed (e.g. `pip install jsonargparse[all]`): `pip install jsonargparse[all]`
- OS (e.g., Linux): macOS


Pull request description:
Fix: Optional pydantic model failing to parse with __pydantic_private__ error
## What does this PR do?

Fixes #521

## Before submitting

- [x] Did you read the [contributing guideline](https://github.com/omni-us/jsonargparse/blob/main/CONTRIBUTING.rst)?
- [n/a] Did you update **the documentation**? (readme and public docstrings)
- [x] Did you write **unit tests** such that there is 100% coverage on related code? (required for bug fixes and new features)
- [x] Did you verify that new and existing **tests pass locally**?
- [x] Did you make sure that all changes preserve **backward compatibility**?
- [x] Did you update **the CHANGELOG**? (not for typos, docs, test updates, or minor internal changes/refactors)


Patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 0b49968..51f6ecb 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -40,6 +40,8 @@ Fixed
 - List of union of classes not accepted by ``add_subclass_arguments`` in
   ``python>=3.11`` (`#522
   <https://github.com/omni-us/jsonargparse/pull/522>`__).
+- Optional pydantic model failing to parse with `__pydantic_private__` error
+  (`#521 <https://github.com/omni-us/jsonargparse/issues/521>`__).
 
 
 v4.29.0 (2024-05-24)
diff --git a/jsonargparse/_signatures.py b/jsonargparse/_signatures.py
index adc3494..c66871d 100644
--- a/jsonargparse/_signatures.py
+++ b/jsonargparse/_signatures.py
@@ -16,6 +16,7 @@ from ._common import (
     is_dataclass_like,
     is_subclass,
 )
+from ._namespace import Namespace
 from ._optionals import get_doc_short_description, is_pydantic_model, pydantic_support
 from ._parameter_resolvers import (
     ParamData,
@@ -29,7 +30,7 @@ from ._typehints import (
     get_subclasses_from_type,
     is_optional,
 )
-from ._util import get_private_kwargs, iter_to_set_str
+from ._util import NoneType, get_private_kwargs, iter_to_set_str
 from .typing import register_pydantic_type
 
 __all__ = [
@@ -51,7 +52,7 @@ class SignatureArguments(LoggerProperty):
         nested_key: Optional[str] = None,
         as_group: bool = True,
         as_positional: bool = False,
-        default: Optional[LazyInitBaseClass] = None,
+        default: Optional[Union[dict, Namespace, LazyInitBaseClass]] = None,
         skip: Optional[Set[Union[str, int]]] = None,
         instantiate: bool = True,
         fail_untyped: bool = True,
@@ -67,7 +68,7 @@ class SignatureArguments(LoggerProperty):
             nested_key: Key for nested namespace.
             as_group: Whether arguments should be added to a new argument group.
             as_positional: Whether to add required parameters as positional arguments.
-            default: Default value used to override parameter defaults. Must be lazy_instance.
+            default: Default value used to override parameter defaults.
             skip: Names of parameters or number of positionals that should be skipped.
             instantiate: Whether the class group should be instantiated by :code:`instantiate_classes`.
             fail_untyped: Whether to raise exception if a required parameter does not have a type.
@@ -81,9 +82,14 @@ class SignatureArguments(LoggerProperty):
             ValueError: When there are required parameters without at least one valid type.
         """
         if not inspect.isclass(get_generic_origin(get_unaliased_type(theclass))):
-            raise ValueError(f'Expected "theclass" parameter to be a class type, got: {theclass}.')
-        if default and not (isinstance(default, LazyInitBaseClass) and isinstance(default, theclass)):
-            raise ValueError(f'Expected "default" parameter to be a lazy instance of the class, got: {default}.')
+            raise ValueError(f"Expected 'theclass' parameter to be a class type, got: {theclass}")
+        if not (
+            isinstance(default, (NoneType, dict, Namespace))
+            or (isinstance(default, LazyInitBaseClass) and isinstance(default, theclass))
+        ):
+            raise ValueError(
+                f"Expected 'default' parameter to be a dict, Namespace or lazy instance of the class, got: {default}"
+            )
         linked_targets = get_private_kwargs(kwargs, linked_targets=None)
 
         added_args = self._add_signature_arguments(
@@ -102,9 +108,13 @@ class SignatureArguments(LoggerProperty):
         if default:
             skip = skip or set()
             prefix = nested_key + "." if nested_key else ""
-            defaults = default.lazy_get_init_args()
+            defaults = default
+            if isinstance(default, LazyInitBaseClass):
+                defaults = default.lazy_get_init_args().as_dict()
+            elif isinstance(default, Namespace):
+                defaults = default.as_dict()
             if defaults:
-                defaults = {prefix + k: v for k, v in defaults.__dict__.items() if k not in skip}
+                defaults = {prefix + k: v for k, v in defaults.items() if k not in skip}
                 self.set_defaults(**defaults)  # type: ignore[attr-defined]
 
         return added_args
diff --git a/jsonargparse/_typehints.py b/jsonargparse/_typehints.py
index 37a6090..afcc291 100644
--- a/jsonargparse/_typehints.py
+++ b/jsonargparse/_typehints.py
@@ -960,7 +960,7 @@ def adapt_typehints(
     elif is_dataclass_like(typehint):
         if isinstance(prev_val, (dict, Namespace)):
             assert isinstance(sub_add_kwargs, dict)
-            sub_add_kwargs["default"] = lazy_instance(typehint, **prev_val)
+            sub_add_kwargs["default"] = prev_val
         parser = ActionTypeHint.get_class_parser(typehint, sub_add_kwargs=sub_add_kwargs)
         if instantiate_classes:
             init_args = parser.instantiate_classes(val)


Test patch:
diff --git a/jsonargparse_tests/test_dataclass_like.py b/jsonargparse_tests/test_dataclass_like.py
index 995f740..9d884c4 100644
--- a/jsonargparse_tests/test_dataclass_like.py
+++ b/jsonargparse_tests/test_dataclass_like.py
@@ -680,6 +680,10 @@ if pydantic_support:
         class PydanticAnnotatedField(pydantic.BaseModel):
             p1: annotated[int, pydantic.Field(default=2, ge=1, le=8)]  # type: ignore[valid-type]
 
+    class OptionalPydantic:
+        def __init__(self, a: Optional[PydanticModel] = None):
+            self.a = a
+
 
 def none(x):
     return x
@@ -797,6 +801,22 @@ class TestPydantic:
         cfg = parser.parse_args(["--data", '{"p3": {"p1": 1.0}}'])
         assert cfg.data == Namespace(p3=Namespace(p1=1.0, p2="-"))
 
+    def test_optional_pydantic_model(self, parser):
+        parser.add_argument("--b", type=OptionalPydantic)
+        parser.add_argument("--cfg", action="config")
+        cfg = parser.parse_args([f"--b={__name__}.OptionalPydantic"])
+        assert cfg.b.class_path == f"{__name__}.OptionalPydantic"
+        assert cfg.b.init_args == Namespace(a=None)
+        config = {
+            "b": {
+                "class_path": f"{__name__}.OptionalPydantic",
+                "init_args": {"a": {"p1": "x"}},
+            }
+        }
+        cfg = parser.parse_args([f"--cfg={config}"])
+        assert cfg.b.class_path == f"{__name__}.OptionalPydantic"
+        assert cfg.b.init_args == Namespace(a=Namespace(p1="x", p2=3))
+
 
 # attrs tests
 
diff --git a/jsonargparse_tests/test_signatures.py b/jsonargparse_tests/test_signatures.py
index 6bcc6f2..fbb2a3d 100644
--- a/jsonargparse_tests/test_signatures.py
+++ b/jsonargparse_tests/test_signatures.py
@@ -80,7 +80,7 @@ class Class3(Class2):
 def test_add_class_failure_not_a_class(parser):
     with pytest.raises(ValueError) as ctx:
         parser.add_class_arguments("Not a class")
-    ctx.match('Expected "theclass" parameter to be a class')
+    ctx.match("Expected 'theclass' parameter to be a class")
 
 
 def test_add_class_failure_positional_without_type(parser):