Repository: marshmallow-code/apispec (python)

Issue:
Incorrect handling of marshmallow nested schema with allow_none=True in OpenAPI 3.1
If I define a marshmallow schema like this:
```python
from marshmallow import fields, Schema

class FooSchema(Schema):
    bar = fields.Integer(required=True)

class MySchema(Schema):
    foo = fields.Nested(FooSchema, allow_none=True)
```
it behaves as I intend - `MySchema` can load objects with a `foo` field that either matches `FooSchema` or is null, but `FooSchema` itself doesn't allow null values:
```python
>>> MySchema().loads('{"foo": {"bar": 1}}')
{'foo': {'bar': 1}}
>>> MySchema().loads('{"foo": null}')
{'foo': None}
>>> FooSchema().loads('null')
marshmallow.exceptions.ValidationError: {'_schema': ['Invalid input type.']}
```

But if I generate an API spec with this schema, using openapi version 3.1:
```python
from apispec import APISpec
from apispec.ext.marshmallow import MarshmallowPlugin

spec = APISpec(
    title="My API Spec",
    version="1.0.0",
    openapi_version="3.1.0",
    plugins=[MarshmallowPlugin()],
)
spec.components.schema("MySchema", schema=MySchema)
```
the resulting API spec is not equivalent to my marshmallow schema:
```python
>>> print(json.dumps(spec.to_dict()["components"]["schemas"], indent=2))
{
  "Foo": {
    "type": "object",
    "properties": {
      "bar": {
        "type": "integer"
      }
    },
    "required": [
      "bar"
    ]
  },
  "MySchema": {
    "type": "object",
    "properties": {
      "foo": {
        "type": [
          "null"
        ],
        "allOf": [
          {
            "$ref": "#/components/schemas/Foo"
          }
        ]
      }
    }
  }
}
```
This spec says that the value of `foo` _must_ be null, AND must be a `Foo` object, so in fact there is no valid input value here.

I think a correct spec in this instance should contain:
```
{
  "Foo": {
    "type": "object",
    "properties": {
      "bar": {
        "type": "integer"
      }
    },
    "required": [
      "bar"
    ]
  },
  "MySchema": {
    "type": "object",
    "properties": {
      "foo": {
        "anyOf": [
          {
            "$ref": "#/components/schemas/Foo"
          },
          {
            "type": "null"
          }
        ]
      }
    }
  }
}
```

I'm using apispec 6.3.0 and marshmallow 3.19.0

Pull request description:
Fix nested nullable for 3.1.0 (updated)
same as #898 with changelog and authors update


closes #845
closes #833 


Patch:
diff --git a/AUTHORS.rst b/AUTHORS.rst
index 0ab09da..16fb499 100644
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -81,4 +81,5 @@ Contributors (chronological)
 - Sebastien Lovergne `@TheBigRoomXXL <https://github.com/TheBigRoomXXL>`_
 - Luna Lovegood `@duchuyvp <https://github.com/duchuyvp>`_
 - Tobias Kolditz `@kolditz-senec <https://github.com/kolditz-senec>`_
-- Christian Proud `@cjproud <https://github.com/cjproud>`_
\ No newline at end of file
+- Christian Proud `@cjproud <https://github.com/cjproud>`_
+- `<https://github.com/kolditz-senec >`_
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index dab6aed..ed754c0 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,10 +1,19 @@
 Changelog
 ---------
 
+6.6.1 (unreleased)
+******************
+
+Bug fixes:
+
+- ``MarshmallowPlugin``: Fix handling of ``Nested`` fields with 
+  ``allow_none=True`` (:issue:`833`). Thanks :user:`jc-harrison`
+  for reporting and :user:`kolditz-senec` for the PR.
+
 6.6.0 (2024-03-15)
 ******************
 
-Featyres:
+Features:
 
 - Add IP fields to `DEFAULT_FIELD_MAPPING (:pr:`892`) to document format.
   Thanks :user:`cjproud` for the PR.
diff --git a/src/apispec/ext/marshmallow/field_converter.py b/src/apispec/ext/marshmallow/field_converter.py
index c3b1065..4e17aa2 100644
--- a/src/apispec/ext/marshmallow/field_converter.py
+++ b/src/apispec/ext/marshmallow/field_converter.py
@@ -103,7 +103,6 @@ class FieldConverterMixin:
             self.field2choices,
             self.field2read_only,
             self.field2write_only,
-            self.field2nullable,
             self.field2range,
             self.field2length,
             self.field2pattern,
@@ -115,6 +114,7 @@ class FieldConverterMixin:
             self.dict2properties,
             self.timedelta2properties,
             self.datetime2properties,
+            self.field2nullable,
         ]
 
     def map_to_openapi_type(self, field_cls, *args):
@@ -302,8 +302,15 @@ class FieldConverterMixin:
                 attributes["x-nullable"] = True
             elif self.openapi_version.minor < 1:
                 attributes["nullable"] = True
+                if "$ref" in ret:
+                    attributes["allOf"] = [{"$ref": ret.pop("$ref")}]
             else:
-                attributes["type"] = [*make_type_list(ret.get("type")), "null"]
+                if "$ref" in ret:
+                    attributes["anyOf"] = [{"$ref": ret.pop("$ref")}, {"type": "null"}]
+                elif "allOf" in ret:
+                    attributes["anyOf"] = [*ret.pop("allOf"), {"type": "null"}]
+                else:
+                    attributes["type"] = [*make_type_list(ret.get("type")), "null"]
         return attributes
 
     def field2range(self, field: marshmallow.fields.Field, ret) -> dict:


Test patch:
diff --git a/tests/test_ext_marshmallow_field.py b/tests/test_ext_marshmallow_field.py
index 1f85fc3..20e079d 100644
--- a/tests/test_ext_marshmallow_field.py
+++ b/tests/test_ext_marshmallow_field.py
@@ -3,7 +3,7 @@ import re
 from enum import Enum
 
 import pytest
-from marshmallow import fields, validate
+from marshmallow import Schema, fields, validate
 
 from .schemas import CategorySchema, CustomIntegerField, CustomList, CustomStringField
 from .utils import build_ref, get_schemas
@@ -199,6 +199,43 @@ def test_field_with_allow_none(spec_fixture):
         assert res["type"] == ["string", "null"]
 
 
+@pytest.mark.parametrize("spec_fixture", ("2.0", "3.0.0", "3.1.0"), indirect=True)
+def test_nested_nullable(spec_fixture):
+    class Child(Schema):
+        name = fields.Str()
+
+    field = fields.Nested(Child, allow_none=True)
+    res = spec_fixture.openapi.field2property(field)
+    version = spec_fixture.openapi.openapi_version
+    if version.major < 3:
+        assert res == {"$ref": "#/definitions/Child", "x-nullable": True}
+    elif version.major == 3 and version.minor < 1:
+        assert res == {
+            "nullable": True,
+            "allOf": [{"$ref": "#/components/schemas/Child"}],
+        }
+    else:
+        assert res == {
+            "anyOf": [{"$ref": "#/components/schemas/Child"}, {"type": "null"}]
+        }
+
+
+@pytest.mark.parametrize("spec_fixture", ("2.0", "3.0.0", "3.1.0"), indirect=True)
+def test_nullable_pluck(spec_fixture):
+    class Example(Schema):
+        name = fields.Str()
+
+    field = fields.Pluck(Example, "name", allow_none=True)
+    res = spec_fixture.openapi.field2property(field)
+    version = spec_fixture.openapi.openapi_version
+    if version.major < 3:
+        assert res == {"type": "string", "x-nullable": True}
+    elif version.major == 3 and version.minor < 1:
+        assert res == {"type": "string", "nullable": True}
+    else:
+        assert res == {"type": ["string", "null"]}
+
+
 def test_field_with_dump_only(spec_fixture):
     field = fields.Str(dump_only=True)
     res = spec_fixture.openapi.field2property(field)