Repository: great-expectations/great_expectations
Pull request #7009 (2023-01-31 16:04:20)

[BUGFIX ] ZEP - pandas serde fix
Follow-up to #6780 


## Changes proposed in this pull request:
- [x] change default serialization method behavior to `exclude_unset` fields by default
    * modify `__fields_set__` as needed to make this work.
- [x] change the `_load_asset_subtype` validator to be no longer be a `pre` validator and also validate `each`
- [x] thorough config serialization testing of pandas data assets


### Definition of Done
Please delete options that are not relevant.

- [x] My code follows the Great Expectations [style guide](https://docs.greatexpectations.io/docs/contributing/style_guides/code_style)
- [x] I have performed a [self-review](https://docs.greatexpectations.io/docs/contributing/contributing_checklist) of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I have made corresponding changes to the documentation
- [x] I have added [unit tests](https://docs.greatexpectations.io/docs/contributing/contributing_test#writing-unit-and-integration-tests) where applicable and made sure that new and existing tests are passing.
- [x] I have run any local integration tests and made sure that nothing is broken.


netlify[bot]: ### <span aria-hidden="true">👷</span> Deploy Preview for *niobium-lead-7998* processing.


|  Name | Link |
|---------------------------------|------------------------|
|<span aria-hidden="true">🔨</span> Latest commit | 7289cda404fe86fa814003a5ad3e3b23f59843f3 |
|<span aria-hidden="true">🔍</span> Latest deploy log | https://app.netlify.com/sites/niobium-lead-7998/deploys/63d93c07dab951000862a3c7 |
codesee-maps[bot]: <details>
<summary><b>👇 <a href="https://app.codesee.io/r/reviews?pr=7009&src=https%3A%2F%2Fgithub.com%2Fgreat-expectations%2Fgreat_expectations">Click on the image</a> for a new way to code review</b>
</summary>

  - Make big changes easier — review code in small groups of related files
  - Know where to start — see the whole change at a glance
  - Take a code tour — explore the change with an interactive tour
  - Make comments and review — all fully sync’ed with github

    [Try it now!](https://app.codesee.io/r/reviews?pr=7009&src=https%3A%2F%2Fgithub.com%2Fgreat-expectations%2Fgreat_expectations)
</details>

[![Review these changes using an interactive CodeSee Map](https://s3.us-east-2.amazonaws.com/maps.codesee.io/images/github/great-expectations/great_expectations/7009/5606e02d/7289cda404fe86fa814003a5ad3e3b23f59843f3.svg)](https://app.codesee.io/r/reviews?pr=7009&src=https%3A%2F%2Fgithub.com%2Fgreat-expectations%2Fgreat_expectations)

#### Legend
![CodeSee Map Legend](https://app.codesee.io/Legend_v2.png)

Patch:
diff --git a/great_expectations/experimental/datasources/constants.py b/great_expectations/experimental/datasources/constants.py
new file mode 100644
index 000000000..e77b9cd80
--- /dev/null
+++ b/great_expectations/experimental/datasources/constants.py
@@ -0,0 +1,6 @@
+from typing_extensions import Final
+
+# this field must be added to `__fields_set__` before pydantic model serialization
+# methods are called. Otherwise it could be excluded.
+# https://docs.pydantic.dev/usage/exporting_models/#modeldict
+_FIELD_ALWAYS_SET: Final[str] = "assets"
diff --git a/great_expectations/experimental/datasources/dynamic_pandas.py b/great_expectations/experimental/datasources/dynamic_pandas.py
index b3a7b3eb2..8fd3e3339 100644
--- a/great_expectations/experimental/datasources/dynamic_pandas.py
+++ b/great_expectations/experimental/datasources/dynamic_pandas.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import enum
 import functools
 import inspect
 import logging
@@ -49,6 +50,14 @@ except ImportError:
     IndexLabel = Union[Hashable, Sequence[Hashable]]
     StorageOptions = Optional[Dict[str, Any]]
 
+try:
+    from pandas._libs.lib import _NoDefault
+except ImportError:
+
+    class _NoDefault(enum.Enum):  # type: ignore[no-redef]
+        no_default = "NO_DEFAULT"
+
+
 logger = logging.getLogger(__file__)
 
 DataFrameFactoryFn: TypeAlias = Callable[..., pd.DataFrame]
@@ -180,6 +189,8 @@ def _get_default_value(
 ) -> object:
     if param.default is inspect.Parameter.empty:
         default = ...
+    elif param.default is _NoDefault.no_default:
+        default = None
     else:
         default = param.default
     return default
@@ -256,9 +267,12 @@ def _create_pandas_asset_model(
     model_base: M,
     type_field: Tuple[Union[Type, str], str],
     fields_dict: Dict[str, _FieldSpec],
+    extra: pydantic.Extra,
 ) -> M:
     """https://docs.pydantic.dev/usage/models/#dynamic-model-creation"""
     model = pydantic.create_model(model_name, __base__=model_base, type=type_field, **fields_dict)  # type: ignore[call-overload] # FieldSpec is a tuple
+    # can't set both __base__ & __config__ when dynamically creating model
+    model.__config__.extra = extra
     return model
 
 
@@ -284,6 +298,7 @@ def _generate_data_asset_models(
                 model_base=base_model_class,
                 type_field=(f"Literal['{type_name}']", type_name),
                 fields_dict=fields,
+                extra=pydantic.Extra.forbid,
             )
             logger.debug(f"{model_name}\n{pf(fields)}")
         except NameError as err:
diff --git a/great_expectations/experimental/datasources/experimental_base_model.py b/great_expectations/experimental/datasources/experimental_base_model.py
index 6111e0bec..7ea24777c 100644
--- a/great_expectations/experimental/datasources/experimental_base_model.py
+++ b/great_expectations/experimental/datasources/experimental_base_model.py
@@ -20,6 +20,8 @@ from typing import (
 import pydantic
 from ruamel.yaml import YAML
 
+from great_expectations.experimental.datasources.constants import _FIELD_ALWAYS_SET
+
 if TYPE_CHECKING:
     MappingIntStrAny = Mapping[Union[int, str], Any]
     AbstractSetIntStr = AbstractSet[Union[int, str]]
@@ -37,6 +39,19 @@ _Self = TypeVar("_Self", bound="ExperimentalBaseModel")
 
 
 class ExperimentalBaseModel(pydantic.BaseModel):
+    """
+    Base model for most ZEP pydantic models.
+
+    Adds yaml dumping and parsing methods.
+
+    Extra fields are not allowed.
+
+    Serialization methods default to `exclude_unset = True` to prevent serializing
+    configs full of mostly unset default values.
+    Also prevents passing along unset kwargs to BatchSpec.
+    https://docs.pydantic.dev/usage/exporting_models/
+    """
+
     class Config:
         extra = pydantic.Extra.forbid
 
@@ -49,11 +64,37 @@ class ExperimentalBaseModel(pydantic.BaseModel):
         return config
 
     @overload
-    def yaml(self, stream_or_path: Union[StringIO, None] = None, **yaml_kwargs) -> str:
+    def yaml(
+        self,
+        stream_or_path: Union[StringIO, None] = None,
+        *,
+        include: Union[AbstractSetIntStr, MappingIntStrAny, None] = ...,
+        exclude: Union[AbstractSetIntStr, MappingIntStrAny, None] = ...,
+        by_alias: bool = ...,
+        exclude_unset: bool = ...,
+        exclude_defaults: bool = ...,
+        exclude_none: bool = ...,
+        encoder: Union[Callable[[Any], Any], None] = ...,
+        models_as_dict: bool = ...,
+        **yaml_kwargs,
+    ) -> str:
         ...
 
     @overload
-    def yaml(self, stream_or_path: pathlib.Path, **yaml_kwargs) -> pathlib.Path:
+    def yaml(
+        self,
+        stream_or_path: pathlib.Path,
+        *,
+        include: Union[AbstractSetIntStr, MappingIntStrAny, None] = ...,
+        exclude: Union[AbstractSetIntStr, MappingIntStrAny, None] = ...,
+        by_alias: bool = ...,
+        exclude_unset: bool = ...,
+        exclude_defaults: bool = ...,
+        exclude_none: bool = ...,
+        encoder: Union[Callable[[Any], Any], None] = ...,
+        models_as_dict: bool = ...,
+        **yaml_kwargs,
+    ) -> pathlib.Path:
         ...
 
     def yaml(
@@ -63,8 +104,7 @@ class ExperimentalBaseModel(pydantic.BaseModel):
         include: Union[AbstractSetIntStr, MappingIntStrAny, None] = None,
         exclude: Union[AbstractSetIntStr, MappingIntStrAny, None] = None,
         by_alias: bool = False,
-        skip_defaults: Union[bool, None] = None,
-        exclude_unset: bool = False,
+        exclude_unset: bool = True,
         exclude_defaults: bool = False,
         exclude_none: bool = False,
         encoder: Union[Callable[[Any], Any], None] = None,
@@ -86,7 +126,6 @@ class ExperimentalBaseModel(pydantic.BaseModel):
             include=include,
             exclude=exclude,
             by_alias=by_alias,
-            skip_defaults=skip_defaults,
             exclude_unset=exclude_unset,
             exclude_defaults=exclude_defaults,
             exclude_none=exclude_none,
@@ -99,14 +138,53 @@ class ExperimentalBaseModel(pydantic.BaseModel):
             return stream_or_path
         return stream_or_path.getvalue()
 
+    def json(
+        self,
+        *,
+        include: AbstractSetIntStr | MappingIntStrAny | None = None,
+        exclude: AbstractSetIntStr | MappingIntStrAny | None = None,
+        by_alias: bool = False,
+        # deprecated - use exclude_unset instead
+        skip_defaults: bool | None = None,
+        # Default to True to prevent serializing long configs full of unset default values
+        exclude_unset: bool = True,
+        exclude_defaults: bool = False,
+        exclude_none: bool = False,
+        encoder: Callable[[Any], Any] | None = None,
+        models_as_dict: bool = True,
+        **dumps_kwargs: Any,
+    ) -> str:
+        """
+        Generate a JSON representation of the model, `include` and `exclude` arguments
+        as per `dict()`.
+
+        `encoder` is an optional function to supply as `default` to json.dumps(), other
+        arguments as per `json.dumps()`.
+
+        Deviates from pydantic `exclude_unset` `True` by default instead of `False` by
+        default.
+        """
+        self.__fields_set__.add(_FIELD_ALWAYS_SET)
+        return super().json(
+            include=include,
+            exclude=exclude,
+            by_alias=by_alias,
+            skip_defaults=skip_defaults,
+            exclude_unset=exclude_unset,
+            exclude_defaults=exclude_defaults,
+            exclude_none=exclude_none,
+            encoder=encoder,
+            models_as_dict=models_as_dict,
+            **dumps_kwargs,
+        )
+
     def _json_dict(
         self,
         *,
         include: Union[AbstractSetIntStr, MappingIntStrAny, None] = None,
         exclude: Union[AbstractSetIntStr, MappingIntStrAny, None] = None,
         by_alias: bool = False,
-        skip_defaults: Union[bool, None] = None,
-        exclude_unset: bool = False,
+        exclude_unset: bool = True,
         exclude_defaults: bool = False,
         exclude_none: bool = False,
         encoder: Union[Callable[[Any], Any], None] = None,
@@ -122,7 +200,6 @@ class ExperimentalBaseModel(pydantic.BaseModel):
                 include=include,
                 exclude=exclude,
                 by_alias=by_alias,
-                skip_defaults=skip_defaults,
                 exclude_unset=exclude_unset,
                 exclude_defaults=exclude_defaults,
                 exclude_none=exclude_none,
@@ -132,5 +209,36 @@ class ExperimentalBaseModel(pydantic.BaseModel):
             )
         )
 
+    def dict(
+        self,
+        *,
+        include: AbstractSetIntStr | MappingIntStrAny | None = None,
+        exclude: AbstractSetIntStr | MappingIntStrAny | None = None,
+        by_alias: bool = False,
+        # Default to True to prevent serializing long configs full of unset default values
+        exclude_unset: bool = True,
+        exclude_defaults: bool = False,
+        exclude_none: bool = False,
+        # deprecated - use exclude_unset instead
+        skip_defaults: bool | None = None,
+    ) -> dict[str, Any]:
+        """
+        Generate a dictionary representation of the model, optionally specifying which
+        fields to include or exclude.
+
+        Deviates from pydantic `exclude_unset` `True` by default instead of `False` by
+        default.
+        """
+        self.__fields_set__.add(_FIELD_ALWAYS_SET)
+        return super().dict(
+            include=include,
+            exclude=exclude,
+            by_alias=by_alias,
+            exclude_unset=exclude_unset,
+            exclude_defaults=exclude_defaults,
+            exclude_none=exclude_none,
+            skip_defaults=skip_defaults,
+        )
+
     def __str__(self):
         return self.yaml()
diff --git a/great_expectations/experimental/datasources/interfaces.py b/great_expectations/experimental/datasources/interfaces.py
index e7eaeac3e..d301ab62a 100644
--- a/great_expectations/experimental/datasources/interfaces.py
+++ b/great_expectations/experimental/datasources/interfaces.py
@@ -26,6 +26,7 @@ from typing_extensions import ClassVar, TypeAlias, TypeGuard
 
 import great_expectations.exceptions as ge_exceptions
 from great_expectations.core.id_dict import BatchSpec
+from great_expectations.experimental.datasources.constants import _FIELD_ALWAYS_SET
 from great_expectations.experimental.datasources.experimental_base_model import (
     ExperimentalBaseModel,
 )
@@ -345,22 +346,32 @@ class Datasource(
     _cached_execution_engine_kwargs: Dict[str, Any] = pydantic.PrivateAttr({})
     _execution_engine: ExecutionEngine | None = pydantic.PrivateAttr(None)
 
-    @pydantic.validator("assets", pre=True)
-    def _load_asset_subtype(cls, v: Dict[str, dict]):
-        logger.info(f"Loading 'assets' ->\n{pf(v, depth=3)}")
-        loaded_assets: Dict[str, DataAssetType] = {}
+    @pydantic.validator("assets", each_item=True)
+    @classmethod
+    def _load_asset_subtype(
+        cls: Type[Datasource[DataAssetType]], data_asset: DataAsset
+    ) -> DataAssetType:
+        """
+        Some `data_asset` may be loaded as a less specific asset subtype different than
+        what was intended.
+        If a more specific subtype is needed the `data_asset` will be converted to a
+        more specific `DataAsset`.
+        """
+        logger.info(f"Loading '{data_asset.name}' asset ->\n{pf(data_asset, depth=4)}")
+        asset_type_name: str = data_asset.type
+        asset_type: Type[DataAssetType] = _SourceFactories.type_lookup[asset_type_name]
+
+        if asset_type is type(data_asset):
+            # asset is already the intended type
+            return data_asset
 
-        # TODO (kilo59): catch key errors
-        for asset_name, config in v.items():
-            asset_type_name: str = config["type"]
-            asset_type: Type[DataAssetType] = _SourceFactories.type_lookup[
-                asset_type_name
-            ]
-            logger.debug(f"Instantiating '{asset_type_name}' as {asset_type}")
-            loaded_assets[asset_name] = asset_type(**config)
+        # strip out asset default kwargs
+        kwargs = data_asset.dict(exclude_unset=True)
+        logger.debug(f"{asset_type_name} - kwargs\n{pf(kwargs)}")
 
-        logger.debug(f"Loaded 'assets' ->\n{repr(loaded_assets)}")
-        return loaded_assets
+        asset_of_intended_type = asset_type(**kwargs)
+        logger.debug(f"{asset_type_name} - {repr(asset_of_intended_type)}")
+        return asset_of_intended_type
 
     def _execution_engine_type(self) -> Type[ExecutionEngine]:
         """Returns the execution engine to be used"""
@@ -414,6 +425,9 @@ class Datasource(
         asset._datasource = self
         asset.test_connection()
         self.assets[asset.name] = asset
+        # pydantic needs to know that an asset has been set so that it doesn't get excluded
+        # when dumping to dict, json, yaml etc.
+        self.__fields_set__.add(_FIELD_ALWAYS_SET)
         return asset
 
     # Abstract Methods
diff --git a/great_expectations/experimental/datasources/pandas_datasource.py b/great_expectations/experimental/datasources/pandas_datasource.py
index c8beeabb2..63a059185 100644
--- a/great_expectations/experimental/datasources/pandas_datasource.py
+++ b/great_expectations/experimental/datasources/pandas_datasource.py
@@ -4,7 +4,7 @@ import copy
 import logging
 import pathlib
 import re
-from typing import TYPE_CHECKING, Dict, List, Optional, Pattern, Tuple, Type, Union
+from typing import TYPE_CHECKING, Dict, List, Optional, Pattern, Set, Tuple, Type, Union
 
 import pydantic
 from typing_extensions import ClassVar, Literal
@@ -36,6 +36,14 @@ class PandasDatasourceError(Exception):
 
 
 class _DataFrameAsset(DataAsset):
+    # Pandas specific class attrs
+    _EXCLUDE_FROM_READER_OPTIONS: ClassVar[Set[str]] = {
+        "name",
+        "base_directory",
+        "regex",
+        "order_by",
+    }
+
     # Pandas specific attributes
     base_directory: pathlib.Path
     regex: Pattern
@@ -45,6 +53,16 @@ class _DataFrameAsset(DataAsset):
         default="batch_request_param_"
     )
 
+    class Config:
+        """
+        Need to allow extra fields for the base type because pydantic will first create
+        an instance of `_DataFrameAsset` before we select and create the more specific
+        asset subtype.
+        Each specific subtype should `forbid` extra fields.
+        """
+
+        extra = pydantic.Extra.allow
+
     def test_connection(self) -> None:
         """Test the connection for the CSVAsset.
 
@@ -159,13 +177,7 @@ class _DataFrameAsset(DataAsset):
                 path=str(path),
                 reader_method=f"read_{self.type}",
                 reader_options=self.dict(
-                    exclude_unset=True,
-                    exclude={  # TODO: don't hardcode
-                        "name",
-                        "base_directory",
-                        "regex",
-                        "order_by",
-                    },
+                    exclude_unset=True, exclude=self._EXCLUDE_FROM_READER_OPTIONS
                 ),
             )
             execution_engine: ExecutionEngine = self.datasource.get_execution_engine()
@@ -239,7 +251,16 @@ class PandasDatasource(Datasource):
     # instance attributes
     type: Literal["pandas"] = "pandas"
     name: str
-    assets: Dict[str, Union[CSVAsset, ExcelAsset, ParquetAsset, JSONAsset]] = {}  # type: ignore[valid-type]
+    assets: Dict[  # type: ignore[valid-type]
+        str,
+        Union[
+            _DataFrameAsset,
+            CSVAsset,
+            ExcelAsset,
+            ParquetAsset,
+            JSONAsset,
+        ],
+    ] = {}
 
     @property
     def execution_engine_type(self) -> Type[ExecutionEngine]:


Test patch:
diff --git a/tests/experimental/datasources/test_config.py b/tests/experimental/datasources/test_config.py
index 6a76db469..e13e0a811 100644
--- a/tests/experimental/datasources/test_config.py
+++ b/tests/experimental/datasources/test_config.py
@@ -1,6 +1,7 @@
 import functools
 import json
 import pathlib
+import re
 from pprint import pformat as pf
 from typing import Callable, List
 from unittest import mock
@@ -11,6 +12,7 @@ import pytest
 from great_expectations.data_context import FileDataContext
 from great_expectations.experimental.datasources.config import GxConfig
 from great_expectations.experimental.datasources.interfaces import Datasource
+from great_expectations.experimental.datasources.sources import _SourceFactories
 from great_expectations.experimental.datasources.sql_datasource import (
     ColumnSplitter,
     SqlYearMonthSplitter,
@@ -69,7 +71,28 @@ PG_COMPLEX_CONFIG_DICT = {
                     "type": "table",
                 },
             },
-        }
+        },
+        "my_pandas_ds": {
+            "type": "pandas",
+            "name": "my_pandas_ds",
+            "assets": {
+                "my_csv_asset": {
+                    "name": "my_csv_asset",
+                    "type": "csv",
+                    "base_directory": __file__,
+                    "regex": r"yellow_tripdata_sample_(?P<year>\d{4})-(?P<month>\d{2}).csv",
+                    "sep": "|",
+                    "names": ["col1", "col2"],
+                },
+                "my_json_asset": {
+                    "name": "my_json_asset",
+                    "type": "json",
+                    "base_directory": __file__,
+                    "regex": r"yellow_tripdata_sample_(?P<year>\d{4})-(?P<month>\d{2}).json",
+                    "orient": "records",
+                },
+            },
+        },
     }
 }
 PG_COMPLEX_CONFIG_JSON = json.dumps(PG_COMPLEX_CONFIG_DICT)
@@ -118,6 +141,63 @@ COMBINED_ZEP_AND_OLD_STYLE_CFG_DICT = {
 }
 
 
+@pytest.mark.parametrize(
+    "asset_dict", [{"type": "json", "orient": "records"}, {"type": "csv", "sep": "|"}]
+)
+class TestExcludeUnsetAssetFields:
+    """
+    Ensure that DataAsset fields are excluded from serialization if they have not be explicitly set.
+
+    We are trying to ensure that our configs aren't filled with default values from DataAssets that
+    users never set.
+    """
+
+    def test_from_datasource(self, asset_dict: dict):
+        ds_mapping = {"csv": "pandas", "json": "pandas"}
+
+        ds_type_: str = ds_mapping[asset_dict["type"]]
+        ds_class = _SourceFactories.type_lookup[ds_type_]
+
+        # fill in required args
+        asset_dict.update(
+            {
+                "name": "my_asset",
+                "base_directory": pathlib.Path(__file__),
+                "regex": re.compile(r"sample_(?P<year>\d{4})-(?P<month>\d{2}).csv"),
+            }
+        )
+        asset_name = asset_dict["name"]
+        ds_dict = {"name": "my_ds", "assets": {asset_name: asset_dict}}
+        datasource: Datasource = ds_class.parse_obj(ds_dict)
+        assert asset_dict == datasource.dict()["assets"][asset_name]
+
+    def test_from_gx_config(self, asset_dict: dict):
+        """
+        Ensure that unset fields are excluded even when being parsed by the the top-level `GxConfig` class.
+        """
+        # fill in required args
+        asset_dict.update(
+            {
+                "name": "my_asset",
+                "base_directory": pathlib.Path(__file__),
+                "regex": re.compile(r"sample_(?P<year>\d{4})-(?P<month>\d{2}).csv"),
+            }
+        )
+        asset_name = asset_dict["name"]
+        ds_dict = {
+            "name": "my_ds",
+            "type": "pandas",
+            "assets": {asset_name: asset_dict},
+        }
+        gx_config = GxConfig.parse_obj({"xdatasources": {"my_ds": ds_dict}})
+
+        gx_config_dict = gx_config.dict()
+        print(f"gx_config_dict\n{pf(gx_config_dict)}")
+        assert (
+            asset_dict == gx_config_dict["xdatasources"]["my_ds"]["assets"][asset_name]
+        )
+
+
 @pytest.mark.parametrize(
     ["load_method", "input_"],
     [
@@ -190,7 +270,7 @@ def test_catch_bad_top_level_config(
     [
         p(
             {"name": "missing `table_name`", "type": "table"},
-            ("xdatasources", "assets", "table_name"),
+            ("xdatasources", "assets", "missing `table_name`", "table_name"),
             "field required",
             id="missing `table_name`",
         ),
@@ -204,7 +284,13 @@ def test_catch_bad_top_level_config(
                     "column_name": "foo",
                 },
             },
-            ("xdatasources", "assets", "column_splitter", "method_name"),
+            (
+                "xdatasources",
+                "assets",
+                "unknown splitter",
+                "column_splitter",
+                "method_name",
+            ),
             "unexpected value; permitted: 'split_on_year_and_month'",
             id="unknown splitter method",
         ),
@@ -219,7 +305,14 @@ def test_catch_bad_top_level_config(
                     "param_names": ["year", "month", "INVALID"],
                 },
             },
-            ("xdatasources", "assets", "column_splitter", "param_names", 2),
+            (
+                "xdatasources",
+                "assets",
+                "bad splitter param",
+                "column_splitter",
+                "param_names",
+                2,
+            ),
             "unexpected value; permitted: 'year', 'month'",
             id="invalid splitter param_name",
         ),
@@ -338,7 +431,7 @@ def test_json_config_round_trip(
     pp(re_loaded)
     assert re_loaded
 
-    assert from_json_gx_config == re_loaded
+    assert from_json_gx_config.dict() == re_loaded.dict()
 
 
 def test_yaml_config_round_trip(
@@ -352,7 +445,7 @@ def test_yaml_config_round_trip(
     pp(re_loaded)
     assert re_loaded
 
-    assert from_yaml_gx_config == re_loaded
+    assert from_yaml_gx_config.dict() == re_loaded.dict()
 
 
 def test_yaml_file_config_round_trip(
diff --git a/tests/experimental/datasources/test_pandas_datasource.py b/tests/experimental/datasources/test_pandas_datasource.py
index 330a6d7be..9ca911be4 100644
--- a/tests/experimental/datasources/test_pandas_datasource.py
+++ b/tests/experimental/datasources/test_pandas_datasource.py
@@ -2,9 +2,10 @@ from __future__ import annotations
 
 import logging
 import pathlib
+import re
 from dataclasses import dataclass
 from pprint import pformat as pf
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Type
 
 import pydantic
 import pytest
@@ -14,7 +15,9 @@ import great_expectations.exceptions as ge_exceptions
 import great_expectations.execution_engine.pandas_execution_engine
 from great_expectations.experimental.datasources.pandas_datasource import (
     CSVAsset,
+    JSONAsset,
     PandasDatasource,
+    _DataFrameAsset,
 )
 
 if TYPE_CHECKING:
@@ -91,7 +94,7 @@ class TestDynamicPandasAssets:
             for t in assets_field.type_.__args__  # accessing the `Union` members with `__args__`
         }
 
-        assert asset_class_names == asset_field_union_members
+        assert asset_class_names.issubset(asset_field_union_members)
 
     @pytest.mark.parametrize(
         "method_name",
@@ -129,6 +132,32 @@ class TestDynamicPandasAssets:
 
         assert type_name in asset_class_names
 
+    @pytest.mark.parametrize(
+        ["asset_model", "extra_kwargs"],
+        [
+            (CSVAsset, {"sep": "|", "names": ["col1", "col2", "col3"]}),
+            (JSONAsset, {"orient": "records", "encoding_errors": "strict"}),
+        ],
+    )
+    def test_data_asset_defaults(
+        self,
+        asset_model: Type[_DataFrameAsset],
+        extra_kwargs: dict,
+    ):
+        """
+        Test that an asset dictionary can be dumped with only the original passed keys
+        present.
+        """
+        kwargs: dict[str, Any] = {
+            "name": "test",
+            "base_directory": pathlib.Path(__file__),
+            "regex": re.compile(r"yellow_tripdata_sample_(\d{4})-(\d{2})"),
+        }
+        kwargs.update(extra_kwargs)
+        print(f"extra_kwargs\n{pf(extra_kwargs)}")
+        asset_instance = asset_model(**kwargs)
+        assert asset_instance.dict() == kwargs
+
     @pytest.mark.parametrize(
         "extra_kwargs",
         [
@@ -385,15 +414,15 @@ def test_pandas_sorter(
 
     ordered_years = reversed(years) if "-year" in order_by else years
     ordered_months = reversed(months) if "-month" in order_by else months
-    if "year" in order_by[0]:
+    if "year" in order_by[0]:  # type: ignore[operator]
         ordered = [
-            TimeRange(key="year", range=ordered_years),
-            TimeRange(key="month", range=ordered_months),
+            TimeRange(key="year", range=ordered_years),  # type: ignore[arg-type]
+            TimeRange(key="month", range=ordered_months),  # type: ignore[arg-type]
         ]
     else:
         ordered = [
-            TimeRange(key="month", range=ordered_months),
-            TimeRange(key="year", range=ordered_years),
+            TimeRange(key="month", range=ordered_months),  # type: ignore[arg-type]
+            TimeRange(key="year", range=ordered_years),  # type: ignore[arg-type]
         ]
 
     batch_index = -1