Repository: great-expectations/great_expectations
Pull request #7056 (2023-02-06 02:01:46)

[BUGFIX] zep - always serialize type field
https://github.com/great-expectations/great_expectations/pull/7009 changed the default behavior of all ZEP class serialization methods to exclude any fields that haven't been explicitly set.
Without this change, we would end up with configs full of default field values.
However, we always need the value of `type` in order to determine the correct asset class/type to use.

## Changes proposed in this pull request:
- always serialize the `Datasource` and `DataAsset` `type` field

netlify[bot]: ### <span aria-hidden="true">👷</span> Deploy Preview for *niobium-lead-7998* processing.


|  Name | Link |
|---------------------------------|------------------------|
|<span aria-hidden="true">🔨</span> Latest commit | e4e6a3326a403f1cd4f94082767f2fbd874a0d7e |
|<span aria-hidden="true">🔍</span> Latest deploy log | https://app.netlify.com/sites/niobium-lead-7998/deploys/63e05f8c2a3cb4000876c6b9 |
codesee-maps[bot]: <details>
<summary><b>👇 <a href="https://app.codesee.io/r/reviews?pr=7056&src=https%3A%2F%2Fgithub.com%2Fgreat-expectations%2Fgreat_expectations">Click on the image</a> for a new way to code review</b>
</summary>

  - Make big changes easier — review code in small groups of related files
  - Know where to start — see the whole change at a glance
  - Take a code tour — explore the change with an interactive tour
  - Make comments and review — all fully sync’ed with github

    [Try it now!](https://app.codesee.io/r/reviews?pr=7056&src=https%3A%2F%2Fgithub.com%2Fgreat-expectations%2Fgreat_expectations)
</details>

[![Review these changes using an interactive CodeSee Map](https://s3.us-east-2.amazonaws.com/maps.codesee.io/images/github/great-expectations/great_expectations/7056/2869cc7c/e4e6a3326a403f1cd4f94082767f2fbd874a0d7e.svg)](https://app.codesee.io/r/reviews?pr=7056&src=https%3A%2F%2Fgithub.com%2Fgreat-expectations%2Fgreat_expectations)

#### Legend
![CodeSee Map Legend](https://app.codesee.io/Legend_v2.png)

Patch:
diff --git a/great_expectations/experimental/datasources/constants.py b/great_expectations/experimental/datasources/constants.py
index e77b9cd80..0f366c03c 100644
--- a/great_expectations/experimental/datasources/constants.py
+++ b/great_expectations/experimental/datasources/constants.py
@@ -1,6 +1,11 @@
+from typing import Set
+
 from typing_extensions import Final
 
-# this field must be added to `__fields_set__` before pydantic model serialization
+# these fields must be added to `__fields_set__` before pydantic model serialization
 # methods are called. Otherwise it could be excluded.
 # https://docs.pydantic.dev/usage/exporting_models/#modeldict
-_FIELD_ALWAYS_SET: Final[str] = "assets"
+_FIELDS_ALWAYS_SET: Final[Set[str]] = {
+    "assets",
+    "type",
+}
diff --git a/great_expectations/experimental/datasources/experimental_base_model.py b/great_expectations/experimental/datasources/experimental_base_model.py
index 7ea24777c..0300842d5 100644
--- a/great_expectations/experimental/datasources/experimental_base_model.py
+++ b/great_expectations/experimental/datasources/experimental_base_model.py
@@ -20,7 +20,7 @@ from typing import (
 import pydantic
 from ruamel.yaml import YAML
 
-from great_expectations.experimental.datasources.constants import _FIELD_ALWAYS_SET
+from great_expectations.experimental.datasources.constants import _FIELDS_ALWAYS_SET
 
 if TYPE_CHECKING:
     MappingIntStrAny = Mapping[Union[int, str], Any]
@@ -164,7 +164,7 @@ class ExperimentalBaseModel(pydantic.BaseModel):
         Deviates from pydantic `exclude_unset` `True` by default instead of `False` by
         default.
         """
-        self.__fields_set__.add(_FIELD_ALWAYS_SET)
+        self.__fields_set__.update(_FIELDS_ALWAYS_SET)
         return super().json(
             include=include,
             exclude=exclude,
@@ -229,7 +229,7 @@ class ExperimentalBaseModel(pydantic.BaseModel):
         Deviates from pydantic `exclude_unset` `True` by default instead of `False` by
         default.
         """
-        self.__fields_set__.add(_FIELD_ALWAYS_SET)
+        self.__fields_set__.update(_FIELDS_ALWAYS_SET)
         return super().dict(
             include=include,
             exclude=exclude,
diff --git a/great_expectations/experimental/datasources/filesystem_data_asset.py b/great_expectations/experimental/datasources/filesystem_data_asset.py
index 6835ad0bc..8138d6be1 100644
--- a/great_expectations/experimental/datasources/filesystem_data_asset.py
+++ b/great_expectations/experimental/datasources/filesystem_data_asset.py
@@ -33,6 +33,7 @@ class _FilesystemDataAsset(DataAsset):
         "base_directory",
         "regex",
         "order_by",
+        "type",
     }
 
     # Filesystem specific attributes
diff --git a/great_expectations/experimental/datasources/interfaces.py b/great_expectations/experimental/datasources/interfaces.py
index e5df5af27..b5637ec31 100644
--- a/great_expectations/experimental/datasources/interfaces.py
+++ b/great_expectations/experimental/datasources/interfaces.py
@@ -26,7 +26,7 @@ from typing_extensions import TypeAlias, TypeGuard
 
 import great_expectations.exceptions as ge_exceptions
 from great_expectations.core.id_dict import BatchSpec
-from great_expectations.experimental.datasources.constants import _FIELD_ALWAYS_SET
+from great_expectations.experimental.datasources.constants import _FIELDS_ALWAYS_SET
 from great_expectations.experimental.datasources.experimental_base_model import (
     ExperimentalBaseModel,
 )
@@ -432,7 +432,7 @@ class Datasource(
         self.assets[asset.name] = asset
         # pydantic needs to know that an asset has been set so that it doesn't get excluded
         # when dumping to dict, json, yaml etc.
-        self.__fields_set__.add(_FIELD_ALWAYS_SET)
+        self.__fields_set__.update(_FIELDS_ALWAYS_SET)
         return asset
 
     # Abstract Methods


Test patch:
diff --git a/tests/experimental/datasources/test_config.py b/tests/experimental/datasources/test_config.py
index da5e5ebd5..855102359 100644
--- a/tests/experimental/datasources/test_config.py
+++ b/tests/experimental/datasources/test_config.py
@@ -441,6 +441,7 @@ def test_yaml_config_round_trip(
     assert re_loaded
 
     assert from_yaml_gx_config.dict() == re_loaded.dict()
+    assert dumped == re_loaded.yaml()
 
 
 def test_yaml_file_config_round_trip(
diff --git a/tests/experimental/datasources/test_pandas_datasource.py b/tests/experimental/datasources/test_pandas_datasource.py
index 1c4d062a4..4a71770e8 100644
--- a/tests/experimental/datasources/test_pandas_datasource.py
+++ b/tests/experimental/datasources/test_pandas_datasource.py
@@ -195,7 +195,7 @@ class TestDynamicPandasAssets:
         kwargs.update(extra_kwargs)
         print(f"extra_kwargs\n{pf(extra_kwargs)}")
         asset_instance = asset_model(**kwargs)
-        assert asset_instance.dict() == kwargs
+        assert asset_instance.dict(exclude={"type"}) == kwargs
 
     @pytest.mark.parametrize(
         "extra_kwargs",
diff --git a/tests/experimental/datasources/test_postgres_datasource.py b/tests/experimental/datasources/test_postgres_datasource.py
index cc9bd5a8a..9718fa078 100644
--- a/tests/experimental/datasources/test_postgres_datasource.py
+++ b/tests/experimental/datasources/test_postgres_datasource.py
@@ -642,6 +642,8 @@ def test_data_source_json_has_properties(create_source: CreateSourceFixture):
         source_json = source.json(indent=4, sort_keys=True)
         print(source_json)
         assert '"order_by": ' in source_json
+        # type should be in dumped json even if not explicitly set
+        assert f'"type": "{asset.type}"'
 
 
 @pytest.mark.unit
@@ -656,6 +658,8 @@ def test_data_source_yaml_has_properties(create_source: CreateSourceFixture):
         asset.add_sorters(["year", "month"])
         source_str = source.__str__()
         assert "order_by:" in source_str
+        # type should be in dumped str even if not explicitly set
+        assert f"type: {asset.type}" in source_str
 
 
 @pytest.mark.unit
@@ -671,6 +675,8 @@ def test_datasource_dict_has_properties(create_source):
         source_dict = source.dict()
         pprint(source_dict)
         assert isinstance(source_dict["assets"]["my_asset"]["order_by"], list)
+        # type should be in dumped dict even if not explicitly set
+        assert "type" in source_dict["assets"]["my_asset"]
 
 
 @pytest.mark.unit