Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/docs/sphinx_api_docs_source/public_api_report.py b/docs/sphinx_api_docs_source/public_api_report.py
--- a/docs/sphinx_api_docs_source/public_api_report.py
+++ b/docs/sphinx_api_docs_source/public_api_report.py
@@ -1844,7 +1844,7 @@ def main():
     # any methods or classes you are adding to documentation with the @public_api
     # decorator and any relevant "new" or "deprecated" public api decorators.
     # If the actual is lower than the threshold, please reduce the threshold.
-    PUBLIC_API_MISSING_THRESHOLD = 101  # TODO: reduce this number again once this works for the Fluent DS dynamic methods
+    PUBLIC_API_MISSING_THRESHOLD = 100  # TODO: reduce this number again once this works for the Fluent DS dynamic methods
     if len(printable_definitions) != PUBLIC_API_MISSING_THRESHOLD:
         error_msg_prefix = f"There are {len(printable_definitions)} items missing from the public API, we currently allow {PUBLIC_API_MISSING_THRESHOLD}."
         if len(printable_definitions) > PUBLIC_API_MISSING_THRESHOLD:
diff --git a/great_expectations/datasource/fluent/config.py b/great_expectations/datasource/fluent/config.py
--- a/great_expectations/datasource/fluent/config.py
+++ b/great_expectations/datasource/fluent/config.py
@@ -1,16 +1,33 @@
 """POC for loading config."""
 from __future__ import annotations
 
+import json
 import logging
 import pathlib
+from io import StringIO
 from pprint import pformat as pf
-from typing import TYPE_CHECKING, Dict, List, Type, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    ClassVar,
+    Dict,
+    List,
+    Set,
+    Tuple,
+    Type,
+    Union,
+    overload,
+)
 
 from pydantic import Extra, Field, ValidationError, validator
+from ruamel.yaml import YAML
 from typing_extensions import Final
 
+from great_expectations.core._docs_decorators import public_api
 from great_expectations.datasource.fluent.fluent_base_model import (
     FluentBaseModel,
+    _recursively_set_config_value,
 )
 from great_expectations.datasource.fluent.interfaces import (
     Datasource,  # noqa: TCH001
@@ -24,10 +41,22 @@
 if TYPE_CHECKING:
     from pydantic.error_wrappers import ErrorDict as PydanticErrorDict
 
+    from great_expectations.core.config_provider import _ConfigurationProvider
+    from great_expectations.datasource.fluent.fluent_base_model import (
+        AbstractSetIntStr,
+        MappingIntStrAny,
+    )
+
 
 logger = logging.getLogger(__name__)
 
 
+yaml = YAML(typ="safe")
+# NOTE (kilo59): the following settings appear to be what we use in existing codebase
+yaml.indent(mapping=2, sequence=4, offset=2)
+yaml.default_flow_style = False
+
+
 _FLUENT_STYLE_DESCRIPTION: Final[str] = "Fluent Datasources"
 
 _MISSING_FLUENT_DATASOURCES_ERRORS: Final[List[PydanticErrorDict]] = [
@@ -46,6 +75,14 @@ class GxConfig(FluentBaseModel):
         ..., description=_FLUENT_STYLE_DESCRIPTION
     )
 
+    _EXCLUDE_FROM_DATASOURCE_SERIALIZATION: ClassVar[Set[str]] = {
+        "name",  # The "name" field is set in validation upon deserialization from configuration key; hence, it should not be serialized.
+    }
+
+    _EXCLUDE_FROM_DATA_ASSET_SERIALIZATION: ClassVar[Set[str]] = {
+        "name",  # The "name" field is set in validation upon deserialization from configuration key; hence, it should not be serialized.
+    }
+
     @property
     def datasources(self) -> Dict[str, Datasource]:
         return self.fluent_datasources
@@ -53,6 +90,7 @@ def datasources(self) -> Dict[str, Datasource]:
     class Config:
         extra = Extra.ignore  # ignore any old style config keys
 
+    # noinspection PyNestedDecorators
     @validator("fluent_datasources", pre=True)
     @classmethod
     def _load_datasource_subtype(cls, v: Dict[str, dict]):
@@ -139,3 +177,239 @@ def parse_yaml(
 
         # noinspection PyTypeChecker
         return super().parse_yaml(f)
+
+    @overload
+    def yaml(
+        self,
+        stream_or_path: Union[StringIO, None] = None,
+        *,
+        include: Union[AbstractSetIntStr, MappingIntStrAny, None] = ...,
+        exclude: Union[AbstractSetIntStr, MappingIntStrAny, None] = ...,
+        by_alias: bool = ...,
+        exclude_unset: bool = ...,
+        exclude_defaults: bool = ...,
+        exclude_none: bool = ...,
+        encoder: Union[Callable[[Any], Any], None] = ...,
+        models_as_dict: bool = ...,
+        **yaml_kwargs,
+    ) -> str:
+        ...
+
+    @overload
+    def yaml(
+        self,
+        stream_or_path: pathlib.Path,
+        *,
+        include: Union[AbstractSetIntStr, MappingIntStrAny, None] = ...,
+        exclude: Union[AbstractSetIntStr, MappingIntStrAny, None] = ...,
+        by_alias: bool = ...,
+        exclude_unset: bool = ...,
+        exclude_defaults: bool = ...,
+        exclude_none: bool = ...,
+        encoder: Union[Callable[[Any], Any], None] = ...,
+        models_as_dict: bool = ...,
+        **yaml_kwargs,
+    ) -> pathlib.Path:
+        ...
+
+    @public_api
+    def yaml(
+        self,
+        stream_or_path: Union[StringIO, pathlib.Path, None] = None,
+        *,
+        include: Union[AbstractSetIntStr, MappingIntStrAny, None] = None,
+        exclude: Union[AbstractSetIntStr, MappingIntStrAny, None] = None,
+        by_alias: bool = False,
+        exclude_unset: bool = True,
+        exclude_defaults: bool = False,
+        exclude_none: bool = False,
+        encoder: Union[Callable[[Any], Any], None] = None,
+        models_as_dict: bool = True,
+        **yaml_kwargs,
+    ) -> Union[str, pathlib.Path]:
+        """
+        Serialize the config object as yaml.
+        Writes to a file if a `pathlib.Path` is provided.
+        Else it writes to a stream and returns a yaml string.
+        """
+        if stream_or_path is None:
+            stream_or_path = StringIO()
+
+        intermediate_json_dict = self._json_dict(
+            include=include,
+            exclude=exclude,
+            by_alias=by_alias,
+            exclude_unset=exclude_unset,
+            exclude_defaults=exclude_defaults,
+            exclude_none=exclude_none,
+            encoder=encoder,
+            models_as_dict=models_as_dict,
+        )
+        intermediate_json_dict = self._exclude_name_fields_from_fluent_datasources(
+            config=intermediate_json_dict
+        )
+        yaml.dump(intermediate_json_dict, stream=stream_or_path, **yaml_kwargs)
+
+        if isinstance(stream_or_path, pathlib.Path):
+            return stream_or_path
+
+        return stream_or_path.getvalue()
+
+    @public_api
+    def json(
+        self,
+        *,
+        include: AbstractSetIntStr | MappingIntStrAny | None = None,
+        exclude: AbstractSetIntStr | MappingIntStrAny | None = None,
+        by_alias: bool = False,
+        # deprecated - use exclude_unset instead
+        skip_defaults: bool | None = None,
+        # Default to True to prevent serializing long configs full of unset default values
+        exclude_unset: bool = True,
+        exclude_defaults: bool = False,
+        exclude_none: bool = False,
+        encoder: Callable[[Any], Any] | None = None,
+        models_as_dict: bool = True,
+        **dumps_kwargs: Any,
+    ) -> str:
+        """
+        Generate a JSON representation of the model, `include` and `exclude` arguments
+        as per `dict()`.
+        `encoder` is an optional function to supply as `default` to json.dumps(), other
+        arguments as per `json.dumps()`.
+        Deviates from pydantic `exclude_unset` `True` by default instead of `False` by
+        default.
+        """
+        intermediate_json_dict = self._json_dict(
+            include=include,
+            exclude=exclude,
+            by_alias=by_alias,
+            exclude_unset=exclude_unset,
+            exclude_defaults=exclude_defaults,
+            exclude_none=exclude_none,
+            encoder=encoder,
+            models_as_dict=models_as_dict,
+        )
+        intermediate_json_dict = self._exclude_name_fields_from_fluent_datasources(
+            config=intermediate_json_dict
+        )
+        return self.__config__.json_dumps(
+            intermediate_json_dict, default=encoder, **dumps_kwargs
+        )
+
+    def _json_dict(
+        self,
+        *,
+        include: Union[AbstractSetIntStr, MappingIntStrAny, None] = None,
+        exclude: Union[AbstractSetIntStr, MappingIntStrAny, None] = None,
+        by_alias: bool = False,
+        exclude_unset: bool = True,
+        exclude_defaults: bool = False,
+        exclude_none: bool = False,
+        encoder: Union[Callable[[Any], Any], None] = None,
+        models_as_dict: bool = True,
+        **dumps_kwargs,
+    ) -> dict:
+        """
+        JSON compatible dictionary. All complex types removed.
+        Prefer `.dict()` or `.json()`
+        """
+        return json.loads(
+            super().json(
+                include=include,
+                exclude=exclude,
+                by_alias=by_alias,
+                exclude_unset=exclude_unset,
+                exclude_defaults=exclude_defaults,
+                exclude_none=exclude_none,
+                encoder=encoder,
+                models_as_dict=models_as_dict,
+                **dumps_kwargs,
+            )
+        )
+
+    @public_api
+    def dict(
+        self,
+        *,
+        include: AbstractSetIntStr | MappingIntStrAny | None = None,
+        exclude: AbstractSetIntStr | MappingIntStrAny | None = None,
+        by_alias: bool = False,
+        # Default to True to prevent serializing long configs full of unset default values
+        exclude_unset: bool = True,
+        exclude_defaults: bool = False,
+        exclude_none: bool = False,
+        # deprecated - use exclude_unset instead
+        skip_defaults: bool | None = None,
+        # custom
+        config_provider: _ConfigurationProvider | None = None,
+    ) -> dict[str, Any]:
+        """
+        Generate a dictionary representation of the model, optionally specifying which
+        fields to include or exclude.
+        Deviates from pydantic `exclude_unset` `True` by default instead of `False` by
+        default.
+        """
+        intermediate_dict: dict = super().dict(
+            include=include,
+            exclude=exclude,
+            by_alias=by_alias,
+            exclude_unset=exclude_unset,
+            exclude_defaults=exclude_defaults,
+            exclude_none=exclude_none,
+            skip_defaults=skip_defaults,
+        )
+        result: dict = self._exclude_name_fields_from_fluent_datasources(
+            config=intermediate_dict
+        )
+
+        if config_provider:
+            logger.info(
+                f"{self.__class__.__name__}.dict() - substituting config values"
+            )
+            _recursively_set_config_value(data=result, config_provider=config_provider)
+
+        return result
+
+    def _exclude_name_fields_from_fluent_datasources(
+        self, config: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        if "fluent_datasources" in config:
+            fluent_datasources: dict = config["fluent_datasources"]
+
+            datasource_name: str
+            datasource_config: dict
+            for datasource_name, datasource_config in fluent_datasources.items():
+                datasource_config = _exclude_fields_from_serialization(
+                    source_dict=datasource_config,
+                    exclusions=self._EXCLUDE_FROM_DATASOURCE_SERIALIZATION,
+                )
+                if "assets" in datasource_config:
+                    data_assets: dict = datasource_config["assets"]
+                    data_asset_name: str
+                    data_asset_config: dict
+                    data_assets = {
+                        data_asset_name: _exclude_fields_from_serialization(
+                            source_dict=data_asset_config,
+                            exclusions=self._EXCLUDE_FROM_DATA_ASSET_SERIALIZATION,
+                        )
+                        for data_asset_name, data_asset_config in data_assets.items()
+                    }
+                    datasource_config["assets"] = data_assets
+
+                fluent_datasources[datasource_name] = datasource_config
+
+        return config
+
+
+def _exclude_fields_from_serialization(
+    source_dict: Dict[str, Any], exclusions: Set[str]
+) -> Dict[str, Any]:
+    element: Tuple[str, Any]
+    # noinspection PyTypeChecker
+    return dict(
+        filter(
+            lambda element: element[0] not in exclusions,
+            source_dict.items(),
+        )
+    )
diff --git a/great_expectations/datasource/fluent/fluent_base_model.py b/great_expectations/datasource/fluent/fluent_base_model.py
--- a/great_expectations/datasource/fluent/fluent_base_model.py
+++ b/great_expectations/datasource/fluent/fluent_base_model.py
@@ -22,6 +22,7 @@
 import pydantic
 from ruamel.yaml import YAML
 
+from great_expectations.core._docs_decorators import public_api
 from great_expectations.datasource.fluent.config_str import ConfigStr
 from great_expectations.datasource.fluent.constants import _FIELDS_ALWAYS_SET
 
@@ -101,6 +102,7 @@ def yaml(
     ) -> pathlib.Path:
         ...
 
+    @public_api
     def yaml(
         self,
         stream_or_path: Union[StringIO, pathlib.Path, None] = None,
@@ -142,6 +144,7 @@ def yaml(
             return stream_or_path
         return stream_or_path.getvalue()
 
+    @public_api
     def json(
         self,
         *,
@@ -213,6 +216,7 @@ def _json_dict(
             )
         )
 
+    @public_api
     def dict(
         self,
         *,


Test patch:
diff --git a/tests/datasource/fluent/test_config.py b/tests/datasource/fluent/test_config.py
--- a/tests/datasource/fluent/test_config.py
+++ b/tests/datasource/fluent/test_config.py
@@ -141,10 +141,8 @@
     "fluent_datasources": {
         DEFAULT_PANDAS_DATASOURCE_NAME: {
             "type": "pandas",
-            "name": DEFAULT_PANDAS_DATASOURCE_NAME,
             "assets": {
                 DEFAULT_PANDAS_DATA_ASSET_NAME: {
-                    "name": DEFAULT_PANDAS_DATA_ASSET_NAME,
                     "type": "csv",
                     "filepath_or_buffer": CSV_PATH
                     / "yellow_tripdata_sample_2018-04.csv",
@@ -152,7 +150,6 @@
                     "names": ["col1", "col2"],
                 },
                 "my_csv_asset": {
-                    "name": "my_csv_asset",
                     "type": "csv",
                     "filepath_or_buffer": CSV_PATH
                     / "yellow_tripdata_sample_2018-04.csv",
@@ -190,13 +187,15 @@ class TestExcludeUnsetAssetFields:
     """
 
     def test_from_datasource(self, asset_dict: dict):
+        asset_dict_config = copy.deepcopy(asset_dict)
+
         ds_mapping = {"csv": "pandas_filesystem", "json": "pandas_filesystem"}
 
-        ds_type_: str = ds_mapping[asset_dict["type"]]
+        ds_type_: str = ds_mapping[asset_dict_config["type"]]
         ds_class = _SourceFactories.type_lookup[ds_type_]
 
         # fill in required args
-        asset_dict.update(
+        asset_dict_config.update(
             {
                 "name": "my_asset",
                 "batching_regex": re.compile(
@@ -204,14 +203,14 @@ def test_from_datasource(self, asset_dict: dict):
                 ),
             }
         )
-        asset_name = asset_dict["name"]
+        asset_name = asset_dict_config["name"]
         ds_dict = {
             "name": "my_ds",
             "base_directory": pathlib.Path(__file__),
-            "assets": {asset_name: asset_dict},
+            "assets": {asset_name: asset_dict_config},
         }
         datasource: Datasource = ds_class.parse_obj(ds_dict)
-        assert asset_dict == datasource.dict()["assets"][asset_name]
+        assert asset_dict_config == datasource.dict()["assets"][asset_name]
 
     def test_from_gx_config(self, asset_dict: dict):
         """
@@ -220,18 +219,17 @@ def test_from_gx_config(self, asset_dict: dict):
         # fill in required args
         asset_dict.update(
             {
-                "name": "my_asset",
                 "batching_regex": re.compile(
                     r"sample_(?P<year>\d{4})-(?P<month>\d{2}).csv"
                 ),
             }
         )
-        asset_name = asset_dict["name"]
+        asset_dict_config = copy.deepcopy(asset_dict)
+
         ds_dict = {
-            "name": "my_ds",
             "type": "pandas_filesystem",
             "base_directory": pathlib.Path(__file__),
-            "assets": {asset_name: asset_dict},
+            "assets": {"my_asset": asset_dict_config},
         }
         gx_config = GxConfig.parse_obj({"fluent_datasources": {"my_ds": ds_dict}})
 
@@ -239,7 +237,7 @@ def test_from_gx_config(self, asset_dict: dict):
         print(f"gx_config_dict\n{pf(gx_config_dict)}")
         assert (
             asset_dict
-            == gx_config_dict["fluent_datasources"]["my_ds"]["assets"][asset_name]
+            == gx_config_dict["fluent_datasources"]["my_ds"]["assets"]["my_asset"]
         )
 
 
@@ -432,6 +430,11 @@ def test_dict_config_round_trip(
     dumped: dict = from_dict_gx_config.dict()
     print(f"  Dumped Dict ->\n\n{pf(dumped)}\n")
 
+    assert "name" not in dumped["fluent_datasources"].keys()
+    for datasource in dumped["fluent_datasources"].values():
+        for asset in datasource["assets"].values():
+            assert "name" not in asset.keys()
+
     re_loaded: GxConfig = GxConfig.parse_obj(dumped)
     pp(re_loaded)
     assert re_loaded
@@ -533,6 +536,10 @@ def test_custom_sorter_serialization(
 
 def test_dict_default_pandas_config_round_trip(inject_engine_lookup_double):
     # the default data asset should be dropped, but one named asset should remain
+    datasource_without_default_pandas_data_asset_config_dict = copy.deepcopy(
+        DEFAULT_PANDAS_DATASOURCE_AND_DATA_ASSET_CONFIG_DICT
+    )
+
     from_dict_default_pandas_config = GxConfig.parse_obj(
         DEFAULT_PANDAS_DATASOURCE_AND_DATA_ASSET_CONFIG_DICT
     )
@@ -546,9 +553,6 @@ def test_dict_default_pandas_config_round_trip(inject_engine_lookup_double):
     dumped: dict = from_dict_default_pandas_config.dict()
     print(f"  Dumped Dict ->\n\n{pf(dumped)}\n")
 
-    datasource_without_default_pandas_data_asset_config_dict = copy.deepcopy(
-        DEFAULT_PANDAS_DATASOURCE_AND_DATA_ASSET_CONFIG_DICT
-    )
     datasource_without_default_pandas_data_asset_config_dict["fluent_datasources"][
         DEFAULT_PANDAS_DATASOURCE_NAME
     ]["assets"].pop(DEFAULT_PANDAS_DATA_ASSET_NAME)