Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/experimental/datasources/file_path_data_asset.py b/great_expectations/experimental/datasources/file_path_data_asset.py
new file mode 100644
--- /dev/null
+++ b/great_expectations/experimental/datasources/file_path_data_asset.py
@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import logging
+from typing import ClassVar, Dict, List, Optional, Pattern, Set
+
+import pydantic
+
+import great_expectations.exceptions as ge_exceptions
+from great_expectations.experimental.datasources.data_asset.data_connector.regex_parser import (
+    RegExParser,
+)
+from great_expectations.experimental.datasources.interfaces import (
+    Batch,
+    BatchRequest,
+    BatchRequestOptions,
+    DataAsset,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class _FilePathDataAsset(DataAsset):
+    _EXCLUDE_FROM_READER_OPTIONS: ClassVar[Set[str]] = {
+        "name",
+        "regex",
+        "order_by",
+        "type",
+    }
+
+    # General file-path DataAsset pertaining attributes.
+    regex: Pattern
+
+    # Internal attributes
+    _unnamed_regex_param_prefix: str = pydantic.PrivateAttr(
+        default="batch_request_param_"
+    )
+    _regex_parser: RegExParser = pydantic.PrivateAttr()
+
+    _all_group_name_to_group_index_mapping: Dict[str, int] = pydantic.PrivateAttr()
+    _all_group_index_to_group_name_mapping: Dict[int, str] = pydantic.PrivateAttr()
+    _all_group_names: List[str] = pydantic.PrivateAttr()
+
+    class Config:
+        """
+        Need to allow extra fields for the base type because pydantic will first create
+        an instance of `_FilesystemDataAsset` before we select and create the more specific
+        asset subtype.
+        Each specific subtype should `forbid` extra fields.
+        """
+
+        extra = pydantic.Extra.allow
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        self._regex_parser = RegExParser(
+            regex_pattern=self.regex,
+            unnamed_regex_group_prefix=self._unnamed_regex_param_prefix,
+        )
+
+        self._all_group_name_to_group_index_mapping = (
+            self._regex_parser.get_all_group_name_to_group_index_mapping()
+        )
+        self._all_group_index_to_group_name_mapping = (
+            self._regex_parser.get_all_group_index_to_group_name_mapping()
+        )
+        self._all_group_names = self._regex_parser.get_all_group_names()
+
+    def batch_request_options_template(
+        self,
+    ) -> BatchRequestOptions:
+        idx: int
+        return {idx: None for idx in self._all_group_names}
+
+    def get_batch_request(
+        self, options: Optional[BatchRequestOptions] = None
+    ) -> BatchRequest:
+        if options:
+            for option, value in options.items():
+                if (
+                    option in self._all_group_name_to_group_index_mapping
+                    and not isinstance(value, str)
+                ):
+                    raise ge_exceptions.InvalidBatchRequestError(
+                        f"All regex matching options must be strings. The value of '{option}' is "
+                        f"not a string: {value}"
+                    )
+
+        return super().get_batch_request(options)
+
+    def test_connection(self) -> None:
+        """Test the connection for the DataAsset.
+
+        Raises:
+            TestConnectionError: If the connection test fails.
+        """
+        raise NotImplementedError(
+            """One needs to implement "test_connection" on a _FilePathDataAsset subclass."""
+        )
+
+    def get_batch_list_from_batch_request(
+        self, batch_request: BatchRequest
+    ) -> List[Batch]:
+        raise NotImplementedError
+
+    def _get_reader_method(self) -> str:
+        raise NotImplementedError(
+            """One needs to explicitly provide "reader_method" for File-Path style DataAsset extensions as temporary \
+work-around, until "type" naming convention and method for obtaining 'reader_method' from it are established."""
+        )
+
+    def _get_reader_options_include(self) -> Set[str] | None:
+        raise NotImplementedError(
+            """One needs to explicitly provide set(str)-valued reader options for "pydantic.BaseModel.dict()" method \
+to use as its "include" directive for File-Path style DataAsset processing."""
+        )
diff --git a/great_expectations/experimental/datasources/filesystem_data_asset.py b/great_expectations/experimental/datasources/filesystem_data_asset.py
--- a/great_expectations/experimental/datasources/filesystem_data_asset.py
+++ b/great_expectations/experimental/datasources/filesystem_data_asset.py
@@ -3,20 +3,17 @@
 import copy
 import logging
 import pathlib
-from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Pattern, Set, Tuple
-
-import pydantic
+from typing import TYPE_CHECKING, ClassVar, List, Optional, Set, Tuple
 
 import great_expectations.exceptions as ge_exceptions
 from great_expectations.core.batch_spec import PathBatchSpec
-from great_expectations.experimental.datasources.data_asset.data_connector.regex_parser import (
-    RegExParser,
+from great_expectations.experimental.datasources.file_path_data_asset import (
+    _FilePathDataAsset,
 )
 from great_expectations.experimental.datasources.interfaces import (
     Batch,
     BatchRequest,
     BatchRequestOptions,
-    DataAsset,
     TestConnectionError,
 )
 
@@ -29,65 +26,26 @@
 logger = logging.getLogger(__name__)
 
 
-class _FilesystemDataAsset(DataAsset):
-    # Pandas specific class attrs
-    _EXCLUDE_FROM_READER_OPTIONS: ClassVar[Set[str]] = {
-        "name",
+class _FilesystemDataAsset(_FilePathDataAsset):
+    _EXCLUDE_FROM_READER_OPTIONS: ClassVar[
+        Set[str]
+    ] = _FilePathDataAsset._EXCLUDE_FROM_READER_OPTIONS | {
         "base_directory",
-        "regex",
-        "order_by",
-        "type",
     }
 
     # Filesystem specific attributes
     base_directory: pathlib.Path
-    regex: Pattern
-
-    # Internal attributes
-    _unnamed_regex_param_prefix: str = pydantic.PrivateAttr(
-        default="batch_request_param_"
-    )
-    _regex_parser: RegExParser = pydantic.PrivateAttr()
-
-    _all_group_name_to_group_index_mapping: Dict[str, int] = pydantic.PrivateAttr()
-    _all_group_index_to_group_name_mapping: Dict[int, str] = pydantic.PrivateAttr()
-    _all_group_names: List[str] = pydantic.PrivateAttr()
-
-    class Config:
-        """
-        Need to allow extra fields for the base type because pydantic will first create
-        an instance of `_FilesystemDataAsset` before we select and create the more specific
-        asset subtype.
-        Each specific subtype should `forbid` extra fields.
-        """
-
-        extra = pydantic.Extra.allow
-
-    def __init__(self, **data):
-        super().__init__(**data)
-        self._regex_parser = RegExParser(
-            regex_pattern=self.regex,
-            unnamed_regex_group_prefix=self._unnamed_regex_param_prefix,
-        )
-
-        self._all_group_name_to_group_index_mapping = (
-            self._regex_parser.get_all_group_name_to_group_index_mapping()
-        )
-        self._all_group_index_to_group_name_mapping = (
-            self._regex_parser.get_all_group_index_to_group_name_mapping()
-        )
-        self._all_group_names = self._regex_parser.get_all_group_names()
 
     def _get_reader_method(self) -> str:
         raise NotImplementedError(
-            """One needs to explicitly provide "reader_method" for File-Path style DataAsset extensions as temporary \
+            """One needs to explicitly provide "reader_method" for Filesystem DataAsset extensions as temporary \
 work-around, until "type" naming convention and method for obtaining 'reader_method' from it are established."""
         )
 
     def _get_reader_options_include(self) -> Set[str] | None:
         raise NotImplementedError(
             """One needs to explicitly provide set(str)-valued reader options for "pydantic.BaseModel.dict()" method \
-to use as its "include" directive for File-Path style DataAsset processing."""
+to use as its "include" directive for Filesystem style DataAsset processing."""
         )
 
     def test_connection(self) -> None:
diff --git a/great_expectations/experimental/datasources/pandas_datasource.py b/great_expectations/experimental/datasources/pandas_datasource.py
--- a/great_expectations/experimental/datasources/pandas_datasource.py
+++ b/great_expectations/experimental/datasources/pandas_datasource.py
@@ -126,9 +126,9 @@ def add_csv_asset(
         """
         asset = CSVAsset(
             name=name,
-            base_directory=base_directory,
-            regex=regex,
-            order_by=order_by or [],
+            base_directory=base_directory,  # type: ignore[arg-type]
+            regex=regex,  # type: ignore[arg-type]  # type: ignore[arg-type]
+            order_by=order_by or [],  # type: ignore[arg-type]
             **kwargs,
         )
         return self.add_asset(asset)
@@ -152,9 +152,9 @@ def add_excel_asset(
         """
         asset = ExcelAsset(
             name=name,
-            base_directory=base_directory,
-            regex=regex,
-            order_by=order_by or [],
+            base_directory=base_directory,  # type: ignore[arg-type]
+            regex=regex,  # type: ignore[arg-type]
+            order_by=order_by or [],  # type: ignore[arg-type]
             **kwargs,
         )
         return self.add_asset(asset)
@@ -178,9 +178,9 @@ def add_json_asset(
         """
         asset = JSONAsset(
             name=name,
-            base_directory=base_directory,
-            regex=regex,
-            order_by=order_by or [],
+            base_directory=base_directory,  # type: ignore[arg-type]
+            regex=regex,  # type: ignore[arg-type]
+            order_by=order_by or [],  # type: ignore[arg-type]
             **kwargs,
         )
         return self.add_asset(asset)
@@ -204,9 +204,9 @@ def add_parquet_asset(
         """
         asset = ParquetAsset(
             name=name,
-            base_directory=base_directory,
-            regex=regex,
-            order_by=order_by or [],
+            base_directory=base_directory,  # type: ignore[arg-type]
+            regex=regex,  # type: ignore[arg-type]
+            order_by=order_by or [],  # type: ignore[arg-type]
             **kwargs,
         )
         return self.add_asset(asset)


Test patch:
diff --git a/tests/experimental/datasources/test_pandas_datasource.py b/tests/experimental/datasources/test_pandas_datasource.py
--- a/tests/experimental/datasources/test_pandas_datasource.py
+++ b/tests/experimental/datasources/test_pandas_datasource.py
@@ -141,7 +141,7 @@ def test_minimal_validation(self, asset_class: Type[_FilesystemDataAsset]):
         This is also a proxy for testing that the dynamic pydantic model creation was successful.
         """
         with pytest.raises(pydantic.ValidationError) as exc_info:
-            asset_class(
+            asset_class(  # type: ignore[call-arg]
                 name="test",
                 base_directory=pathlib.Path(__file__),
                 regex=re.compile(r"yellow_tripdata_sample_(\d{4})-(\d{2})"),
@@ -243,10 +243,10 @@ def test_add_csv_asset_to_datasource(
 @pytest.mark.unit
 def test_construct_csv_asset_directly(csv_path: pathlib.Path):
     # noinspection PyTypeChecker
-    asset = CSVAsset(
+    asset = CSVAsset(  # type: ignore[call-arg]
         name="csv_asset",
         base_directory=csv_path,
-        regex=r"yellow_tripdata_sample_(\d{4})-(\d{2}).csv",
+        regex=r"yellow_tripdata_sample_(\d{4})-(\d{2}).csv",  # type: ignore[arg-type]
     )
     assert asset.name == "csv_asset"
     assert asset.base_directory == csv_path
@@ -489,7 +489,7 @@ def datasource_test_connection_error_messages(
     pandas_datasource: PandasDatasource, request
 ) -> tuple[PandasDatasource, TestConnectionError]:
     base_directory, regex, test_connection_error = request.param()
-    csv_asset = CSVAsset(
+    csv_asset = CSVAsset(  # type: ignore[call-arg]
         name="csv_asset",
         base_directory=base_directory,
         regex=regex,
diff --git a/tests/experimental/datasources/test_schemas.py b/tests/experimental/datasources/test_schemas.py
--- a/tests/experimental/datasources/test_schemas.py
+++ b/tests/experimental/datasources/test_schemas.py
@@ -1,6 +1,6 @@
 import json
 import sys
-from typing import Type
+from typing import Any, Type
 
 import pandas
 import pydantic
@@ -38,6 +38,23 @@ def test_vcs_schemas_match(zep_ds_or_asset_model: Type[pydantic.BaseModel]):
     Note: if the installed version of pandas doesn't match the one used in the standard
     test pipeline, the test will be marked a `xfail` because the schemas will not match.
     """
+
+    def _sort_required_lists(schema_as_dict: dict) -> None:
+        """Someties "required" lists come unsorted, causing misleading assertion failures; this corrects the issue.
+
+        Args:
+            schema_as_dict: source dictionary (will be modified "in-situ")
+
+        """
+        key: str
+        value: Any
+        for key, value in schema_as_dict.items():
+            if key == "required":
+                schema_as_dict[key] = sorted(value)
+
+            if isinstance(value, dict):
+                _sort_required_lists(schema_as_dict=value)
+
     if _PANDAS_SCHEMA_VERSION != PANDAS_VERSION:
         pytest.xfail(reason=f"schema generated with pandas {_PANDAS_SCHEMA_VERSION}")
 
@@ -59,6 +76,11 @@ def test_vcs_schemas_match(zep_ds_or_asset_model: Type[pydantic.BaseModel]):
 
     json_str = schema_path.read_text().rstrip()
 
+    schema_as_dict = json.loads(json_str)
+    _sort_required_lists(schema_as_dict=schema_as_dict)
+    zep_ds_or_asset_model_as_dict = zep_ds_or_asset_model.schema()
+    _sort_required_lists(schema_as_dict=zep_ds_or_asset_model_as_dict)
+
     assert (
-        json.loads(json_str) == zep_ds_or_asset_model.schema()
+        schema_as_dict == zep_ds_or_asset_model_as_dict
     ), "Schemas are out of sync. Run `invoke schema --sync`. Also check your pandas version."