Repository: great-expectations/great_expectations

Issue:
Usage of "pandas_convert_dtypes" in experimental Fabric data source and assets breaking with use in Fabric notebooks 
Overview:
Cannot successfully execute a checkpoint that validates any type of Fabric data source asset using the (experimental) Fabric data source and Fabric data assets. 

The apparent cause is the explicit usage of the "pandas_convert_dtypes" parameter in library file "experimental/datasource/fabric.py" when calling semantic-link functions (e.g. evaluate_dax(), evaluate_measure(), read_table()) as well as their their GX wrapper classes.

This specific parameter appears to have been removed by the maintainers of the semantic-link package from version 0.3.6, due to underlying improvements in the library making them redundant ([source](https://community.fabric.microsoft.com/t5/General-Discussion/Read-from-semantic-models-Semantic-Link-No-Longer-Working/m-p/3543442)). 

This issue impacts GX Fabric assets when using semantic-link versions 0.3.6 or later (current @ 0.4.0).
This does not impact when using semantic-link versions 0.3.5 or earlier. 

Steps to reproduce:
1) Import GX (current @ 0.18.3) into any Fabric notebook, while also using semantic-link 0.3.6 or later.
2) Add any Fabric data sources and assets to the GX context.
3) Configure any validation of the Fabric assets into a checkpoint definition.
4) Try and run the checkpoint using checkpoint.run().

Stacktrace example of the failed "checkpoint.run()" is below:
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[40], line 1
----> 1 result = checkpoint.run()

File /nfs4/pyenv-9b3b2969-a3b9-443f-9c90-b39a05a68ff9/lib/python3.10/site-packages/great_expectations/core/usage_statistics/usage_statistics.py:260, in usage_statistics_enabled_method.<locals>.usage_statistics_wrapped_method(*args, **kwargs)
    257         args_payload = args_payload_fn(*args, **kwargs) or {}
    258         nested_update(event_payload, args_payload)
--> 260     result = func(*args, **kwargs)
    261     message["success"] = True
    262 except Exception:

File /nfs4/pyenv-9b3b2969-a3b9-443f-9c90-b39a05a68ff9/lib/python3.10/site-packages/great_expectations/checkpoint/checkpoint.py:305, in BaseCheckpoint.run(self, template_name, run_name_template, expectation_suite_name, batch_request, validator, action_list, evaluation_parameters, runtime_configuration, validations, profilers, run_id, run_name, run_time, result_format, expectation_suite_ge_cloud_id)
    303 if len(validations) > 0:
    304     for idx, validation_dict in enumerate(validations):
--> 305         self._run_validation(
    306             substituted_runtime_config=substituted_runtime_config,
    307             async_validation_operator_results=async_validation_operator_results,
    308             async_executor=async_executor,
    309             result_format=result_format,
    310             run_id=run_id,
    311             idx=idx,
    312             validation_dict=validation_dict,
    313         )
    314 else:
    315     self._run_validation(
    316         substituted_runtime_config=substituted_runtime_config,
    317         async_validation_operator_results=async_validation_operator_results,
   (...)
    320         run_id=run_id,
    321     )

File /nfs4/pyenv-9b3b2969-a3b9-443f-9c90-b39a05a68ff9/lib/python3.10/site-packages/great_expectations/checkpoint/checkpoint.py:480, in BaseCheckpoint._run_validation(self, substituted_runtime_config, async_validation_operator_results, async_executor, result_format, run_id, idx, validation_dict)
    475 if include_rendered_content is None:
    476     include_rendered_content = (
    477         self._data_context._determine_if_expectation_validation_result_include_rendered_content()
    478     )
--> 480 validator: Validator = self._validator or self.data_context.get_validator(
    481     batch_request=batch_request,
    482     expectation_suite_name=expectation_suite_name
    483     if not self._using_cloud_context
    484     else None,
    485     expectation_suite_ge_cloud_id=(
    486         expectation_suite_ge_cloud_id if self._using_cloud_context else None
    487     ),
    488     include_rendered_content=include_rendered_content,
    489 )
    491 action_list: Sequence[ActionDict] | None = substituted_validation_dict.get(
    492     "action_list"
    493 )
    494 runtime_configuration_validation = substituted_validation_dict.get(
    495     "runtime_configuration", {}
    496 )

File /nfs4/pyenv-9b3b2969-a3b9-443f-9c90-b39a05a68ff9/lib/python3.10/site-packages/great_expectations/data_context/data_context/abstract_data_context.py:2366, in AbstractDataContext.get_validator(***failed resolving arguments***)
   2362         batch_request_list = [batch_request]  # type: ignore[list-item]
   2364     for batch_request in batch_request_list:
   2365         batch_list.extend(
-> 2366             self.get_batch_list(
   2367                 datasource_name=datasource_name,
   2368                 data_connector_name=data_connector_name,
   2369                 data_asset_name=data_asset_name,
   2370                 batch_request=batch_request,
   2371                 batch_data=batch_data,
   2372                 data_connector_query=data_connector_query,
   2373                 batch_identifiers=batch_identifiers,
   2374                 limit=limit,
   2375                 index=index,
   2376                 custom_filter_function=custom_filter_function,
   2377                 sampling_method=sampling_method,
   2378                 sampling_kwargs=sampling_kwargs,
   2379                 splitter_method=splitter_method,
   2380                 splitter_kwargs=splitter_kwargs,
   2381                 runtime_parameters=runtime_parameters,
   2382                 query=query,
   2383                 path=path,
   2384                 batch_filter_parameters=batch_filter_parameters,
   2385                 batch_spec_passthrough=batch_spec_passthrough,
   2386                 **kwargs,
   2387             )
   2388         )
   2390 return self.get_validator_using_batch_list(
   2391     expectation_suite=expectation_suite,  # type: ignore[arg-type]
   2392     batch_list=batch_list,
   2393     include_rendered_content=include_rendered_content,
   2394 )

File /nfs4/pyenv-9b3b2969-a3b9-443f-9c90-b39a05a68ff9/lib/python3.10/site-packages/great_expectations/core/usage_statistics/usage_statistics.py:260, in usage_statistics_enabled_method.<locals>.usage_statistics_wrapped_method(*args, **kwargs)
    257         args_payload = args_payload_fn(*args, **kwargs) or {}
    258         nested_update(event_payload, args_payload)
--> 260     result = func(*args, **kwargs)
    261     message["success"] = True
    262 except Exception:

File /nfs4/pyenv-9b3b2969-a3b9-443f-9c90-b39a05a68ff9/lib/python3.10/site-packages/great_expectations/data_context/data_context/abstract_data_context.py:2535, in AbstractDataContext.get_batch_list(self, datasource_name, data_connector_name, data_asset_name, batch_request, batch_data, data_connector_query, batch_identifiers, limit, index, custom_filter_function, sampling_method, sampling_kwargs, splitter_method, splitter_kwargs, runtime_parameters, query, path, batch_filter_parameters, batch_spec_passthrough, batch_request_options, **kwargs)
   2457 @public_api
   2458 @usage_statistics_enabled_method(
   2459     event_name=UsageStatsEvents.DATA_CONTEXT_GET_BATCH_LIST,
   (...)
   2484     **kwargs: Optional[dict],
   2485 ) -> List[Batch]:
   2486     """Get the list of zero or more batches, based on a variety of flexible input types.
   2487 
   2488     `get_batch_list` is the main user-facing API for getting batches.
   (...)
   2533 
   2534     """
-> 2535     return self._get_batch_list(
   2536         datasource_name=datasource_name,
   2537         data_connector_name=data_connector_name,
   2538         data_asset_name=data_asset_name,
   2539         batch_request=batch_request,
   2540         batch_data=batch_data,
   2541         data_connector_query=data_connector_query,
   2542         batch_identifiers=batch_identifiers,
   2543         limit=limit,
   2544         index=index,
   2545         custom_filter_function=custom_filter_function,
   2546         sampling_method=sampling_method,
   2547         sampling_kwargs=sampling_kwargs,
   2548         splitter_method=splitter_method,
   2549         splitter_kwargs=splitter_kwargs,
   2550         runtime_parameters=runtime_parameters,
   2551         query=query,
   2552         path=path,
   2553         batch_filter_parameters=batch_filter_parameters,
   2554         batch_spec_passthrough=batch_spec_passthrough,
   2555         batch_request_options=batch_request_options,
   2556         **kwargs,
   2557     )

File /nfs4/pyenv-9b3b2969-a3b9-443f-9c90-b39a05a68ff9/lib/python3.10/site-packages/great_expectations/data_context/data_context/abstract_data_context.py:2616, in AbstractDataContext._get_batch_list(self, datasource_name, data_connector_name, data_asset_name, batch_request, batch_data, data_connector_query, batch_identifiers, limit, index, custom_filter_function, sampling_method, sampling_kwargs, splitter_method, splitter_kwargs, runtime_parameters, query, path, batch_filter_parameters, batch_spec_passthrough, batch_request_options, **kwargs)
   2609 if not datasource:
   2610     raise gx_exceptions.DatasourceError(
   2611         datasource_name,
   2612         "The given datasource could not be retrieved from the DataContext; "
   2613         "please confirm that your configuration is accurate.",
   2614     )
-> 2616 return datasource.get_batch_list_from_batch_request(batch_request=result)

File /nfs4/pyenv-9b3b2969-a3b9-443f-9c90-b39a05a68ff9/lib/python3.10/site-packages/great_expectations/datasource/fluent/interfaces.py:474, in Datasource.get_batch_list_from_batch_request(self, batch_request)
    464 """A list of batches that correspond to the BatchRequest.
    465 
    466 Args:
   (...)
    471     A list of batches that match the options specified in the batch request.
    472 """
    473 data_asset = self.get_asset(batch_request.data_asset_name)
--> 474 return data_asset.get_batch_list_from_batch_request(batch_request)

File /nfs4/pyenv-9b3b2969-a3b9-443f-9c90-b39a05a68ff9/lib/python3.10/site-packages/great_expectations/experimental/datasource/fabric.py:100, in _PowerBIAsset.get_batch_list_from_batch_request(self, batch_request)
     98 # TODO: update get_batch_data_and_markers types
     99 execution_engine: PandasExecutionEngine = self.datasource.get_execution_engine()
--> 100 data, markers = execution_engine.get_batch_data_and_markers(
    101     batch_spec=batch_spec
    102 )
    104 # batch_definition (along with batch_spec and markers) is only here to satisfy a
    105 # legacy constraint when computing usage statistics in a validator. We hope to remove
    106 # it in the future.
    107 # imports are done inline to prevent a circular dependency with core/batch.py
    108 from great_expectations.core import IDDict

File /nfs4/pyenv-9b3b2969-a3b9-443f-9c90-b39a05a68ff9/lib/python3.10/site-packages/great_expectations/execution_engine/pandas_execution_engine.py:360, in PandasExecutionEngine.get_batch_data_and_markers(self, batch_spec)
    358         elif isinstance(batch_spec, FabricBatchSpec):
    359             reader_fn = batch_spec.get_reader_function()
--> 360             df = reader_fn(**batch_spec.reader_options)
    362         else:
    363             raise gx_exceptions.BatchSpecError(
    364                 f"""batch_spec must be of type RuntimeDataBatchSpec, PandasBatchSpec, PathBatchSpec, S3BatchSpec, AzureBatchSpec or FabricBatchSpec \
    365 not {batch_spec.__class__.__name__}"""
    366             )

File /nfs4/pyenv-9b3b2969-a3b9-443f-9c90-b39a05a68ff9/lib/python3.10/site-packages/sempy/_utils/_log.py:262, in mds_log.<locals>.get_wrapper.<locals>.log_decorator_wrapper(*args, **kwargs)
    259     extra.update(extractor.get_initialization_mds_fields())
    261     s = inspect.signature(func)
--> 262     arg_dict = s.bind(*args, **kwargs).arguments
    264 except Exception:
    265     extractor.logger.error(message, extra, exc_info=True)

File ~/cluster-env/trident_env/lib/python3.10/inspect.py:3186, in Signature.bind(self, *args, **kwargs)
   3181 def bind(self, /, *args, **kwargs):
   3182     """Get a BoundArguments object, that maps the passed `args`
   3183     and `kwargs` to the function's signature.  Raises `TypeError`
   3184     if the passed arguments can not be bound.
   3185     """
-> 3186     return self._bind(args, kwargs)

File ~/cluster-env/trident_env/lib/python3.10/inspect.py:3175, in Signature._bind(self, args, kwargs, partial)
   3173         arguments[kwargs_param.name] = kwargs
   3174     else:
-> 3175         raise TypeError(
   3176             'got an unexpected keyword argument {arg!r}'.format(
   3177                 arg=next(iter(kwargs))))
   3179 return self._bound_arguments_cls(self, arguments)

TypeError: got an unexpected keyword argument 'pandas_convert_dtypes'

Expected behavior is that a well defined validation checkpoint is successfully executed against Fabric data assets and results are returned.

 - Operating System: N/A
 - Great Expectations Version: 0.18.3
 - Data Source: Fabric Semantic Model
 - Cloud environment: Fabric PySpark (with semantic-link version 0.3.6 or later)

The Fabric assets are obviously currently experimental functionality and some breakage is to be anticipated.

Thanks!


Discussion:

Resolving patch:
diff --git a/great_expectations/experimental/datasource/fabric.py b/great_expectations/experimental/datasource/fabric.py
--- a/great_expectations/experimental/datasource/fabric.py
+++ b/great_expectations/experimental/datasource/fabric.py
@@ -49,6 +49,7 @@
 SortersDefinition: TypeAlias = List[Union[Sorter, str, dict]]
 
 _REQUIRED_FABRIC_SERVICE: Final[str] = "Microsoft.ProjectArcadia"
+Mode: TypeAlias = Literal["xmla", "rest", "onelake"]
 
 
 class _PowerBIAsset(DataAsset):
@@ -186,7 +187,6 @@ class PowerBIDax(_PowerBIAsset):
 
     type: Literal["powerbi_dax"] = "powerbi_dax"
     dax_string: str
-    pandas_convert_dtypes: bool = True
 
 
 @public_api
@@ -201,7 +201,6 @@ class PowerBIMeasure(_PowerBIAsset):
     filters: Optional[Dict[str, List[str]]] = None
     fully_qualified_columns: Optional[bool] = None
     num_rows: Optional[int] = None
-    pandas_convert_dtypes: bool = True
     use_xmla: bool = False
 
 
@@ -216,7 +215,7 @@ class PowerBITable(_PowerBIAsset):
     fully_qualified_columns: bool = False
     num_rows: Optional[int] = None
     multiindex_hierarchies: bool = False
-    pandas_convert_dtypes: bool = True
+    mode: Mode = "xmla"
 
 
 # This improves our error messages by providing a more specific type for pydantic to validate against
@@ -293,13 +292,12 @@ def test_connection(self, test_assets: bool = True) -> None:
                 asset.test_connection()
 
     @public_api
-    def add_powerbi_dax_asset(  # noqa: PLR0913
+    def add_powerbi_dax_asset(
         self,
         name: str,
         dax_string: str,
         order_by: Optional[SortersDefinition] = None,
         batch_metadata: Optional[BatchMetadata] = None,
-        pandas_convert_dtypes: bool = True,
     ) -> PowerBIDax:
         """Adds a PowerBIDax asset to this datasource.
 
@@ -318,7 +316,6 @@ def add_powerbi_dax_asset(  # noqa: PLR0913
             order_by=order_by_sorters,
             batch_metadata=batch_metadata or {},
             dax_string=dax_string,
-            pandas_convert_dtypes=pandas_convert_dtypes,
         )
         return self._add_asset(asset)
 
@@ -333,7 +330,6 @@ def add_powerbi_measure_asset(  # noqa: PLR0913
         filters: Optional[Dict[str, List[str]]] = None,
         fully_qualified_columns: Optional[bool] = None,
         num_rows: Optional[int] = None,
-        pandas_convert_dtypes: bool = True,
         use_xmla: bool = False,
     ) -> PowerBIMeasure:
         """Adds a PowerBIMeasure asset to this datasource.
@@ -357,7 +353,6 @@ def add_powerbi_measure_asset(  # noqa: PLR0913
             filters=filters,
             fully_qualified_columns=fully_qualified_columns,
             num_rows=num_rows,
-            pandas_convert_dtypes=pandas_convert_dtypes,
             use_xmla=use_xmla,
         )
         return self._add_asset(asset)
@@ -372,7 +367,7 @@ def add_powerbi_table_asset(  # noqa: PLR0913
         fully_qualified_columns: bool = False,
         num_rows: Optional[int] = None,
         multiindex_hierarchies: bool = False,
-        pandas_convert_dtypes: bool = True,
+        mode: Mode = "xmla",
     ) -> PowerBITable:
         """Adds a PowerBITable asset to this datasource.
 
@@ -395,7 +390,7 @@ def add_powerbi_table_asset(  # noqa: PLR0913
             fully_qualified_columns=fully_qualified_columns,
             num_rows=num_rows,
             multiindex_hierarchies=multiindex_hierarchies,
-            pandas_convert_dtypes=pandas_convert_dtypes,
+            mode=mode,
         )
         return self._add_asset(asset)
 


Test patch: