Repository: great-expectations/great_expectations

Issue:
How to pass reader_method to ConfiguredAssetFilesystemDataConnector
I am unable to find instructions on how to configure reader_method or other batch_spec parameters using a ConfiguredAssetFilesystemDataConnector.

I have tried variations similar to the following:

```
class_name: Datasource
execution_engine:
  class_name: SparkDFExecutionEngine
data_connectors:
  stats:
    class_name: ConfiguredAssetFilesystemDataConnector
    base_directory: base_directory
    glob_directive: "**/"
    assets:
      logo_events:
        pattern: subdir/subdir/year=(\d{4})/week=(\d{2})
        group_names:
          - year
          - week
 #       batch_spec_passthrough:
 #          reader_method: parquet
```

Because I am using Spark, it is important to me to be able to provide full directories as my path parameter, which understandably foils the reader_method detection. I'm observing: `BatchKwargsError: Unable to determine reader method from path: subdir/subdir/year=2020/week=18`

I can't seem to provide the passthrough.




Discussion:

Resolving patch:
diff --git a/great_expectations/datasource/data_connector/asset/asset.py b/great_expectations/datasource/data_connector/asset/asset.py
--- a/great_expectations/datasource/data_connector/asset/asset.py
+++ b/great_expectations/datasource/data_connector/asset/asset.py
@@ -21,6 +21,7 @@ def __init__(
         prefix: str = None,
         delimiter: str = None,
         max_keys: int = None,
+        batch_spec_passthrough: dict = None,
     ):
         self._name = name
         self._base_directory = base_directory
@@ -34,6 +35,7 @@ def __init__(
         self._prefix = prefix
         self._delimiter = delimiter
         self._max_keys = max_keys
+        self._batch_spec_passthrough = batch_spec_passthrough or {}
 
     @property
     def name(self) -> str:
@@ -70,3 +72,7 @@ def delimiter(self) -> str:
     @property
     def max_keys(self) -> int:
         return self._max_keys
+
+    @property
+    def batch_spec_passthrough(self) -> dict:
+        return self._batch_spec_passthrough
diff --git a/great_expectations/datasource/data_connector/configured_asset_file_path_data_connector.py b/great_expectations/datasource/data_connector/configured_asset_file_path_data_connector.py
--- a/great_expectations/datasource/data_connector/configured_asset_file_path_data_connector.py
+++ b/great_expectations/datasource/data_connector/configured_asset_file_path_data_connector.py
@@ -3,12 +3,13 @@
 from typing import Dict, List, Optional, Union
 
 import great_expectations.exceptions as ge_exceptions
-from great_expectations.core.batch import BatchDefinition
+from great_expectations.core.batch import BatchDefinition, BatchRequest
 from great_expectations.data_context.util import instantiate_class_from_config
 from great_expectations.datasource.data_connector.asset.asset import Asset
 from great_expectations.datasource.data_connector.file_path_data_connector import (
     FilePathDataConnector,
 )
+from great_expectations.datasource.types import PathBatchSpec
 from great_expectations.execution_engine import ExecutionEngine
 
 logger = logging.getLogger(__name__)
@@ -230,3 +231,22 @@ def _get_data_reference_list_for_asset(self, asset: Optional[Asset]) -> List[str
 
     def _get_full_file_path_for_asset(self, path: str, asset: Optional[Asset]) -> str:
         raise NotImplementedError
+
+    def build_batch_spec(self, batch_definition: BatchDefinition) -> PathBatchSpec:
+        """
+        Build BatchSpec from batch_definition by calling DataConnector's build_batch_spec function.
+
+        Args:
+            batch_definition (BatchDefinition): to be used to build batch_spec
+
+        Returns:
+            BatchSpec built from batch_definition
+        """
+        batch_spec = super().build_batch_spec(batch_definition=batch_definition)
+
+        if batch_definition.data_asset_name in self.assets:
+            batch_spec.update(
+                self.assets[batch_definition.data_asset_name].batch_spec_passthrough
+            )
+
+        return PathBatchSpec(batch_spec)
diff --git a/great_expectations/datasource/data_connector/inferred_asset_filesystem_data_connector.py b/great_expectations/datasource/data_connector/inferred_asset_filesystem_data_connector.py
--- a/great_expectations/datasource/data_connector/inferred_asset_filesystem_data_connector.py
+++ b/great_expectations/datasource/data_connector/inferred_asset_filesystem_data_connector.py
@@ -2,6 +2,7 @@
 from pathlib import Path
 from typing import List, Optional
 
+from great_expectations.core.batch import BatchDefinition, BatchRequest
 from great_expectations.datasource.data_connector import (
     InferredAssetFilePathDataConnector,
 )
@@ -9,6 +10,7 @@
     get_filesystem_one_level_directory_glob_path_list,
     normalize_directory_path,
 )
+from great_expectations.datasource.types import PathBatchSpec
 from great_expectations.execution_engine import ExecutionEngine
 
 logger = logging.getLogger(__name__)
@@ -36,6 +38,7 @@ def __init__(
         default_regex: Optional[dict] = None,
         glob_directive: str = "*",
         sorters: Optional[list] = None,
+        batch_spec_passthrough: dict = None,
     ):
         """
         Base class for DataConnectors that connect to filesystem-like data. This class supports the configuration of default_regex
@@ -61,6 +64,7 @@ def __init__(
 
         self._base_directory = base_directory
         self._glob_directive = glob_directive
+        self._batch_spec_passthrough = batch_spec_passthrough or {}
 
     def _get_data_reference_list(
         self, data_asset_name: Optional[str] = None
@@ -90,3 +94,19 @@ def base_directory(self):
             dir_path=self._base_directory,
             root_directory_path=self.data_context_root_directory,
         )
+
+    def build_batch_spec(self, batch_definition: BatchDefinition) -> PathBatchSpec:
+        """
+        Build BatchSpec from batch_definition by calling DataConnector's build_batch_spec function.
+
+        Args:
+            batch_definition (BatchDefinition): to be used to build batch_spec
+
+        Returns:
+            BatchSpec built from batch_definition
+        """
+        batch_spec = super().build_batch_spec(batch_definition=batch_definition)
+
+        batch_spec.update(self._batch_spec_passthrough)
+
+        return PathBatchSpec(batch_spec)


Test patch:
diff --git a/tests/datasource/data_connector/test_data_connector.py b/tests/datasource/data_connector/test_data_connector.py
--- a/tests/datasource/data_connector/test_data_connector.py
+++ b/tests/datasource/data_connector/test_data_connector.py
@@ -1,4 +1,8 @@
 # TODO: <Alex>This module should be broken up -- please see suggestions below.</Alex>
+import json
+import os
+
+import pandas as pd
 import pytest
 from ruamel.yaml import YAML
 
diff --git a/tests/datasource/test_new_datasource.py b/tests/datasource/test_new_datasource.py
--- a/tests/datasource/test_new_datasource.py
+++ b/tests/datasource/test_new_datasource.py
@@ -710,3 +710,91 @@ def test_get_available_data_asset_names_with_single_partition_file_data_connecto
 
 def test_get_available_data_asset_names_with_caching():
     pass
+
+
+def test__data_source_batch_spec_passthrough(tmp_path_factory):
+    base_directory = str(
+        tmp_path_factory.mktemp("data_connector_batch_spec_passthrough")
+    )
+    with open(
+        os.path.join(base_directory, "csv_with_extra_header_rows.csv"), "w"
+    ) as f_:
+        f_.write(
+            """--- extra ---
+--- extra ---
+x,y
+0,a
+1,b
+2,c
+3,d
+4,e
+5,f
+6,g
+7,h
+8,i
+9,j
+"""
+        )
+
+    my_datasource: Datasource = instantiate_class_from_config(
+        yaml.load(
+            f"""
+class_name: Datasource
+
+execution_engine:
+    class_name: PandasExecutionEngine
+
+data_connectors:
+
+    my_configured_data_connector:
+        module_name: great_expectations.datasource.data_connector
+        class_name: ConfiguredAssetFilesystemDataConnector
+
+        base_directory: {base_directory}
+        glob_directive: "*.csv"
+        default_regex:
+            pattern: "(.*)"
+            group_names:
+                - file_name
+        assets:
+            stuff:
+                batch_spec_passthrough:
+                    reader_options:
+                        skiprows: 2
+
+
+    my_inferred_data_connector:
+        module_name: great_expectations.datasource.data_connector
+        class_name: InferredAssetFilesystemDataConnector
+        base_directory: {base_directory}
+        batch_spec_passthrough:
+            reader_options:
+                skiprows: 2
+
+        default_regex:
+            pattern: (.*)\.csv
+            group_names:
+                - data_asset_name
+    """,
+        ),
+        runtime_environment={"name": "my_datasource"},
+        config_defaults={"module_name": "great_expectations.datasource"},
+    )
+
+    report_obj = my_datasource.self_check()
+
+    print(json.dumps(report_obj, indent=2))
+
+    assert (
+        report_obj["data_connectors"]["my_configured_data_connector"][
+            "example_data_reference"
+        ]["n_rows"]
+        == 10
+    )
+
+    assert (
+        report_obj["data_connectors"]["my_inferred_data_connector"][
+            "example_data_reference"
+        ]["n_rows"]
+        == 10
+    )