Repository: great-expectations/great_expectations

Issue:
S3 prefixes get broken when running on Windows
**Describe the bug**
On Windows when referencing an S3 data source, the prefix is suffixed with ``\`` instead of ``/``, resulting in query always failing.

**To Reproduce**
Create a ``InferredAssetS3DataConnector`` data connector, and pass any valid prefix (probably needs to be non-empty). Then, attempt to load a file from that connector.

**Expected behavior**
The file gets loaded properly.

**Error**
```
...\lib\site-packages\great_expectations\datasource\data_connector\util.py in list_s3_keys(s3, query_options, iterator_dict, recursive)
    452 
    453     if not any(key in s3_objects_info for key in ["Contents", "CommonPrefixes"]):
--> 454         raise ValueError("S3 query may not have been configured correctly.")
    455 
    456     if "Contents" in s3_objects_info:

ValueError: S3 query may not have been configured correctly.
```

**Fix**
In ``inferred_asset_s3_data_connector.py``, line 82 is:

```
        self._prefix = FilePathDataConnector.sanitize_prefix(prefix)
```

This ensures that the file path is always a folder, by appending a slash to the end. However, on Windows, the slash is ``\``, and thus you end up with a prefix like ``path/to/some/folder\``, which results in an S3 query with no results, hence the above stacktrace.

The fix is to ensure that only the correct slash can be appended to the prefix, regardless of the host OS. For example:

```
        self._prefix = prefix.rstrip('/') + '/'
```

or something to that effect.

**Environment (please complete the following information):**
 - Operating System: Windows
 - Great Expectations Version: 0.14.1



Discussion:
Howdy @scnerd, thanks for reaching out. I'm bringing this up with the team, we'll let keep you informed.

Resolving patch:
diff --git a/great_expectations/datasource/data_connector/configured_asset_s3_data_connector.py b/great_expectations/datasource/data_connector/configured_asset_s3_data_connector.py
--- a/great_expectations/datasource/data_connector/configured_asset_s3_data_connector.py
+++ b/great_expectations/datasource/data_connector/configured_asset_s3_data_connector.py
@@ -13,9 +13,6 @@
 from great_expectations.datasource.data_connector.configured_asset_file_path_data_connector import (
     ConfiguredAssetFilePathDataConnector,
 )
-from great_expectations.datasource.data_connector.file_path_data_connector import (
-    FilePathDataConnector,
-)
 from great_expectations.datasource.data_connector.util import list_s3_keys
 from great_expectations.execution_engine import ExecutionEngine
 
@@ -82,7 +79,7 @@ def __init__(
             batch_spec_passthrough=batch_spec_passthrough,
         )
         self._bucket = bucket
-        self._prefix = FilePathDataConnector.sanitize_prefix(prefix)
+        self._prefix = self.sanitize_prefix_for_s3(prefix)
         self._delimiter = delimiter
         self._max_keys = max_keys
 
@@ -96,6 +93,26 @@ def __init__(
                 "Unable to load boto3 (it is required for ConfiguredAssetS3DataConnector)."
             )
 
+    @staticmethod
+    def sanitize_prefix_for_s3(text: str) -> str:
+        """
+        Takes in a given user-prefix and cleans it to work with file-system traversal methods
+        (i.e. add '/' to the end of a string meant to represent a directory)
+
+        Customized for S3 paths, ignoring the path separator used by the host OS
+        """
+        text = text.strip()
+        if not text:
+            return text
+
+        path_parts = text.split("/")
+        if not path_parts:  # Empty prefix
+            return text
+        elif "." in path_parts[-1]:  # File, not folder
+            return text
+        else:  # Folder, should have trailing /
+            return text.rstrip("/") + "/"
+
     def build_batch_spec(self, batch_definition: BatchDefinition) -> S3BatchSpec:
         """
         Build BatchSpec from batch_definition by calling DataConnector's build_batch_spec function.
diff --git a/great_expectations/datasource/data_connector/inferred_asset_s3_data_connector.py b/great_expectations/datasource/data_connector/inferred_asset_s3_data_connector.py
--- a/great_expectations/datasource/data_connector/inferred_asset_s3_data_connector.py
+++ b/great_expectations/datasource/data_connector/inferred_asset_s3_data_connector.py
@@ -4,9 +4,7 @@
 import great_expectations.exceptions as ge_exceptions
 from great_expectations.core.batch import BatchDefinition
 from great_expectations.core.batch_spec import PathBatchSpec, S3BatchSpec
-from great_expectations.datasource.data_connector.file_path_data_connector import (
-    FilePathDataConnector,
-)
+from great_expectations.datasource.data_connector import ConfiguredAssetS3DataConnector
 
 try:
     import boto3
@@ -79,7 +77,7 @@ def __init__(
         )
 
         self._bucket = bucket
-        self._prefix = FilePathDataConnector.sanitize_prefix(prefix)
+        self._prefix = ConfiguredAssetS3DataConnector.sanitize_prefix_for_s3(prefix)
         self._delimiter = delimiter
         self._max_keys = max_keys
 


Test patch:
diff --git a/tests/datasource/data_connector/test_configured_asset_s3_data_connector.py b/tests/datasource/data_connector/test_configured_asset_s3_data_connector.py
--- a/tests/datasource/data_connector/test_configured_asset_s3_data_connector.py
+++ b/tests/datasource/data_connector/test_configured_asset_s3_data_connector.py
@@ -1,4 +1,5 @@
 import json
+import os
 from typing import List
 from unittest import mock
 
@@ -17,7 +18,10 @@
     IDDict,
 )
 from great_expectations.data_context.util import instantiate_class_from_config
-from great_expectations.datasource.data_connector import ConfiguredAssetS3DataConnector
+from great_expectations.datasource.data_connector import (
+    ConfiguredAssetS3DataConnector,
+    FilePathDataConnector,
+)
 from great_expectations.execution_engine import PandasExecutionEngine
 
 yaml = YAML()
@@ -1117,3 +1121,31 @@ def test_example_with_explicit_data_asset_names():
         )
         == 5
     )
+
+
+def test_sanitize_prefix_behaves_the_same_as_local_files():
+    def check_sameness(prefix, expected_output):
+        s3_sanitized = ConfiguredAssetS3DataConnector.sanitize_prefix_for_s3(prefix)
+        file_system_sanitized = FilePathDataConnector.sanitize_prefix(prefix)
+        if os.sep == "\\":  # Fix to ensure tests work on Windows
+            file_system_sanitized = file_system_sanitized.replace("\\", "/")
+
+        assert file_system_sanitized == expected_output, (
+            f"Expected output does not match original sanitization behavior, got "
+            f"{file_system_sanitized} instead of {expected_output}"
+        )
+        assert (
+            s3_sanitized == expected_output == file_system_sanitized
+        ), f'S3 sanitized result is incorrect, "{s3_sanitized} instead of {expected_output}'
+
+    # Copy of all samples from tests/datasource/data_connector/test_file_path_data_connector.py
+    check_sameness("foo/", "foo/")
+    check_sameness("bar", "bar/")
+    check_sameness("baz.txt", "baz.txt")
+    check_sameness("a/b/c/baz.txt", "a/b/c/baz.txt")
+
+    # A couple additional checks
+    check_sameness("a/b/c", "a/b/c/")
+    check_sameness("a.x/b/c", "a.x/b/c/")
+    check_sameness("path/to/folder.something/", "path/to/folder.something/")
+    check_sameness("path/to/folder.something", "path/to/folder.something")