Repository: meltano/sdk
Pull request #2785 (2024-11-30 01:08:22)

fix: Use FS-specific `listdir` in folder tap


<!-- readthedocs-preview meltano-sdk start -->
----
📚 Documentation preview 📚: https://meltano-sdk--2785.org.readthedocs.build/en/2785/

<!-- readthedocs-preview meltano-sdk end -->

codspeed-hq[bot]: ## [CodSpeed Performance Report](https://codspeed.io/meltano/sdk/branches/edgarrmondragon%2Ffix%2Flistdir-folder-tap)

### Merging #2785 will **not alter performance**

<sub>Comparing <code>edgarrmondragon/fix/listdir-folder-tap</code> (1eb5b39) with <code>main</code> (3ad4615)</sub>



### Summary

`✅ 6` untouched benchmarks  




codecov[bot]: ## [Codecov](https://app.codecov.io/gh/meltano/sdk/pull/2785?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 90.79%. Comparing base [(`3ad4615`)](https://app.codecov.io/gh/meltano/sdk/commit/3ad46154de72389834a9225a72b0531b44cc1443?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano) to head [(`1eb5b39`)](https://app.codecov.io/gh/meltano/sdk/commit/1eb5b39b7a35bf586ba23525af2fa358f93e44d9?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano).

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #2785      +/-   ##
==========================================
- Coverage   90.88%   90.79%   -0.10%     
==========================================
  Files          62       62              
  Lines        5170     5170              
  Branches      667      667              
==========================================
- Hits         4699     4694       -5     
- Misses        330      333       +3     
- Partials      141      143       +2     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/meltano/sdk/pull/2785?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano).


Patch:
diff --git a/singer_sdk/contrib/filesystem/tap.py b/singer_sdk/contrib/filesystem/tap.py
index 58ca673d..ff0c9881 100644
--- a/singer_sdk/contrib/filesystem/tap.py
+++ b/singer_sdk/contrib/filesystem/tap.py
@@ -5,11 +5,12 @@ from __future__ import annotations
 import enum
 import functools
 import logging
-import os
 import typing as t
 from pathlib import Path
 
 import fsspec
+import fsspec.implementations
+import fsspec.implementations.dirfs
 
 import singer_sdk.typing as th
 from singer_sdk import Tap
@@ -138,6 +139,11 @@ class FolderTap(Tap, t.Generic[_T]):
         """Folder read mode."""
         return ReadMode(self.config["read_mode"])
 
+    @functools.cached_property
+    def path(self) -> str:
+        """Return the path to the directory."""
+        return self.config["path"]  # type: ignore[no-any-return]
+
     @functools.cached_property
     def fs(self) -> fsspec.AbstractFileSystem:
         """Return the filesystem object.
@@ -147,13 +153,18 @@ class FolderTap(Tap, t.Generic[_T]):
         """
         protocol = self.config["filesystem"]
         if protocol != "local" and protocol not in self.config:  # pragma: no cover
-            msg = "Filesytem configuration is missing"
+            msg = "Filesystem configuration is missing"
             raise ConfigValidationError(
                 msg,
                 errors=[f"Missing configuration for filesystem {protocol}"],
             )
-        logger.info("Instatiating filesystem inteface: '%s'", protocol)
-        return fsspec.filesystem(protocol, **self.config.get(protocol, {}))
+        logger.info("Instantiating filesystem interface: '%s'", protocol)
+
+        return fsspec.implementations.dirfs.DirFileSystem(
+            path=self.path,
+            target_protocol=protocol,
+            target_options=self.config.get(protocol),
+        )
 
     def discover_streams(self) -> list:
         """Return a list of discovered streams.
@@ -162,11 +173,9 @@ class FolderTap(Tap, t.Generic[_T]):
             ValueError: If the path does not exist or is not a directory.
         """
         # A directory for now, but could be a glob pattern.
-        path: str = self.config["path"]
-
-        if not self.fs.exists(path) or not self.fs.isdir(path):  # pragma: no cover
+        if not self.fs.exists(".") or not self.fs.isdir("."):  # pragma: no cover
             # Raise a more specific error if the path is not a directory.
-            msg = f"Path {path} does not exist or is not a directory"
+            msg = f"Path {self.path} does not exist or is not a directory"
             raise ValueError(msg)
 
         # One stream per file
@@ -174,12 +183,13 @@ class FolderTap(Tap, t.Generic[_T]):
             return [
                 self.default_stream_class(
                     tap=self,
-                    name=file_path_to_stream_name(member),
-                    filepaths=[os.path.join(path, member)],  # noqa: PTH118
+                    name=file_path_to_stream_name(member["name"]),
+                    filepaths=[member["name"]],
                     filesystem=self.fs,
                 )
-                for member in os.listdir(path)
-                if member.endswith(self.valid_extensions)
+                for member in self.fs.listdir(".")
+                if member["type"] == "file"
+                and member["name"].endswith(self.valid_extensions)
             ]
 
         # Merge
@@ -188,9 +198,10 @@ class FolderTap(Tap, t.Generic[_T]):
                 tap=self,
                 name=self.config["stream_name"],
                 filepaths=[
-                    os.path.join(path, member)  # noqa: PTH118
-                    for member in os.listdir(path)
-                    if member.endswith(self.valid_extensions)
+                    member["name"]
+                    for member in self.fs.listdir(".")
+                    if member["type"] == "file"
+                    and member["name"].endswith(self.valid_extensions)
                 ],
                 filesystem=self.fs,
             )


Test patch:
diff --git a/singer_sdk/testing/templates.py b/singer_sdk/testing/templates.py
index 8a21f639..716caaf0 100644
--- a/singer_sdk/testing/templates.py
+++ b/singer_sdk/testing/templates.py
@@ -103,8 +103,8 @@ class TestTemplate:
         Raises:
             ValueError: if Test instance does not have `name` and `type` properties.
         """
-        if not self.name or not self.plugin_type:
-            msg = "Test must have 'name' and 'type' properties."
+        if not self.name or not self.plugin_type:  # pragma: no cover
+            msg = "Test must have 'name' and 'plugin_type' properties."
             raise ValueError(msg)
 
         self.config = config
diff --git a/tests/samples/test_tap_csv.py b/tests/samples/test_tap_csv.py
index cb16e0e0..217022f2 100644
--- a/tests/samples/test_tap_csv.py
+++ b/tests/samples/test_tap_csv.py
@@ -1,11 +1,15 @@
 from __future__ import annotations
 
 import datetime
+import typing as t
 
 import pytest
 
 from samples.sample_tap_csv.sample_tap_csv import SampleTapCSV
-from singer_sdk.testing import SuiteConfig, get_tap_test_class
+from singer_sdk.testing import SuiteConfig, TapTestRunner, get_tap_test_class
+
+if t.TYPE_CHECKING:
+    from samples.sample_tap_csv.client import CSVStream
 
 _TestCSVMerge = get_tap_test_class(
     tap_class=SampleTapCSV,
@@ -44,7 +48,7 @@ STATE = {
         "customers": {
             "partitions": [
                 {
-                    "context": {"_sdc_path": "fixtures/csv/customers.csv"},
+                    "context": {"_sdc_path": "./customers.csv"},
                     "replication_key": "_sdc_modified_at",
                     "replication_key_value": FUTURE.isoformat(),
                 }
@@ -53,7 +57,7 @@ STATE = {
         "employees": {
             "partitions": [
                 {
-                    "context": {"_sdc_path": "fixtures/csv/employees.csv"},
+                    "context": {"_sdc_path": "./employees.csv"},
                     "replication_key": "_sdc_modified_at",
                     "replication_key_value": FUTURE.isoformat(),
                 }
@@ -76,10 +80,27 @@ _TestCSVOneStreamPerFileIncremental = get_tap_test_class(
 
 
 class TestCSVOneStreamPerFileIncremental(_TestCSVOneStreamPerFileIncremental):
-    @pytest.mark.xfail(reason="No records are extracted", strict=True)
-    def test_tap_stream_transformed_catalog_schema_matches_record(self, stream: str):
-        super().test_tap_stream_transformed_catalog_schema_matches_record(stream)
-
-    @pytest.mark.xfail(reason="No records are extracted", strict=True)
-    def test_tap_stream_returns_record(self, stream: str):
-        super().test_tap_stream_returns_record(stream)
+    def test_tap_stream_transformed_catalog_schema_matches_record(
+        self,
+        config: SuiteConfig,
+        resource: t.Any,
+        runner: TapTestRunner,
+        stream: CSVStream,
+    ):
+        with pytest.warns(UserWarning):
+            super().test_tap_stream_transformed_catalog_schema_matches_record(
+                config,
+                resource,
+                runner,
+                stream,
+            )
+
+    def test_tap_stream_returns_record(
+        self,
+        config: SuiteConfig,
+        resource: t.Any,
+        runner: TapTestRunner,
+        stream: CSVStream,
+    ):
+        with pytest.warns(UserWarning):
+            super().test_tap_stream_returns_record(config, resource, runner, stream)