Repository: meltano/sdk
Pull request #2130 (2024-01-02 16:54:43)

refactor: Allow loading stream schemas from `importlib.resources.abc.Traversable` types


<!-- readthedocs-preview meltano-sdk start -->
----
📚 Documentation preview 📚: https://meltano-sdk--2130.org.readthedocs.build/en/2130/

<!-- readthedocs-preview meltano-sdk end -->

codspeed-hq[bot]: ## [CodSpeed Performance Report](https://codspeed.io/meltano/sdk/branches/edgarrmondragon/refactor/allow-loading-importlib-traversable-types)

### Merging #2130 will **not alter performance**

<sub>Comparing <code>edgarrmondragon/refactor/allow-loading-importlib-traversable-types</code> (16d8a17) with <code>main</code> (5624ef0)</sub>



### Summary

`✅ 6` untouched benchmarks





codecov[bot]: ## [Codecov](https://app.codecov.io/gh/meltano/sdk/pull/2130?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Comparison is base [(`5624ef0`)](https://app.codecov.io/gh/meltano/sdk/commit/5624ef0a60c299eab062d7b9d82472a265a6df65?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano) 87.76% compared to head [(`37c47fa`)](https://app.codecov.io/gh/meltano/sdk/pull/2130?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano) 87.76%.


<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff           @@
##             main    #2130   +/-   ##
=======================================
  Coverage   87.76%   87.76%           
=======================================
  Files          60       60           
  Lines        4946     4946           
  Branches     1007     1007           
=======================================
  Hits         4341     4341           
  Misses        424      424           
  Partials      181      181           
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/meltano/sdk/pull/2130?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano).


Patch:
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
index 8fc4f083..772e4b90 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
@@ -21,6 +21,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.8,<4"
+importlib-resources = { version = "==6.1.*", python = "<3.9" }
 singer-sdk = { version="~=0.34.1" }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py
index dae2269d..21316908 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py
@@ -2,10 +2,7 @@
 
 from __future__ import annotations
 
-{% if cookiecutter.auth_method in ("OAuth2", "JWT") -%}
 import sys
-{% endif -%}
-from pathlib import Path
 from typing import Any, Callable, Iterable
 
 import requests
@@ -49,8 +46,15 @@ else:
 
 {% endif -%}
 
+if sys.version_info >= (3, 9):
+    import importlib.resources as importlib_resources
+else:
+    import importlib_resources
+
 _Auth = Callable[[requests.PreparedRequest], requests.PreparedRequest]
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
+
+# TODO: Delete this is if not using json files for schema definition
+SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
 
 class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream):
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py
index 8272cbc2..69c955e6 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py
@@ -2,15 +2,21 @@
 
 from __future__ import annotations
 
+import sys
 import typing as t
-from pathlib import Path
 
 from singer_sdk import typing as th  # JSON Schema typing helpers
 
 from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream
 
+if sys.version_info >= (3, 9):
+    import importlib.resources as importlib_resources
+else:
+    import importlib_resources
+
+
 # TODO: Delete this is if not using json files for schema definition
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
+SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
 
 {%- if cookiecutter.stream_type == "GraphQL" %}
diff --git a/poetry.lock b/poetry.lock
index 4a1ddef8..d9d49f97 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -3046,4 +3046,4 @@ testing = ["pytest", "pytest-durations"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.7.1"
-content-hash = "bd5ad4eb7d109f3d184dcfb99e6808e6209776dfc741b86e3ba3ac3369d64449"
+content-hash = "b31a1c8736111f0a95b99d84aec5444b73c6e11ffb4e7a59d15c490257f69b8a"
diff --git a/pyproject.toml b/pyproject.toml
index 34f3fca8..cb7965d0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,7 +46,7 @@ click = "~=8.0"
 cryptography = ">=3.4.6"
 fs = ">=2.4.16"
 importlib-metadata = {version = "<7.0.0", python = "<3.12"}
-importlib-resources = {version = ">=5.12.0", markers = "python_version < \"3.9\""}
+importlib-resources = {version = ">=5.12.0", python = "<3.9"}
 inflection = ">=0.5.1"
 joblib = ">=1.0.1"
 jsonpath-ng = ">=1.5.3"
diff --git a/samples/aapl/aapl.py b/samples/aapl/aapl.py
index ecbf032d..7784a34f 100644
--- a/samples/aapl/aapl.py
+++ b/samples/aapl/aapl.py
@@ -3,11 +3,16 @@
 from __future__ import annotations
 
 import json
-from pathlib import Path
+import sys
 
 from singer_sdk import Stream, Tap
 
-PROJECT_DIR = Path(__file__).parent
+if sys.version_info < (3, 9):
+    import importlib_resources
+else:
+    import importlib.resources as importlib_resources
+
+PROJECT_DIR = importlib_resources.files("samples.aapl")
 
 
 class AAPL(Stream):
diff --git a/samples/sample_tap_countries/countries_streams.py b/samples/sample_tap_countries/countries_streams.py
index 708e1678..3b68a557 100644
--- a/samples/sample_tap_countries/countries_streams.py
+++ b/samples/sample_tap_countries/countries_streams.py
@@ -9,12 +9,12 @@ See the online explorer and query builder here:
 from __future__ import annotations
 
 import abc
-from pathlib import Path
 
 from singer_sdk import typing as th
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.streams.graphql import GraphQLStream
 
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
+SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
 
 class CountriesAPIStream(GraphQLStream, metaclass=abc.ABCMeta):
diff --git a/samples/sample_tap_gitlab/gitlab_graphql_streams.py b/samples/sample_tap_gitlab/gitlab_graphql_streams.py
index b29fbc13..30396461 100644
--- a/samples/sample_tap_gitlab/gitlab_graphql_streams.py
+++ b/samples/sample_tap_gitlab/gitlab_graphql_streams.py
@@ -6,13 +6,12 @@
 
 from __future__ import annotations
 
-from pathlib import Path
-
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.streams import GraphQLStream
 
 SITE_URL = "https://gitlab.com/graphql"
 
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
+SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
 
 class GitlabGraphQLStream(GraphQLStream):
diff --git a/samples/sample_tap_gitlab/gitlab_rest_streams.py b/samples/sample_tap_gitlab/gitlab_rest_streams.py
index 1480a017..1db62909 100644
--- a/samples/sample_tap_gitlab/gitlab_rest_streams.py
+++ b/samples/sample_tap_gitlab/gitlab_rest_streams.py
@@ -3,9 +3,9 @@
 from __future__ import annotations
 
 import typing as t
-from pathlib import Path
 
 from singer_sdk.authenticators import SimpleAuthenticator
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.pagination import SimpleHeaderPaginator
 from singer_sdk.streams.rest import RESTStream
 from singer_sdk.typing import (
@@ -17,7 +17,7 @@ from singer_sdk.typing import (
     StringType,
 )
 
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
+SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
 DEFAULT_URL_BASE = "https://gitlab.com/api/v4"
 
diff --git a/samples/sample_tap_google_analytics/ga_tap_stream.py b/samples/sample_tap_google_analytics/ga_tap_stream.py
index 04c3a253..5bd0503f 100644
--- a/samples/sample_tap_google_analytics/ga_tap_stream.py
+++ b/samples/sample_tap_google_analytics/ga_tap_stream.py
@@ -4,14 +4,14 @@ from __future__ import annotations
 
 import datetime
 import typing as t
-from pathlib import Path
 
 from singer_sdk.authenticators import OAuthJWTAuthenticator
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.streams import RESTStream
 
 GOOGLE_OAUTH_ENDPOINT = "https://oauth2.googleapis.com/token"
 GA_OAUTH_SCOPES = "https://www.googleapis.com/auth/analytics.readonly"
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
+SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
 
 
 class GoogleJWTAuthenticator(OAuthJWTAuthenticator):
diff --git a/singer_sdk/helpers/_compat.py b/singer_sdk/helpers/_compat.py
index cc84576a..c9a7df6c 100644
--- a/singer_sdk/helpers/_compat.py
+++ b/singer_sdk/helpers/_compat.py
@@ -12,15 +12,22 @@ else:
     from importlib import metadata
     from typing import final  # noqa: ICN003
 
-if sys.version_info < (3, 12):
-    from importlib_metadata import entry_points
+if sys.version_info < (3, 9):
+    import importlib_resources
 else:
-    from importlib.metadata import entry_points
+    from importlib import resources as importlib_resources
 
 if sys.version_info < (3, 9):
-    import importlib_resources as resources
+    from importlib_resources.abc import Traversable
+elif sys.version_info < (3, 12):
+    from importlib.abc import Traversable
 else:
-    from importlib import resources
+    from importlib.resources.abc import Traversable
+
+if sys.version_info < (3, 12):
+    from importlib_metadata import entry_points
+else:
+    from importlib.metadata import entry_points
 
 if sys.version_info < (3, 11):
     from backports.datetime_fromisoformat import MonkeyPatch
@@ -34,9 +41,10 @@ time_fromisoformat = datetime.time.fromisoformat
 __all__ = [
     "metadata",
     "final",
-    "resources",
     "entry_points",
     "datetime_fromisoformat",
     "date_fromisoformat",
     "time_fromisoformat",
+    "importlib_resources",
+    "Traversable",
 ]
diff --git a/singer_sdk/streams/core.py b/singer_sdk/streams/core.py
index afcc1c01..459b9e76 100644
--- a/singer_sdk/streams/core.py
+++ b/singer_sdk/streams/core.py
@@ -54,6 +54,7 @@ from singer_sdk.mapper import RemoveRecordTransform, SameRecordTransform, Stream
 if t.TYPE_CHECKING:
     import logging
 
+    from singer_sdk.helpers._compat import Traversable
     from singer_sdk.tap_base import Tap
 
 # Replication methods
@@ -136,7 +137,7 @@ class Stream(metaclass=abc.ABCMeta):
         self._replication_key: str | None = None
         self._primary_keys: t.Sequence[str] | None = None
         self._state_partitioning_keys: list[str] | None = None
-        self._schema_filepath: Path | None = None
+        self._schema_filepath: Path | Traversable | None = None
         self._metadata: singer.MetadataMapping | None = None
         self._mask: singer.SelectionMask | None = None
         self._schema: dict
@@ -160,7 +161,7 @@ class Stream(metaclass=abc.ABCMeta):
                 raise ValueError(msg)
 
         if self.schema_filepath:
-            self._schema = json.loads(Path(self.schema_filepath).read_text())
+            self._schema = json.loads(self.schema_filepath.read_text())
 
         if not self.schema:
             msg = (
@@ -421,7 +422,7 @@ class Stream(metaclass=abc.ABCMeta):
         return utc_now() if self.is_timestamp_replication_key else None
 
     @property
-    def schema_filepath(self) -> Path | None:
+    def schema_filepath(self) -> Path | Traversable | None:
         """Get path to schema file.
 
         Returns:


Test patch:
diff --git a/singer_sdk/testing/runners.py b/singer_sdk/testing/runners.py
index f6e135fe..c5ec10a1 100644
--- a/singer_sdk/testing/runners.py
+++ b/singer_sdk/testing/runners.py
@@ -8,11 +8,15 @@ import json
 import typing as t
 from collections import defaultdict
 from contextlib import redirect_stderr, redirect_stdout
-from pathlib import Path
 
 from singer_sdk import Tap, Target
 from singer_sdk.testing.config import SuiteConfig
 
+if t.TYPE_CHECKING:
+    from pathlib import Path
+
+    from singer_sdk.helpers._compat import Traversable
+
 
 class SingerTestRunner(metaclass=abc.ABCMeta):
     """Base Singer Test Runner."""
@@ -197,7 +201,7 @@ class TargetTestRunner(SingerTestRunner):
         target_class: type[Target],
         config: dict | None = None,
         suite_config: SuiteConfig | None = None,
-        input_filepath: Path | None = None,
+        input_filepath: Path | Traversable | None = None,
         input_io: io.StringIO | None = None,
         **kwargs: t.Any,
     ) -> None:
@@ -242,9 +246,7 @@ class TargetTestRunner(SingerTestRunner):
             if self.input_io:
                 self._input = self.input_io
             elif self.input_filepath:
-                self._input = Path(self.input_filepath).open(  # noqa: SIM115
-                    encoding="utf8",
-                )
+                self._input = self.input_filepath.open(encoding="utf8")
         return t.cast(t.IO[str], self._input)
 
     @target_input.setter
diff --git a/singer_sdk/testing/templates.py b/singer_sdk/testing/templates.py
index 0f01e3f4..4a16feb0 100644
--- a/singer_sdk/testing/templates.py
+++ b/singer_sdk/testing/templates.py
@@ -5,12 +5,12 @@ from __future__ import annotations
 import contextlib
 import typing as t
 import warnings
-from pathlib import Path
 
-from singer_sdk.helpers._compat import resources
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.testing import target_test_streams
 
 if t.TYPE_CHECKING:
+    from singer_sdk.helpers._compat import Traversable
     from singer_sdk.streams import Stream
 
     from .config import SuiteConfig
@@ -322,14 +322,14 @@ class TargetFileTestTemplate(TargetTestTemplate):
         """
         # get input from file
         if getattr(self, "singer_filepath", None):
-            assert Path(
-                self.singer_filepath,
-            ).exists(), f"Singer file {self.singer_filepath} does not exist."
+            assert (
+                self.singer_filepath.is_file()
+            ), f"Singer file {self.singer_filepath} does not exist."
             runner.input_filepath = self.singer_filepath
         super().run(config, resource, runner)
 
     @property
-    def singer_filepath(self) -> Path:
+    def singer_filepath(self) -> Traversable:
         """Get path to singer JSONL formatted messages file.
 
         Files will be sourced from `./target_test_streams/<test name>.singer`.
@@ -337,4 +337,4 @@ class TargetFileTestTemplate(TargetTestTemplate):
         Returns:
             The expected Path to this tests singer file.
         """
-        return resources.files(target_test_streams).joinpath(f"{self.name}.singer")  # type: ignore[no-any-return]
+        return importlib_resources.files(target_test_streams) / f"{self.name}.singer"  # type: ignore[no-any-return]
diff --git a/tests/samples/test_target_csv.py b/tests/samples/test_target_csv.py
index e55aa3cb..a0a3f349 100644
--- a/tests/samples/test_target_csv.py
+++ b/tests/samples/test_target_csv.py
@@ -15,6 +15,7 @@ from click.testing import CliRunner
 from samples.sample_mapper.mapper import StreamTransform
 from samples.sample_tap_countries.countries_tap import SampleTapCountries
 from samples.sample_target_csv.csv_target import SampleTargetCSV
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.testing import (
     get_target_test_class,
     sync_end_to_end,
@@ -146,7 +147,9 @@ def test_target_batching():
     }
 
 
-SAMPLE_FILENAME = Path(__file__).parent / Path("./resources/messages.jsonl")
+SAMPLE_FILENAME = (
+    importlib_resources.files("tests.samples") / "resources/messages.jsonl"
+)
 EXPECTED_OUTPUT = """"id"	"name"
 1	"Chris"
 2	"Mike"