Repository: meltano/sdk
Pull request #2651 (2024-09-06 17:36:43)

feat(taps): SQL taps now emit schemas with `maxLength` when applicable


<!-- readthedocs-preview meltano-sdk start -->
----
📚 Documentation preview 📚: https://meltano-sdk--2651.org.readthedocs.build/en/2651/

<!-- readthedocs-preview meltano-sdk end -->

codspeed-hq[bot]: ## [CodSpeed Performance Report](https://codspeed.io/meltano/sdk/branches/edgarrmondragon/feat/schema-max-length)

### Merging #2651 will **not alter performance**

<sub>Comparing <code>edgarrmondragon/feat/schema-max-length</code> (39a0ed5) with <code>main</code> (7ea1422)</sub>



### Summary

`✅ 6` untouched benchmarks





codecov[bot]: ## [Codecov](https://app.codecov.io/gh/meltano/sdk/pull/2651?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 89.82%. Comparing base [(`7ea1422`)](https://app.codecov.io/gh/meltano/sdk/commit/7ea1422907ea7ee6ca396127a20a6a29fee776d2?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano) to head [(`39a0ed5`)](https://app.codecov.io/gh/meltano/sdk/commit/39a0ed59e2e8a4e82544a8a014af6e294960c710?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano).

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #2651      +/-   ##
==========================================
- Coverage   89.86%   89.82%   -0.04%     
==========================================
  Files          58       58              
  Lines        4884     4886       +2     
  Branches      959      960       +1     
==========================================
  Hits         4389     4389              
- Misses        344      346       +2     
  Partials      151      151              
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/meltano/sdk/pull/2651?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=meltano).


Patch:
diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py
index 697a74cb..949ec523 100644
--- a/singer_sdk/connectors/sql.py
+++ b/singer_sdk/connectors/sql.py
@@ -166,15 +166,14 @@ class SQLToJSONSchema:
         return th.NumberType.type_dict  # type: ignore[no-any-return]
 
     @to_jsonschema.register
-    def string_to_jsonschema(self, column_type: sa.types.String) -> dict:  # noqa: ARG002, PLR6301
+    def string_to_jsonschema(self, column_type: sa.types.String) -> dict:  # noqa: PLR6301
         """Return a JSON Schema representation of a generic string type.
 
         Args:
             column_type (:column_type:`String`): The column type.
         """
-        # TODO: Enable support for maxLength.
-        # if sa_type.length:
-        #     return StringType(max_length=sa_type.length).type_dict  # noqa: ERA001
+        if column_type.length:
+            return th.StringType(max_length=column_type.length).type_dict  # type: ignore[no-any-return]
         return th.StringType.type_dict  # type: ignore[no-any-return]
 
     @to_jsonschema.register
diff --git a/singer_sdk/streams/sql.py b/singer_sdk/streams/sql.py
index 2877a505..de6447cd 100644
--- a/singer_sdk/streams/sql.py
+++ b/singer_sdk/streams/sql.py
@@ -210,8 +210,6 @@ class SQLStream(Stream, metaclass=abc.ABCMeta):
 
         with self.connector._connect() as conn:  # noqa: SLF001
             for record in conn.execute(query).mappings():
-                # TODO: Standardize record mapping type
-                # https://github.com/meltano/sdk/issues/2096
                 transformed_record = self.post_process(dict(record))
                 if transformed_record is None:
                     # Record filtered out during post_process()


Test patch:
diff --git a/tests/core/test_connector_sql.py b/tests/core/test_connector_sql.py
index 5b866366..f76f3052 100644
--- a/tests/core/test_connector_sql.py
+++ b/tests/core/test_connector_sql.py
@@ -405,7 +405,6 @@ def test_fully_qualified_name_empty_error():
         pytest.param(
             sa.types.VARCHAR(length=127),
             {"type": ["string"], "maxLength": 127},
-            marks=pytest.mark.xfail,
             id="varchar-length",
         ),
         pytest.param(sa.types.TEXT(), {"type": ["string"]}, id="text"),
diff --git a/tests/samples/conftest.py b/tests/samples/conftest.py
index 90cb80de..6580c0d0 100644
--- a/tests/samples/conftest.py
+++ b/tests/samples/conftest.py
@@ -29,14 +29,15 @@ def _sqlite_sample_db(sqlite_connector):
                     f"""
                     CREATE TABLE t{t} (
                         c1 int PRIMARY KEY NOT NULL,
-                        c2 varchar(10) NOT NULL
+                        c2 varchar(10) NOT NULL,
+                        c3 text NOT NULL
                     )
                     """
                 ),
             )
             for x in range(100):
                 conn.execute(
-                    sa.text(f"INSERT INTO t{t} VALUES ({x}, 'x={x}')"),  # noqa: S608
+                    sa.text(f"INSERT INTO t{t} VALUES ({x}, 'x={x}', 'y={x}')"),  # noqa: S608
                 )
 
 
diff --git a/tests/samples/test_tap_sqlite.py b/tests/samples/test_tap_sqlite.py
index b019d944..ff83de33 100644
--- a/tests/samples/test_tap_sqlite.py
+++ b/tests/samples/test_tap_sqlite.py
@@ -70,7 +70,7 @@ def test_sqlite_discovery(sqlite_sample_tap: SQLTap):
     sqlite_sample_tap.sync_all()
     stream = t.cast(SQLStream, sqlite_sample_tap.streams["main-t1"])
     schema = stream.schema
-    assert len(schema["properties"]) == 2
+    assert len(schema["properties"]) == 3
     assert stream.name == stream.tap_stream_id == "main-t1"
 
     md_map = MetadataMapping.from_iterable(stream.catalog_entry["metadata"])
@@ -90,13 +90,17 @@ def test_sqlite_discovery(sqlite_sample_tap: SQLTap):
 def test_sqlite_input_catalog(sqlite_sample_tap: SQLTap):
     sqlite_sample_tap.sync_all()
     stream = t.cast(SQLStream, sqlite_sample_tap.streams["main-t1"])
-    assert len(stream.schema["properties"]) == 2
-    assert len(stream.stream_maps[0].transformed_schema["properties"]) == 2
+    assert len(stream.schema["properties"]) == 3
+    assert len(stream.stream_maps[0].transformed_schema["properties"]) == 3
 
     for schema in [stream.schema, stream.stream_maps[0].transformed_schema]:
-        assert len(schema["properties"]) == 2
+        assert len(schema["properties"]) == 3
         assert schema["properties"]["c1"] == {"type": ["integer"]}
-        assert schema["properties"]["c2"] == {"type": ["string", "null"]}
+        assert schema["properties"]["c2"] == {
+            "type": ["string", "null"],
+            "maxLength": 10,
+        }
+        assert schema["properties"]["c3"] == {"type": ["string", "null"]}
         assert stream.name == stream.tap_stream_id == "main-t1"
 
     md_map = MetadataMapping.from_iterable(stream.catalog_entry["metadata"])