Repository: meltano/meltano

Issue:
Legacy python 3.6 compatibility code
Migrated from GitLab: https://gitlab.com/meltano/meltano/-/issues/3172

Originally created by @pandemicsyn on 2022-01-20 14:00:40

---

With the dropping of python 3.6 support in [#2991](https://gitlab.com/meltano/meltano/-/issues/2991), I also had a very quick search through the codebase to find comments that referenced 3.6 compatibility hacks:

```
(melty-3.8) ➜  meltano git:(2991-python-3-6-eol) ✗ grep "3\.6" * -r | egrep -v "\.js|.vue|.lock|\.svg|.node_modules|\.log|Binary file|CHANGELOG"
src/meltano/core/plugin_install_service.py:        # TODO: use dataclasses.dataclass for this when 3.6 support is dropped
tests/meltano/cli/test_run.py:            # or is hack to work around python 3.6 failures
tests/meltano/cli/test_run.py:            # or is hack to work around python 3.6 failures
tests/meltano/cli/test_run.py:            # or is hack to work around python 3.6 failures
tests/meltano/cli/test_run.py:            # or is hack to work around python 3.6 failures
tests/meltano/cli/test_run.py:            # or is hack to work around python 3.6 failures

# and heres some other code paths that reference 3.7
src/meltano/core/block/extract_load.py:    with suppress(AttributeError):  # `wait_closed` is Python 3.7+
src/meltano/core/logging/utils.py:            with suppress(AttributeError):  # `wait_closed` is Python 3.7+
src/meltano/core/runner/singer.py:            with suppress(AttributeError):  # `wait_closed` is Python 3.7+
src/meltano/core/utils/__init__.py:    # and inspired by Python 3.7's `asyncio.run`
```

There's probably a few more here and there, but these are the most obvious ones that pop'd up in my quick searches.


Discussion:
[View 1 previous comment from the original issue on GitLab](https://gitlab.com/meltano/meltano/-/issues/3172)

Resolving patch:
diff --git a/src/meltano/core/logging/utils.py b/src/meltano/core/logging/utils.py
--- a/src/meltano/core/logging/utils.py
+++ b/src/meltano/core/logging/utils.py
@@ -6,7 +6,6 @@
 import logging
 import os
 import sys
-from contextlib import suppress
 from logging import config as logging_config
 
 import structlog
@@ -204,9 +203,7 @@ async def _write_line_writer(writer, line):
             writer.write(line)
             await writer.drain()
         except (BrokenPipeError, ConnectionResetError):
-            with suppress(AttributeError):  # `wait_closed` is Python 3.7+
-                await writer.wait_closed()
-
+            await writer.wait_closed()
             return False
     else:
         writer.writeline(line.decode())
diff --git a/src/meltano/core/plugin_install_service.py b/src/meltano/core/plugin_install_service.py
--- a/src/meltano/core/plugin_install_service.py
+++ b/src/meltano/core/plugin_install_service.py
@@ -7,6 +7,7 @@
 import logging
 import os
 import sys
+from dataclasses import dataclass
 from enum import Enum
 from multiprocessing import cpu_count
 from typing import Any, Callable, Iterable, Mapping
@@ -53,52 +54,42 @@ class PluginInstallStatus(Enum):
     WARNING = "warning"
 
 
+@dataclass(frozen=True)
 class PluginInstallState:
-    """A message reporting the progress of installing a plugin."""
+    """A message reporting the progress of installing a plugin.
 
-    def __init__(
-        self,
-        plugin: ProjectPlugin,
-        reason: PluginInstallReason,
-        status: PluginInstallStatus,
-        message: str | None = None,
-        details: str | None = None,
-    ):
-        """Initialize PluginInstallState instance.
+    plugin: Plugin related to this install state.
+    reason: Reason for plugin install.
+    status: Status of plugin install.
+    message: Formatted install state message.
+    details: Extra details relating to install (including error details if failed).
+    """
 
-        Args:
-            plugin: Plugin related to this install state.
-            reason: Reason for plugin install.
-            status: Status of plugin install.
-            message: Formatted install state message.
-            details: Extra details relating to install (including error details if failed).
-        """
-        # TODO: use dataclasses.dataclass for this when 3.6 support is dropped
-        self.plugin = plugin
-        self.reason = reason
-        self.status = status
-        self.message = message
-        self.details = details
-
-    @property
+    plugin: ProjectPlugin
+    reason: PluginInstallReason
+    status: PluginInstallStatus
+    message: str | None = None
+    details: str | None = None
+
+    @cached_property
     def successful(self):
         """Plugin install success status.
 
         Returns:
-            'True' if plugin install successful.
+            `True` if plugin install successful.
         """
         return self.status in {PluginInstallStatus.SUCCESS, PluginInstallStatus.SKIPPED}
 
-    @property
+    @cached_property
     def skipped(self):
         """Plugin install skipped status.
 
         Returns:
-            'True' if the installation was skipped / not needed.
+            `True` if the installation was skipped / not needed.
         """
         return self.status == PluginInstallStatus.SKIPPED
 
-    @property
+    @cached_property
     def verb(self) -> str:
         """Verb form of status.
 
diff --git a/src/meltano/core/runner/singer.py b/src/meltano/core/runner/singer.py
--- a/src/meltano/core/runner/singer.py
+++ b/src/meltano/core/runner/singer.py
@@ -4,7 +4,6 @@
 import logging
 import subprocess
 import sys
-from contextlib import suppress
 
 from meltano.core.elt_context import ELTContext
 from meltano.core.logging import capture_subprocess_output
@@ -175,8 +174,7 @@ async def invoke(  # noqa: WPS210, WPS213, WPS217, WPS231, WPS238
 
             # Close target stdin so process can complete naturally
             p_target.stdin.close()
-            with suppress(AttributeError):  # `wait_closed` is Python 3.7+
-                await p_target.stdin.wait_closed()
+            await p_target.stdin.wait_closed()
 
             # Wait for all buffered target output to be processed
             await asyncio.wait([target_stdout_future, target_stderr_future])


Test patch:
diff --git a/tests/meltano/cli/test_run.py b/tests/meltano/cli/test_run.py
--- a/tests/meltano/cli/test_run.py
+++ b/tests/meltano/cli/test_run.py
@@ -172,7 +172,7 @@ def event_matches(self, event: str) -> bool:
             True if the event was found, False otherwise.
         """
         for line in self.seen_events:
-            matches = line.get("event") == event
+            matches = line["event"] == event
             if matches:
                 return True
 
@@ -187,7 +187,7 @@ def find_by_event(self, event: str) -> list[dict] | None:
         """
         matches = []
         for line in self.seen_events:
-            match = line.get("event") == event
+            match = line["event"] == event
             if match:
                 matches.append(line)
         return matches
@@ -266,7 +266,7 @@ def test_run_parsing_failures(
             assert matcher.event_matches(
                 "All ExtractLoadBlocks validated, starting execution."
             )
-            assert matcher.find_by_event("Block run completed.")[0].get("success")
+            assert matcher.find_by_event("Block run completed.")[0]["success"]
 
     @pytest.mark.backend("sqlite")
     @mock.patch(
@@ -308,10 +308,10 @@ def test_run_basic_invocations(
             )
             target_stop_event = matcher.find_by_event("target done")
             assert len(target_stop_event) == 1
-            assert target_stop_event[0].get("name") == target.name
-            assert target_stop_event[0].get("cmd_type") == "elb"
-            assert target_stop_event[0].get("stdio") == "stderr"
-            assert matcher.find_by_event("Block run completed.")[0].get("success")
+            assert target_stop_event[0]["name"] == target.name
+            assert target_stop_event[0]["cmd_type"] == "elb"
+            assert target_stop_event[0]["stdio"] == "stderr"
+            assert matcher.find_by_event("Block run completed.")[0]["success"]
 
         # Verify that a vanilla command plugin (dbt:run) run works
         invoke_async = AsyncMock(side_effect=(dbt_process,))  # dbt run
@@ -329,10 +329,10 @@ def test_run_basic_invocations(
             )
             dbt_start_event = matcher.find_by_event("dbt done")
             assert len(dbt_start_event) == 1
-            assert dbt_start_event[0].get("name") == "dbt"
-            assert dbt_start_event[0].get("cmd_type") == "command"
-            assert dbt_start_event[0].get("stdio") == "stderr"
-            assert matcher.find_by_event("Block run completed.")[0].get("success")
+            assert dbt_start_event[0]["name"] == "dbt"
+            assert dbt_start_event[0]["cmd_type"] == "command"
+            assert dbt_start_event[0]["stdio"] == "stderr"
+            assert matcher.find_by_event("Block run completed.")[0]["success"]
 
     @pytest.mark.backend("sqlite")
     @mock.patch(
@@ -363,7 +363,7 @@ def test_run_custom_suffix_command_option(
             assert result.exit_code == 0
 
             matcher = EventMatcher(result.stderr)
-            assert matcher.find_by_event("Block run completed.")[0].get("success")
+            assert matcher.find_by_event("Block run completed.")[0]["success"]
 
             job_logging_service.get_latest_log(
                 f"dev:{tap.name}-to-{target.name}:test-suffix"
@@ -440,7 +440,7 @@ def test_run_custom_suffix_active_environment(
             assert result.exit_code == 0
 
             matcher = EventMatcher(result.stderr)
-            assert matcher.find_by_event("Block run completed.")[0].get("success")
+            assert matcher.find_by_event("Block run completed.")[0]["success"]
 
             job_logging_service.get_latest_log(
                 f"dev:{tap.name}-to-{target.name}:${expected_suffix}"
@@ -486,14 +486,14 @@ def test_run_multiple_commands(
             assert len(command_add_events) == 2
             assert invoke_async.call_count == 2
 
-            assert command_add_events[0].get("command_name") == "test"
+            assert command_add_events[0]["command_name"] == "test"
             assert invoke_async.mock_calls[0][2]["command"] == "test"
 
-            assert command_add_events[1].get("command_name") == "run"
+            assert command_add_events[1]["command_name"] == "run"
             assert invoke_async.mock_calls[1][2]["command"] == "run"
 
-            assert matcher.find_by_event("Block run completed.")[0].get("success")
-            assert matcher.find_by_event("Block run completed.")[1].get("success")
+            assert matcher.find_by_event("Block run completed.")[0]["success"]
+            assert matcher.find_by_event("Block run completed.")[1]["success"]
 
     @pytest.mark.backend("sqlite")
     @mock.patch(
@@ -529,36 +529,36 @@ def test_run_complex_invocations(
 
             # make sure mapper was found and at its expected positions
             for ev in matcher.find_by_event("found block"):
-                if ev.get("block_type") == "mappers":
-                    assert ev.get("index") == 1
+                if ev["block_type"] == "mappers":
+                    assert ev["index"] == 1
 
             assert (
-                matcher.find_by_event("found PluginCommand")[0].get("plugin_type")
+                matcher.find_by_event("found PluginCommand")[0]["plugin_type"]
                 == "transformers"
             )  # dbt
 
             completed_events = matcher.find_by_event("Block run completed.")
             assert len(completed_events) == 2
             for event in completed_events:
-                assert event.get("success")
+                assert event["success"]
 
             tap_stop_event = matcher.find_by_event("tap done")
             assert len(tap_stop_event) == 1
-            assert tap_stop_event[0].get("name") == tap.name
-            assert tap_stop_event[0].get("cmd_type") == "elb"
-            assert tap_stop_event[0].get("stdio") == "stderr"
+            assert tap_stop_event[0]["name"] == tap.name
+            assert tap_stop_event[0]["cmd_type"] == "elb"
+            assert tap_stop_event[0]["stdio"] == "stderr"
 
             target_stop_event = matcher.find_by_event("target done")
             assert len(target_stop_event) == 1
-            assert target_stop_event[0].get("name") == target.name
-            assert target_stop_event[0].get("cmd_type") == "elb"
-            assert target_stop_event[0].get("stdio") == "stderr"
+            assert target_stop_event[0]["name"] == target.name
+            assert target_stop_event[0]["cmd_type"] == "elb"
+            assert target_stop_event[0]["stdio"] == "stderr"
 
             dbt_done_event = matcher.find_by_event("dbt done")
             assert len(dbt_done_event) == 1
-            assert dbt_done_event[0].get("name") == "dbt"
-            assert dbt_done_event[0].get("cmd_type") == "command"
-            assert dbt_done_event[0].get("stdio") == "stderr"
+            assert dbt_done_event[0]["name"] == "dbt"
+            assert dbt_done_event[0]["cmd_type"] == "command"
+            assert dbt_done_event[0]["stdio"] == "stderr"
 
     @pytest.mark.backend("sqlite")
     @mock.patch(
@@ -599,29 +599,29 @@ def test_run_plugin_command_failure(
                 "found ExtractLoadBlocks set"
             )  # tap/target pair
             assert (
-                matcher.find_by_event("found PluginCommand")[0].get("plugin_type")
+                matcher.find_by_event("found PluginCommand")[0]["plugin_type"]
                 == "transformers"
             )  # dbt
 
             completed_events = matcher.find_by_event("Block run completed.")
             assert len(completed_events) == 2
             for event in completed_events:
-                if event.get("block_type") == "ExtractLoadBlocks":
-                    assert event.get("success")
-                elif event.get("block_type") == "InvokerCommand":
-                    assert event.get("success") is False
+                if event["block_type"] == "ExtractLoadBlocks":
+                    assert event["success"]
+                elif event["block_type"] == "InvokerCommand":
+                    assert event["success"] is False
 
             tap_stop_event = matcher.find_by_event("tap done")
             assert len(tap_stop_event) == 1
-            assert tap_stop_event[0].get("name") == tap.name
-            assert tap_stop_event[0].get("cmd_type") == "elb"
-            assert tap_stop_event[0].get("stdio") == "stderr"
+            assert tap_stop_event[0]["name"] == tap.name
+            assert tap_stop_event[0]["cmd_type"] == "elb"
+            assert tap_stop_event[0]["stdio"] == "stderr"
 
             target_stop_event = matcher.find_by_event("target done")
             assert len(target_stop_event) == 1
-            assert target_stop_event[0].get("name") == target.name
-            assert target_stop_event[0].get("cmd_type") == "elb"
-            assert target_stop_event[0].get("stdio") == "stderr"
+            assert target_stop_event[0]["name"] == target.name
+            assert target_stop_event[0]["cmd_type"] == "elb"
+            assert target_stop_event[0]["stdio"] == "stderr"
 
             assert not matcher.event_matches("dbt done")
             assert matcher.event_matches("dbt starting")
@@ -670,32 +670,28 @@ def test_run_elb_tap_failure(
             matcher = EventMatcher(result.stderr)
             assert matcher.event_matches("found ExtractLoadBlocks set")
             assert (
-                matcher.find_by_event("found PluginCommand")[0].get("plugin_type")
+                matcher.find_by_event("found PluginCommand")[0]["plugin_type"]
                 == "transformers"
             )
 
             completed_events = matcher.find_by_event("Block run completed.")
             assert len(completed_events) == 1
-            assert completed_events[0].get("success") is False
+            assert completed_events[0]["success"] is False
 
-            # or is hack to work around python 3.6 failures
-            assert (
-                completed_events[0].get("err") == "RunnerError('Extractor failed',)"
-                or "RunnerError('Extractor failed')"
-            )
-            assert completed_events[0].get("exit_codes").get("extractors") == 1
+            assert completed_events[0]["err"] == "RunnerError('Extractor failed')"
+            assert completed_events[0]["exit_codes"]["extractors"] == 1
 
             tap_stop_event = matcher.find_by_event("tap failure")
             assert len(tap_stop_event) == 1
-            assert tap_stop_event[0].get("name") == tap.name
-            assert tap_stop_event[0].get("cmd_type") == "elb"
-            assert tap_stop_event[0].get("stdio") == "stderr"
+            assert tap_stop_event[0]["name"] == tap.name
+            assert tap_stop_event[0]["cmd_type"] == "elb"
+            assert tap_stop_event[0]["stdio"] == "stderr"
 
             target_stop_event = matcher.find_by_event("target done")
             assert len(target_stop_event) == 1
-            assert target_stop_event[0].get("name") == target.name
-            assert target_stop_event[0].get("cmd_type") == "elb"
-            assert target_stop_event[0].get("stdio") == "stderr"
+            assert target_stop_event[0]["name"] == target.name
+            assert target_stop_event[0]["cmd_type"] == "elb"
+            assert target_stop_event[0]["stdio"] == "stderr"
 
             # dbt should not have run at all
             assert not matcher.event_matches("dbt starting")
@@ -767,21 +763,17 @@ async def target_wait_mock():
                 "found ExtractLoadBlocks set"
             )  # tap/target pair
             assert (
-                matcher.find_by_event("found PluginCommand")[0].get("plugin_type")
+                matcher.find_by_event("found PluginCommand")[0]["plugin_type"]
                 == "transformers"
             )  # dbt
 
             completed_events = matcher.find_by_event("Block run completed.")
             # there should only be one completed event
             assert len(completed_events) == 1
-            assert completed_events[0].get("success") is False
+            assert completed_events[0]["success"] is False
 
-            # or is hack to work around python 3.6 failures
-            assert (
-                completed_events[0].get("err") == "RunnerError('Loader failed',)"
-                or "RunnerError('Loader failed')"
-            )
-            assert completed_events[0].get("exit_codes").get("loaders") == 1
+            assert completed_events[0]["err"] == "RunnerError('Loader failed')"
+            assert completed_events[0]["exit_codes"]["loaders"] == 1
 
             # the tap should NOT have finished, we'll have a write of the SCHEMA message and then nothing further:
             assert matcher.event_matches("SCHEMA")
@@ -790,9 +782,9 @@ async def target_wait_mock():
 
             target_stop_event = matcher.find_by_event("target failure")
             assert len(target_stop_event) == 1
-            assert target_stop_event[0].get("name") == target.name
-            assert target_stop_event[0].get("cmd_type") == "elb"
-            assert target_stop_event[0].get("stdio") == "stderr"
+            assert target_stop_event[0]["name"] == target.name
+            assert target_stop_event[0]["cmd_type"] == "elb"
+            assert target_stop_event[0]["stdio"] == "stderr"
 
             # dbt should not have run at all
             assert not matcher.event_matches("dbt starting")
@@ -842,32 +834,28 @@ def test_run_elb_target_failure_after_tap_finished(  # noqa: WPS118
                 "found ExtractLoadBlocks set"
             )  # tap/target pair
             assert (
-                matcher.find_by_event("found PluginCommand")[0].get("plugin_type")
+                matcher.find_by_event("found PluginCommand")[0]["plugin_type"]
                 == "transformers"
             )  # dbt
 
             completed_events = matcher.find_by_event("Block run completed.")
             # there should only be one completed event
             assert len(completed_events) == 1
-            assert completed_events[0].get("success") is False
-            # or is hack to work around python 3.6 failures
-            assert (
-                completed_events[0].get("err") == "RunnerError('Loader failed',)"
-                or "RunnerError('Loader failed')"
-            )
-            assert completed_events[0].get("exit_codes").get("loaders") == 1
+            assert completed_events[0]["success"] is False
+            assert completed_events[0]["err"] == "RunnerError('Loader failed')"
+            assert completed_events[0]["exit_codes"]["loaders"] == 1
 
             tap_stop_event = matcher.find_by_event("tap done")
             assert len(tap_stop_event) == 1
-            assert tap_stop_event[0].get("name") == tap.name
-            assert tap_stop_event[0].get("cmd_type") == "elb"
-            assert tap_stop_event[0].get("stdio") == "stderr"
+            assert tap_stop_event[0]["name"] == tap.name
+            assert tap_stop_event[0]["cmd_type"] == "elb"
+            assert tap_stop_event[0]["stdio"] == "stderr"
 
             target_stop_event = matcher.find_by_event("target failure")
             assert len(target_stop_event) == 1
-            assert target_stop_event[0].get("name") == target.name
-            assert target_stop_event[0].get("cmd_type") == "elb"
-            assert target_stop_event[0].get("stdio") == "stderr"
+            assert target_stop_event[0]["name"] == target.name
+            assert target_stop_event[0]["cmd_type"] == "elb"
+            assert target_stop_event[0]["stdio"] == "stderr"
 
             # dbt should not have run at all
             assert not matcher.event_matches("dbt starting")
@@ -923,33 +911,31 @@ def test_run_elb_tap_and_target_failed(
             matcher = EventMatcher(result.stderr)
             assert matcher.event_matches("found ExtractLoadBlocks set")
             assert (
-                matcher.find_by_event("found PluginCommand")[0].get("plugin_type")
+                matcher.find_by_event("found PluginCommand")[0]["plugin_type"]
                 == "transformers"
             )
 
             completed_events = matcher.find_by_event("Block run completed.")
             assert len(completed_events) == 1
-            assert completed_events[0].get("success") is False
+            assert completed_events[0]["success"] is False
 
-            # or is hack to work around python 3.6 failures
             assert (
-                completed_events[0].get("err")
-                == "RunnerError('Extractor and loader failed',)"
-                or "RunnerError('Extractor and loader failed')"
+                completed_events[0]["err"]
+                == "RunnerError('Extractor and loader failed')"
             )
-            assert completed_events[0].get("exit_codes").get("loaders") == 1
+            assert completed_events[0]["exit_codes"]["loaders"] == 1
 
             tap_stop_event = matcher.find_by_event("tap failure")
             assert len(tap_stop_event) == 1
-            assert tap_stop_event[0].get("name") == tap.name
-            assert tap_stop_event[0].get("cmd_type") == "elb"
-            assert tap_stop_event[0].get("stdio") == "stderr"
+            assert tap_stop_event[0]["name"] == tap.name
+            assert tap_stop_event[0]["cmd_type"] == "elb"
+            assert tap_stop_event[0]["stdio"] == "stderr"
 
             target_stop_event = matcher.find_by_event("target failure")
             assert len(target_stop_event) == 1
-            assert target_stop_event[0].get("name") == target.name
-            assert target_stop_event[0].get("cmd_type") == "elb"
-            assert target_stop_event[0].get("stdio") == "stderr"
+            assert target_stop_event[0]["name"] == target.name
+            assert target_stop_event[0]["cmd_type"] == "elb"
+            assert target_stop_event[0]["stdio"] == "stderr"
 
             # dbt should not have run at all
             assert not matcher.event_matches("dbt starting")
@@ -1015,13 +1001,11 @@ async def wait_mock():
 
             # there should only be one completed event
             assert len(completed_events) == 1
-            assert completed_events[0].get("success") is False
+            assert completed_events[0]["success"] is False
 
-            # or is hack to work around python 3.6 failures
             assert (
-                completed_events[0].get("err")
-                == "RunnerError('Output line length limit exceeded',)"
-                or "RunnerError('Output line length limit exceeded')"
+                completed_events[0]["err"]
+                == "RunnerError('Output line length limit exceeded')"
             )
 
     @pytest.mark.backend("sqlite")
@@ -1204,13 +1188,13 @@ def test_run_elb_mapper_failure(
             matcher = EventMatcher(result.stderr)
             assert matcher.event_matches("found ExtractLoadBlocks set")
             assert (
-                matcher.find_by_event("found PluginCommand")[0].get("plugin_type")
+                matcher.find_by_event("found PluginCommand")[0]["plugin_type"]
                 == "transformers"
             )
 
             completed_events = matcher.find_by_event("Block run completed.")
             assert len(completed_events) == 1
-            assert completed_events[0].get("success") is False
+            assert completed_events[0]["success"] is False
 
             # the tap should have completed successfully
             matcher.event_matches("tap done")
@@ -1221,16 +1205,16 @@ def test_run_elb_mapper_failure(
             # the failed block should have been the mapper
             mapper_stop_event = matcher.find_by_event("mapper failure")
             assert len(mapper_stop_event) == 1
-            assert mapper_stop_event[0].get("name") == mapper.name
-            assert mapper_stop_event[0].get("cmd_type") == "elb"
-            assert mapper_stop_event[0].get("stdio") == "stderr"
+            assert mapper_stop_event[0]["name"] == mapper.name
+            assert mapper_stop_event[0]["cmd_type"] == "elb"
+            assert mapper_stop_event[0]["stdio"] == "stderr"
 
             # target should have attempted to complete
             target_stop_event = matcher.find_by_event("target done")
             assert len(target_stop_event) == 1
-            assert target_stop_event[0].get("name") == target.name
-            assert target_stop_event[0].get("cmd_type") == "elb"
-            assert target_stop_event[0].get("stdio") == "stderr"
+            assert target_stop_event[0]["name"] == target.name
+            assert target_stop_event[0]["cmd_type"] == "elb"
+            assert target_stop_event[0]["stdio"] == "stderr"
 
             # dbt should not have run at all
             assert not matcher.event_matches("dbt starting")