Repository: meltano/meltano

Issue:
Remove Lockfiles Feature Flag and use feature by default



Discussion:
Thanks for raising this, @edgarrmondragon! I've added to the project board.

Any other feature flags we should toggle? Does it make sense to widen this to cover any others as well?
@aaronsteers 

> Any other feature flags we should toggle? Does it make sense to widen this to cover any others as well?

I think for the moment the other two (Uvicorn and Scheduled jobs in the API) are good to keep, unless @pandemicsyn thinks different 😄 
> Uvicorn and Scheduled jobs in the API

Thanks, @edgarrmondragon. For my part, I think (?) those should both stay off by default for now.
> Uvicorn and Scheduled jobs in the API

> Thanks, @edgarrmondragon. For my part, I think (?) those should both stay off by default for now.

+1
@edgarrmondragon @aaronsteers quick clarification: I'm reading this to mean that we want to *remove the feature flag entirely along with the checks around the relevant code paths*. Is that correct or do we just want to leave the feature flag in place but have it turned on by default so users can still toggle it off?

Resolving patch:
diff --git a/src/meltano/cli/add.py b/src/meltano/cli/add.py
--- a/src/meltano/cli/add.py
+++ b/src/meltano/cli/add.py
@@ -12,8 +12,6 @@
 from meltano.core.project import Project
 from meltano.core.project_add_service import ProjectAddService
 from meltano.core.project_plugins_service import ProjectPluginsService
-from meltano.core.project_settings_service import ProjectSettingsService
-from meltano.core.settings_service import FeatureFlags
 
 from . import cli
 from .params import pass_project
@@ -82,7 +80,6 @@ def add(
         plugin_names = [as_name]
 
     plugins_service = ProjectPluginsService(project)
-    settings_service = ProjectSettingsService(project)
 
     if flags["custom"]:
         if plugin_type in {
@@ -104,31 +101,22 @@ def add(
 
     plugins: List[ProjectPlugin] = []
     tracker = LegacyTracker(project)
-
-    with settings_service.feature_flag(
-        FeatureFlags.LOCKFILES,
-        raise_error=False,
-    ) as lock:
-        for plugin in plugin_names:
-            plugins.append(
-                add_plugin(
-                    project,
-                    plugin_type,
-                    plugin,
-                    inherit_from=inherit_from,
-                    variant=variant,
-                    custom=flags["custom"],
-                    add_service=add_service,
-                    lock=lock,
-                )
+    for plugin in plugin_names:
+        plugins.append(
+            add_plugin(
+                project,
+                plugin_type,
+                plugin,
+                inherit_from=inherit_from,
+                variant=variant,
+                custom=flags["custom"],
+                add_service=add_service,
             )
-            tracker.track_meltano_add(plugin_type=plugin_type, plugin_name=plugin)
+        )
+        tracker.track_meltano_add(plugin_type=plugin_type, plugin_name=plugin)
 
         required_plugins = add_required_plugins(
-            project,
-            plugins,
-            add_service=add_service,
-            lock=lock,
+            project, plugins, add_service=add_service
         )
     plugins.extend(required_plugins)
 
diff --git a/src/meltano/core/project_plugins_service.py b/src/meltano/core/project_plugins_service.py
--- a/src/meltano/core/project_plugins_service.py
+++ b/src/meltano/core/project_plugins_service.py
@@ -11,7 +11,6 @@
 from meltano.core.plugin.base import VariantNotFoundError
 from meltano.core.plugin_lock_service import PluginLockService
 from meltano.core.project_settings_service import ProjectSettingsService
-from meltano.core.settings_service import FeatureFlags
 
 from .config_service import ConfigService
 from .plugin import PluginRef, PluginType
@@ -461,7 +460,6 @@ def find_parent(
             error: If the parent plugin is not found.
         """
         error = None
-
         if plugin.inherit_from and not plugin.is_variant_set:
             try:
                 return (
@@ -473,23 +471,16 @@ def find_parent(
             except PluginNotFoundError as inherited_exc:
                 error = inherited_exc
 
-        with self.settings_service.feature_flag(
-            FeatureFlags.LOCKFILES,
-            raise_error=False,
-        ) as allowed:
-            if allowed:
-                try:
-                    return (
-                        self.locked_definition_service.get_base_plugin(
-                            plugin,
-                            variant_name=plugin.variant,
-                        ),
-                        DefinitionSource.LOCKFILE,
-                    )
-                except PluginNotFoundError as lockfile_exc:
-                    error = lockfile_exc
-
-            logger.debug("Lockfile is feature-flagged", status=allowed)
+        try:
+            return (
+                self.locked_definition_service.get_base_plugin(
+                    plugin,
+                    variant_name=plugin.variant,
+                ),
+                DefinitionSource.LOCKFILE,
+            )
+        except PluginNotFoundError as lockfile_exc:
+            error = lockfile_exc
 
         if self._use_discovery_yaml:
             try:
diff --git a/src/meltano/core/settings_service.py b/src/meltano/core/settings_service.py
--- a/src/meltano/core/settings_service.py
+++ b/src/meltano/core/settings_service.py
@@ -29,7 +29,6 @@
 class FeatureFlags(Enum):
     """Available Meltano Feature Flags."""
 
-    LOCKFILES = "lock_files"
     ENABLE_UVICORN = "enable_uvicorn"
     ENABLE_API_SCHEDULED_JOB_LIST = "enable_api_scheduled_job_list"
     STRICT_ENV_VAR_MODE = "strict_env_var_mode"


Test patch:
diff --git a/tests/meltano/cli/test_add.py b/tests/meltano/cli/test_add.py
--- a/tests/meltano/cli/test_add.py
+++ b/tests/meltano/cli/test_add.py
@@ -1,4 +1,5 @@
 import os
+import shutil
 from unittest import mock
 
 import pytest
@@ -165,6 +166,10 @@ def test_add_files_with_updates(
         project_plugins_service,
         plugin_settings_service_factory,
     ):
+        # if plugin is locked, we actually wouldn't expect it to update.
+        # So we must remove lockfile
+        shutil.rmtree("plugins/files", ignore_errors=True)
+
         result = cli_runner.invoke(cli, ["add", "files", "airflow"])
         assert_cli_runner(result)
 
@@ -210,8 +215,9 @@ def test_add_files_without_updates(
     def test_add_files_that_already_exists(
         self, project, cli_runner, project_plugins_service
     ):
+        # dbt lockfile was created in an upstream test. Need to remove.
+        shutil.rmtree(project.root_dir("plugins/files"), ignore_errors=True)
         project.root_dir("transform/dbt_project.yml").write_text("Exists!")
-
         result = cli_runner.invoke(cli, ["add", "files", "dbt"])
         assert_cli_runner(result)
 
diff --git a/tests/meltano/cli/test_upgrade.py b/tests/meltano/cli/test_upgrade.py
--- a/tests/meltano/cli/test_upgrade.py
+++ b/tests/meltano/cli/test_upgrade.py
@@ -1,3 +1,4 @@
+import shutil
 from unittest import mock
 
 from asserts import assert_cli_runner
@@ -70,6 +71,8 @@ def test_upgrade_files(
         # Update file if changed
         file_path.write_text("Overwritten!")
 
+        # The behavior being tested assumes that the file is not locked.
+        shutil.rmtree(project.root_dir("plugins/files"), ignore_errors=True)
         result = cli_runner.invoke(cli, ["upgrade", "files"])
         assert_cli_runner(result)
 
diff --git a/tests/meltano/core/test_connection_service.py b/tests/meltano/core/test_connection_service.py
--- a/tests/meltano/core/test_connection_service.py
+++ b/tests/meltano/core/test_connection_service.py
@@ -5,6 +5,10 @@
 from meltano.core.project_plugins_service import PluginAlreadyAddedException
 
 
+# This is dead code that likely should have been removed
+# as part of the wider m5o deprecation.
+# A later issue should remove this.
+@pytest.mark.skip
 class TestConnectionService:
     @pytest.mark.parametrize(
         "loader,analyze_params",
diff --git a/tests/meltano/core/test_elt_context.py b/tests/meltano/core/test_elt_context.py
--- a/tests/meltano/core/test_elt_context.py
+++ b/tests/meltano/core/test_elt_context.py
@@ -33,7 +33,6 @@ def assert_transform_env(transform, env):
     assert env["MELTANO_TRANSFORM_NAME"] == transform.name
     assert env["MELTANO_TRANSFORM_NAMESPACE"] == transform.namespace
     assert env["MELTANO_TRANSFORM_VARIANT"] == Variant.ORIGINAL_NAME
-    assert env["MELTANO_TRANSFORM__PACKAGE_NAME"] == "dbt_mock"
 
 
 def assert_transformer_env(transformer, env):
diff --git a/tests/meltano/core/test_project_add_service.py b/tests/meltano/core/test_project_add_service.py
--- a/tests/meltano/core/test_project_add_service.py
+++ b/tests/meltano/core/test_project_add_service.py
@@ -103,8 +103,8 @@ def test_add_inherited(
             "inherit_from": "tap-mock-inherited",
         }
 
-        assert hub_request_counter["/extractors/index"] == 3
-        assert hub_request_counter["/extractors/tap-mock--meltano"] == 3
+        assert hub_request_counter["/extractors/index"] == 1
+        assert hub_request_counter["/extractors/tap-mock--meltano"] == 1
         assert len(hub_request_counter) == 2
 
     def test_lockfile_inherited(
@@ -156,6 +156,6 @@ def test_lockfile_inherited(
         matches = list(parent_path.parent.glob("tap-mock-inherited-new*"))
         assert not matches
 
-        assert hub_request_counter["/extractors/index"] == 2
-        assert hub_request_counter["/extractors/tap-mock--meltano"] == 2
+        assert hub_request_counter["/extractors/index"] == 1
+        assert hub_request_counter["/extractors/tap-mock--meltano"] == 1
         assert len(hub_request_counter) == 2
diff --git a/tests/meltano/core/test_project_plugins_service.py b/tests/meltano/core/test_project_plugins_service.py
--- a/tests/meltano/core/test_project_plugins_service.py
+++ b/tests/meltano/core/test_project_plugins_service.py
@@ -1,4 +1,5 @@
 import json
+import shutil
 from copy import deepcopy
 
 import pytest
@@ -9,7 +10,6 @@
 from meltano.core.plugin_discovery_service import LockedDefinitionService
 from meltano.core.project import Project
 from meltano.core.project_plugins_service import DefinitionSource, ProjectPluginsService
-from meltano.core.settings_service import FeatureFlags
 
 
 @pytest.fixture
@@ -87,7 +87,26 @@ def test_get_plugin(
         subject._use_cache = False
         assert subject.get_plugin(tap) is not plugin
 
-    def test_get_parent(
+    def test_get_parent_from_lockfile(
+        self,
+        subject: ProjectPluginsService,
+        tap: ProjectPlugin,
+        locked_definition_service: LockedDefinitionService,
+        modified_lockfile,
+    ):
+        expected = locked_definition_service.find_base_plugin(
+            plugin_type=PluginType.EXTRACTORS,
+            plugin_name="tap-mock",
+            variant="meltano",
+        )
+
+        result, source = subject.find_parent(tap)
+        assert source == DefinitionSource.LOCKFILE
+        assert result == expected
+        assert result.settings == expected.settings
+        assert result.settings[-1].name == "foo"
+
+    def test_get_parent_no_lockfiles(
         self,
         subject: ProjectPluginsService,
         tap,
@@ -96,6 +115,9 @@ def test_get_parent(
         alternative_target,
         plugin_discovery_service,
     ):
+
+        # The behavior being tested here assumes that no lockfiles exist.
+        shutil.rmtree(subject.project.root_dir("plugins"), ignore_errors=True)
         # name="tap-mock", variant="meltano"
         # Shadows base plugin with correct variant
         assert subject.get_parent(tap) == plugin_discovery_service.find_base_plugin(
@@ -133,36 +155,6 @@ def test_get_parent(
         with pytest.raises(PluginParentNotFoundError):
             assert subject.get_parent(nonexistent_parent)
 
-    def test_get_parent_from_lockfile(
-        self,
-        subject: ProjectPluginsService,
-        tap: ProjectPlugin,
-        locked_definition_service: LockedDefinitionService,
-        modified_lockfile,
-    ):
-        expected = locked_definition_service.find_base_plugin(
-            plugin_type=PluginType.EXTRACTORS,
-            plugin_name="tap-mock",
-            variant="meltano",
-        )
-
-        # Feature flag off means definition is not retrieved from lockfile
-        subject.settings_service.set(FeatureFlags.LOCKFILES.setting_name, False)
-        result_no_ff, source = subject.find_parent(tap)
-        assert source == DefinitionSource.DISCOVERY
-        assert result_no_ff == expected
-        assert len(result_no_ff.settings) == 13  # noqa: WPS432
-
-        # Feature flag on means definition is indeed retrieved from lockfile
-        subject.settings_service.set(FeatureFlags.LOCKFILES.setting_name, True)
-        result, source = subject.find_parent(tap)
-        assert source == DefinitionSource.LOCKFILE
-        assert result == expected
-        assert result.settings == expected.settings
-        assert result.settings[-1].name == "foo"
-
-        subject.settings_service.set(FeatureFlags.LOCKFILES.setting_name, False)
-
     def test_update_plugin(self, subject, tap):
         # update a tap with a random value
         tap.config["test"] = 42