Repository: meltano/meltano

Issue:
feat: consider clearing the catalog cache when `--full-refresh` present
This has come up in slack a bunch of times but most recently in https://meltano.slack.com/archives/C01TCRBBJD7/p1689207554179589

Lots of users run into this where the catalog is being cached so the output theyre seeing is not as expected, we end up recommending that they clear their `.meltano/run/tap-x` directory so it regenerates the catalog. If someone runs with the `--full-refresh` flag its probably because something in the source changed so they need to re-run the replication but the cached catalog is blocking those changes from propogating.


Related to:
- https://github.com/meltano/meltano/issues/6292
- https://github.com/meltano/meltano/issues/6763
- https://github.com/meltano/meltano/issues/2856
- https://github.com/meltano/meltano/issues/2848




Discussion:

Resolving patch:
diff --git a/src/meltano/cli/__init__.py b/src/meltano/cli/__init__.py
--- a/src/meltano/cli/__init__.py
+++ b/src/meltano/cli/__init__.py
@@ -10,7 +10,6 @@
 from meltano.cli import (  # noqa: WPS235
     add,
     config,
-    discovery,
     docs,
     dragon,
     elt,
@@ -44,7 +43,6 @@
 cli.add_command(cloud)
 cli.add_command(compile_module.compile_command)
 cli.add_command(config.config)
-cli.add_command(discovery.discover)
 cli.add_command(docs.docs)
 cli.add_command(dragon.dragon)
 cli.add_command(elt.elt)
diff --git a/src/meltano/cli/discovery.py b/src/meltano/cli/discovery.py
deleted file mode 100644
--- a/src/meltano/cli/discovery.py
+++ /dev/null
@@ -1,67 +0,0 @@
-"""Discoverable Plugins CLI."""
-
-from __future__ import annotations
-
-import typing as t
-
-import click
-
-from meltano.cli.params import pass_project
-from meltano.cli.utils import InstrumentedCmd
-from meltano.core.plugin import PluginType
-
-if t.TYPE_CHECKING:
-    from meltano.core.project import Project
-
-
-@click.command(
-    cls=InstrumentedCmd,
-    short_help="List the available plugins in Meltano Hub and their variants.",
-)
-@click.argument(
-    "plugin_type",
-    type=click.Choice([*list(PluginType), "all"]),
-    default="all",
-)
-@pass_project()
-def discover(  # noqa: C901
-    project: Project,
-    plugin_type: str,
-):
-    """
-    List the available discoverable plugins and their variants.
-
-    \b\nRead more at https://docs.meltano.com/reference/command-line-interface#discover
-    """
-    if plugin_type == "all":
-        plugin_types = [
-            plugin_type for plugin_type in list(PluginType) if plugin_type.discoverable
-        ]
-    else:
-        plugin_types = [PluginType.from_cli_argument(plugin_type)]
-
-    for idx, discovered_plugin_type in enumerate(plugin_types):
-        if idx > 0:
-            click.echo()
-
-        click.secho(f"{str(discovered_plugin_type).capitalize()}", fg="green")
-
-        try:
-            plugin_type_index = project.hub_service.get_plugins_of_type(
-                discovered_plugin_type,
-            )
-        except Exception:
-            click.secho(
-                f"Can not retrieve {discovered_plugin_type} from the Hub",
-                fg="yellow",
-                err=True,
-            )
-            continue
-
-        for plugin_name, plugin in plugin_type_index.items():
-            click.echo(plugin_name, nl=False)
-
-            if len(plugin.variants) > 1:
-                click.echo(f", variants: {', '.join(plugin.variant_labels)}")
-            else:
-                click.echo()


Test patch:
diff --git a/tests/meltano/cli/test_cli.py b/tests/meltano/cli/test_cli.py
--- a/tests/meltano/cli/test_cli.py
+++ b/tests/meltano/cli/test_cli.py
@@ -64,7 +64,7 @@ def test_activate_project(self, test_cli_project, cli_runner, pushd):
         project = test_cli_project
 
         pushd(project.root)
-        cli_runner.invoke(cli, ["discover"])
+        cli_runner.invoke(cli, ["hub", "ping"])
 
         assert Project._default is not None
         assert Project._default.root == project.root
@@ -87,7 +87,7 @@ def test_activate_project_readonly_env(
         monkeypatch.setenv(PROJECT_READONLY_ENV, "true")
         assert Project._default is None
         pushd(test_cli_project.root)
-        cli_runner.invoke(cli, ["discover"])
+        cli_runner.invoke(cli, ["hub", "ping"])
         assert Project._default.readonly
 
     @pytest.mark.order(2)
@@ -100,7 +100,7 @@ def test_activate_project_readonly_dotenv(
         test_cli_project.settings.set("project_readonly", True)
         assert Project._default is None
         pushd(test_cli_project.root)
-        cli_runner.invoke(cli, ["discover"])
+        cli_runner.invoke(cli, ["hub", "ping"])
         assert Project._default.readonly
 
     def test_environment_precedence(
@@ -200,7 +200,7 @@ def test_lower_null_environment_overrides_default(
         pushd(project_files_cli.root)
         cli_runner.invoke(
             cli,
-            ["--environment", "null", "discover"],
+            ["--environment", "null", "hub", "ping"],
         )
         assert Project._default.environment is None
 
@@ -214,7 +214,7 @@ def test_upper_null_environment_overrides_default(
         pushd(project_files_cli.root)
         cli_runner.invoke(
             cli,
-            ["--environment", "NULL", "discover"],
+            ["--environment", "NULL", "hub", "ping"],
         )
         assert Project._default.environment is None
 
@@ -228,7 +228,7 @@ def test_no_environment_overrides_default(
         pushd(project_files_cli.root)
         cli_runner.invoke(
             cli,
-            ["--no-environment", "discover"],
+            ["--no-environment", "hub", "ping"],
         )
         assert Project._default.environment is None
 
@@ -242,7 +242,7 @@ def test_no_environment_and_null_environment_overrides_default(  # noqa: WPS118
         pushd(project_files_cli.root)
         cli_runner.invoke(
             cli,
-            ["--no-environment", "--environment", "null", "discover"],
+            ["--no-environment", "--environment", "null", "hub", "ping"],
         )
         assert Project._default.environment is None
 
diff --git a/tests/meltano/cli/test_discover.py b/tests/meltano/cli/test_discover.py
deleted file mode 100644
--- a/tests/meltano/cli/test_discover.py
+++ /dev/null
@@ -1,64 +0,0 @@
-from __future__ import annotations
-
-from collections import Counter
-
-import mock
-
-from asserts import assert_cli_runner
-from meltano.cli import cli
-from meltano.core.plugin.base import PluginType
-from meltano.core.project import Project
-
-
-class TestCliDiscover:
-    def test_discover(
-        self,
-        project: Project,
-        cli_runner,
-        hub_request_counter: Counter,
-    ):
-        adapter = project.hub_service.session.get_adapter(
-            project.hub_service.hub_api_url,
-        )
-
-        with mock.patch("requests.adapters.HTTPAdapter.send", adapter.send):
-            result = cli_runner.invoke(cli, ["discover"])
-
-        assert_cli_runner(result)
-
-        for plugin_type in PluginType:
-            request_count = 1 if plugin_type.discoverable else 0
-            assert hub_request_counter[f"/{plugin_type}/index"] == request_count
-
-        assert "Extractors" in result.stdout
-        assert "tap-gitlab" in result.stdout
-        assert "tap-mock" in result.stdout
-        assert "tap-mock, variants: meltano (default), singer-io" in result.stdout
-
-        assert "Loaders" in result.stdout
-        assert "target-jsonl" in result.stdout
-        assert "target-mock" in result.stdout
-
-    def test_discover_extractors(
-        self,
-        project: Project,
-        cli_runner,
-        hub_request_counter: Counter,
-    ):
-        adapter = project.hub_service.session.get_adapter(
-            project.hub_service.hub_api_url,
-        )
-
-        with mock.patch("requests.adapters.HTTPAdapter.send", adapter.send):
-            result = cli_runner.invoke(cli, ["discover", "extractors"])
-
-        assert_cli_runner(result)
-
-        assert hub_request_counter["/extractors/index"] == 1
-        assert hub_request_counter["/loaders/index"] == 0
-
-        assert "Extractors" in result.stdout
-        assert "tap-gitlab" in result.stdout
-        assert "tap-mock" in result.stdout
-
-        assert "Loaders" not in result.stdout
diff --git a/tests/meltano/core/hub/test_meltano_hub_service.py b/tests/meltano/core/hub/test_meltano_hub_service.py
--- a/tests/meltano/core/hub/test_meltano_hub_service.py
+++ b/tests/meltano/core/hub/test_meltano_hub_service.py
@@ -9,7 +9,7 @@
 from requests.adapters import BaseAdapter
 
 from meltano.cli import cli
-from meltano.cli.discovery import discover
+from meltano.cli.hub import hub
 from meltano.core.hub.client import HubConnectionError, HubPluginVariantNotFoundError
 from meltano.core.plugin.base import PluginType, Variant
 from meltano.core.plugin.error import PluginNotFoundError
@@ -130,12 +130,12 @@ def test_server_error(self, project: Project):
     def test_request_headers(self, project: Project):
         with mock.patch("click.get_current_context") as get_context:
             get_context.return_value = click.Context(
-                discover,
-                info_name="discover",
+                hub,
+                info_name="hub",
                 parent=click.Context(cli, info_name="meltano"),
             )
             request = project.hub_service._build_request("GET", "https://example.com")
-            assert request.headers["X-Meltano-Command"] == "meltano discover"
+            assert request.headers["X-Meltano-Command"] == "meltano hub"
 
         with mock.patch("click.get_current_context") as get_context:
             get_context.return_value = None