Repository: meltano/meltano

Issue:
Emit deprecation warning when `elt` command is used instead of `el`
Also emit a deprecation warning when the transform option is used.

The `el` command would have to exist before this could be completed.

Relates to:
- https://github.com/meltano/meltano/discussions/6273


Discussion:
Since https://github.com/meltano/meltano/issues/7576 is in the v3 Milestone and the `elt` might behave unexpectedly for users of `--transform` (see https://github.com/meltano/meltano/issues/7548#issuecomment-1513382766), do we want to add this to the v3 Milestone too?

@WillDaSilva @tayloramurphy
Sounds good to me @edgarrmondragon. We'd want to target `main` with the addition of the new `el` command, and likely with the addition of the deprecation warning too.

Resolving patch:
diff --git a/src/meltano/cli/__init__.py b/src/meltano/cli/__init__.py
--- a/src/meltano/cli/__init__.py
+++ b/src/meltano/cli/__init__.py
@@ -47,6 +47,7 @@
 cli.add_command(discovery.discover)
 cli.add_command(docs.docs)
 cli.add_command(dragon.dragon)
+cli.add_command(elt.el)
 cli.add_command(elt.elt)
 cli.add_command(environment.meltano_environment)
 cli.add_command(hub.hub)
diff --git a/src/meltano/cli/elt.py b/src/meltano/cli/elt.py
--- a/src/meltano/cli/elt.py
+++ b/src/meltano/cli/elt.py
@@ -42,55 +42,137 @@
 logger = structlog_stdlib.get_logger(__name__)
 
 
+class ELOptions:
+    """Namespace for all options shared by the `el` and `elt` commands."""
+
+    extractor = click.argument("extractor")
+    loader = click.argument("loader")
+    transform = click.option(
+        "--transform",
+        type=click.Choice(["skip", "only", "run"]),
+    )
+    dry = click.option("--dry", help="Do not actually run.", is_flag=True)
+    full_refresh = click.option(
+        "--full-refresh",
+        help="Perform a full refresh (ignore state left behind by any previous runs).",
+        is_flag=True,
+    )
+    select = click.option(
+        "--select",
+        "-s",
+        help="Select only these specific entities for extraction.",
+        multiple=True,
+        default=[],
+    )
+    exclude = click.option(
+        "--exclude",
+        "-e",
+        help="Exclude these specific entities from extraction.",
+        multiple=True,
+        default=[],
+    )
+    catalog = click.option("--catalog", help="Extractor catalog file.")
+    state = click.option("--state", help="Extractor state file.")
+    dump = click.option(
+        "--dump",
+        type=click.Choice(DUMPABLES.keys()),
+        help="Dump content of pipeline-specific generated file.",
+    )
+    state_id = click.option(
+        "--state-id",
+        envvar="MELTANO_STATE_ID",
+        help="A custom string to identify the job.",
+    )
+    force = click.option(
+        "--force",
+        "-f",
+        help=(
+            "Force a new run even when a pipeline with the same state ID is "
+            "already running."
+        ),
+        is_flag=True,
+    )
+
+
 @click.command(
     cls=PartialInstrumentedCmd,
-    short_help="Run an ELT pipeline to Extract, Load, and Transform data.",
+    short_help="Run an EL pipeline to Extract and Load data.",
     environment_behavior=CliEnvironmentBehavior.environment_optional_use_default,
 )
-@click.argument("extractor")
-@click.argument("loader")
-@click.option("--transform", type=click.Choice(["skip", "only", "run"]))
-@click.option("--dry", help="Do not actually run.", is_flag=True)
-@click.option(
-    "--full-refresh",
-    help="Perform a full refresh (ignore state left behind by any previous runs).",
-    is_flag=True,
-)
-@click.option(
-    "--select",
-    "-s",
-    help="Select only these specific entities for extraction.",
-    multiple=True,
-    default=[],
-)
-@click.option(
-    "--exclude",
-    "-e",
-    help="Exclude these specific entities from extraction.",
-    multiple=True,
-    default=[],
-)
-@click.option("--catalog", help="Extractor catalog file.")
-@click.option("--state", help="Extractor state file.")
-@click.option(
-    "--dump",
-    type=click.Choice(DUMPABLES.keys()),
-    help="Dump content of pipeline-specific generated file.",
-)
-@click.option(
-    "--state-id",
-    envvar="MELTANO_STATE_ID",
-    help="A custom string to identify the job.",
-)
-@click.option(
-    "--force",
-    "-f",
-    help=(
-        "Force a new run even when a pipeline with the same state ID is "
-        "already running."
-    ),
-    is_flag=True,
+@ELOptions.extractor
+@ELOptions.loader
+@ELOptions.dry
+@ELOptions.full_refresh
+@ELOptions.select
+@ELOptions.exclude
+@ELOptions.catalog
+@ELOptions.state
+@ELOptions.dump
+@ELOptions.state_id
+@ELOptions.force
+@click.pass_context
+@pass_project(migrate=True)
+@click_run_async
+async def el(  # WPS408
+    project: Project,
+    ctx: click.Context,
+    extractor: str,
+    loader: str,
+    dry: bool,
+    full_refresh: bool,
+    select: list[str],
+    exclude: list[str],
+    catalog: str,
+    state: str,
+    dump: str,
+    state_id: str,
+    force: bool,
+):
+    """
+    Run an EL pipeline to Extract and Load data.
+
+    meltano el '<extractor_name>' '<loader_name>'
+
+    extractor_name: extractor to be used in this pipeline.
+    loader_name: loader to be used in this pipeline.
+
+    \b\nRead more at https://docs.meltano.com/reference/command-line-interface#el
+    """
+    await _run_el_command(
+        project,
+        ctx,
+        extractor,
+        loader,
+        None,
+        dry,
+        full_refresh,
+        select,
+        exclude,
+        catalog,
+        state,
+        dump,
+        state_id,
+        force,
+    )
+
+
+@click.command(
+    cls=PartialInstrumentedCmd,
+    short_help="Run an ELT pipeline to Extract, Load, and Transform data.",
+    environment_behavior=CliEnvironmentBehavior.environment_optional_use_default,
 )
+@ELOptions.extractor
+@ELOptions.loader
+@ELOptions.transform
+@ELOptions.dry
+@ELOptions.full_refresh
+@ELOptions.select
+@ELOptions.exclude
+@ELOptions.catalog
+@ELOptions.state
+@ELOptions.dump
+@ELOptions.state_id
+@ELOptions.force
 @click.pass_context
 @pass_project(migrate=True)
 @click_run_async
@@ -120,6 +202,41 @@ async def elt(  # WPS408
 
     \b\nRead more at https://docs.meltano.com/reference/command-line-interface#elt
     """
+    logger.warn("The `elt` command is deprecated in favor of `el`")
+    await _run_el_command(
+        project,
+        ctx,
+        extractor,
+        loader,
+        transform,
+        dry,
+        full_refresh,
+        select,
+        exclude,
+        catalog,
+        state,
+        dump,
+        state_id,
+        force,
+    )
+
+
+async def _run_el_command(
+    project: Project,
+    ctx: click.Context,
+    extractor: str,
+    loader: str,
+    transform: str | None,
+    dry: bool,
+    full_refresh: bool,
+    select: list[str],
+    exclude: list[str],
+    catalog: str,
+    state: str,
+    dump: str,
+    state_id: str,
+    force: bool,
+):
     if platform.system() == "Windows":
         raise CliError(
             "ELT command not supported on Windows. Please use the run command "
@@ -132,8 +249,8 @@ async def elt(  # WPS408
     # We no longer set a default choice for transform, so that we can detect
     # explicit usages of the `--transform` option if transform is `None` we
     # still need manually default to skip after firing the tracking event above
-    if not transform:
-        transform = "skip"
+    # TODO: Remove the transform option entirely in a future release
+    transform = transform or "skip"
 
     select_filter = [*select, *(f"!{entity}" for entity in exclude)]
 
@@ -288,6 +405,10 @@ async def _run_elt(
                 await _run_extract_load(log, elt_context, output_logger)
 
             if elt_context.transformer:
+                log.warn(
+                    "The option to run a transformation is deprecated and will be "
+                    "removed in a future version.",
+                )
                 await _run_transform(log, elt_context, output_logger)
             else:
                 log.info("Transformation skipped.")


Test patch:
diff --git a/docs/docs/contribute/tests.md b/docs/docs/contribute/tests.md
--- a/docs/docs/contribute/tests.md
+++ b/docs/docs/contribute/tests.md
@@ -44,17 +44,17 @@ the bellow in `docs/example-library/transition-from-elt-to-run/index.md`:
 ````markdown
 # Example of how to transition from `meltano elt` to `meltano run`
 
-This example shows how to transition an `elt` task with a custom state-id to a `job` executed via `run`.
+This example shows how to transition an `el` or `elt` task with a custom state-id to a `job` executed via `run`.
 To follow along with this example, download link to meltano yml to a fresh project and run:
 
 ```
 meltano install
 ```
 
-Then assuming you had an `elt` job invoked like so:
+Then assuming you had an `el` job invoked like so:
 
 ```shell
-meltano elt --state-id=my-custom-id tap-gitlab target-postgres
+meltano el --state-id=my-custom-id tap-gitlab target-postgres
 ```
 
 You would first need to rename the id to match meltano's internal pattern:
@@ -76,7 +76,7 @@ Our integration framework will then parse this markdown, searching for code fenc
 
 ```shell
 meltano install
-meltano elt --state-id=my-custom-id tap-gitlab target-postgres
+meltano el --state-id=my-custom-id tap-gitlab target-postgres
 meltano state copy my-custom-id tap-gitlab-to-target-postgres
 meltano job add my-new-job --task="tap-gitlab target-postgres"
 meltano run my-new-job
diff --git a/tests/meltano/cli/test_elt.py b/tests/meltano/cli/test_elt.py
--- a/tests/meltano/cli/test_elt.py
+++ b/tests/meltano/cli/test_elt.py
@@ -220,6 +220,7 @@ def test_elt_windows(
 class TestCliEltScratchpadOne:
     @pytest.mark.backend("sqlite")
     @pytest.mark.usefixtures("use_test_log_config", "project")
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
     def test_elt(
         self,
         cli_runner,
@@ -228,12 +229,13 @@ def test_elt(
         tap_process,
         target_process,
         job_logging_service,
+        command: str,
     ):
-        result = cli_runner.invoke(cli, ["elt"])
+        result = cli_runner.invoke(cli, [command])
         assert result.exit_code == 2
 
-        state_id = "pytest_test_elt"
-        args = ["elt", "--state-id", state_id, tap.name, target.name]
+        state_id = f"pytest_test_{command}"
+        args = [command, "--state-id", state_id, tap.name, target.name]
 
         # exit cleanly when everything is fine
         create_subprocess_exec = AsyncMock(side_effect=(tap_process, target_process))
@@ -301,6 +303,7 @@ def test_elt(
 
     @pytest.mark.backend("sqlite")
     @pytest.mark.usefixtures("use_test_log_config", "project")
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
     def test_elt_debug_logging(
         self,
         cli_runner,
@@ -310,9 +313,10 @@ def test_elt_debug_logging(
         target_process,
         job_logging_service,
         monkeypatch,
+        command: str,
     ):
-        state_id = "pytest_test_elt_debug"
-        args = ["elt", "--state-id", state_id, tap.name, target.name]
+        state_id = f"pytest_test_{command}_debug"
+        args = [command, "--state-id", state_id, tap.name, target.name]
 
         job_logging_service.delete_all_logs(state_id)
 
@@ -418,17 +422,19 @@ def test_elt_debug_logging(
 
             full_result = result.stdout + result.stderr
 
-            # We expect a difference of 1 line because the cli emits one log
-            # line not found in the log.
+            # We expect a difference of 2 lines because the cli emits two log
+            # lines not found in the log.
             # we already test the redirect handler in test_output_logger,
             # so we'll just verify that the # of lines matches.
-            assert len(log.splitlines()) == len(full_result.splitlines()) - 1
+            log_diff = 1 if command == "el" else 2
+            assert len(log.splitlines()) == len(full_result.splitlines()) - log_diff
             # and just to be safe - check if these debug mode only strings show up
             assert "target-mock (out)" in log
             assert "tap-mock (out)" in log
 
     @pytest.mark.backend("sqlite")
     @pytest.mark.usefixtures("use_test_log_config", "project")
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
     def test_elt_tap_failure(
         self,
         cli_runner,
@@ -437,9 +443,10 @@ def test_elt_tap_failure(
         tap_process,
         target_process,
         job_logging_service,
+        command: str,
     ):
-        state_id = "pytest_test_elt"
-        args = ["elt", "--state-id", state_id, tap.name, target.name]
+        state_id = f"pytest_test_{command}"
+        args = [command, "--state-id", state_id, tap.name, target.name]
 
         tap_process.wait.return_value = 1
         tap_process.stderr.readline.side_effect = (
@@ -488,6 +495,7 @@ def test_elt_tap_failure(
 
     @pytest.mark.backend("sqlite")
     @pytest.mark.usefixtures("use_test_log_config", "project")
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
     def test_elt_target_failure_before_tap_finishes(
         self,
         cli_runner,
@@ -496,9 +504,10 @@ def test_elt_target_failure_before_tap_finishes(
         tap_process,
         target_process,
         job_logging_service,
+        command: str,
     ):
-        state_id = "pytest_test_elt"
-        args = ["elt", "--state-id", state_id, tap.name, target.name]
+        state_id = f"pytest_test_{command}"
+        args = [command, "--state-id", state_id, tap.name, target.name]
 
         # Have `tap_process.wait` take 2s to make sure the target can fail
         # before tap finishes
@@ -567,6 +576,7 @@ async def target_wait_mock():
 
     @pytest.mark.backend("sqlite")
     @pytest.mark.usefixtures("use_test_log_config", "project")
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
     def test_elt_target_failure_after_tap_finishes(
         self,
         cli_runner,
@@ -575,9 +585,10 @@ def test_elt_target_failure_after_tap_finishes(
         tap_process,
         target_process,
         job_logging_service,
+        command: str,
     ):
-        state_id = "pytest_test_elt"
-        args = ["elt", "--state-id", state_id, tap.name, target.name]
+        state_id = f"pytest_test_{command}"
+        args = [command, "--state-id", state_id, tap.name, target.name]
 
         target_process.wait.return_value = 1
         target_process.stderr.readline.side_effect = (
@@ -626,6 +637,7 @@ def test_elt_target_failure_after_tap_finishes(
 
     @pytest.mark.backend("sqlite")
     @pytest.mark.usefixtures("use_test_log_config", "project")
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
     def test_elt_tap_and_target_failure(
         self,
         cli_runner,
@@ -634,9 +646,10 @@ def test_elt_tap_and_target_failure(
         tap_process,
         target_process,
         job_logging_service,
+        command: str,
     ):
-        state_id = "pytest_test_elt"
-        args = ["elt", "--state-id", state_id, tap.name, target.name]
+        state_id = f"pytest_test_{command}"
+        args = [command, "--state-id", state_id, tap.name, target.name]
 
         tap_process.wait.return_value = 1
         tap_process.stderr.readline.side_effect = (
@@ -694,6 +707,7 @@ def test_elt_tap_and_target_failure(
 
     @pytest.mark.backend("sqlite")
     @pytest.mark.usefixtures("use_test_log_config", "project")
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
     def test_elt_tap_line_length_limit_error(
         self,
         cli_runner,
@@ -702,9 +716,10 @@ def test_elt_tap_line_length_limit_error(
         tap_process,
         target_process,
         job_logging_service,
+        command: str,
     ):
-        state_id = "pytest_test_elt"
-        args = ["elt", "--state-id", state_id, tap.name, target.name]
+        state_id = f"pytest_test_{command}"
+        args = [command, "--state-id", state_id, tap.name, target.name]
 
         # Raise a `ValueError` wrapping a `LimitOverrunError`, like
         # `StreamReader.readline` does:
@@ -768,6 +783,7 @@ async def wait_mock():
 
     @pytest.mark.backend("sqlite")
     @pytest.mark.usefixtures("use_test_log_config", "project")
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
     def test_elt_output_handler_error(
         self,
         cli_runner,
@@ -775,9 +791,10 @@ def test_elt_output_handler_error(
         target,
         tap_process,
         target_process,
+        command: str,
     ):
-        state_id = "pytest_test_elt"
-        args = ["elt", "--state-id", state_id, tap.name, target.name]
+        state_id = f"pytest_test_{command}"
+        args = [command, "--state-id", state_id, tap.name, target.name]
 
         exc = Exception("Failed to read from target stderr.")
         target_process.stderr.readline.side_effect = exc
@@ -812,9 +829,10 @@ async def wait_mock():
                 Exception("Failed to read from target stderr."),
             )
 
-    def test_elt_already_running(self, cli_runner, tap, target, session):
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
+    def test_elt_already_running(self, cli_runner, tap, target, session, command: str):
         state_id = "already_running"
-        args = ["elt", "--state-id", state_id, tap.name, target.name]
+        args = [command, "--state-id", state_id, tap.name, target.name]
 
         existing_job = Job(job_name=state_id, state=State.RUNNING)
         existing_job.save(session)
@@ -829,20 +847,22 @@ def test_elt_already_running(self, cli_runner, tap, target, session):
                 result.exception,
             )
 
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
     def test_dump_catalog(
         self,
         cli_runner,
         project,
         tap,
         target,
+        command: str,
     ):
         catalog = {"streams": []}
         with project.root.joinpath("catalog.json").open("w") as catalog_file:
             json.dump(catalog, catalog_file)
 
-        state_id = "pytest_test_elt"
+        state_id = f"pytest_test_{command}"
         args = [
-            "elt",
+            command,
             "--state-id",
             state_id,
             tap.name,
@@ -859,20 +879,22 @@ def test_dump_catalog(
         assert json.loads(result.stdout) == catalog
 
     @pytest.mark.usefixtures("session")
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
     def test_dump_state(
         self,
         cli_runner,
         project,
         tap,
         target,
+        command: str,
     ):
         state = {"success": True}
         with project.root.joinpath("state.json").open("w") as state_file:
             json.dump(state, state_file)
 
-        state_id = "pytest_test_elt"
+        state_id = f"pytest_test_{command}"
         args = [
-            "elt",
+            command,
             "--state-id",
             state_id,
             tap.name,
@@ -893,16 +915,18 @@ def test_dump_state(
             assert json.loads(result.stdout) == state
 
     @pytest.mark.usefixtures("project")
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
     def test_dump_extractor_config(
         self,
         cli_runner,
         tap,
         target,
         plugin_settings_service_factory,
+        command: str,
     ):
-        state_id = "pytest_test_elt"
+        state_id = f"pytest_test_{command}"
         args = [
-            "elt",
+            command,
             "--state-id",
             state_id,
             tap.name,
@@ -926,16 +950,18 @@ def test_dump_extractor_config(
             )
 
     @pytest.mark.usefixtures("project")
+    @pytest.mark.parametrize("command", ("elt", "el"), ids=["elt", "el"])
     def test_dump_loader_config(
         self,
         cli_runner,
         tap,
         target,
         plugin_settings_service_factory,
+        command: str,
     ):
-        state_id = "pytest_test_elt"
+        state_id = f"pytest_test_{command}"
         args = [
-            "elt",
+            command,
             "--state-id",
             state_id,
             tap.name,
@@ -1136,6 +1162,12 @@ def test_elt_transform_only(
             assert_log_lines(
                 result.stdout + result.stderr,
                 [
+                    LogEntry(
+                        None,
+                        None,
+                        "The `elt` command is deprecated in favor of `el`",
+                        "warning",
+                    ),
                     LogEntry("meltano", None, "Extract & load skipped.", "info"),
                     LogEntry("meltano", None, "Running transformation...", "info"),
                     LogEntry("meltano", None, "Transformation complete!", "info"),