Repository: dbt-labs/dbt-core

Issue:
[CT-2205] [Feature] Support supply params in dbtRunner.invoke to overwrite flags
# Feature description
During `dbtRunner.invoke`, user should be able to supply a configuration set that used for that invocation with the runner.

```Python
# this set of params will overwrite attributes that exists on Flags object during initialization of Flags.
# Anything set here will be visible to the structured logs and exposed in jinja context.
params = {"fail_fast": True, 'version_check': False}

# initialize
from dbt.cli.main import dbtRunner  
dbt = dbtRunner()

# this will fail fast, and not check version
dbt.invoke(['run'], params=params)
```

# Requirement
- params will overwrite all other formats of configs(env var, cli specified)(during initialization of Flags)
- Add a structure logging event that include the names of params that got passed in via the params in dbt.invoke
- Add tests for params that we know `dbt-server` would want to set/overwrite.
  - log_format
  - log_path
  - version_check
  - project_dir
  - profiles_dir
  - profile_name
  - target_dir
- Make sure `dbt-server` is unblocked
  
[CT-2205] [Feature] Support supply params in dbtRunner.invoke to overwrite flags
# Feature description
During `dbtRunner.invoke`, user should be able to supply a configuration set that used for that invocation with the runner.

```Python
# this set of params will overwrite attributes that exists on Flags object during initialization of Flags.
# Anything set here will be visible to the structured logs and exposed in jinja context.
params = {"fail_fast": True, 'version_check': False}

# initialize
from dbt.cli.main import dbtRunner  
dbt = dbtRunner()

# this will fail fast, and not check version
dbt.invoke(['run'], params=params)
```

# Requirement
- params will overwrite all other formats of configs(env var, cli specified)(during initialization of Flags)
- Add a structure logging event that include the names of params that got passed in via the params in dbt.invoke
- Add tests for params that we know `dbt-server` would want to set/overwrite.
  - log_format
  - log_path
  - version_check
  - project_dir
  - profiles_dir
  - profile_name
  - target_dir
- Make sure `dbt-server` is unblocked
  


Discussion:
By doing it this way, anything set by `params` will still be visible (name + value) to the structured logs ([MainReportArgs](https://github.com/dbt-labs/dbt-core/blob/0ef9931d197c996ccd4e3caf641729176eceda99/core/dbt/cli/requires.py#L41)), exposed in jinja context ([docs](https://docs.getdbt.com/reference/dbt-jinja-functions/flags)) and in the `args` dict of `run_results.json`.

Let's confirm that this is acceptable for the intended use case. If it isn't acceptable, we should proceed with this work as currently scoped, and open a new issue to address `SensitiveFlags` (whose values should be excluded/scrubbed from logging, artifacts, Jinja context).
By doing it this way, anything set by `params` will still be visible (name + value) to the structured logs ([MainReportArgs](https://github.com/dbt-labs/dbt-core/blob/0ef9931d197c996ccd4e3caf641729176eceda99/core/dbt/cli/requires.py#L41)), exposed in jinja context ([docs](https://docs.getdbt.com/reference/dbt-jinja-functions/flags)) and in the `args` dict of `run_results.json`.

Let's confirm that this is acceptable for the intended use case. If it isn't acceptable, we should proceed with this work as currently scoped, and open a new issue to address `SensitiveFlags` (whose values should be excluded/scrubbed from logging, artifacts, Jinja context).

Resolving patch:
diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py
--- a/core/dbt/cli/flags.py
+++ b/core/dbt/cli/flags.py
@@ -112,7 +112,10 @@ def assign_params(ctx, params_assigned_from_default, deprecated_env_vars):
                         # param already set via its deprecated but still respected env var
                         continue
 
-                    if param_name not in EXPECTED_DUPLICATE_PARAMS:
+                    if (
+                        param_name not in EXPECTED_DUPLICATE_PARAMS
+                        and ctx.get_parameter_source(param_name) != "kwargs"
+                    ):
                         raise Exception(
                             f"Duplicate flag names found in click command: {param_name}"
                         )
diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py
--- a/core/dbt/cli/main.py
+++ b/core/dbt/cli/main.py
@@ -2,27 +2,28 @@
 from typing import Callable, List, Tuple, Optional
 
 import click
+
 from dbt.cli import requires, params as p
-from dbt.config.project import Project
 from dbt.config.profile import Profile
+from dbt.config.project import Project
 from dbt.contracts.graph.manifest import Manifest
 from dbt.events.base_types import EventMsg
+from dbt.task.build import BuildTask
 from dbt.task.clean import CleanTask
 from dbt.task.compile import CompileTask
-from dbt.task.deps import DepsTask
 from dbt.task.debug import DebugTask
+from dbt.task.deps import DepsTask
+from dbt.task.freshness import FreshnessTask
+from dbt.task.generate import GenerateTask
+from dbt.task.init import InitTask
+from dbt.task.list import ListTask
 from dbt.task.run import RunTask
+from dbt.task.run_operation import RunOperationTask
+from dbt.task.seed import SeedTask
 from dbt.task.serve import ServeTask
 from dbt.task.show import ShowTask
-from dbt.task.test import TestTask
 from dbt.task.snapshot import SnapshotTask
-from dbt.task.seed import SeedTask
-from dbt.task.list import ListTask
-from dbt.task.freshness import FreshnessTask
-from dbt.task.run_operation import RunOperationTask
-from dbt.task.build import BuildTask
-from dbt.task.generate import GenerateTask
-from dbt.task.init import InitTask
+from dbt.task.test import TestTask
 
 
 class dbtUsageException(Exception):
@@ -47,7 +48,7 @@ def __init__(
         self.manifest = manifest
         self.callbacks = callbacks
 
-    def invoke(self, args: List[str]) -> Tuple[Optional[List], bool]:
+    def invoke(self, args: List[str], **kwargs) -> Tuple[Optional[List], bool]:
         try:
             dbt_ctx = cli.make_context(cli.name, args)
             dbt_ctx.obj = {
@@ -56,6 +57,12 @@ def invoke(self, args: List[str]) -> Tuple[Optional[List], bool]:
                 "manifest": self.manifest,
                 "callbacks": self.callbacks,
             }
+
+            for key, value in kwargs.items():
+                dbt_ctx.params[key] = value
+                # Hack to set parameter source to custom string
+                dbt_ctx.set_parameter_source(key, "kwargs")  # type: ignore
+
             return cli.invoke(dbt_ctx)
         except requires.HandledExit as e:
             return (e.result, e.success)


Test patch:
diff --git a/tests/functional/dbt_runner/test_dbt_runner.py b/tests/functional/dbt_runner/test_dbt_runner.py
new file mode 100644
--- /dev/null
+++ b/tests/functional/dbt_runner/test_dbt_runner.py
@@ -0,0 +1,69 @@
+from unittest import mock
+
+import pytest
+
+from dbt.cli.main import dbtRunner, dbtUsageException
+from dbt.exceptions import DbtProjectError
+
+
+class TestDbtRunner:
+    @pytest.fixture
+    def dbt(self) -> dbtRunner:
+        return dbtRunner()
+
+    def test_group_invalid_option(self, dbt: dbtRunner) -> None:
+        with pytest.raises(dbtUsageException):
+            dbt.invoke(["--invalid-option"])
+
+    def test_command_invalid_option(self, dbt: dbtRunner) -> None:
+        with pytest.raises(dbtUsageException):
+            dbt.invoke(["deps", "--invalid-option"])
+
+    def test_command_mutually_exclusive_option(self, dbt: dbtRunner) -> None:
+        with pytest.raises(dbtUsageException):
+            dbt.invoke(["--warn-error", "--warn-error-options", '{"include": "all"}', "deps"])
+
+    def test_invalid_command(self, dbt: dbtRunner) -> None:
+        with pytest.raises(dbtUsageException):
+            dbt.invoke(["invalid-command"])
+
+    def test_invoke_version(self, dbt: dbtRunner) -> None:
+        dbt.invoke(["--version"])
+
+    def test_callbacks(self) -> None:
+        mock_callback = mock.MagicMock()
+        dbt = dbtRunner(callbacks=[mock_callback])
+        # the `debug` command is one of the few commands wherein you don't need
+        # to have a project to run it and it will emit events
+        dbt.invoke(["debug"])
+        mock_callback.assert_called()
+
+    def test_invoke_kwargs(self, project, dbt):
+        results, success = dbt.invoke(
+            ["run"],
+            log_format="json",
+            log_path="some_random_path",
+            version_check=False,
+            profile_name="some_random_profile_name",
+            target_dir="some_random_target_dir",
+        )
+        assert results.args["log_format"] == "json"
+        assert results.args["log_path"] == "some_random_path"
+        assert results.args["version_check"] is False
+        assert results.args["profile_name"] == "some_random_profile_name"
+        assert results.args["target_dir"] == "some_random_target_dir"
+
+    def test_invoke_kwargs_project_dir(self, project, dbt):
+        with pytest.raises(
+            DbtProjectError,
+            match="No dbt_project.yml found at expected path some_random_project_dir",
+        ):
+            dbt.invoke(["run"], project_dir="some_random_project_dir")
+
+    def test_invoke_kwargs_profiles_dir(self, project, dbt):
+        with pytest.raises(DbtProjectError, match="Could not find profile named 'test'"):
+            dbt.invoke(["run"], profiles_dir="some_random_profiles_dir")
+
+    def test_invoke_kwargs_and_flags(self, project, dbt):
+        results, success = dbt.invoke(["--log-format=text", "run"], log_format="json")
+        assert results.args["log_format"] == "json"
diff --git a/tests/unit/test_dbt_runner.py b/tests/unit/test_dbt_runner.py
deleted file mode 100644
--- a/tests/unit/test_dbt_runner.py
+++ /dev/null
@@ -1,37 +0,0 @@
-import pytest
-
-from dbt.cli.main import dbtRunner, dbtUsageException
-from unittest import mock
-
-
-class TestDbtRunner:
-    @pytest.fixture
-    def dbt(self) -> dbtRunner:
-        return dbtRunner()
-
-    def test_group_invalid_option(self, dbt: dbtRunner) -> None:
-        with pytest.raises(dbtUsageException):
-            dbt.invoke(["--invalid-option"])
-
-    def test_command_invalid_option(self, dbt: dbtRunner) -> None:
-        with pytest.raises(dbtUsageException):
-            dbt.invoke(["deps", "--invalid-option"])
-
-    def test_command_mutually_exclusive_option(self, dbt: dbtRunner) -> None:
-        with pytest.raises(dbtUsageException):
-            dbt.invoke(["--warn-error", "--warn-error-options", '{"include": "all"}', "deps"])
-
-    def test_invalid_command(self, dbt: dbtRunner) -> None:
-        with pytest.raises(dbtUsageException):
-            dbt.invoke(["invalid-command"])
-
-    def test_invoke_version(self, dbt: dbtRunner) -> None:
-        dbt.invoke(["--version"])
-
-    def test_callbacks(self) -> None:
-        mock_callback = mock.MagicMock()
-        dbt = dbtRunner(callbacks=[mock_callback])
-        # the `debug` command is one of the few commands wherein you don't need
-        # to have a project to run it and it will emit events
-        dbt.invoke(["debug"])
-        mock_callback.assert_called()