Repository: kedro-org/kedro

Issue:
Add an option to run a specific namespace in `KedroSession` run and `kedro run`
## Description

Currently there's a way to select different sections from a pipeline to run, but selecting by namespace is missing. We should add a way for users to run a specific subpipeline (namespace) by executing this in the terminal:

```
kedro run --namespace data_engineering.preprocessing
```

```
kedro run -ns data_engineering
```


This will run all nodes whose namespace starts with `data_engineering.preprocessing`, regardless of depth. In order to implement this, we'd need to provide a similar option in `KedroSession`.
The `Pipeline` object already has the relevant filter `only_nodes_with_namespace` which can be used for implementing this functionality.

## Context

This change is important to providing a nice way to convert Kedro pipelines into orchestrator pipelines. Currently most conversions of Kedro pipelines happen on a node to node mapping,
which is far from ideal. Orchestrator run pipelines tend to be much less granular than Kedro pipelines and this 1-to-1 translation produces suboptimal executions. By enabling running pipelines
by namespace, we will make it easier to create much more clever deployment plugins, which will take advantage of the topology of the underlying pipelines better.

## Possible Implementation

To be decided by the team.

## Possible Alternatives


Discussion:
This should be `multiple=False` just like pipeline

Resolving patch:
diff --git a/kedro/framework/cli/project.py b/kedro/framework/cli/project.py
--- a/kedro/framework/cli/project.py
+++ b/kedro/framework/cli/project.py
@@ -55,6 +55,7 @@
 override the loaded ones."""
 PIPELINE_ARG_HELP = """Name of the registered pipeline to run.
 If not set, the '__default__' pipeline is run."""
+NAMESPACE_ARG_HELP = """Name of the node namespace to run."""
 PARAMS_ARG_HELP = """Specify extra parameters that you want to pass
 to the context initialiser. Items must be separated by comma, keys - by colon or equals sign,
 example: param1=value1,param2=value2. Each parameter is split by the first comma,
@@ -392,6 +393,7 @@ def activate_nbstripout(
     callback=_split_load_versions,
 )
 @click.option("--pipeline", "-p", type=str, default=None, help=PIPELINE_ARG_HELP)
+@click.option("--namespace", "-ns", type=str, default=None, help=NAMESPACE_ARG_HELP)
 @click.option(
     "--config",
     "-c",
@@ -411,7 +413,7 @@ def activate_nbstripout(
     help=PARAMS_ARG_HELP,
     callback=_split_params,
 )
-# pylint: disable=too-many-arguments,unused-argument, too-many-locals
+# pylint: disable=too-many-arguments,unused-argument,too-many-locals
 def run(
     tag,
     tags,
@@ -430,6 +432,7 @@ def run(
     config,
     conf_source,
     params,
+    namespace,
 ):
     """Run the pipeline."""
 
@@ -459,4 +462,5 @@ def run(
             to_outputs=to_outputs,
             load_versions=load_version,
             pipeline_name=pipeline,
+            namespace=namespace,
         )
diff --git a/kedro/framework/session/session.py b/kedro/framework/session/session.py
--- a/kedro/framework/session/session.py
+++ b/kedro/framework/session/session.py
@@ -313,6 +313,7 @@ def run(  # pylint: disable=too-many-arguments,too-many-locals
         from_inputs: Iterable[str] = None,
         to_outputs: Iterable[str] = None,
         load_versions: Dict[str, str] = None,
+        namespace: str = None,
     ) -> Dict[str, Any]:
         """Runs the pipeline with a specified runner.
 
@@ -336,6 +337,7 @@ def run(  # pylint: disable=too-many-arguments,too-many-locals
                 used as an end point of the new ``Pipeline``.
             load_versions: An optional flag to specify a particular dataset
                 version timestamp to load.
+            namespace: The namespace of the nodes that is being run.
         Raises:
             ValueError: If the named or `__default__` pipeline is not
                 defined by `register_pipelines`.
@@ -382,6 +384,7 @@ def run(  # pylint: disable=too-many-arguments,too-many-locals
             node_names=node_names,
             from_inputs=from_inputs,
             to_outputs=to_outputs,
+            node_namespace=namespace,
         )
 
         record_data = {
@@ -398,6 +401,7 @@ def run(  # pylint: disable=too-many-arguments,too-many-locals
             "load_versions": load_versions,
             "extra_params": extra_params,
             "pipeline_name": pipeline_name,
+            "namespace": namespace,
             "runner": getattr(runner, "__name__", str(runner)),
         }
 


Test patch:
diff --git a/tests/framework/cli/test_cli.py b/tests/framework/cli/test_cli.py
--- a/tests/framework/cli/test_cli.py
+++ b/tests/framework/cli/test_cli.py
@@ -473,6 +473,7 @@ def test_run_successfully(
             to_outputs=[],
             load_versions={},
             pipeline_name=None,
+            namespace=None,
         )
 
         runner = fake_session.run.call_args_list[0][1]["runner"]
@@ -512,6 +513,7 @@ def test_run_specific_nodes(
             to_outputs=[],
             load_versions={},
             pipeline_name=None,
+            namespace=None,
         )
 
         runner = fake_session.run.call_args_list[0][1]["runner"]
@@ -551,6 +553,7 @@ def test_run_with_tags(
             to_outputs=[],
             load_versions={},
             pipeline_name=None,
+            namespace=None,
         )
 
         runner = fake_session.run.call_args_list[0][1]["runner"]
@@ -562,9 +565,12 @@ def test_run_with_pipeline_filters(
     ):
         from_nodes = ["--from-nodes", "splitting_data"]
         to_nodes = ["--to-nodes", "training_model"]
+        namespace = ["--namespace", "fake_namespace"]
         tags = ["--tags", "de"]
         result = CliRunner().invoke(
-            fake_project_cli, ["run", *from_nodes, *to_nodes, *tags], obj=fake_metadata
+            fake_project_cli,
+            ["run", *from_nodes, *to_nodes, *tags, *namespace],
+            obj=fake_metadata,
         )
         assert not result.exit_code
 
@@ -578,6 +584,7 @@ def test_run_with_pipeline_filters(
             to_outputs=[],
             load_versions={},
             pipeline_name=None,
+            namespace="fake_namespace",
         )
 
         runner = fake_session.run.call_args_list[0][1]["runner"]
@@ -601,6 +608,7 @@ def test_run_successfully_parallel(
             to_outputs=[],
             load_versions={},
             pipeline_name=None,
+            namespace=None,
         )
 
         runner = fake_session.run.call_args_list[0][1]["runner"]
@@ -640,6 +648,7 @@ def test_run_with_config(
             to_outputs=[],
             load_versions={},
             pipeline_name="pipeline1",
+            namespace=None,
         )
 
     @mark.parametrize(
@@ -683,6 +692,7 @@ def test_run_with_params_in_config(
             to_outputs=[],
             load_versions={},
             pipeline_name="pipeline1",
+            namespace=None,
         )
         mock_session_create.assert_called_once_with(
             env=mocker.ANY, conf_source=None, extra_params=expected
@@ -782,6 +792,7 @@ def test_reformat_load_versions(
             to_outputs=[],
             load_versions={ds: t},
             pipeline_name=None,
+            namespace=None,
         )
 
     @mark.parametrize(
@@ -821,6 +832,7 @@ def test_split_load_versions(
             to_outputs=[],
             load_versions=lv_dict,
             pipeline_name=None,
+            namespace=None,
         )
 
     def test_fail_reformat_load_versions(self, fake_project_cli, fake_metadata):
@@ -896,6 +908,7 @@ def test_safe_split_option_arguments(
             to_outputs=[],
             load_versions={},
             pipeline_name=None,
+            namespace=None,
         )
 
     def test_run_with_alternative_conf_source(self, fake_project_cli, fake_metadata):
@@ -950,6 +963,7 @@ def test_both_node_flags(
             to_outputs=[],
             load_versions={},
             pipeline_name=None,
+            namespace=None,
         )
 
     def test_both_tag_flags(
@@ -978,6 +992,7 @@ def test_both_tag_flags(
             to_outputs=[],
             load_versions={},
             pipeline_name=None,
+            namespace=None,
         )
 
     def test_both_load_version_flags(
@@ -1006,4 +1021,5 @@ def test_both_load_version_flags(
             to_outputs=[],
             load_versions=lv_dict,
             pipeline_name=None,
+            namespace=None,
         )
diff --git a/tests/framework/session/test_session.py b/tests/framework/session/test_session.py
--- a/tests/framework/session/test_session.py
+++ b/tests/framework/session/test_session.py
@@ -623,6 +623,7 @@ def test_run(
             "load_versions": None,
             "extra_params": {},
             "pipeline_name": fake_pipeline_name,
+            "namespace": None,
             "runner": mock_runner.__name__,
         }
 
@@ -692,6 +693,7 @@ def test_run_multiple_times(  # pylint: disable=too-many-locals
             "load_versions": None,
             "extra_params": {},
             "pipeline_name": fake_pipeline_name,
+            "namespace": None,
             "runner": mock_runner.__name__,
         }
 
@@ -779,6 +781,7 @@ def test_run_exception(  # pylint: disable=too-many-locals
             "load_versions": None,
             "extra_params": {},
             "pipeline_name": fake_pipeline_name,
+            "namespace": None,
             "runner": mock_runner.__name__,
         }
 
@@ -846,6 +849,7 @@ def test_run_broken_pipeline_multiple_times(  # pylint: disable=too-many-locals
             "load_versions": None,
             "extra_params": {},
             "pipeline_name": fake_pipeline_name,
+            "namespace": None,
             "runner": broken_runner.__name__,
         }