Repository: kedro-org/kedro

Issue:
Update kedro pipeline create to use new /conf file structure 
## Description
Currently, the `kedro pipeline create` command creates a new /parameters folder and adds a {pipeline-name}.yml inside (or just adds the file to the parameters folder if it aready exists. 

## Context
To make this more consistent and so the global parameters.yml doesn't just randomly sit outside the directories, we want to maintian all the pipeline specific config files directly in the /conf with a specific naming convention. 

Before
```
├── conf
    └── base
        └── parameters         
	    ├── data_science.yml     #blank files
        ├── catalog.yml              #blank files   
	└── parameters.yml           #blank files
```

After
```
├── conf
    └── base
	├── catalog_data_science.yml          #blank files
	└── parameters_data_science.yml       #blank files
        ├── catalog.yml          #blank files   
	└── parameters.yml       #blank files
```
(This is outdated - edited by Nok)


## Possible Implementation

calling the updated `kedro pipeline create` should: 
1. add `parameters_{pipeline_name}.yml` to `/conf`
2. add `catalog_{pipeline_name}.yml` to `/conf`
3. add a pipeline diretory called {pipeline_name} to the `/pipelines` with empty `nodes.py` and `pipeline.py` files 

Currently, it also adds a `/tests` directory, but we want to remove this from the command because of work on #2149 

# Final decision - see comments for the full details
> Following discussion with @merelcht @deepyaman @astrojuanlu @DimedS we decided to move forward with this work with the following updates:
> 
> calling `kedro pipeline create` will:
> 
> * add `parameters_{pipeline_name}.yml` to /conf
> * add a pipeline directory called `{pipeline_name}` to the `/pipelines` with empty `nodes.py` and `pipeline.py` files
> * No `catalog_pipeline_name.yml`
> it will no longer add a /tests directory it will also no longer add a `catalog_{pipeline_name}.yml` to /conf, this will instead be delegated to the `kedro catalog create` command which will also need updating as a follow up to this. #2844

- [x] #2752
- [x] #2753
- [x] #2844


Discussion:

Resolving patch:
diff --git a/kedro/framework/cli/micropkg.py b/kedro/framework/cli/micropkg.py
--- a/kedro/framework/cli/micropkg.py
+++ b/kedro/framework/cli/micropkg.py
@@ -613,9 +613,16 @@ def _package_micropkg(
     )
     # as the source distribution will only contain parameters, we aren't listing other
     # config files not to confuse users and avoid useless file copies
+    # collect configs to package not only from parameters folder, but from core conf folder also
+    # because parameters had been moved from foldername to yml filename
     configs_to_package = _find_config_files(
         package_conf,
-        [f"parameters*/**/{micropkg_name}.yml", f"parameters*/**/{micropkg_name}/**/*"],
+        [
+            f"**/parameters_{micropkg_name}.yml",
+            f"**/{micropkg_name}/**/*",
+            f"parameters*/**/{micropkg_name}.yml",
+            f"parameters*/**/{micropkg_name}/**/*",
+        ],
     )
 
     source_paths = (package_source, package_tests, configs_to_package)
diff --git a/kedro/framework/cli/pipeline.py b/kedro/framework/cli/pipeline.py
--- a/kedro/framework/cli/pipeline.py
+++ b/kedro/framework/cli/pipeline.py
@@ -140,10 +140,14 @@ def delete_pipeline(
     pipeline_artifacts = _get_pipeline_artifacts(metadata, pipeline_name=name, env=env)
 
     files_to_delete = [
-        pipeline_artifacts.pipeline_conf / confdir / f"{name}.yml"
+        pipeline_artifacts.pipeline_conf / filepath
         for confdir in ("parameters", "catalog")
-        if (pipeline_artifacts.pipeline_conf / confdir / f"{name}.yml").is_file()
+        # Since we remove nesting in 'parameters' and 'catalog' folders,
+        # we want to also del the old project's structure for backward compatibility
+        for filepath in (Path(f"{confdir}_{name}.yml"), Path(confdir) / f"{name}.yml")
+        if (pipeline_artifacts.pipeline_conf / filepath).is_file()
     ]
+
     dirs_to_delete = [
         path
         for path in (pipeline_artifacts.pipeline_dir, pipeline_artifacts.pipeline_tests)


Test patch:
diff --git a/tests/framework/cli/micropkg/test_micropkg_package.py b/tests/framework/cli/micropkg/test_micropkg_package.py
--- a/tests/framework/cli/micropkg/test_micropkg_package.py
+++ b/tests/framework/cli/micropkg/test_micropkg_package.py
@@ -33,7 +33,7 @@ def assert_sdist_contents_correct(
             f"{package_name}-{version}/{package_name}/README.md",
             f"{package_name}-{version}/{package_name}/nodes.py",
             f"{package_name}-{version}/{package_name}/pipeline.py",
-            f"{package_name}-{version}/{package_name}/config/parameters/{package_name}.yml",
+            f"{package_name}-{version}/{package_name}/config/parameters_{package_name}.yml",
             f"{package_name}-{version}/tests/__init__.py",
             f"{package_name}-{version}/tests/test_pipeline.py",
         }
@@ -354,9 +354,9 @@ def test_package_modular_pipeline_with_nested_parameters(
         assert (
             "retail-0.1/retail/config/parameters/retail/params1.yml" in sdist_contents
         )
-        assert "retail-0.1/retail/config/parameters/retail.yml" in sdist_contents
+        assert "retail-0.1/retail/config/parameters_retail.yml" in sdist_contents
         assert (
-            "retail-0.1/retail/config/parameters/retail_banking.yml"
+            "retail-0.1/retail/config/parameters_retail_banking.yml"
             not in sdist_contents
         )
 
@@ -424,7 +424,7 @@ def test_package_pipeline_with_deep_nested_parameters(
             "retail-0.1/retail/config/parameters/retail/deep/params1.yml"
             in sdist_contents
         )
-        assert "retail-0.1/retail/config/parameters/retail.yml" in sdist_contents
+        assert "retail-0.1/retail/config/parameters_retail.yml" in sdist_contents
         assert "retail-0.1/retail/config/parameters/deep/retail.yml" in sdist_contents
         assert (
             "retail-0.1/retail/config/parameters/a/b/c/d/retail/params3.yml"
diff --git a/tests/framework/cli/micropkg/test_micropkg_pull.py b/tests/framework/cli/micropkg/test_micropkg_pull.py
--- a/tests/framework/cli/micropkg/test_micropkg_pull.py
+++ b/tests/framework/cli/micropkg/test_micropkg_pull.py
@@ -115,8 +115,7 @@ def test_pull_local_sdist(
             fake_repo_path
             / settings.CONF_SOURCE
             / config_env
-            / "parameters"
-            / f"{pipeline_name}.yml"
+            / f"parameters_{pipeline_name}.yml"
         )
 
         self.assert_package_files_exist(source_dest)
@@ -160,8 +159,7 @@ def test_pull_local_sdist_compare(
             fake_repo_path
             / settings.CONF_SOURCE
             / "base"
-            / "parameters"
-            / f"{PIPELINE_NAME}.yml"
+            / f"parameters_{PIPELINE_NAME}.yml"
         )
 
         sdist_file = (
@@ -189,8 +187,7 @@ def test_pull_local_sdist_compare(
             fake_repo_path
             / settings.CONF_SOURCE
             / config_env
-            / "parameters"
-            / f"{pipeline_name}.yml"
+            / f"parameters_{pipeline_name}.yml"
         )
 
         assert not filecmp.dircmp(source_path, source_dest).diff_files
@@ -237,8 +234,7 @@ def test_micropkg_pull_same_alias_package_name(
             fake_repo_path
             / settings.CONF_SOURCE
             / config_env
-            / "parameters"
-            / f"{pipeline_name}.yml"
+            / f"parameters_{pipeline_name}.yml"
         )
 
         self.assert_package_files_exist(source_dest)
@@ -287,8 +283,7 @@ def test_micropkg_pull_nested_destination(
             fake_repo_path
             / settings.CONF_SOURCE
             / config_env
-            / "parameters"
-            / f"{pipeline_name}.yml"
+            / f"parameters_{pipeline_name}.yml"
         )
 
         self.assert_package_files_exist(source_dest)
@@ -451,8 +446,7 @@ def test_pull_tests_missing(
             fake_repo_path
             / settings.CONF_SOURCE
             / "base"
-            / "parameters"
-            / f"{PIPELINE_NAME}.yml"
+            / f"parameters_{PIPELINE_NAME}.yml"
         )
         # Make sure the files actually deleted before pulling from the sdist file.
         assert not source_path.exists()
@@ -480,8 +474,7 @@ def test_pull_tests_missing(
             fake_repo_path
             / settings.CONF_SOURCE
             / config_env
-            / "parameters"
-            / f"{pipeline_name}.yml"
+            / f"parameters_{pipeline_name}.yml"
         )
 
         self.assert_package_files_exist(source_dest)
@@ -509,8 +502,7 @@ def test_pull_config_missing(
             fake_repo_path
             / settings.CONF_SOURCE
             / "base"
-            / "parameters"
-            / f"{PIPELINE_NAME}.yml"
+            / f"parameters_{PIPELINE_NAME}.yml"
         )
         source_params_config.unlink()
         call_micropkg_package(fake_project_cli, fake_metadata)
@@ -544,8 +536,7 @@ def test_pull_config_missing(
             fake_repo_path
             / settings.CONF_SOURCE
             / config_env
-            / "parameters"
-            / f"{pipeline_name}.yml"
+            / f"parameters_{pipeline_name}.yml"
         )
 
         self.assert_package_files_exist(source_dest)
@@ -586,8 +577,7 @@ def test_pull_from_pypi(
             fake_repo_path
             / settings.CONF_SOURCE
             / "base"
-            / "parameters"
-            / f"{PIPELINE_NAME}.yml"
+            / f"parameters_{PIPELINE_NAME}.yml"
         )
         # Make sure the files actually deleted before pulling from pypi.
         assert not source_path.exists()
@@ -645,8 +635,7 @@ def get_all(self, name, failobj=None):  # pylint: disable=unused-argument
             fake_repo_path
             / settings.CONF_SOURCE
             / config_env
-            / "parameters"
-            / f"{pipeline_name}.yml"
+            / f"parameters_{pipeline_name}.yml"
         )
 
         self.assert_package_files_exist(source_dest)
diff --git a/tests/framework/cli/pipeline/test_pipeline.py b/tests/framework/cli/pipeline/test_pipeline.py
--- a/tests/framework/cli/pipeline/test_pipeline.py
+++ b/tests/framework/cli/pipeline/test_pipeline.py
@@ -1,4 +1,3 @@
-import os
 import shutil
 from pathlib import Path
 
@@ -20,14 +19,17 @@
 def make_pipelines(request, fake_repo_path, fake_package_path, mocker):
     source_path = fake_package_path / "pipelines" / PIPELINE_NAME
     tests_path = fake_repo_path / "src" / "tests" / "pipelines" / PIPELINE_NAME
-    conf_path = fake_repo_path / settings.CONF_SOURCE / request.param / "parameters"
+    conf_path = fake_repo_path / settings.CONF_SOURCE / request.param
+    # old conf structure for 'pipeline delete' command backward compatibility
+    old_conf_path = conf_path / "parameters"
 
-    for path in (source_path, tests_path, conf_path):
+    for path in (source_path, tests_path, conf_path, old_conf_path):
         path.mkdir(parents=True, exist_ok=True)
 
-    (conf_path / f"{PIPELINE_NAME}.yml").touch()
     (tests_path / "test_pipe.py").touch()
     (source_path / "pipe.py").touch()
+    (conf_path / f"parameters_{PIPELINE_NAME}.yml").touch()
+    (old_conf_path / f"{PIPELINE_NAME}.yml").touch()
 
     yield
     mocker.stopall()
@@ -67,8 +69,8 @@ def test_create_pipeline(  # pylint: disable=too-many-locals
         # config
         conf_env = env or "base"
         conf_dir = (fake_repo_path / settings.CONF_SOURCE / conf_env).resolve()
-        actual_configs = list(conf_dir.glob(f"**/{PIPELINE_NAME}.yml"))
-        expected_configs = [conf_dir / "parameters" / f"{PIPELINE_NAME}.yml"]
+        actual_configs = list(conf_dir.glob(f"**/*{PIPELINE_NAME}.yml"))
+        expected_configs = [conf_dir / f"parameters_{PIPELINE_NAME}.yml"]
         assert actual_configs == expected_configs
 
         # tests
@@ -92,7 +94,7 @@ def test_create_pipeline_skip_config(
         assert f"Pipeline '{PIPELINE_NAME}' was successfully created." in result.output
 
         conf_dirs = list((fake_repo_path / settings.CONF_SOURCE).rglob(PIPELINE_NAME))
-        assert conf_dirs == []  # no configs created for the pipeline
+        assert not conf_dirs  # no configs created for the pipeline
 
         test_dir = fake_repo_path / "src" / "tests" / "pipelines" / PIPELINE_NAME
         assert test_dir.is_dir()
@@ -117,13 +119,12 @@ def test_catalog_and_params(  # pylint: disable=too-many-locals
                 "filepath": "data/01_raw/iris.csv",
             }
         }
-        catalog_file = conf_dir / "catalog" / f"{PIPELINE_NAME}.yml"
-        catalog_file.parent.mkdir()
+        catalog_file = conf_dir / f"catalog_{PIPELINE_NAME}.yml"
         with catalog_file.open("w") as f:
             yaml.dump(catalog_dict, f)
 
         # write pipeline parameters
-        params_file = conf_dir / "parameters" / f"{PIPELINE_NAME}.yml"
+        params_file = conf_dir / f"parameters_{PIPELINE_NAME}.yml"
         assert params_file.is_file()
         params_dict = {"params_from_pipeline": {"p1": [1, 2, 3], "p2": None}}
         with params_file.open("w") as f:
@@ -143,8 +144,7 @@ def test_skip_copy(self, fake_repo_path, fake_project_cli, fake_metadata):
                 fake_repo_path
                 / settings.CONF_SOURCE
                 / "base"
-                / dirname
-                / f"{PIPELINE_NAME}.yml"
+                / f"{dirname}_{PIPELINE_NAME}.yml"
             )
             path.parent.mkdir(exist_ok=True)
             path.touch()
@@ -166,7 +166,7 @@ def test_skip_copy(self, fake_repo_path, fake_project_cli, fake_metadata):
 
         assert result.exit_code == 0
         assert "__init__.py': SKIPPED" in result.output
-        assert f"parameters{os.sep}{PIPELINE_NAME}.yml': SKIPPED" in result.output
+        assert f"parameters_{PIPELINE_NAME}.yml': SKIPPED" in result.output
         assert result.output.count("SKIPPED") == 2  # only 2 files skipped
 
     def test_failed_copy(
@@ -269,17 +269,15 @@ def test_delete_pipeline(
 
         source_path = fake_package_path / "pipelines" / PIPELINE_NAME
         tests_path = fake_repo_path / "src" / "tests" / "pipelines" / PIPELINE_NAME
-        params_path = (
-            fake_repo_path
-            / settings.CONF_SOURCE
-            / expected_conf
-            / "parameters"
-            / f"{PIPELINE_NAME}.yml"
-        )
+        conf_path = fake_repo_path / settings.CONF_SOURCE / expected_conf
+        params_path = conf_path / f"parameters_{PIPELINE_NAME}.yml"
+        # old params structure for 'pipeline delete' command backward compatibility
+        old_params_path = conf_path / "parameters" / f"{PIPELINE_NAME}.yml"
 
         assert f"Deleting '{source_path}': OK" in result.output
         assert f"Deleting '{tests_path}': OK" in result.output
         assert f"Deleting '{params_path}': OK" in result.output
+        assert f"Deleting '{old_params_path}': OK" in result.output
 
         assert f"Pipeline '{PIPELINE_NAME}' was successfully deleted." in result.output
         assert (
@@ -290,6 +288,7 @@ def test_delete_pipeline(
         assert not source_path.exists()
         assert not tests_path.exists()
         assert not params_path.exists()
+        assert not params_path.exists()
 
     def test_delete_pipeline_skip(
         self, fake_repo_path, fake_project_cli, fake_metadata, fake_package_path
@@ -309,8 +308,7 @@ def test_delete_pipeline_skip(
             fake_repo_path
             / settings.CONF_SOURCE
             / "base"
-            / "parameters"
-            / f"{PIPELINE_NAME}.yml"
+            / f"parameters_{PIPELINE_NAME}.yml"
         )
 
         assert f"Deleting '{source_path}'" not in result.output
@@ -401,8 +399,7 @@ def test_pipeline_delete_confirmation(
             fake_repo_path
             / settings.CONF_SOURCE
             / "base"
-            / "parameters"
-            / f"{PIPELINE_NAME}.yml"
+            / f"parameters_{PIPELINE_NAME}.yml"
         )
 
         assert "The following paths will be removed:" in result.output
@@ -442,8 +439,7 @@ def test_pipeline_delete_confirmation_skip(
             fake_repo_path
             / settings.CONF_SOURCE
             / "base"
-            / "parameters"
-            / f"{PIPELINE_NAME}.yml"
+            / f"parameters_{PIPELINE_NAME}.yml"
         )
 
         assert "The following paths will be removed:" in result.output