Repository: kedro-org/kedro

Issue:
Add logging when computing micropackage version
## Description

My team and I had a hard time debugging our micropackages versioning. It is not easy to find out why the package version that is picked is not the one that we expect. 

If Kedro was a bit more verbose at startup, it would help to find out what is wrong in our project structure or setup (in our case, it was because some dependencies were not installed in the python environment. But the module loading was failing silently so we could not see that pandas was missing and that this was the cause for the wrong versioning of the micropackage)

## Context

I need to micropackage different modular pipelines with different version numbers. Each version number is written in `src/<project_name>/pipelines/<pipeline_name>/__init__.py`

## Possible Implementation

Add some logging in `kedro/framework/cli/micropkg.py`:

```

def _get_default_version(metadata: ProjectMetadata, micropkg_module_path: str) -> str:
    # default to micropkg package version
    try:
        micropkg_module = import_module(
            f"{metadata.package_name}.{micropkg_module_path}"
        )
        return micropkg_module.__version__  # type: ignore
    except (AttributeError, ModuleNotFoundError):
        logger.warning(f"Micropackage version not found in '{metadata.package_name}.{micropkg_module_path}', will take the top-level one in '{metadata.package_name}')
        # if micropkg version doesn't exist, take the project one
        project_module = import_module(f"{metadata.package_name}")
        return project_module.__version__  # type: ignore
```

## Possible Alternative

Additionnally to adding somme logging in `kedro/framework/cli/micropkg.py`, we could stop catching the exception if micropackage module cannot be loaded, so that the error is not silenced:

```
def _get_default_version(metadata: ProjectMetadata, micropkg_module_path: str) -> str:
    # default to micropkg package version
    micropkg_module = import_module(
        f"{metadata.package_name}.{micropkg_module_path}"
    )
    try:
        return micropkg_module.__version__  # type: ignore
    except AttributeError:
        logger.warning(f"Micropackage version not found in '{metadata.package_name}.{micropkg_module_path}', will take the top-level one in '{metadata.package_name}')
        # if micropkg version doesn't exist, take the project one
        project_module = import_module(f"{metadata.package_name}")
        return project_module.__version__  # type: ignore
```




Discussion:
That's a very reasonable request. Thanks for flagging it @clotildeguinard . I'll add it to the backlog, but we would also be very happy to accept a PR for this. It might be faster to get the change in that way 🙂 

Resolving patch:
diff --git a/kedro/framework/cli/pipeline.py b/kedro/framework/cli/pipeline.py
--- a/kedro/framework/cli/pipeline.py
+++ b/kedro/framework/cli/pipeline.py
@@ -90,10 +90,17 @@ def pipeline():
     is_flag=True,
     help="Skip creation of config files for the new pipeline(s).",
 )
+@click.option(
+    "template_path",
+    "-t",
+    "--template",
+    type=click.Path(file_okay=False, dir_okay=True, exists=True, path_type=Path),
+    help="Path to cookiecutter template to use for pipeline(s). Will override any local templates.",
+)
 @env_option(help="Environment to create pipeline configuration in. Defaults to `base`.")
 @click.pass_obj  # this will pass the metadata as first argument
 def create_pipeline(
-    metadata: ProjectMetadata, name, skip_config, env, **kwargs
+    metadata: ProjectMetadata, name, template_path, skip_config, env, **kwargs
 ):  # noqa: unused-argument
     """Create a new modular pipeline by providing a name."""
     package_dir = metadata.source_dir / metadata.package_name
@@ -107,7 +114,19 @@ def create_pipeline(
             f"Make sure it exists in the project configuration."
         )
 
-    result_path = _create_pipeline(name, package_dir / "pipelines")
+    # Precedence for template_path is: command line > project templates/pipeline dir > global default
+    # If passed on the CLI, click will verify that the path exists so no need to check again
+    if template_path is None:
+        # No path provided on the CLI, try `PROJECT_PATH/templates/pipeline`
+        template_path = Path(metadata.project_path / "templates" / "pipeline")
+
+        if not template_path.exists():
+            # and if that folder doesn't exist fall back to the global default
+            template_path = Path(kedro.__file__).parent / "templates" / "pipeline"
+
+    click.secho(f"Using pipeline template at: '{template_path}'")
+
+    result_path = _create_pipeline(name, template_path, package_dir / "pipelines")
     _copy_pipeline_tests(name, result_path, package_dir)
     _copy_pipeline_configs(result_path, project_conf_path, skip_config, env=env)
     click.secho(f"\nPipeline '{name}' was successfully created.\n", fg="green")
@@ -191,12 +210,11 @@ def _echo_deletion_warning(message: str, **paths: list[Path]):
         click.echo(indent(paths_str, " " * 2))
 
 
-def _create_pipeline(name: str, output_dir: Path) -> Path:
+def _create_pipeline(name: str, template_path: Path, output_dir: Path) -> Path:
     with _filter_deprecation_warnings():
         # noqa: import-outside-toplevel
         from cookiecutter.main import cookiecutter
 
-    template_path = Path(kedro.__file__).parent / "templates" / "pipeline"
     cookie_context = {"pipeline_name": name, "kedro_version": kedro.__version__}
 
     click.echo(f"Creating the pipeline '{name}': ", nl=False)


Test patch:
diff --git a/tests/framework/cli/pipeline/conftest.py b/tests/framework/cli/pipeline/conftest.py
--- a/tests/framework/cli/pipeline/conftest.py
+++ b/tests/framework/cli/pipeline/conftest.py
@@ -1,10 +1,24 @@
+import json
 import shutil
+from pathlib import Path
 
 import pytest
 
 from kedro.framework.project import settings
 
 
+def _write_json(filepath: Path, content: dict):
+    filepath.parent.mkdir(parents=True, exist_ok=True)
+    json_str = json.dumps(content, indent=4)
+    filepath.write_text(json_str)
+
+
+def _write_dummy_file(filepath: Path, content: str = ""):
+    filepath.parent.mkdir(parents=True, exist_ok=True)
+    with filepath.open("w") as f:
+        f.write(content)
+
+
 @pytest.fixture(autouse=True)
 def cleanup_micropackages(fake_repo_path, fake_package_path):
     packages = {p.name for p in fake_package_path.iterdir() if p.is_dir()}
@@ -82,3 +96,39 @@ def cleanup_pyproject_toml(fake_repo_path):
     yield
 
     pyproject_toml.write_text(existing_toml)
+
+
+@pytest.fixture()
+def fake_local_template_dir(fake_repo_path):
+    """Set up a local template directory. This won't be functional we're just testing the actual layout works.
+
+    Note that this is not scoped to module because we don't want to have this folder present in most of the tests,
+    so we will tear it down every time.
+    """
+    template_path = fake_repo_path / Path("templates")
+    pipeline_template_path = template_path / Path("pipeline")
+    cookiecutter_template_path = (
+        pipeline_template_path / "{{ cookiecutter.pipeline_name }}"
+    )
+
+    cookiecutter_template_path.mkdir(parents=True)
+
+    # Create the absolute bare minimum files
+    cookiecutter_json = {
+        "pipeline_name": "default",
+    }
+    _write_json(pipeline_template_path / "cookiecutter.json", cookiecutter_json)
+    _write_dummy_file(
+        cookiecutter_template_path / "pipeline_{{ cookiecutter.pipeline_name }}.py",
+    )
+    _write_dummy_file(cookiecutter_template_path / "__init__.py", "")
+    _write_dummy_file(
+        cookiecutter_template_path
+        / r"config/parameters/{{ cookiecutter.pipeline_name }}.yml",
+    )
+    _write_dummy_file(
+        cookiecutter_template_path / r"tests/test_{{ cookiecutter.pipeline_name }}.py",
+    )
+    yield template_path.resolve()
+
+    shutil.rmtree(template_path)
diff --git a/tests/framework/cli/pipeline/test_pipeline.py b/tests/framework/cli/pipeline/test_pipeline.py
--- a/tests/framework/cli/pipeline/test_pipeline.py
+++ b/tests/framework/cli/pipeline/test_pipeline.py
@@ -79,6 +79,78 @@ def test_create_pipeline(  # pylint: disable=too-many-locals
         actual_files = {f.name for f in test_dir.iterdir()}
         assert actual_files == expected_files
 
+    @pytest.mark.parametrize("env", [None, "local"])
+    def test_create_pipeline_template(  # pylint: disable=too-many-locals
+        self,
+        fake_repo_path,
+        fake_project_cli,
+        fake_metadata,
+        env,
+        fake_package_path,
+        fake_local_template_dir,
+    ):
+        pipelines_dir = fake_package_path / "pipelines"
+        assert pipelines_dir.is_dir()
+
+        assert not (pipelines_dir / PIPELINE_NAME).exists()
+
+        cmd = ["pipeline", "create", PIPELINE_NAME]
+        cmd += ["-e", env] if env else []
+        result = CliRunner().invoke(fake_project_cli, cmd, obj=fake_metadata)
+
+        assert (
+            f"Using pipeline template at: '{fake_repo_path / 'templates'}"
+            in result.output
+        )
+        assert f"Creating the pipeline '{PIPELINE_NAME}': OK" in result.output
+        assert f"Location: '{pipelines_dir / PIPELINE_NAME}'" in result.output
+        assert f"Pipeline '{PIPELINE_NAME}' was successfully created." in result.output
+
+        # Dummy pipeline rendered correctly
+        assert (pipelines_dir / PIPELINE_NAME / f"pipeline_{PIPELINE_NAME}.py").exists()
+
+        assert result.exit_code == 0
+
+    @pytest.mark.parametrize("env", [None, "local"])
+    def test_create_pipeline_template_command_line_override(  # pylint: disable=too-many-locals
+        self,
+        fake_repo_path,
+        fake_project_cli,
+        fake_metadata,
+        env,
+        fake_package_path,
+        fake_local_template_dir,
+    ):
+        pipelines_dir = fake_package_path / "pipelines"
+        assert pipelines_dir.is_dir()
+
+        assert not (pipelines_dir / PIPELINE_NAME).exists()
+
+        # Rename the local template dir to something else so we know the command line flag is taking precedence
+        try:
+            # Can skip if already there but copytree has a dirs_exist_ok flag in >python 3.8 only
+            shutil.copytree(fake_local_template_dir, fake_repo_path / "local_templates")
+        except FileExistsError:
+            pass
+
+        cmd = ["pipeline", "create", PIPELINE_NAME]
+        cmd += ["-t", str(fake_repo_path / "local_templates/pipeline")]
+        cmd += ["-e", env] if env else []
+        result = CliRunner().invoke(fake_project_cli, cmd, obj=fake_metadata)
+
+        assert (
+            f"Using pipeline template at: '{fake_repo_path / 'local_templates'}"
+            in result.output
+        )
+        assert f"Creating the pipeline '{PIPELINE_NAME}': OK" in result.output
+        assert f"Location: '{pipelines_dir / PIPELINE_NAME}'" in result.output
+        assert f"Pipeline '{PIPELINE_NAME}' was successfully created." in result.output
+
+        # Dummy pipeline rendered correctly
+        assert (pipelines_dir / PIPELINE_NAME / f"pipeline_{PIPELINE_NAME}.py").exists()
+
+        assert result.exit_code == 0
+
     @pytest.mark.parametrize("env", [None, "local"])
     def test_create_pipeline_skip_config(
         self, fake_repo_path, fake_project_cli, fake_metadata, env