Repository: getindata/kedro-kubeflow
Pull request #269 (2024-09-02 06:53:51)

Fix e2e test for kedro 19
#### Description

`Includes fix for successfully testing the pipeline e2e in kubeflow` 

##### PR Checklist
- [x] Tests added
- [x] [Changelog](CHANGELOG.md) updated 



Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 55c27e8..d4d65b5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,10 +2,10 @@
 
 ## [Unreleased]
 
-- Added support for python 3.11, 3.12 is still blocked by kedro-mlflow 
+- Added support for python 3.11. Python 3.12 is still blocked by kedro-mlflow
 - Deprecated EnvTemplatedConfigLoader, use OmegaConfigLoader with oc.env resolver
 - Updated dependencies updated mlflow to 2.3.2
-- Add support for `kedro>0.18.4,<=0.19.6`
+- Added support for `kedro>0.18.4,<=0.19.6`
 
 ## [0.7.4] - 2023-02-27
 
diff --git a/docs/source/01_introduction/01_intro.md b/docs/source/01_introduction/01_intro.md
index c8e3ca1..de8d44e 100644
--- a/docs/source/01_introduction/01_intro.md
+++ b/docs/source/01_introduction/01_intro.md
@@ -16,6 +16,6 @@ it can be started on any Kubernetes cluster. The code to execute is stored insid
 docker images that cover not only the source itself, but also all the libraries and 
 entire execution environment. Portability is also one of the key Kedro qualities, as 
 the pipelines must be versionable and packageable. Kedro, with 
-[Kedro-docker](https://github.com/quantumblacklabs/kedro-docker) plugin does a fantastic 
+[Kedro-Docker](https://github.com/kedro-org/kedro-plugins/tree/main/kedro-docker) plugin does a fantastic 
 job to achieve this and Kubeflow looks like a nice add-on to run the pipelines 
 on powerful remote Kubernetes clusters.
diff --git a/kedro_kubeflow/context_helper.py b/kedro_kubeflow/context_helper.py
index 26d5111..add049a 100644
--- a/kedro_kubeflow/context_helper.py
+++ b/kedro_kubeflow/context_helper.py
@@ -77,7 +77,7 @@ class ContextHelper(object):
 
     @property
     def project_name(self):
-        return self._metadata.project_path.name
+        return self._metadata.project_name
 
     @property
     @lru_cache()
diff --git a/kedro_kubeflow/generators/one_pod_pipeline_generator.py b/kedro_kubeflow/generators/one_pod_pipeline_generator.py
index 1cef2e8..4961961 100644
--- a/kedro_kubeflow/generators/one_pod_pipeline_generator.py
+++ b/kedro_kubeflow/generators/one_pod_pipeline_generator.py
@@ -24,7 +24,7 @@ class OnePodPipelineGenerator(object):
         self.context: KedroContext = context
         dsl.ContainerOp._DISABLE_REUSABLE_COMPONENT_WARNING = True
         self.run_config = config.run_config
-        self.catalog = context.config_loader.get("catalog*")
+        self.catalog = context.config_loader.get("catalog")
 
     def generate_pipeline(self, pipeline, image, image_pull_policy):
         merged_params = merge_namespaced_params_to_dict(self.context.params)
diff --git a/kedro_kubeflow/generators/pod_per_node_pipeline_generator.py b/kedro_kubeflow/generators/pod_per_node_pipeline_generator.py
index 7242c87..9550aa1 100644
--- a/kedro_kubeflow/generators/pod_per_node_pipeline_generator.py
+++ b/kedro_kubeflow/generators/pod_per_node_pipeline_generator.py
@@ -27,7 +27,7 @@ class PodPerNodePipelineGenerator(object):
         self.context: KedroContext = context
         dsl.ContainerOp._DISABLE_REUSABLE_COMPONENT_WARNING = True
         self.run_config = config.run_config
-        self.catalog = context.config_loader.get("catalog*")
+        self.catalog = context.config_loader.get("catalog")
 
     def configure_max_cache_staleness(self, kfp_ops):
         if self.run_config.max_cache_staleness not in [None, ""]:
@@ -128,7 +128,7 @@ class PodPerNodePipelineGenerator(object):
                         "run "
                         f"--env {self.context.env} "
                         f"--pipeline {pipeline} "
-                        f"--node {node.name} "
+                        f"--nodes {node.name} "
                         f"--config config.yaml"
                     ),
                     arguments=create_arguments_from_parameters(params.keys()),


Test patch:
diff --git a/tests/e2e/catalog.yml b/tests/e2e/catalog.yml
index 1ddd0f8..a712ea0 100644
--- a/tests/e2e/catalog.yml
+++ b/tests/e2e/catalog.yml
@@ -1,57 +1,46 @@
 companies:
   type: pandas.CSVDataSet
   filepath: data/01_raw/companies.csv
-  layer: raw
 
 reviews:
   type: pandas.CSVDataSet
   filepath: data/01_raw/reviews.csv
-  layer: raw
 
 shuttles:
   type: pandas.ExcelDataSet
   filepath: data/01_raw/shuttles.xlsx
-  layer: raw
   load_args:
     engine: openpyxl
 
 model_input_table:
   type: pandas.CSVDataSet
-  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${run_id}/03_primary/model_input_table.csv
-  layer: primary
+  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${oc.env:KEDRO_CONFIG_RUN_ID,unknown}/03_primary/model_input_table.csv
 
 preprocessed_companies:
   type: pandas.CSVDataSet
-  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${run_id}/02_intermediate/preprocessed_companies.csv
-  layer: intermediate
+  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${oc.env:KEDRO_CONFIG_RUN_ID,unknown}/02_intermediate/preprocessed_companies.csv
 
 preprocessed_shuttles:
   type: pandas.CSVDataSet
-  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${run_id}/02_intermediate/preprocessed_shuttles.csv
-  layer: intermediate
+  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${oc.env:KEDRO_CONFIG_RUN_ID,unknown}/02_intermediate/preprocessed_shuttles.csv
 
 X_train:
   type: pickle.PickleDataSet
-  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${run_id}/05_model_input/X_train.pickle
-  layer: model_input
+  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${oc.env:KEDRO_CONFIG_RUN_ID,unknown}/05_model_input/X_train.pickle
 
 y_train:
   type: pickle.PickleDataSet
-  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${run_id}/05_model_input/y_train.pickle
-  layer: model_input
+  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${oc.env:KEDRO_CONFIG_RUN_ID,unknown}/05_model_input/y_train.pickle
 
 X_test:
   type: pickle.PickleDataSet
-  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${run_id}/05_model_input/X_test.pickle
-  layer: model_input
+  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${oc.env:KEDRO_CONFIG_RUN_ID,unknown}/05_model_input/X_test.pickle
 
 y_test:
   type: pickle.PickleDataSet
-  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${run_id}/05_model_input/y_test.pickle
-  layer: model_input
+  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${oc.env:KEDRO_CONFIG_RUN_ID,unknown}/05_model_input/y_test.pickle
 
 regressor:
   type: pickle.PickleDataSet
-  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${run_id}/06_models/regressor.pickle
+  filepath: gs://gid-ml-ops-sandbox-plugin-tests/kube_${oc.env:KEDRO_CONFIG_RUN_ID,unknown}/06_models/regressor.pickle
   versioned: true
-  layer: models
diff --git a/tests/test_context_helper.py b/tests/test_context_helper.py
index 6cc0ac7..63761a1 100644
--- a/tests/test_context_helper.py
+++ b/tests/test_context_helper.py
@@ -28,7 +28,7 @@ class TestContextHelper(unittest.TestCase, MinimalConfigMixin):
 
     def test_project_name(self):
         metadata = Mock()
-        metadata.project_path = Path("test_project")
+        metadata.project_name = "test_project"
 
         helper = ContextHelper.init(metadata, "test")
         assert helper.project_name == "test_project"