Repository: getindata/kedro-kubeflow
Pull request #74 (2021-09-27 14:02:59)

Release 0.4.3
Bump version and CHANGELOG for next release.


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ea918f1..fd560ce 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,11 @@
 
 ## [Unreleased]
 
+## [0.4.3] - 2021-09-27
+
+-   Kedro environment used by `kedro kubeflow` invocation is passed to the steps
+-   A flag to skip steps output artifacts registration in Kubeflow Metadata
+
 ## [0.4.2] - 2021-08-19
 
 -   Improved Vertex scheduling: removal of stale schedules
@@ -79,7 +84,9 @@
 -   Method to schedule runs for most recent version of given pipeline `kedro kubeflow schedule` 
 -   Shortcut to open UI for pipelines using `kedro kubeflow ui` 
 
-[Unreleased]: https://github.com/getindata/kedro-kubeflow/compare/0.4.2...HEAD
+[Unreleased]: https://github.com/getindata/kedro-kubeflow/compare/0.4.3...HEAD
+
+[0.4.3]: https://github.com/getindata/kedro-kubeflow/compare/0.4.2...0.4.3
 
 [0.4.2]: https://github.com/getindata/kedro-kubeflow/compare/0.4.1...0.4.2
 
diff --git a/kedro_kubeflow/__init__.py b/kedro_kubeflow/__init__.py
index f650bcd..1a8b535 100644
--- a/kedro_kubeflow/__init__.py
+++ b/kedro_kubeflow/__init__.py
@@ -1,3 +1,3 @@
 """kedro_kubeflow."""
 
-version = "0.4.2"
+version = "0.4.3"
diff --git a/kedro_kubeflow/config.py b/kedro_kubeflow/config.py
index 4c6e540..4ba8105 100644
--- a/kedro_kubeflow/config.py
+++ b/kedro_kubeflow/config.py
@@ -40,6 +40,11 @@ run_config:
   # See https://en.wikipedia.org/wiki/ISO_8601 in section 'Duration'
   #max_cache_staleness: P0D
 
+  # Set to false to disable kfp artifacts exposal
+  # This setting can be useful if you don't want to store
+  # intermediate results in the MLMD
+  #store_kedro_outputs_as_kfp_artifacts: True
+
   # Optional volume specification (only for non vertex-ai)
   volume:
 
@@ -195,6 +200,12 @@ class RunConfig(Config):
     def wait_for_completion(self):
         return bool(self._get_or_default("wait_for_completion", False))
 
+    @property
+    def store_kedro_outputs_as_kfp_artifacts(self):
+        return bool(
+            self._get_or_default("store_kedro_outputs_as_kfp_artifacts", True)
+        )
+
     @property
     def max_cache_staleness(self):
         return str(self._get_or_default("max_cache_staleness", None))
diff --git a/kedro_kubeflow/generator.py b/kedro_kubeflow/generator.py
index f66811d..dc52862 100644
--- a/kedro_kubeflow/generator.py
+++ b/kedro_kubeflow/generator.py
@@ -138,6 +138,8 @@ class PipelineGenerator(object):
                     command=["kedro"],
                     arguments=[
                         "kubeflow",
+                        "--env",
+                        self.context.env,
                         "mlflow-start",
                         dsl.RUN_ID_PLACEHOLDER,
                     ],
@@ -176,6 +178,8 @@ class PipelineGenerator(object):
                     command=["kedro"],
                     arguments=[
                         "run",
+                        "--env",
+                        self.context.env,
                         "--params",
                         params,
                         "--pipeline",
@@ -191,6 +195,7 @@ class PipelineGenerator(object):
                         for output in node.outputs
                         if output in self.catalog
                         and "filepath" in self.catalog[output]
+                        and self.run_config.store_kedro_outputs_as_kfp_artifacts
                     },
                 ),
                 image_pull_policy,
diff --git a/setup.cfg b/setup.cfg
index d69a577..4fe8eb0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.2
+current_version = 0.4.3
 
 [bumpversion:file:setup.py]
 
diff --git a/setup.py b/setup.py
index dc28b6a..44acc60 100644
--- a/setup.py
+++ b/setup.py
@@ -12,6 +12,7 @@ INSTALL_REQUIRES = [
     "tabulate>=0.8.7",
     "semver~=2.10",
     "google-auth<2.0dev",
+    "protobuf<3.18.0",
 ]
 
 # Dev Requirements
@@ -36,7 +37,7 @@ EXTRA_REQUIRE = {
 
 setup(
     name="kedro-kubeflow",
-    version="0.4.2",
+    version="0.4.3",
     description="Kedro plugin with Kubeflow support",
     long_description=README,
     long_description_content_type="text/markdown",


Test patch:
diff --git a/tests/test_generator.py b/tests/test_generator.py
index 7d2d72f..92cf41a 100644
--- a/tests/test_generator.py
+++ b/tests/test_generator.py
@@ -149,6 +149,8 @@ class TestGenerator(unittest.TestCase):
         assert init_step.image == "unittest-image"
         assert init_step.args == [
             "kubeflow",
+            "--env",
+            "unittests",
             "mlflow-start",
             "{{workflow.uid}}",
         ]
@@ -201,6 +203,8 @@ class TestGenerator(unittest.TestCase):
             args = dsl_pipeline.ops[node_name].container.args
             assert args == [
                 "run",
+                "--env",
+                "unittests",
                 "--params",
                 "param1:{{pipelineparam:op=;name=param1}},"
                 "param2:{{pipelineparam:op=;name=param2}}",
@@ -290,6 +294,29 @@ class TestGenerator(unittest.TestCase):
         outputs2 = dsl_pipeline.ops["node2"].file_outputs
         assert len(outputs2) == 0  # output "C" is missing in the catalog
 
+    def test_should_skip_artifact_registration_if_requested(self):
+        # given
+        self.create_generator(
+            catalog={
+                "B": {
+                    "type": "pandas.CSVDataSet",
+                    "filepath": "data/02_intermediate/b.csv",
+                }
+            },
+            config={"store_kedro_outputs_as_kfp_artifacts": False},
+        )
+
+        # when
+        pipeline = self.generator_under_test.generate_pipeline(
+            "pipeline", "unittest-image", "Always"
+        )
+        with kfp.dsl.Pipeline(None) as dsl_pipeline:
+            pipeline()
+
+        # then
+        outputs1 = dsl_pipeline.ops["node1"].file_outputs
+        assert len(outputs1) == 0
+
     def test_should_skip_volume_removal_if_requested(self):
         # given
         self.create_generator(config={"volume": {"keep": True}})
@@ -312,6 +339,7 @@ class TestGenerator(unittest.TestCase):
             "obj",
             (object,),
             {
+                "env": "unittests",
                 "params": params,
                 "config_loader": config_loader,
                 "pipelines": {