Repository: getindata/kedro-kubeflow
Pull request #43 (2021-01-18 08:13:16)

GitHub actions
#### Description

Adds `--with-github-actions` flag to the `init` command to generate sample github actions

##### PR Checklist
- [x] Tests added
- [x] [Changelog](CHANGELOG.md) updated 



Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c296cdf..f657194 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@
 named `KEDRO_CONFIG_<NAME>` can be referenced in configuration files as `${name}`
 - Added IAP authentication support for MLflow
 - Increased test coverage for the CLI
+- Creating github actions template with `kedro kubeflow init --with-github-actions`
 
 ### Fixed
 
diff --git a/docs/source/02_installation/02_configuration.md b/docs/source/02_installation/02_configuration.md
index dae02f6..f82415f 100644
--- a/docs/source/02_installation/02_configuration.md
+++ b/docs/source/02_installation/02_configuration.md
@@ -20,7 +20,7 @@ run_config:
   experiment_name: Kubeflow Plugin Demo
 
   # Name of the run for run-once
-  run_name: Kubeflow Plugin Demo
+  run_name: Kubeflow Plugin Demo Run
 
   # Flag indicating if the run-once should wait for the pipeline to finish
   wait_for_completion: False
@@ -36,12 +36,16 @@ run_config:
     # classes
     size: 1Gi
 
-    # Access mode of the volume used to exchange data. ReadWriteOnce doesn't
-    # allos multiple nodes to bind the volume at the same time, but may be
-    # the only option on some environments. Default value: ReadWriteMany
-    #access_modes: [ReadWriteOnce]
+    # Access mode of the volume used to exchange data. ReadWriteMany is
+    # preferred, but it is not supported on some environements (like GKE)
+    # Default value: ReadWriteOnce
+    #access_modes: [ReadWriteMany]
 
     # Flag indicating if the data-volume-init step (copying raw data to the
     # fresh volume) should be skipped
     skip_init: False
+
+    # Allows to specify user executing pipelines within containers
+    # Default: root user (to avoid issues with volumes in GKE)
+    owner: 0
 ```
diff --git a/docs/source/03_getting_started/04_continuous_deployment.md b/docs/source/03_getting_started/04_continuous_deployment.md
new file mode 100644
index 0000000..a3a8699
--- /dev/null
+++ b/docs/source/03_getting_started/04_continuous_deployment.md
@@ -0,0 +1,24 @@
+# Continuous Deployment
+
+With kedro pipelines started on the remote Kubeflow Pipelnes clusters, changes in the code require re-building docker images and (sometimes) changing the pipeline structure. To simplify this workflow, Kedro-kubeflow plugin is capable of creating configuration for the most popular CI/CD automation tools.
+
+The autogenerated configuration defines these actions:
+
+* on any new push to the repository - image is re-built and the pipeline is started using `run-once`,
+* on merge to master - image is re-built, the pipeline is registered in the Pipelines and scheduled to execute on the daily basis.
+
+The behaviour and parameters (like schedule expression) can be adjusted by editing the genrated files. The configuration assumes that Google Container Registry is used to store the images, but users can freely adapt it to any (private or public) docker images registry.
+
+## Github Actions
+
+If the Kedro project is stored on github (either in private or public repository), Github Actions can be used to automate the Continuous Deployment. To configure the repository, go to Settings->Secrets and add there:
+
+* `GKE_PROJECT`: ID of the google project.
+* `GKE_SA_KEY`: service account key, encoded with base64 (this service account must have access to push images into registry),
+* `IAP_CLIENT_ID`: id of the IAP proxy client to communicate with rest APIs.
+
+Next, re-configure the project using
+
+    kedro kubeflow init --with-github-actions https://<endpoint_name>.endpoints.<project-name>.cloud.goog/pipelines
+
+This command will generate Github Actions in `.github/workflows` directory. Then push the code to any branch and go to "Actions" tab in Github interface.
diff --git a/docs/source/03_getting_started/index.rst b/docs/source/03_getting_started/index.rst
index 6ca9450..2027467 100644
--- a/docs/source/03_getting_started/index.rst
+++ b/docs/source/03_getting_started/index.rst
@@ -7,3 +7,4 @@ Getting started
    Quickstart <01_quickstart.md>
    Google Cloud Platform support <02_gcp.md>
    Kedro-Mlflow integration <03_mlflow.md>
+   Continuous Deployment <04_continuous_deployment.md>
diff --git a/kedro_kubeflow/cli.py b/kedro_kubeflow/cli.py
index 5568359..01d884e 100644
--- a/kedro_kubeflow/cli.py
+++ b/kedro_kubeflow/cli.py
@@ -171,15 +171,21 @@ def schedule(ctx, experiment_name: str, cron_expression: str):
 
 @kubeflow_group.command()
 @click.argument("kfp_url", type=str)
+@click.option("--with-github-actions", is_flag=True, default=False)
 @click.pass_context
-def init(ctx, kfp_url: str):
+def init(ctx, kfp_url: str, with_github_actions: bool):
     """Initializes configuration for the plugin"""
     context_helper = ctx.obj["context_helper"]
-    image = (
-        context_helper.context.project_path.name
-    )  # default from kedro-docker
+    project_name = context_helper.context.project_path.name
+    if with_github_actions:
+        image = f"gcr.io/${{google_project_id}}/{project_name}:${{commit_id}}"
+        run_name = f"{project_name}:${{commit_id}}"
+    else:
+        image = project_name
+        run_name = project_name
+
     sample_config = PluginConfig.sample_config(
-        url=kfp_url, image=image, project=context_helper.project_name
+        url=kfp_url, image=image, project=project_name, run_name=run_name
     )
     config_path = Path.cwd().joinpath("conf/base/kubeflow.yaml")
     with open(config_path, "w") as f:
@@ -187,6 +193,13 @@ def init(ctx, kfp_url: str):
 
     click.echo(f"Configuration generated in {config_path}")
 
+    if with_github_actions:
+        PluginConfig.initialize_github_actions(
+            project_name,
+            where=Path.cwd(),
+            templates_dir=Path(__file__).parent / "templates",
+        )
+
 
 @kubeflow_group.command(hidden=True)
 @click.argument("kubeflow_run_id", type=str)
diff --git a/kedro_kubeflow/config.py b/kedro_kubeflow/config.py
index a08c122..eff5ea2 100644
--- a/kedro_kubeflow/config.py
+++ b/kedro_kubeflow/config.py
@@ -1,3 +1,5 @@
+import os
+
 from kedro.config import MissingConfigException
 
 DEFAULT_CONFIG_TEMPLATE = """
@@ -18,7 +20,7 @@ run_config:
   experiment_name: {project}
 
   # Name of the run for run-once
-  run_name: {project}
+  run_name: {run_name}
 
   # Flag indicating if the run-once should wait for the pipeline to finish
   wait_for_completion: False
@@ -34,16 +36,17 @@ run_config:
     # classes
     size: 1Gi
 
-    # Access mode of the volume used to exchange data. ReadWriteOnce doesn't
-    # allos multiple nodes to bind the volume at the same time, but may be
-    # the only option on some environments. Default value: ReadWriteMany
-    #access_modes: [ReadWriteOnce]
+    # Access mode of the volume used to exchange data. ReadWriteMany is
+    # preferred, but it is not supported on some environements (like GKE)
+    # Default value: ReadWriteOnce
+    #access_modes: [ReadWriteMany]
 
     # Flag indicating if the data-volume-init step (copying raw data to the
     # fresh volume) should be skipped
     skip_init: False
 
-    ## Allows to specify user executing pipelines within containers
+    # Allows to specify user executing pipelines within containers
+    # Default: root user (to avoid issues with volumes in GKE)
     owner: 0
 """
 
@@ -81,7 +84,7 @@ class VolumeConfig(Config):
 
     @property
     def access_modes(self):
-        return self._get_or_default("access_modes", ["ReadWriteMany"])
+        return self._get_or_default("access_modes", ["ReadWriteOnce"])
 
     @property
     def skip_init(self):
@@ -89,7 +92,7 @@ class VolumeConfig(Config):
 
     @property
     def owner(self):
-        return self._get_or_default("owner", None)
+        return self._get_or_default("owner", 0)
 
     def _get_prefix(self):
         return "run_config.volume."
@@ -141,3 +144,12 @@ class PluginConfig(Config):
     @staticmethod
     def sample_config(**kwargs):
         return DEFAULT_CONFIG_TEMPLATE.format(**kwargs)
+
+    @staticmethod
+    def initialize_github_actions(project_name, where, templates_dir):
+        os.makedirs(where / ".github/workflows", exist_ok=True)
+        for template in ["on-merge-to-master.yml", "on-push.yml"]:
+            file_path = where / ".github/workflows" / template
+            template_file = templates_dir / f"github-{template}"
+            with open(template_file, "r") as tfile, open(file_path, "w") as f:
+                f.write(tfile.read().format(project_name=project_name))
diff --git a/kedro_kubeflow/templates/github-on-merge-to-master.yml b/kedro_kubeflow/templates/github-on-merge-to-master.yml
new file mode 100644
index 0000000..a02e3a8
--- /dev/null
+++ b/kedro_kubeflow/templates/github-on-merge-to-master.yml
@@ -0,0 +1,49 @@
+name: On merge to master - build, register and schedule
+
+on:
+  push:
+    branches:
+      - master
+
+env:
+  PROJECT_ID: ${{{{ secrets.GKE_PROJECT }}}}
+  IAP_CLIENT_ID: ${{{{ secrets.IAP_CLIENT_ID }}}}
+  PROJECT_NAME: {project_name}
+
+jobs:
+  build-and-run:
+    name: Build image and run on Kubeflow
+    runs-on: ubuntu-20.04
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - uses: google-github-actions/setup-gcloud@v0.2.0
+      with:
+        service_account_key: ${{{{ secrets.GKE_SA_KEY }}}}
+        project_id: ${{{{ secrets.GKE_PROJECT }}}}
+        export_default_credentials: true
+
+    - name: Build
+      run: |-
+        pip3 install kedro-docker
+        gcloud --quiet auth configure-docker
+        docker pull "gcr.io/$PROJECT_ID/$PROJECT_NAME:latest" || true        
+        /home/runner/.local/bin/kedro docker build --image "gcr.io/$PROJECT_ID/$PROJECT_NAME:$GITHUB_SHA" --docker-args "--cache-from gcr.io/$PROJECT_ID/$PROJECT_NAME:latest" --gid 0
+
+    - name: Publish
+      run: |-
+        docker push "gcr.io/$PROJECT_ID/$PROJECT_NAME:$GITHUB_SHA"
+        docker tag "gcr.io/$PROJECT_ID/$PROJECT_NAME:$GITHUB_SHA" "gcr.io/$PROJECT_ID/$PROJECT_NAME:latest"
+        docker push "gcr.io/$PROJECT_ID/$PROJECT_NAME:latest"
+
+    - name: Deploy pipeline
+      run: |-
+        pip3 install kedro-kubeflow
+        /home/runner/.local/bin/kedro install
+        export KEDRO_CONFIG_GOOGLE_PROJECT_ID=$PROJECT_ID
+        export KEDRO_CONFIG_COMMIT_ID=$GITHUB_SHA
+        /home/runner/.local/bin/kedro kubeflow upload-pipeline
+        /home/runner/.local/bin/kedro kubeflow list-pipelines
+        /home/runner/.local/bin/kedro kubeflow schedule -c '0 0 4 * * *' # 04:00:00 each day
diff --git a/kedro_kubeflow/templates/github-on-push.yml b/kedro_kubeflow/templates/github-on-push.yml
new file mode 100644
index 0000000..0fe4e9e
--- /dev/null
+++ b/kedro_kubeflow/templates/github-on-push.yml
@@ -0,0 +1,47 @@
+name: On push - build image and run on Kubeflow
+
+on:
+  push:
+    branches-ignore:
+      - master
+
+env:
+  PROJECT_ID: ${{{{ secrets.GKE_PROJECT }}}}
+  IAP_CLIENT_ID: ${{{{ secrets.IAP_CLIENT_ID }}}}
+  PROJECT_NAME: {project_name}
+
+jobs:
+  build-and-run:
+    name: Build image and run on Kubeflow
+    runs-on: ubuntu-20.04
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - uses: google-github-actions/setup-gcloud@v0.2.0
+      with:
+        service_account_key: ${{{{ secrets.GKE_SA_KEY }}}}
+        project_id: ${{{{ secrets.GKE_PROJECT }}}}
+        export_default_credentials: true
+
+    - name: Build
+      run: |-
+        pip3 install kedro-docker
+        gcloud --quiet auth configure-docker
+        docker pull "gcr.io/$PROJECT_ID/$PROJECT_NAME:latest" || true        
+        /home/runner/.local/bin/kedro docker build --image "gcr.io/$PROJECT_ID/$PROJECT_NAME:$GITHUB_SHA" --docker-args "--cache-from gcr.io/$PROJECT_ID/$PROJECT_NAME:latest" --gid 0
+
+    - name: Publish
+      run: |-
+        docker push "gcr.io/$PROJECT_ID/$PROJECT_NAME:$GITHUB_SHA"
+        docker tag "gcr.io/$PROJECT_ID/$PROJECT_NAME:$GITHUB_SHA" "gcr.io/$PROJECT_ID/$PROJECT_NAME:latest"
+        docker push "gcr.io/$PROJECT_ID/$PROJECT_NAME:latest"
+
+    - name: Deploy pipeline
+      run: |-
+        pip3 install kedro-kubeflow
+        /home/runner/.local/bin/kedro install
+        export KEDRO_CONFIG_GOOGLE_PROJECT_ID=$PROJECT_ID
+        export KEDRO_CONFIG_COMMIT_ID=$GITHUB_SHA
+        /home/runner/.local/bin/kedro kubeflow run-once


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index de88691..ee026ef 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -143,12 +143,43 @@ class TestPluginCLI(unittest.TestCase):
             path = Path(temp_dir)
             cwd.return_value = path
             os.makedirs(path.joinpath("conf/base"))
-            result = runner.invoke(init, ["http:/kubeflow"], obj=config)
+            result = runner.invoke(init, ["http://kubeflow"], obj=config)
 
             assert result.exit_code == 0
             assert result.output.startswith("Configuration generated in ")
             with open(path.joinpath("conf/base/kubeflow.yaml"), "r") as f:
-                assert "host: http:/kubeflow" in f.read()
+                assert "host: http://kubeflow" in f.read()
+
+    @patch.object(Path, "cwd")
+    def test_init_with_github_actions(self, cwd):
+        context_helper = MagicMock(ContextHelper)
+        context_helper.config = test_config
+        context_helper.context.project_name = "Test Project"
+        context_helper.context.project_path.name = "test_project_path"
+        config = dict(context_helper=context_helper)
+        runner = CliRunner()
+
+        with TemporaryDirectory() as temp_dir:
+            path = Path(temp_dir)
+            cwd.return_value = path
+            os.makedirs(path.joinpath("conf/base"))
+            result = runner.invoke(
+                init, ["--with-github-actions", "http://kubeflow"], obj=config
+            )
+
+            assert result.exit_code == 0
+            on_push_actions = path / ".github" / "workflows" / "on-push.yml"
+            assert on_push_actions.exists()
+            with open(on_push_actions, "r") as f:
+                assert "kedro kubeflow run-once" in f.read()
+            on_merge_actions = (
+                path / ".github" / "workflows" / "on-merge-to-master.yml"
+            )
+            assert on_merge_actions.exists()
+            with open(on_merge_actions, "r") as f:
+                content = f.read()
+                assert "kedro kubeflow upload-pipeline" in content
+                assert "kedro kubeflow schedule" in content
 
     @patch("kedro_mlflow.framework.context.get_mlflow_config")
     @patch("mlflow.start_run")
@@ -171,7 +202,7 @@ class TestPluginCLI(unittest.TestCase):
                 obj=config,
             )
 
-            assert result.output == "Started run: MLFLOW_RUN_ID\n"
+            assert "Started run: MLFLOW_RUN_ID" in result.output
             assert result.exit_code == 0
             with open(run_id_file_path) as f:
                 assert f.read() == "MLFLOW_RUN_ID"
diff --git a/tests/test_mod.py b/tests/test_mod.py
index df32c41..3875827 100644
--- a/tests/test_mod.py
+++ b/tests/test_mod.py
@@ -378,19 +378,22 @@ class TestKubeflowClient(unittest.TestCase):
         assert len(dsl_pipeline.ops) == 4
         volume_spec = dsl_pipeline.ops["data-volume-create"].k8s_resource.spec
         assert volume_spec.resources.requests["storage"] == "1Gi"
-        assert volume_spec.access_modes == ["ReadWriteMany"]
+        assert volume_spec.access_modes == ["ReadWriteOnce"]
         assert volume_spec.storage_class_name is None
         volume_init_spec = dsl_pipeline.ops["data-volume-init"].container
         assert volume_init_spec.image == "unittest-image"
         assert volume_init_spec.image_pull_policy == "IfNotPresent"
-        assert volume_init_spec.security_context is None
+        assert volume_init_spec.security_context.run_as_user == 0
         assert volume_init_spec.args[0].startswith("cp --verbose -r")
         for node_name in ["data-volume-init", "node1", "node2"]:
             volumes = dsl_pipeline.ops[node_name].container.volume_mounts
             assert len(volumes) == 1
             assert volumes[0].name == "data-volume-create"
             assert (
-                dsl_pipeline.ops[node_name].container.security_context is None
+                dsl_pipeline.ops[
+                    node_name
+                ].container.security_context.run_as_user
+                == 0
             )
 
     def test_should_support_inter_steps_volume_with_given_spec(self):