Repository: kubeflow/pipelines

Issue:
[v2 sample test] kaniko build times out / OOM
We've observed significantly high build time outs and OOMs with Kaniko recently.
I've tried several combinations:
1. 1.3.0-debug with/without --snapshotMode=redo + 4GB memory
2. 1.6.0-debug with/without --snapshotMode=redo + 8GB memory https://github.com/kubeflow/pipelines/pull/6226

but none of them run stably in reasonable amount of time.

The memory and timeout issues can be found upstream:
* https://github.com/GoogleContainerTools/kaniko/issues/1680
* https://github.com/GoogleContainerTools/kaniko/issues/1333

but they are both long standing, and no one still maintains the repo actively.


Discussion:
## Proposal

Our major use-case is building images with go applications, so `ko` seems like a natural fit.

ref: introduction ko vs buildpacks vs docker: https://cloud.google.com/blog/topics/developers-practitioners/ship-your-go-applications-faster-cloud-run-ko
TL;DR: both ko and buildpacks build images from language specific source code, `ko` is only for golang, `buildpacks` supports multiple languages. However, under the hood, `buildpacks` calls into docker, so considering build speed, `ko` is the fastest.

Resolving patch:
diff --git a/sdk/python/kfp/compiler/v2_compat.py b/sdk/python/kfp/compiler/v2_compat.py
--- a/sdk/python/kfp/compiler/v2_compat.py
+++ b/sdk/python/kfp/compiler/v2_compat.py
@@ -44,7 +44,7 @@ def update_op(op: dsl.ContainerOp,
   image_name = launcher_image or _DEFAULT_LAUNCHER_IMAGE
   launcher_container = dsl.UserContainer(name="kfp-launcher",
                                          image=image_name,
-                                         command="/bin/mount_launcher.sh",
+                                         command=["launcher", "--copy", "/kfp-launcher/launch"],
                                          mirror_volume_mounts=True)
 
   op.add_init_container(launcher_container)


Test patch:
diff --git a/v2/compiler/argo_test.go b/v2/compiler/argo_test.go
--- a/v2/compiler/argo_test.go
+++ b/v2/compiler/argo_test.go
@@ -56,7 +56,7 @@ func Test_argo_compiler(t *testing.T) {
         - --executor_input_path
         - '{{outputs.parameters.executor-input.path}}'
         command:
-        - /bin/kfp-driver
+        - driver
         image: gcr.io/ml-pipeline/kfp-driver:latest
         name: ""
         resources: {}
@@ -139,7 +139,9 @@ func Test_argo_compiler(t *testing.T) {
           name: kfp-launcher
       initContainers:
       - command:
-        - mount_launcher.sh
+        - launcher-v2
+        - --copy
+        - /kfp-launcher/launch
         image: gcr.io/ml-pipeline/kfp-launcher-v2:latest
         imagePullPolicy: Always
         name: kfp-launcher
@@ -225,7 +227,7 @@ func Test_argo_compiler(t *testing.T) {
         - --context_id_path
         - '{{outputs.parameters.context-id.path}}'
         command:
-        - /bin/kfp-driver
+        - driver
         image: gcr.io/ml-pipeline/kfp-driver:latest
         name: ""
         resources: {}
diff --git a/v2/test/Dockerfile b/v2/test/Dockerfile
--- a/v2/test/Dockerfile
+++ b/v2/test/Dockerfile
@@ -12,26 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-### Build backend compiler binary ###
-# Pin golang to 1.15 because of https://github.com/golang/go/issues/44129.
-# Remove 1.15 pin when the issue gets resolved
-FROM golang:1.15-alpine as builder
-
-# Install tools
-RUN apk add --no-cache make bash
-
-# Download go modules in a separate step for better docker caching.
-WORKDIR /build
-# api package is a dependency of v2/go.mod
-COPY api/go.mod api/go.sum api/
-COPY v2/go.mod v2/go.sum v2/
-RUN cd v2 && go mod download
-# Build the application
-COPY api api
-COPY v2 v2
-WORKDIR /build/v2
-RUN make build/compiler
-
 FROM python:3.7-slim
 
 WORKDIR /workdir
@@ -40,6 +20,5 @@ COPY sdk/python sdk/python
 # relative path in requirements.txt are relative to workdir, so we need to
 # cd to that folder first
 RUN cd v2/test && pip3 install --no-cache-dir -r requirements.txt
-COPY --from=builder /build/v2/build/compiler /bin/kfp-v2-compiler
 # copy all other code
 COPY . .
diff --git a/v2/test/components/build_go.yaml b/v2/test/components/build_go.yaml
new file mode 100644
--- /dev/null
+++ b/v2/test/components/build_go.yaml
@@ -0,0 +1,75 @@
+# Copyright 2021 The Kubeflow Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build KFP v2 go images & binaries
+inputs:
+- name: context
+  type: Folder
+  description: |
+    Input context, it should be a folder.
+- {name: destination, type: URI, description: destination should not contain tag}
+outputs:
+- {name: digest_launcher, type: URI, description: KFP launcher image digest}
+- {name: digest_launcher_v2, type: URI, description: KFP launcher v2 image digest}
+- {name: digest_driver, type: URI, description: KFP driver image digest}
+- {name: backend_compiler, type: Binary, description: KFP v2 backend compiler}
+metadata:
+  annotations:
+    author: Yuan Gong <gongyuan94@gmail.com>
+implementation:
+  container:
+    image: golang:1.16-alpine
+    command:
+    - sh
+    - -exc
+    - |
+        context_path="$0"
+        destination="$1"
+        digest_launcher_output_path="$2"
+        digest_launcher_v2_output_path="$3"
+        digest_driver_output_path="$4"
+        backend_compiler_output_path="$5"
+
+        cd "${context_path}/v2"
+
+        # install dependencies
+        apk update
+        apk add make bash curl
+        ./test/scripts/install-docker-cred-gcr.sh
+
+        # prepare parent folders
+        mkdir -p "$(dirname $digest_launcher_output_path)"
+        mkdir -p "$(dirname $digest_launcher_v2_output_path)"
+        mkdir -p "$(dirname $digest_driver_output_path)"
+        mkdir -p "$(dirname $backend_compiler_output_path)"
+
+        # authenticate to gcr
+        docker-credential-gcr configure-docker
+
+        # build images
+        go mod download || (sleep 10 && go mod download) # try download twice to avoid temporary network failure
+        make install-ko
+        export DEV_IMAGE_PREFIX="${destination}"
+        make image-launcher-dev > "${digest_launcher_output_path}"
+        make image-launcher-v2-dev > "${digest_launcher_v2_output_path}"
+        make image-driver-dev > "${digest_driver_output_path}"
+        # build backend compiler as a binary
+        make build/compiler
+        cp build/compiler "${backend_compiler_output_path}"
+    - inputPath: context
+    - inputValue: destination
+    - outputPath: digest_launcher
+    - outputPath: digest_launcher_v2
+    - outputPath: digest_driver
+    - outputPath: backend_compiler
diff --git a/v2/test/components/run_sample.yaml b/v2/test/components/run_sample.yaml
--- a/v2/test/components/run_sample.yaml
+++ b/v2/test/components/run_sample.yaml
@@ -22,22 +22,42 @@ inputs:
 - {name: Launcher Image, type: URI, default: "gcr.io/ml-pipeline/kfp-launcher:latest"}
 - {name: Launcher v2 Image, type: URI, default: "gcr.io/ml-pipeline/kfp-launcher-v2:latest"}
 - {name: Driver Image, type: URI, default: "gcr.io/ml-pipeline/kfp-driver:latest"}
+- {name: backend_compiler, type: Binary}
 implementation:
   container:
     image: python:3.7-alpine
     command:
-    - python3
-    - -m
+    - sh
+    - -exc
+    - |
+      backend_compiler_path="$0"
+      sample_path="$1"
+      output_dir="$2"
+      host="$3"
+      external_host="$4"
+      launcher_image="$5"
+      launcher_v2_image="$6"
+      driver_image="$7"
+
+      # install kfp v2 backend compiler
+      chmod +x "$backend_compiler_path"
+      cp "$backend_compiler_path" /usr/local/bin/kfp-v2-compiler
+      # run test sample
+      python3 \
+        -u \
+        -m "$sample_path" \
+        --output_directory "$output_dir" \
+        --host "$host" \
+        --external_host "$external_host" \
+        --launcher_image "$launcher_image" \
+        --launcher_v2_image "$launcher_v2_image" \
+        --driver_image "$driver_image"
+      
+    - inputPath: backend_compiler
     - inputValue: Sample Path
-    - --output_directory
     - concat: [{inputValue: GCS Root}, '/', {inputValue: Name}]
-    - --host
     - inputValue: Host
-    - --external_host
     - inputValue: External Host
-    - --launcher_image
     - inputValue: Launcher Image
-    - --launcher_v2_image
     - inputValue: Launcher v2 Image
-    - --driver_image
     - inputValue: Driver Image
diff --git a/v2/test/presubmit-v2-go-test.sh b/v2/test/presubmit-v2-go-test.sh
--- a/v2/test/presubmit-v2-go-test.sh
+++ b/v2/test/presubmit-v2-go-test.sh
@@ -27,12 +27,12 @@ mkdir go1.15.10
 cd go1.15.10
 curl -LO https://dl.google.com/go/go1.15.10.linux-amd64.tar.gz
 tar -xf go1.15.10.linux-amd64.tar.gz
-GO_CMD=/home/prow/go1.15.10/go/bin/go
+export PATH="/home/prow/go1.15.10/go/bin:${PATH}"
 cd /home/prow/go/src/github.com/kubeflow/pipelines/v2
 # 2. Check go modules are tidy
 # Reference: https://github.com/golang/go/issues/27005#issuecomment-564892876
-${GO_CMD} mod download
-${GO_CMD} mod tidy
+go mod download
+go mod tidy
 git diff --exit-code -- go.mod go.sum || (echo "go modules are not tidy, run 'go mod tidy'." && exit 1)
 
 # Note, for tests that use metadata grpc api, port-forward it locally in a separate terminal by:
@@ -49,4 +49,9 @@ trap cleanup EXIT
 kubectl port-forward svc/metadata-grpc-service 8080:8080 -n "$NAMESPACE" & PORT_FORWARD_PID=$!
 # wait for kubectl port forward
 sleep 10
-${GO_CMD} test -v -cover ./...
+go test -v -cover ./...
+
+# verify licenses are up-to-date,  because all license updates must be reviewed by a human.
+../hack/install-go-licenses.sh
+make license-launcher
+git diff --exit-code -- third_party/licenses/launcher.csv || (echo "v2/third_party/licenses/launcher.csv is outdated, refer to https://github.com/kubeflow/pipelines/tree/master/v2#update-licenses for update instructions.")
diff --git a/v2/test/sample_test.py b/v2/test/sample_test.py
--- a/v2/test/sample_test.py
+++ b/v2/test/sample_test.py
@@ -30,6 +30,7 @@
     'components/run_sample.yaml'
 )
 kaniko = kfp.components.load_component_from_file('components/kaniko.yaml')
+build_go = kfp.components.load_component_from_file('components/build_go.yaml')
 
 PIPELINE_TIME_OUT = 40 * 60  # 40 minutes
 
@@ -72,24 +73,20 @@ def build_image(name: str, dockerfile: str) -> kfp.dsl.ContainerOp:
         )  # Always -> retry on both system error and user code failure.
         return task
 
-    # build v2 compatible image
-    build_kfp_launcher_op = build_image(
-        name='kfp-launcher',
-        dockerfile='v2/container/launcher/Dockerfile',
+    # build v2 go images
+    build_go_op = build_go(
+        destination=f'{image_registry}/kfp-',
+        context=download_src_op.outputs['folder'],
     )
+    build_go_op.set_retry(1, policy='Always')
+    build_go_op.container.set_cpu_request('1').set_cpu_limit('2')
+    build_go_op.container.set_memory_request('4Gi').set_memory_limit('4Gi')
+
     # build sample test image
     build_samples_image_op = build_image(
         name='v2-sample-test',
         dockerfile='v2/test/Dockerfile',
     )
-    # build v2 engine images
-    build_kfp_launcher_v2_op = build_image(
-        name='kfp-launcher-v2',
-        dockerfile='v2/container/launcher-v2/Dockerfile'
-    )
-    build_kfp_driver_op = build_image(
-        name='kfp-driver', dockerfile='v2/container/driver/Dockerfile'
-    )
 
     # run test samples in parallel
     with kfp.dsl.ParallelFor(samples_config) as sample:
@@ -98,9 +95,10 @@ def build_image(name: str, dockerfile: str) -> kfp.dsl.ContainerOp:
             sample_path=sample.path,
             gcs_root=gcs_root,
             external_host=kfp_host,
-            launcher_image=build_kfp_launcher_op.outputs['digest'],
-            launcher_v2_image=build_kfp_launcher_v2_op.outputs['digest'],
-            driver_image=build_kfp_driver_op.outputs['digest'],
+            launcher_image=build_go_op.outputs['digest_launcher'],
+            launcher_v2_image=build_go_op.outputs['digest_launcher_v2'],
+            driver_image=build_go_op.outputs['digest_driver'],
+            backend_compiler=build_go_op.outputs['backend_compiler'],
         )
         run_sample_op.container.image = build_samples_image_op.outputs['digest']
         run_sample_op.set_display_name(f'sample_{sample.name}')
diff --git a/v2/test/scripts/install-docker-cred-gcr.sh b/v2/test/scripts/install-docker-cred-gcr.sh
new file mode 100755
--- /dev/null
+++ b/v2/test/scripts/install-docker-cred-gcr.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2021 The Kubeflow Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# doc: https://cloud.google.com/container-registry/docs/advanced-authentication#standalone-helper
+
+VERSION=2.0.0
+OS=linux  # or "darwin" for OSX, "windows" for Windows.
+ARCH=amd64  # or "386" for 32-bit OSs, "arm64" for ARM 64.
+
+curl -fsSL "https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v${VERSION}/docker-credential-gcr_${OS}_${ARCH}-${VERSION}.tar.gz" \
+	| tar xz --to-stdout ./docker-credential-gcr \
+	> /usr/local/bin/docker-credential-gcr && chmod +x /usr/local/bin/docker-credential-gcr