Repository: kubeflow/kfp-tekton (language: python)

Issue:
[v2] MLMD host/port hardcoded, preventing deployment into non-kubeflow namespace
/kind bug

**What steps did you take and what happened:**
When deploying the `v2-integration branch` into a non-kubeflow namespace, connectivity to mlmd fails [here](https://github.com/kubeflow/kfp-tekton/blob/cdb7932cdb129fa038ef4ff95a9dd46d1ab252cc/backend/src/v2/metadata/client.go#L1014). This seems to be because [these fields are never set](https://github.com/kubeflow/kfp-tekton/blob/89b719b7b013ab29eb8c3b4110e0388a057f086e/backend/src/v2/controller/run.go#L280-L283). And instead the [hardcoded mlmd hostport ](https://github.com/kubeflow/kfp-tekton/blob/89b719b7b013ab29eb8c3b4110e0388a057f086e/backend/src/v2/controller/run.go#L213) is used.

Brief skim suggests it's expecting these from the `CustomRun`, coming from [here](https://github.com/kubeflow/kfp-tekton/blob/89d657d2a32eaf709133c8e96992913226155b02/backend/src/v2/compiler/tektoncompiler/dag.go#L239) or [here](https://github.com/kubeflow/kfp-tekton/blob/89d657d2a32eaf709133c8e96992913226155b02/backend/src/v2/compiler/tektoncompiler/container.go#L208), or both. I'm guessing solution here would involve passing in the mlmd host passed into the api server? 


**What did you expect to happen:**
Expect to be able to execute runs with successful connectivity to mlmd.



Discussion / hints:

Fix patch:
diff --git a/backend/src/v2/compiler/tektoncompiler/container.go b/backend/src/v2/compiler/tektoncompiler/container.go
index 0f01451782..8937b0fa2b 100644
--- a/backend/src/v2/compiler/tektoncompiler/container.go
+++ b/backend/src/v2/compiler/tektoncompiler/container.go
@@ -251,6 +251,16 @@ func (c *pipelinerunCompiler) containerDriverTask(name string, inputs *container
 				Name:  paramKubernetesConfig,
 				Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.kubernetesConfig},
 			},
+			// "--mlmd_server_address"
+			{
+				Name:  paramNameMLMDServerHost,
+				Value: pipelineapi.ParamValue{Type: "string", StringVal: GetMLMDHost()},
+			},
+			// "--mlmd_server_port"
+			{
+				Name:  paramNameMLMDServerPort,
+				Value: pipelineapi.ParamValue{Type: "string", StringVal: GetMLMDPort()},
+			},
 			// produce the following outputs:
 			// - execution-id
 			// - executor-input
diff --git a/backend/src/v2/compiler/tektoncompiler/dag.go b/backend/src/v2/compiler/tektoncompiler/dag.go
index a13c6c3238..ea5ba82f55 100644
--- a/backend/src/v2/compiler/tektoncompiler/dag.go
+++ b/backend/src/v2/compiler/tektoncompiler/dag.go
@@ -277,6 +277,16 @@ func (c *pipelinerunCompiler) dagDriverTask(
 				Name:  paramNameIterationIndex,
 				Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.getIterationIndex()},
 			},
+			// "--mlmd_server_address"
+			{
+				Name:  paramNameMLMDServerHost,
+				Value: pipelineapi.ParamValue{Type: "string", StringVal: GetMLMDHost()},
+			},
+			// "--mlmd_server_port"
+			{
+				Name:  paramNameMLMDServerPort,
+				Value: pipelineapi.ParamValue{Type: "string", StringVal: GetMLMDPort()},
+			},
 			// produce the following outputs:
 			// - execution-id
 			// - iteration-count
@@ -323,6 +333,16 @@ func (c *pipelinerunCompiler) dagPubDriverTask(
 				Name:  paramNameDagExecutionId,
 				Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.getParentDagID(c.ExitHandlerScope() || rootDagPub)},
 			},
+			// "--mlmd_server_address"
+			{
+				Name:  paramNameMLMDServerHost,
+				Value: pipelineapi.ParamValue{Type: "string", StringVal: GetMLMDHost()},
+			},
+			// "--mlmd_server_port"
+			{
+				Name:  paramNameMLMDServerPort,
+				Value: pipelineapi.ParamValue{Type: "string", StringVal: GetMLMDPort()},
+			},
 		},
 	}
 	if len(inputs.deps) > 0 {
diff --git a/backend/src/v2/compiler/tektoncompiler/tekton.go b/backend/src/v2/compiler/tektoncompiler/tekton.go
index f818c4cce1..69e1ac7a9a 100644
--- a/backend/src/v2/compiler/tektoncompiler/tekton.go
+++ b/backend/src/v2/compiler/tektoncompiler/tekton.go
@@ -658,6 +658,8 @@ const (
 	paramNameCachedDecision   = "cached_decision"
 	paramNamePodSpecPatchPath = "pod_spec_patch_path"
 	paramNameExecutorInput    = "executor_input"
+	paramNameMLMDServerHost   = "mlmd_server_address"
+	paramNameMLMDServerPort   = "mlmd_server_port"
 	paramKubernetesConfig     = "kubernetes_config" // stores Kubernetes config
 
 	kindPipelineLoop   = "PipelineLoop"
diff --git a/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml
index 77acb167fd..d866b05781 100644
--- a/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml
+++ b/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml
@@ -16,7 +16,7 @@ resources:
 images:
 - name: gcr.io/ml-pipeline/api-server
   newName: docker.io/aipipeline/apiserver-dev
-  newTag: 89d657d2a
+  newTag: 35628146f
 - name: gcr.io/ml-pipeline/persistenceagent
   newName: docker.io/aipipeline/persistenceagent-dev
   newTag: 4a9a4eaf1
diff --git a/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml b/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml
index 6ff1439942..6ec04a3145 100644
--- a/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml
+++ b/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml
@@ -67,7 +67,7 @@ patches:
 images:
 - name: gcr.io/ml-pipeline/api-server
   newName: docker.io/aipipeline/apiserver-dev
-  newTag: 89d657d2a
+  newTag: 35628146f
 - name: gcr.io/ml-pipeline/persistenceagent
   newName: docker.io/aipipeline/persistenceagent-dev
   newTag: 4a9a4eaf1


Test patch:
diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml
index faffa801d8..cdc2640f29 100755
--- a/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml
+++ b/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml
@@ -12,6 +12,197 @@ metadata:
     pipelines.kubeflow.org/v2_component: "true"
 spec:
   pipelineSpec:
+    finally:
+    - name: exithandler
+      params:
+      - name: parent-dag-id
+        value: $(tasks.root-system-dag-driver.results.execution-id)
+      taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: ExitHandler
+        metadata: {}
+        spec:
+          pipelineSpec:
+            params:
+            - name: parent-dag-id
+              type: string
+            tasks:
+            - name: print-op-driver
+              params:
+              - name: type
+                value: CONTAINER
+              - name: pipeline_name
+                value: pipeline-with-exit-handler
+              - name: run_id
+                value: $(context.pipelineRun.uid)
+              - name: dag_execution_id
+                value: $(params.parent-dag-id)
+              - name: component
+                value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}'
+              - name: task
+                value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Exit
+                  handler has worked!"}}}},"taskInfo":{"name":"print-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}'
+              - name: container
+                value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif
+                  ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||
+                  python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1
+                  python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.0-beta.6''
+                  \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp
+                  -d)\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\npython3
+                  -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n","\nimport
+                  kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import
+                  *\n\ndef print_op(message: str):\n    \"\"\"Prints a message.\"\"\"\n    print(message)\n\n"],"image":"python:3.7"}'
+              - name: iteration_index
+                value: ""
+              - name: kubernetes_config
+                value: ""
+              - name: mlmd_server_address
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: mlmd_server_port
+                value: "8080"
+              taskRef:
+                apiVersion: kfp-driver.tekton.dev/v1alpha1
+                kind: KFPDriver
+            - name: print-op
+              params:
+              - name: executor-input
+                value: $(tasks.print-op-driver.results.executor-input)
+              - name: execution-id
+                value: $(tasks.print-op-driver.results.execution-id)
+              - name: run-id
+                value: $(context.pipelineRun.uid)
+              - name: component-spec
+                value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}'
+              - name: pod-spec-patch
+                value: $(tasks.print-op-driver.results.pod-spec-patch)
+              taskSpec:
+                apiVersion: custom.tekton.dev/v1alpha1
+                kind: KFPTask
+                metadata:
+                  annotations:
+                    pipelines.kubeflow.org/v2_pipeline: "true"
+                  labels:
+                    pipelines.kubeflow.org/v2_component: "true"
+                spec:
+                  taskSpec:
+                    params:
+                    - name: executor-input
+                      type: string
+                    - name: execution-id
+                      type: string
+                    - name: run-id
+                      type: string
+                    - name: component-spec
+                      type: string
+                    steps:
+                    - command:
+                      - launcher-v2
+                      - --copy
+                      - /tekton/home/launch
+                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+                      imagePullPolicy: Always
+                      name: kfp-launcher
+                      resources: {}
+                    - args:
+                      - sh
+                      - -c
+                      - |2
+
+                        if ! [ -x "$(command -v pip)" ]; then
+                            python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
+                        fi
+
+                        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
+                      - sh
+                      - -ec
+                      - |
+                        program_path=$(mktemp -d)
+                        printf "%s" "$0" > "$program_path/ephemeral_component.py"
+                        python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+                      - |2+
+
+                        import kfp
+                        from kfp import dsl
+                        from kfp.dsl import *
+                        from typing import *
+
+                        def print_op(message: str):
+                            """Prints a message."""
+                            print(message)
+
+                      - --executor_input
+                      - '{{$}}'
+                      - --function_to_execute
+                      - print_op
+                      command:
+                      - /tekton/home/launch
+                      - --pipeline_name
+                      - pipeline-with-exit-handler
+                      - --run_id
+                      - $(params.run-id)
+                      - --execution_id
+                      - $(params.execution-id)
+                      - --executor_input
+                      - $(params.executor-input)
+                      - --component_spec
+                      - $(params.component-spec)
+                      - --pod_name
+                      - $(KFP_POD_NAME)
+                      - --pod_uid
+                      - $(KFP_POD_UID)
+                      - --mlmd_server_address
+                      - $(METADATA_GRPC_SERVICE_HOST)
+                      - --mlmd_server_port
+                      - $(METADATA_GRPC_SERVICE_PORT)
+                      - --
+                      env:
+                      - name: KFP_POD_NAME
+                        valueFrom:
+                          fieldRef:
+                            fieldPath: metadata.name
+                      - name: KFP_POD_UID
+                        valueFrom:
+                          fieldRef:
+                            fieldPath: metadata.uid
+                      - name: METADATA_GRPC_SERVICE_HOST
+                        value: metadata-grpc-service.kubeflow.svc.cluster.local
+                      - name: METADATA_GRPC_SERVICE_PORT
+                        value: "8080"
+                      - name: ML_PIPELINE_SERVICE_HOST
+                        value: ml-pipeline.kubeflow.svc.cluster.local
+                      - name: ML_PIPELINE_SERVICE_PORT_GRPC
+                        value: "8887"
+                      envFrom:
+                      - configMapRef:
+                          name: metadata-grpc-configmap
+                          optional: true
+                      image: python:3.7
+                      name: user-main
+                      resources: {}
+              when:
+              - input: $(tasks.print-op-driver.results.cached-decision)
+                operator: in
+                values:
+                - "false"
+            - name: root-system-dag-pub-driver
+              params:
+              - name: type
+                value: DAG-PUB
+              - name: pipeline_name
+                value: pipeline-with-exit-handler
+              - name: run_id
+                value: $(context.pipelineRun.uid)
+              - name: dag_execution_id
+                value: $(params.parent-dag-id)
+              - name: mlmd_server_address
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: mlmd_server_port
+                value: "8080"
+              runAfter:
+              - print-op
+              taskRef:
+                apiVersion: kfp-driver.tekton.dev/v1alpha1
+                kind: KFPDriver
     tasks:
     - name: fail-op-driver
       params:
@@ -40,10 +231,15 @@ spec:
           fail_op(message: str):\n    \"\"\"Fails.\"\"\"\n    import sys\n    print(message)\n    sys.exit(1)\n\n"],"image":"python:3.7"}'
       - name: iteration_index
         value: ""
+      - name: kubernetes_config
+        value: ""
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
     - name: fail-op
       params:
       - name: executor-input
@@ -54,107 +250,114 @@ spec:
         value: $(context.pipelineRun.uid)
       - name: component-spec
         value: '{"executorLabel":"exec-fail-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}'
+      - name: pod-spec-patch
+        value: $(tasks.fail-op-driver.results.pod-spec-patch)
       taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: KFPTask
         metadata:
           annotations:
             pipelines.kubeflow.org/v2_pipeline: "true"
           labels:
             pipelines.kubeflow.org/v2_component: "true"
-        params:
-        - name: executor-input
-          type: string
-        - name: execution-id
-          type: string
-        - name: run-id
-          type: string
-        - name: component-spec
-          type: string
-        spec: null
-        steps:
-        - command:
-          - launcher-v2
-          - --copy
-          - /tekton/home/launch
-          image: aipipeline/kfp-launcher-v2:latest
-          imagePullPolicy: Always
-          name: kfp-launcher
-          resources: {}
-        - args:
-          - sh
-          - -c
-          - |2
+        spec:
+          taskSpec:
+            params:
+            - name: executor-input
+              type: string
+            - name: execution-id
+              type: string
+            - name: run-id
+              type: string
+            - name: component-spec
+              type: string
+            steps:
+            - command:
+              - launcher-v2
+              - --copy
+              - /tekton/home/launch
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              imagePullPolicy: Always
+              name: kfp-launcher
+              resources: {}
+            - args:
+              - sh
+              - -c
+              - |2
 
-            if ! [ -x "$(command -v pip)" ]; then
-                python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
-            fi
+                if ! [ -x "$(command -v pip)" ]; then
+                    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
+                fi
 
-            PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
-          - sh
-          - -ec
-          - |
-            program_path=$(mktemp -d)
-            printf "%s" "$0" > "$program_path/ephemeral_component.py"
-            python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
-          - |2+
+                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
+              - sh
+              - -ec
+              - |
+                program_path=$(mktemp -d)
+                printf "%s" "$0" > "$program_path/ephemeral_component.py"
+                python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+              - |2+
 
-            import kfp
-            from kfp import dsl
-            from kfp.dsl import *
-            from typing import *
+                import kfp
+                from kfp import dsl
+                from kfp.dsl import *
+                from typing import *
 
-            def fail_op(message: str):
-                """Fails."""
-                import sys
-                print(message)
-                sys.exit(1)
+                def fail_op(message: str):
+                    """Fails."""
+                    import sys
+                    print(message)
+                    sys.exit(1)
 
-          - --executor_input
-          - '{{$}}'
-          - --function_to_execute
-          - fail_op
-          command:
-          - /tekton/home/launch
-          - --pipeline_name
-          - pipeline-with-exit-handler
-          - --run_id
-          - $(params.run-id)
-          - --execution_id
-          - $(params.execution-id)
-          - --executor_input
-          - $(params.executor-input)
-          - --component_spec
-          - $(params.component-spec)
-          - --runtime
-          - tekton
-          - --pod_name
-          - $(KFP_POD_NAME)
-          - --pod_uid
-          - $(KFP_POD_UID)
-          - --mlmd_server_address
-          - $(METADATA_GRPC_SERVICE_HOST)
-          - --mlmd_server_port
-          - $(METADATA_GRPC_SERVICE_PORT)
-          - --
-          env:
-          - name: KFP_POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: KFP_POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-          - name: METADATA_GRPC_SERVICE_HOST
-            value: metadata-grpc-service.kubeflow.svc.cluster.local
-          - name: METADATA_GRPC_SERVICE_PORT
-            value: "8080"
-          envFrom:
-          - configMapRef:
-              name: metadata-grpc-configmap
-              optional: true
-          image: python:3.7
-          name: user-main
-          resources: {}
+              - --executor_input
+              - '{{$}}'
+              - --function_to_execute
+              - fail_op
+              command:
+              - /tekton/home/launch
+              - --pipeline_name
+              - pipeline-with-exit-handler
+              - --run_id
+              - $(params.run-id)
+              - --execution_id
+              - $(params.execution-id)
+              - --executor_input
+              - $(params.executor-input)
+              - --component_spec
+              - $(params.component-spec)
+              - --pod_name
+              - $(KFP_POD_NAME)
+              - --pod_uid
+              - $(KFP_POD_UID)
+              - --mlmd_server_address
+              - $(METADATA_GRPC_SERVICE_HOST)
+              - --mlmd_server_port
+              - $(METADATA_GRPC_SERVICE_PORT)
+              - --
+              env:
+              - name: KFP_POD_NAME
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.name
+              - name: KFP_POD_UID
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.uid
+              - name: METADATA_GRPC_SERVICE_HOST
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: METADATA_GRPC_SERVICE_PORT
+                value: "8080"
+              - name: ML_PIPELINE_SERVICE_HOST
+                value: ml-pipeline.kubeflow.svc.cluster.local
+              - name: ML_PIPELINE_SERVICE_PORT_GRPC
+                value: "8887"
+              envFrom:
+              - configMapRef:
+                  name: metadata-grpc-configmap
+                  optional: true
+              image: python:3.7
+              name: user-main
+              resources: {}
       when:
       - input: $(tasks.fail-op-driver.results.cached-decision)
         operator: in
@@ -186,10 +389,15 @@ spec:
           print_op(message: str):\n    \"\"\"Prints a message.\"\"\"\n    print(message)\n\n"],"image":"python:3.7"}'
       - name: iteration_index
         value: ""
+      - name: kubernetes_config
+        value: ""
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
     - name: print-op-2
       params:
       - name: executor-input
@@ -200,105 +408,112 @@ spec:
         value: $(context.pipelineRun.uid)
       - name: component-spec
         value: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}'
+      - name: pod-spec-patch
+        value: $(tasks.print-op-2-driver.results.pod-spec-patch)
       taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: KFPTask
         metadata:
           annotations:
             pipelines.kubeflow.org/v2_pipeline: "true"
           labels:
             pipelines.kubeflow.org/v2_component: "true"
-        params:
-        - name: executor-input
-          type: string
-        - name: execution-id
-          type: string
-        - name: run-id
-          type: string
-        - name: component-spec
-          type: string
-        spec: null
-        steps:
-        - command:
-          - launcher-v2
-          - --copy
-          - /tekton/home/launch
-          image: aipipeline/kfp-launcher-v2:latest
-          imagePullPolicy: Always
-          name: kfp-launcher
-          resources: {}
-        - args:
-          - sh
-          - -c
-          - |2
+        spec:
+          taskSpec:
+            params:
+            - name: executor-input
+              type: string
+            - name: execution-id
+              type: string
+            - name: run-id
+              type: string
+            - name: component-spec
+              type: string
+            steps:
+            - command:
+              - launcher-v2
+              - --copy
+              - /tekton/home/launch
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              imagePullPolicy: Always
+              name: kfp-launcher
+              resources: {}
+            - args:
+              - sh
+              - -c
+              - |2
 
-            if ! [ -x "$(command -v pip)" ]; then
-                python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
-            fi
+                if ! [ -x "$(command -v pip)" ]; then
+                    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
+                fi
 
-            PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
-          - sh
-          - -ec
-          - |
-            program_path=$(mktemp -d)
-            printf "%s" "$0" > "$program_path/ephemeral_component.py"
-            python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
-          - |2+
+                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
+              - sh
+              - -ec
+              - |
+                program_path=$(mktemp -d)
+                printf "%s" "$0" > "$program_path/ephemeral_component.py"
+                python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+              - |2+
 
-            import kfp
-            from kfp import dsl
-            from kfp.dsl import *
-            from typing import *
+                import kfp
+                from kfp import dsl
+                from kfp.dsl import *
+                from typing import *
 
-            def print_op(message: str):
-                """Prints a message."""
-                print(message)
+                def print_op(message: str):
+                    """Prints a message."""
+                    print(message)
 
-          - --executor_input
-          - '{{$}}'
-          - --function_to_execute
-          - print_op
-          command:
-          - /tekton/home/launch
-          - --pipeline_name
-          - pipeline-with-exit-handler
-          - --run_id
-          - $(params.run-id)
-          - --execution_id
-          - $(params.execution-id)
-          - --executor_input
-          - $(params.executor-input)
-          - --component_spec
-          - $(params.component-spec)
-          - --runtime
-          - tekton
-          - --pod_name
-          - $(KFP_POD_NAME)
-          - --pod_uid
-          - $(KFP_POD_UID)
-          - --mlmd_server_address
-          - $(METADATA_GRPC_SERVICE_HOST)
-          - --mlmd_server_port
-          - $(METADATA_GRPC_SERVICE_PORT)
-          - --
-          env:
-          - name: KFP_POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: KFP_POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-          - name: METADATA_GRPC_SERVICE_HOST
-            value: metadata-grpc-service.kubeflow.svc.cluster.local
-          - name: METADATA_GRPC_SERVICE_PORT
-            value: "8080"
-          envFrom:
-          - configMapRef:
-              name: metadata-grpc-configmap
-              optional: true
-          image: python:3.7
-          name: user-main
-          resources: {}
+              - --executor_input
+              - '{{$}}'
+              - --function_to_execute
+              - print_op
+              command:
+              - /tekton/home/launch
+              - --pipeline_name
+              - pipeline-with-exit-handler
+              - --run_id
+              - $(params.run-id)
+              - --execution_id
+              - $(params.execution-id)
+              - --executor_input
+              - $(params.executor-input)
+              - --component_spec
+              - $(params.component-spec)
+              - --pod_name
+              - $(KFP_POD_NAME)
+              - --pod_uid
+              - $(KFP_POD_UID)
+              - --mlmd_server_address
+              - $(METADATA_GRPC_SERVICE_HOST)
+              - --mlmd_server_port
+              - $(METADATA_GRPC_SERVICE_PORT)
+              - --
+              env:
+              - name: KFP_POD_NAME
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.name
+              - name: KFP_POD_UID
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.uid
+              - name: METADATA_GRPC_SERVICE_HOST
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: METADATA_GRPC_SERVICE_PORT
+                value: "8080"
+              - name: ML_PIPELINE_SERVICE_HOST
+                value: ml-pipeline.kubeflow.svc.cluster.local
+              - name: ML_PIPELINE_SERVICE_PORT_GRPC
+                value: "8887"
+              envFrom:
+              - configMapRef:
+                  name: metadata-grpc-configmap
+                  optional: true
+              image: python:3.7
+              name: user-main
+              resources: {}
       when:
       - input: $(tasks.print-op-2-driver.results.cached-decision)
         operator: in
@@ -323,12 +538,13 @@ spec:
         value: ""
       - name: iteration_index
         value: "-1"
-      runAfter:
-      - root-system-dag-driver
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
     - name: exit-handler-1-dag-pub-driver
       params:
       - name: type
@@ -339,160 +555,16 @@ spec:
         value: $(context.pipelineRun.uid)
       - name: dag_execution_id
         value: $(tasks.exit-handler-1-dag-driver.results.execution-id)
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       runAfter:
       - fail-op
       - print-op-2
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
-    - name: print-op-driver
-      params:
-      - name: type
-        value: CONTAINER
-      - name: pipeline_name
-        value: pipeline-with-exit-handler
-      - name: run_id
-        value: $(context.pipelineRun.uid)
-      - name: dag_execution_id
-        value: $(tasks.root-system-dag-driver.results.execution-id)
-      - name: component
-        value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}'
-      - name: task
-        value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Exit
-          handler has worked!"}}}},"taskInfo":{"name":"print-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}'
-      - name: container
-        value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif
-          ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3
-          -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1
-          python3 -m pip install --quiet     --no-warn-script-location ''kfp==2.0.0-beta.6''
-          \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\nprintf
-          \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\npython3 -m
-          kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n","\nimport
-          kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef
-          print_op(message: str):\n    \"\"\"Prints a message.\"\"\"\n    print(message)\n\n"],"image":"python:3.7"}'
-      - name: iteration_index
-        value: ""
-      runAfter:
-      - exit-handler-1
-      taskRef:
-        apiVersion: kfp-driver.tekton.dev/v1alpha1
-        kind: KFPDriver
-        name: kfp-driver
-    - name: print-op
-      params:
-      - name: executor-input
-        value: $(tasks.print-op-driver.results.executor-input)
-      - name: execution-id
-        value: $(tasks.print-op-driver.results.execution-id)
-      - name: run-id
-        value: $(context.pipelineRun.uid)
-      - name: component-spec
-        value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}'
-      taskSpec:
-        metadata:
-          annotations:
-            pipelines.kubeflow.org/v2_pipeline: "true"
-          labels:
-            pipelines.kubeflow.org/v2_component: "true"
-        params:
-        - name: executor-input
-          type: string
-        - name: execution-id
-          type: string
-        - name: run-id
-          type: string
-        - name: component-spec
-          type: string
-        spec: null
-        steps:
-        - command:
-          - launcher-v2
-          - --copy
-          - /tekton/home/launch
-          image: aipipeline/kfp-launcher-v2:latest
-          imagePullPolicy: Always
-          name: kfp-launcher
-          resources: {}
-        - args:
-          - sh
-          - -c
-          - |2
-
-            if ! [ -x "$(command -v pip)" ]; then
-                python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
-            fi
-
-            PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
-          - sh
-          - -ec
-          - |
-            program_path=$(mktemp -d)
-            printf "%s" "$0" > "$program_path/ephemeral_component.py"
-            python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
-          - |2+
-
-            import kfp
-            from kfp import dsl
-            from kfp.dsl import *
-            from typing import *
-
-            def print_op(message: str):
-                """Prints a message."""
-                print(message)
-
-          - --executor_input
-          - '{{$}}'
-          - --function_to_execute
-          - print_op
-          command:
-          - /tekton/home/launch
-          - --pipeline_name
-          - pipeline-with-exit-handler
-          - --run_id
-          - $(params.run-id)
-          - --execution_id
-          - $(params.execution-id)
-          - --executor_input
-          - $(params.executor-input)
-          - --component_spec
-          - $(params.component-spec)
-          - --runtime
-          - tekton
-          - --pod_name
-          - $(KFP_POD_NAME)
-          - --pod_uid
-          - $(KFP_POD_UID)
-          - --mlmd_server_address
-          - $(METADATA_GRPC_SERVICE_HOST)
-          - --mlmd_server_port
-          - $(METADATA_GRPC_SERVICE_PORT)
-          - --
-          env:
-          - name: KFP_POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: KFP_POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-          - name: METADATA_GRPC_SERVICE_HOST
-            value: metadata-grpc-service.kubeflow.svc.cluster.local
-          - name: METADATA_GRPC_SERVICE_PORT
-            value: "8080"
-          envFrom:
-          - configMapRef:
-              name: metadata-grpc-configmap
-              optional: true
-          image: python:3.7
-          name: user-main
-          resources: {}
-      when:
-      - input: $(tasks.print-op-driver.results.cached-decision)
-        operator: in
-        values:
-        - "false"
     - name: root-system-dag-driver
       params:
       - name: type
@@ -504,7 +576,7 @@ spec:
       - name: dag_execution_id
         value: "0"
       - name: component
-        value: '{"dag":{"tasks":{"exit-handler-1":{"componentRef":{"name":"comp-exit-handler-1"},"dependentTasks":["root-system-dag-driver"],"inputs":{"parameters":{"pipelinechannel--message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"exit-handler-1"}},"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Exit
+        value: '{"dag":{"tasks":{"exit-handler-1":{"componentRef":{"name":"comp-exit-handler-1"},"inputs":{"parameters":{"pipelinechannel--message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"exit-handler-1"}},"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Exit
           handler has worked!"}}}},"taskInfo":{"name":"print-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}}},"inputDefinitions":{"parameters":{"message":{"defaultValue":"Hello
           World!","parameterType":"STRING"}}}}'
       - name: task
@@ -513,24 +585,11 @@ spec:
         value: '{"parameterValues":{"message":"Hello World!"}}'
       - name: iteration_index
         value: "-1"
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
-    - name: root-system-dag-pub-driver
-      params:
-      - name: type
-        value: DAG-PUB
-      - name: pipeline_name
-        value: pipeline-with-exit-handler
-      - name: run_id
-        value: $(context.pipelineRun.uid)
-      - name: dag_execution_id
-        value: $(tasks.root-system-dag-driver.results.execution-id)
-      runAfter:
-      - print-op
-      taskRef:
-        apiVersion: kfp-driver.tekton.dev/v1alpha1
-        kind: KFPDriver
-        name: kfp-driver
 status: {}
diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml
index def4c33d4e..4b1c7be4f0 100755
--- a/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml
+++ b/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml
@@ -42,13 +42,17 @@ spec:
           str)])\n    return output(json.dumps(experiment_spec_json))\n\n"],"image":"python:3.7"}'
       - name: iteration_index
         value: ""
+      - name: kubernetes_config
+        value: ""
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       runAfter:
       - create-katib-experiment-task
-      - root-system-dag-driver
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
     - name: convert-experiment-spec-to-str
       params:
       - name: executor-input
@@ -59,111 +63,113 @@ spec:
         value: $(context.pipelineRun.uid)
       - name: component-spec
         value: '{"executorLabel":"exec-convert-experiment-spec-to-str","inputDefinitions":{"parameters":{"experiment_spec_json":{"parameterType":"STRUCT"}}},"outputDefinitions":{"parameters":{"experiment_spec_str_output":{"parameterType":"STRING"}}}}'
-      runAfter:
-      - convert-experiment-spec-to-str-driver
+      - name: pod-spec-patch
+        value: $(tasks.convert-experiment-spec-to-str-driver.results.pod-spec-patch)
       taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: KFPTask
         metadata:
           annotations:
             pipelines.kubeflow.org/v2_pipeline: "true"
           labels:
             pipelines.kubeflow.org/v2_component: "true"
-        params:
-        - name: executor-input
-          type: string
-        - name: execution-id
-          type: string
-        - name: run-id
-          type: string
-        - name: component-spec
-          type: string
-        results:
-        - description: execution id
-          name: execution-id
-        - description: executor input
-          name: executor-input
-        spec: null
-        steps:
-        - command:
-          - launcher-v2
-          - --copy
-          - /tekton/home/launch
-          image: gcr.io/ml-pipeline-test/dev/kfp-launcher-v2:latest
-          imagePullPolicy: Always
-          name: kfp-launcher
-          resources: {}
-        - args:
-          - sh
-          - -c
-          - |2
+        spec:
+          taskSpec:
+            params:
+            - name: executor-input
+              type: string
+            - name: execution-id
+              type: string
+            - name: run-id
+              type: string
+            - name: component-spec
+              type: string
+            steps:
+            - command:
+              - launcher-v2
+              - --copy
+              - /tekton/home/launch
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              imagePullPolicy: Always
+              name: kfp-launcher
+              resources: {}
+            - args:
+              - sh
+              - -c
+              - |2
 
-            if ! [ -x "$(command -v pip)" ]; then
-                python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
-            fi
+                if ! [ -x "$(command -v pip)" ]; then
+                    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
+                fi
 
-            PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
-          - sh
-          - -ec
-          - |
-            program_path=$(mktemp -d)
-            printf "%s" "$0" > "$program_path/ephemeral_component.py"
-            python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
-          - |2+
+                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
+              - sh
+              - -ec
+              - |
+                program_path=$(mktemp -d)
+                printf "%s" "$0" > "$program_path/ephemeral_component.py"
+                python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+              - |2+
 
-            import kfp
-            from kfp import dsl
-            from kfp.dsl import *
-            from typing import *
+                import kfp
+                from kfp import dsl
+                from kfp.dsl import *
+                from typing import *
 
-            def convert_experiment_spec_to_str(experiment_spec_json: Dict[str, str])-> NamedTuple('Outputs', [('experiment_spec_str_output', str)]):
-                import json
-                output = NamedTuple('Outputs', [('experiment_spec_str_output', str)])
-                return output(json.dumps(experiment_spec_json))
+                def convert_experiment_spec_to_str(experiment_spec_json: Dict[str, str])-> NamedTuple('Outputs', [('experiment_spec_str_output', str)]):
+                    import json
+                    output = NamedTuple('Outputs', [('experiment_spec_str_output', str)])
+                    return output(json.dumps(experiment_spec_json))
 
-          - --executor_input
-          - '{{$}}'
-          - --function_to_execute
-          - convert_experiment_spec_to_str
-          command:
-          - /tekton/home/launch
-          - --pipeline_name
-          - end-to-end-pipeline
-          - --run_id
-          - $(params.run-id)
-          - --execution_id
-          - $(params.execution-id)
-          - --executor_input
-          - $(params.executor-input)
-          - --component_spec
-          - $(params.component-spec)
-          - --pod_name
-          - $(KFP_POD_NAME)
-          - --pod_uid
-          - $(KFP_POD_UID)
-          - --mlmd_server_address
-          - $(METADATA_GRPC_SERVICE_HOST)
-          - --mlmd_server_port
-          - $(METADATA_GRPC_SERVICE_PORT)
-          - --
-          env:
-          - name: KFP_POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: KFP_POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-          - name: METADATA_GRPC_SERVICE_HOST
-            value: metadata-grpc-service.kubeflow.svc.cluster.local
-          - name: METADATA_GRPC_SERVICE_PORT
-            value: "8080"
-          envFrom:
-          - configMapRef:
-              name: metadata-grpc-configmap
-              optional: true
-          image: python:3.7
-          name: user-main
-          resources: {}
+              - --executor_input
+              - '{{$}}'
+              - --function_to_execute
+              - convert_experiment_spec_to_str
+              command:
+              - /tekton/home/launch
+              - --pipeline_name
+              - end-to-end-pipeline
+              - --run_id
+              - $(params.run-id)
+              - --execution_id
+              - $(params.execution-id)
+              - --executor_input
+              - $(params.executor-input)
+              - --component_spec
+              - $(params.component-spec)
+              - --pod_name
+              - $(KFP_POD_NAME)
+              - --pod_uid
+              - $(KFP_POD_UID)
+              - --mlmd_server_address
+              - $(METADATA_GRPC_SERVICE_HOST)
+              - --mlmd_server_port
+              - $(METADATA_GRPC_SERVICE_PORT)
+              - --
+              env:
+              - name: KFP_POD_NAME
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.name
+              - name: KFP_POD_UID
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.uid
+              - name: METADATA_GRPC_SERVICE_HOST
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: METADATA_GRPC_SERVICE_PORT
+                value: "8080"
+              - name: ML_PIPELINE_SERVICE_HOST
+                value: ml-pipeline.kubeflow.svc.cluster.local
+              - name: ML_PIPELINE_SERVICE_PORT_GRPC
+                value: "8887"
+              envFrom:
+              - configMapRef:
+                  name: metadata-grpc-configmap
+                  optional: true
+              image: python:3.7
+              name: user-main
+              resources: {}
       when:
       - input: $(tasks.convert-experiment-spec-to-str-driver.results.cached-decision)
         operator: in
@@ -203,13 +209,17 @@ spec:
           \" \".join(best_hps)\n\n"],"image":"python:3.7"}'
       - name: iteration_index
         value: ""
+      - name: kubernetes_config
+        value: ""
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       runAfter:
       - create-dataset
-      - root-system-dag-driver
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
     - name: convert-katib-results
       params:
       - name: executor-input
@@ -220,124 +230,126 @@ spec:
         value: $(context.pipelineRun.uid)
       - name: component-spec
         value: '{"executorLabel":"exec-convert-katib-results","inputDefinitions":{"artifacts":{"katib_results":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}'
-      runAfter:
-      - convert-katib-results-driver
+      - name: pod-spec-patch
+        value: $(tasks.convert-katib-results-driver.results.pod-spec-patch)
       taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: KFPTask
         metadata:
           annotations:
             pipelines.kubeflow.org/v2_pipeline: "true"
           labels:
             pipelines.kubeflow.org/v2_component: "true"
-        params:
-        - name: executor-input
-          type: string
-        - name: execution-id
-          type: string
-        - name: run-id
-          type: string
-        - name: component-spec
-          type: string
-        results:
-        - description: execution id
-          name: execution-id
-        - description: executor input
-          name: executor-input
-        spec: null
-        steps:
-        - command:
-          - launcher-v2
-          - --copy
-          - /tekton/home/launch
-          image: gcr.io/ml-pipeline-test/dev/kfp-launcher-v2:latest
-          imagePullPolicy: Always
-          name: kfp-launcher
-          resources: {}
-        - args:
-          - sh
-          - -c
-          - |2
+        spec:
+          taskSpec:
+            params:
+            - name: executor-input
+              type: string
+            - name: execution-id
+              type: string
+            - name: run-id
+              type: string
+            - name: component-spec
+              type: string
+            steps:
+            - command:
+              - launcher-v2
+              - --copy
+              - /tekton/home/launch
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              imagePullPolicy: Always
+              name: kfp-launcher
+              resources: {}
+            - args:
+              - sh
+              - -c
+              - |2
 
-            if ! [ -x "$(command -v pip)" ]; then
-                python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
-            fi
+                if ! [ -x "$(command -v pip)" ]; then
+                    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
+                fi
 
-            PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
-          - sh
-          - -ec
-          - |
-            program_path=$(mktemp -d)
-            printf "%s" "$0" > "$program_path/ephemeral_component.py"
-            python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
-          - |2+
+                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
+              - sh
+              - -ec
+              - |
+                program_path=$(mktemp -d)
+                printf "%s" "$0" > "$program_path/ephemeral_component.py"
+                python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+              - |2+
 
-            import kfp
-            from kfp import dsl
-            from kfp.dsl import *
-            from typing import *
+                import kfp
+                from kfp import dsl
+                from kfp.dsl import *
+                from typing import *
 
-            def convert_katib_results(katib_results: Input[Artifact]) -> str:
-                import json
-                import pprint
-                katib_results_str = ''
-                with open(katib_results.path, 'r') as f:
-                    katib_results_str = f.read()
-                katib_results_json = json.loads(katib_results_str)
-                print("Katib results:")
-                pprint.pprint(katib_results_json)
-                best_hps = []
-                for pa in katib_results_json["currentOptimalTrial"]["parameterAssignments"]:
-                    if pa["name"] == "learning_rate":
-                        best_hps.append("--tf-learning-rate=" + pa["value"])
-                    elif pa["name"] == "batch_size":
-                        best_hps.append("--tf-batch-size=" + pa["value"])
-                print("Best Hyperparameters: {}".format(best_hps))
-                return " ".join(best_hps)
+                def convert_katib_results(katib_results: Input[Artifact]) -> str:
+                    import json
+                    import pprint
+                    katib_results_str = ''
+                    with open(katib_results.path, 'r') as f:
+                        katib_results_str = f.read()
+                    katib_results_json = json.loads(katib_results_str)
+                    print("Katib results:")
+                    pprint.pprint(katib_results_json)
+                    best_hps = []
+                    for pa in katib_results_json["currentOptimalTrial"]["parameterAssignments"]:
+                        if pa["name"] == "learning_rate":
+                            best_hps.append("--tf-learning-rate=" + pa["value"])
+                        elif pa["name"] == "batch_size":
+                            best_hps.append("--tf-batch-size=" + pa["value"])
+                    print("Best Hyperparameters: {}".format(best_hps))
+                    return " ".join(best_hps)
 
-          - --executor_input
-          - '{{$}}'
-          - --function_to_execute
-          - convert_katib_results
-          command:
-          - /tekton/home/launch
-          - --pipeline_name
-          - end-to-end-pipeline
-          - --run_id
-          - $(params.run-id)
-          - --execution_id
-          - $(params.execution-id)
-          - --executor_input
-          - $(params.executor-input)
-          - --component_spec
-          - $(params.component-spec)
-          - --pod_name
-          - $(KFP_POD_NAME)
-          - --pod_uid
-          - $(KFP_POD_UID)
-          - --mlmd_server_address
-          - $(METADATA_GRPC_SERVICE_HOST)
-          - --mlmd_server_port
-          - $(METADATA_GRPC_SERVICE_PORT)
-          - --
-          env:
-          - name: KFP_POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: KFP_POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-          - name: METADATA_GRPC_SERVICE_HOST
-            value: metadata-grpc-service.kubeflow.svc.cluster.local
-          - name: METADATA_GRPC_SERVICE_PORT
-            value: "8080"
-          envFrom:
-          - configMapRef:
-              name: metadata-grpc-configmap
-              optional: true
-          image: python:3.7
-          name: user-main
-          resources: {}
+              - --executor_input
+              - '{{$}}'
+              - --function_to_execute
+              - convert_katib_results
+              command:
+              - /tekton/home/launch
+              - --pipeline_name
+              - end-to-end-pipeline
+              - --run_id
+              - $(params.run-id)
+              - --execution_id
+              - $(params.execution-id)
+              - --executor_input
+              - $(params.executor-input)
+              - --component_spec
+              - $(params.component-spec)
+              - --pod_name
+              - $(KFP_POD_NAME)
+              - --pod_uid
+              - $(KFP_POD_UID)
+              - --mlmd_server_address
+              - $(METADATA_GRPC_SERVICE_HOST)
+              - --mlmd_server_port
+              - $(METADATA_GRPC_SERVICE_PORT)
+              - --
+              env:
+              - name: KFP_POD_NAME
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.name
+              - name: KFP_POD_UID
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.uid
+              - name: METADATA_GRPC_SERVICE_HOST
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: METADATA_GRPC_SERVICE_PORT
+                value: "8080"
+              - name: ML_PIPELINE_SERVICE_HOST
+                value: ml-pipeline.kubeflow.svc.cluster.local
+              - name: ML_PIPELINE_SERVICE_PORT_GRPC
+                value: "8887"
+              envFrom:
+              - configMapRef:
+                  name: metadata-grpc-configmap
+                  optional: true
+              image: python:3.7
+              name: user-main
+              resources: {}
       when:
       - input: $(tasks.convert-katib-results-driver.results.cached-decision)
         operator: in
@@ -361,13 +373,17 @@ spec:
         value: '{"args":["--experiment-name","{{$.inputs.parameters[''experiment_name'']}}","--experiment-namespace","{{$.inputs.parameters[''experiment_namespace'']}}","--experiment-spec","{{$.inputs.parameters[''experiment_spec_json'']}}","--experiment-timeout-minutes","{{$.inputs.parameters[''experiment_timeout_minutes'']}}","--delete-after-done","False","--output-file","{{$.outputs.artifacts[''parameter_set''].path}}"],"command":["python","src/launch_experiment.py"],"image":"docker.io/kubeflowkatib/kubeflow-pipelines-launcher"}'
       - name: iteration_index
         value: ""
+      - name: kubernetes_config
+        value: ""
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       runAfter:
       - convert-experiment-spec-to-str
-      - root-system-dag-driver
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
     - name: create-dataset
       params:
       - name: executor-input
@@ -378,94 +394,96 @@ spec:
         value: $(context.pipelineRun.uid)
       - name: component-spec
         value: '{"executorLabel":"exec-create-dataset","inputDefinitions":{"parameters":{"experiment_name":{"parameterType":"STRING"},"experiment_namespace":{"parameterType":"STRING"},"experiment_spec_json":{"parameterType":"STRING"},"experiment_timeout_minutes":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"parameter_set":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}'
-      runAfter:
-      - create-dataset-driver
+      - name: pod-spec-patch
+        value: $(tasks.create-dataset-driver.results.pod-spec-patch)
       taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: KFPTask
         metadata:
           annotations:
             pipelines.kubeflow.org/v2_pipeline: "true"
           labels:
             pipelines.kubeflow.org/v2_component: "true"
-        params:
-        - name: executor-input
-          type: string
-        - name: execution-id
-          type: string
-        - name: run-id
-          type: string
-        - name: component-spec
-          type: string
-        results:
-        - description: execution id
-          name: execution-id
-        - description: executor input
-          name: executor-input
-        spec: null
-        steps:
-        - command:
-          - launcher-v2
-          - --copy
-          - /tekton/home/launch
-          image: gcr.io/ml-pipeline-test/dev/kfp-launcher-v2:latest
-          imagePullPolicy: Always
-          name: kfp-launcher
-          resources: {}
-        - args:
-          - python
-          - src/launch_experiment.py
-          - --experiment-name
-          - '{{$.inputs.parameters[''experiment_name'']}}'
-          - --experiment-namespace
-          - '{{$.inputs.parameters[''experiment_namespace'']}}'
-          - --experiment-spec
-          - '{{$.inputs.parameters[''experiment_spec_json'']}}'
-          - --experiment-timeout-minutes
-          - '{{$.inputs.parameters[''experiment_timeout_minutes'']}}'
-          - --delete-after-done
-          - "False"
-          - --output-file
-          - '{{$.outputs.artifacts[''parameter_set''].path}}'
-          command:
-          - /tekton/home/launch
-          - --pipeline_name
-          - end-to-end-pipeline
-          - --run_id
-          - $(params.run-id)
-          - --execution_id
-          - $(params.execution-id)
-          - --executor_input
-          - $(params.executor-input)
-          - --component_spec
-          - $(params.component-spec)
-          - --pod_name
-          - $(KFP_POD_NAME)
-          - --pod_uid
-          - $(KFP_POD_UID)
-          - --mlmd_server_address
-          - $(METADATA_GRPC_SERVICE_HOST)
-          - --mlmd_server_port
-          - $(METADATA_GRPC_SERVICE_PORT)
-          - --
-          env:
-          - name: KFP_POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: KFP_POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-          - name: METADATA_GRPC_SERVICE_HOST
-            value: metadata-grpc-service.kubeflow.svc.cluster.local
-          - name: METADATA_GRPC_SERVICE_PORT
-            value: "8080"
-          envFrom:
-          - configMapRef:
-              name: metadata-grpc-configmap
-              optional: true
-          image: docker.io/kubeflowkatib/kubeflow-pipelines-launcher
-          name: user-main
-          resources: {}
+        spec:
+          taskSpec:
+            params:
+            - name: executor-input
+              type: string
+            - name: execution-id
+              type: string
+            - name: run-id
+              type: string
+            - name: component-spec
+              type: string
+            steps:
+            - command:
+              - launcher-v2
+              - --copy
+              - /tekton/home/launch
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              imagePullPolicy: Always
+              name: kfp-launcher
+              resources: {}
+            - args:
+              - python
+              - src/launch_experiment.py
+              - --experiment-name
+              - '{{$.inputs.parameters[''experiment_name'']}}'
+              - --experiment-namespace
+              - '{{$.inputs.parameters[''experiment_namespace'']}}'
+              - --experiment-spec
+              - '{{$.inputs.parameters[''experiment_spec_json'']}}'
+              - --experiment-timeout-minutes
+              - '{{$.inputs.parameters[''experiment_timeout_minutes'']}}'
+              - --delete-after-done
+              - "False"
+              - --output-file
+              - '{{$.outputs.artifacts[''parameter_set''].path}}'
+              command:
+              - /tekton/home/launch
+              - --pipeline_name
+              - end-to-end-pipeline
+              - --run_id
+              - $(params.run-id)
+              - --execution_id
+              - $(params.execution-id)
+              - --executor_input
+              - $(params.executor-input)
+              - --component_spec
+              - $(params.component-spec)
+              - --pod_name
+              - $(KFP_POD_NAME)
+              - --pod_uid
+              - $(KFP_POD_UID)
+              - --mlmd_server_address
+              - $(METADATA_GRPC_SERVICE_HOST)
+              - --mlmd_server_port
+              - $(METADATA_GRPC_SERVICE_PORT)
+              - --
+              env:
+              - name: KFP_POD_NAME
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.name
+              - name: KFP_POD_UID
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.uid
+              - name: METADATA_GRPC_SERVICE_HOST
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: METADATA_GRPC_SERVICE_PORT
+                value: "8080"
+              - name: ML_PIPELINE_SERVICE_HOST
+                value: ml-pipeline.kubeflow.svc.cluster.local
+              - name: ML_PIPELINE_SERVICE_PORT_GRPC
+                value: "8887"
+              envFrom:
+              - configMapRef:
+                  name: metadata-grpc-configmap
+                  optional: true
+              image: docker.io/kubeflowkatib/kubeflow-pipelines-launcher
+              name: user-main
+              resources: {}
       when:
       - input: $(tasks.create-dataset-driver.results.cached-decision)
         operator: in
@@ -536,12 +554,15 @@ spec:
           output(experiment_spec_json)\n\n"],"image":"python:3.8"}'
       - name: iteration_index
         value: ""
-      runAfter:
-      - root-system-dag-driver
+      - name: kubernetes_config
+        value: ""
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
     - name: create-katib-experiment-task
       params:
       - name: executor-input
@@ -552,253 +573,255 @@ spec:
         value: $(context.pipelineRun.uid)
       - name: component-spec
         value: '{"executorLabel":"exec-create-katib-experiment-task","inputDefinitions":{"parameters":{"experiment_name":{"parameterType":"STRING"},"experiment_namespace":{"parameterType":"STRING"},"training_steps":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"experiment_spec_json":{"parameterType":"STRUCT"}}}}'
-      runAfter:
-      - create-katib-experiment-task-driver
+      - name: pod-spec-patch
+        value: $(tasks.create-katib-experiment-task-driver.results.pod-spec-patch)
       taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: KFPTask
         metadata:
           annotations:
             pipelines.kubeflow.org/v2_pipeline: "true"
           labels:
             pipelines.kubeflow.org/v2_component: "true"
-        params:
-        - name: executor-input
-          type: string
-        - name: execution-id
-          type: string
-        - name: run-id
-          type: string
-        - name: component-spec
-          type: string
-        results:
-        - description: execution id
-          name: execution-id
-        - description: executor input
-          name: executor-input
-        spec: null
-        steps:
-        - command:
-          - launcher-v2
-          - --copy
-          - /tekton/home/launch
-          image: gcr.io/ml-pipeline-test/dev/kfp-launcher-v2:latest
-          imagePullPolicy: Always
-          name: kfp-launcher
-          resources: {}
-        - args:
-          - sh
-          - -c
-          - |2
+        spec:
+          taskSpec:
+            params:
+            - name: executor-input
+              type: string
+            - name: execution-id
+              type: string
+            - name: run-id
+              type: string
+            - name: component-spec
+              type: string
+            steps:
+            - command:
+              - launcher-v2
+              - --copy
+              - /tekton/home/launch
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              imagePullPolicy: Always
+              name: kfp-launcher
+              resources: {}
+            - args:
+              - sh
+              - -c
+              - |2
 
-            if ! [ -x "$(command -v pip)" ]; then
-                python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
-            fi
+                if ! [ -x "$(command -v pip)" ]; then
+                    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
+                fi
 
-            PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kubeflow-katib==0.12.0' 'kfp==2.0.0-beta.6' && "$0" "$@"
-          - sh
-          - -ec
-          - |
-            program_path=$(mktemp -d)
-            printf "%s" "$0" > "$program_path/ephemeral_component.py"
-            python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
-          - |2+
+                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kubeflow-katib==0.12.0' 'kfp==2.0.0-beta.6' && "$0" "$@"
+              - sh
+              - -ec
+              - |
+                program_path=$(mktemp -d)
+                printf "%s" "$0" > "$program_path/ephemeral_component.py"
+                python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+              - |2+
 
-            import kfp
-            from kfp import dsl
-            from kfp.dsl import *
-            from typing import *
+                import kfp
+                from kfp import dsl
+                from kfp.dsl import *
+                from typing import *
 
-            def create_katib_experiment_task(experiment_name: str, experiment_namespace: str, training_steps: str
-                                            ) -> NamedTuple('Outputs', [('experiment_spec_json', Dict[str, str])]):
+                def create_katib_experiment_task(experiment_name: str, experiment_namespace: str, training_steps: str
+                                                ) -> NamedTuple('Outputs', [('experiment_spec_json', Dict[str, str])]):
 
-                from kubeflow.katib import ApiClient
-                from kubeflow.katib import V1beta1ExperimentSpec
-                from kubeflow.katib import V1beta1AlgorithmSpec
-                from kubeflow.katib import V1beta1ObjectiveSpec
-                from kubeflow.katib import V1beta1ParameterSpec
-                from kubeflow.katib import V1beta1FeasibleSpace
-                from kubeflow.katib import V1beta1TrialTemplate
-                from kubeflow.katib import V1beta1TrialParameterSpec
+                    from kubeflow.katib import ApiClient
+                    from kubeflow.katib import V1beta1ExperimentSpec
+                    from kubeflow.katib import V1beta1AlgorithmSpec
+                    from kubeflow.katib import V1beta1ObjectiveSpec
+                    from kubeflow.katib import V1beta1ParameterSpec
+                    from kubeflow.katib import V1beta1FeasibleSpace
+                    from kubeflow.katib import V1beta1TrialTemplate
+                    from kubeflow.katib import V1beta1TrialParameterSpec
 
-                # Trial count specification.
-                max_trial_count = 5
-                max_failed_trial_count = 3
-                parallel_trial_count = 2
+                    # Trial count specification.
+                    max_trial_count = 5
+                    max_failed_trial_count = 3
+                    parallel_trial_count = 2
 
-                # Objective specification.
-                objective = V1beta1ObjectiveSpec(
-                    type="minimize",
-                    goal=0.001,
-                    objective_metric_name="loss"
-                )
+                    # Objective specification.
+                    objective = V1beta1ObjectiveSpec(
+                        type="minimize",
+                        goal=0.001,
+                        objective_metric_name="loss"
+                    )
 
-                # Algorithm specification.
-                algorithm = V1beta1AlgorithmSpec(
-                    algorithm_name="random",
-                )
+                    # Algorithm specification.
+                    algorithm = V1beta1AlgorithmSpec(
+                        algorithm_name="random",
+                    )
 
-                # Experiment search space.
-                # In this example we tune learning rate and batch size.
-                parameters = [
-                    V1beta1ParameterSpec(
-                        name="learning_rate",
-                        parameter_type="double",
-                        feasible_space=V1beta1FeasibleSpace(
-                            min="0.01",
-                            max="0.05"
-                        ),
-                    ),
-                    V1beta1ParameterSpec(
-                        name="batch_size",
-                        parameter_type="int",
-                        feasible_space=V1beta1FeasibleSpace(
-                            min="80",
-                            max="100"
+                    # Experiment search space.
+                    # In this example we tune learning rate and batch size.
+                    parameters = [
+                        V1beta1ParameterSpec(
+                            name="learning_rate",
+                            parameter_type="double",
+                            feasible_space=V1beta1FeasibleSpace(
+                                min="0.01",
+                                max="0.05"
+                            ),
                         ),
-                    )
-                ]
+                        V1beta1ParameterSpec(
+                            name="batch_size",
+                            parameter_type="int",
+                            feasible_space=V1beta1FeasibleSpace(
+                                min="80",
+                                max="100"
+                            ),
+                        )
+                    ]
 
-                # Experiment Trial template.
-                # TODO (andreyvelich): Use community image for the mnist example.
-                trial_spec = {
-                    "apiVersion": "kubeflow.org/v1",
-                    "kind": "TFJob",
-                    "spec": {
-                        "tfReplicaSpecs": {
-                            "Chief": {
-                                "replicas": 1,
-                                "restartPolicy": "OnFailure",
-                                "template": {
-                                    "metadata": {
-                                        "annotations": {
-                                            "sidecar.istio.io/inject": "false"
-                                        }
-                                    },
-                                    "spec": {
-                                        "containers": [
-                                            {
-                                                "name": "tensorflow",
-                                                "image": "docker.io/liuhougangxa/tf-estimator-mnist",
-                                                "command": [
-                                                    "python",
-                                                    "/opt/model.py",
-                                                    "--tf-train-steps=" + str(training_steps),
-                                                    "--tf-learning-rate=${trialParameters.learningRate}",
-                                                    "--tf-batch-size=${trialParameters.batchSize}"
-                                                ]
+                    # Experiment Trial template.
+                    # TODO (andreyvelich): Use community image for the mnist example.
+                    trial_spec = {
+                        "apiVersion": "kubeflow.org/v1",
+                        "kind": "TFJob",
+                        "spec": {
+                            "tfReplicaSpecs": {
+                                "Chief": {
+                                    "replicas": 1,
+                                    "restartPolicy": "OnFailure",
+                                    "template": {
+                                        "metadata": {
+                                            "annotations": {
+                                                "sidecar.istio.io/inject": "false"
                                             }
-                                        ]
-                                    }
-                                }
-                            },
-                            "Worker": {
-                                "replicas": 1,
-                                "restartPolicy": "OnFailure",
-                                "template": {
-                                    "metadata": {
-                                        "annotations": {
-                                            "sidecar.istio.io/inject": "false"
+                                        },
+                                        "spec": {
+                                            "containers": [
+                                                {
+                                                    "name": "tensorflow",
+                                                    "image": "docker.io/liuhougangxa/tf-estimator-mnist",
+                                                    "command": [
+                                                        "python",
+                                                        "/opt/model.py",
+                                                        "--tf-train-steps=" + str(training_steps),
+                                                        "--tf-learning-rate=${trialParameters.learningRate}",
+                                                        "--tf-batch-size=${trialParameters.batchSize}"
+                                                    ]
+                                                }
+                                            ]
                                         }
-                                    },
-                                    "spec": {
-                                        "containers": [
-                                            {
-                                                "name": "tensorflow",
-                                                "image": "docker.io/liuhougangxa/tf-estimator-mnist",
-                                                "command": [
-                                                    "python",
-                                                    "/opt/model.py",
-                                                    "--tf-train-steps=" + str(training_steps),
-                                                    "--tf-learning-rate=${trialParameters.learningRate}",
-                                                    "--tf-batch-size=${trialParameters.batchSize}"
-                                                ]
+                                    }
+                                },
+                                "Worker": {
+                                    "replicas": 1,
+                                    "restartPolicy": "OnFailure",
+                                    "template": {
+                                        "metadata": {
+                                            "annotations": {
+                                                "sidecar.istio.io/inject": "false"
                                             }
-                                        ]
+                                        },
+                                        "spec": {
+                                            "containers": [
+                                                {
+                                                    "name": "tensorflow",
+                                                    "image": "docker.io/liuhougangxa/tf-estimator-mnist",
+                                                    "command": [
+                                                        "python",
+                                                        "/opt/model.py",
+                                                        "--tf-train-steps=" + str(training_steps),
+                                                        "--tf-learning-rate=${trialParameters.learningRate}",
+                                                        "--tf-batch-size=${trialParameters.batchSize}"
+                                                    ]
+                                                }
+                                            ]
+                                        }
                                     }
                                 }
                             }
                         }
                     }
-                }
 
-                # Configure parameters for the Trial template.
-                trial_template = V1beta1TrialTemplate(
-                    primary_container_name="tensorflow",
-                    trial_parameters=[
-                        V1beta1TrialParameterSpec(
-                            name="learningRate",
-                            description="Learning rate for the training model",
-                            reference="learning_rate"
-                        ),
-                        V1beta1TrialParameterSpec(
-                            name="batchSize",
-                            description="Batch size for the model",
-                            reference="batch_size"
-                        ),
-                    ],
-                    trial_spec=trial_spec
-                )
+                    # Configure parameters for the Trial template.
+                    trial_template = V1beta1TrialTemplate(
+                        primary_container_name="tensorflow",
+                        trial_parameters=[
+                            V1beta1TrialParameterSpec(
+                                name="learningRate",
+                                description="Learning rate for the training model",
+                                reference="learning_rate"
+                            ),
+                            V1beta1TrialParameterSpec(
+                                name="batchSize",
+                                description="Batch size for the model",
+                                reference="batch_size"
+                            ),
+                        ],
+                        trial_spec=trial_spec
+                    )
 
-                # Create an Experiment from the above parameters.
-                experiment_spec = V1beta1ExperimentSpec(
-                    max_trial_count=max_trial_count,
-                    max_failed_trial_count=max_failed_trial_count,
-                    parallel_trial_count=parallel_trial_count,
-                    objective=objective,
-                    algorithm=algorithm,
-                    parameters=parameters,
-                    trial_template=trial_template
-                )
+                    # Create an Experiment from the above parameters.
+                    experiment_spec = V1beta1ExperimentSpec(
+                        max_trial_count=max_trial_count,
+                        max_failed_trial_count=max_failed_trial_count,
+                        parallel_trial_count=parallel_trial_count,
+                        objective=objective,
+                        algorithm=algorithm,
+                        parameters=parameters,
+                        trial_template=trial_template
+                    )
 
-                # Convert experiment_spec to Dict type.
-                experiment_spec_json = ApiClient().sanitize_for_serialization(experiment_spec)
-                output = NamedTuple('Outputs', [('experiment_spec_json', Dict[str, str])])
-                return output(experiment_spec_json)
+                    # Convert experiment_spec to Dict type.
+                    experiment_spec_json = ApiClient().sanitize_for_serialization(experiment_spec)
+                    output = NamedTuple('Outputs', [('experiment_spec_json', Dict[str, str])])
+                    return output(experiment_spec_json)
 
-          - --executor_input
-          - '{{$}}'
-          - --function_to_execute
-          - create_katib_experiment_task
-          command:
-          - /tekton/home/launch
-          - --pipeline_name
-          - end-to-end-pipeline
-          - --run_id
-          - $(params.run-id)
-          - --execution_id
-          - $(params.execution-id)
-          - --executor_input
-          - $(params.executor-input)
-          - --component_spec
-          - $(params.component-spec)
-          - --pod_name
-          - $(KFP_POD_NAME)
-          - --pod_uid
-          - $(KFP_POD_UID)
-          - --mlmd_server_address
-          - $(METADATA_GRPC_SERVICE_HOST)
-          - --mlmd_server_port
-          - $(METADATA_GRPC_SERVICE_PORT)
-          - --
-          env:
-          - name: KFP_POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: KFP_POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-          - name: METADATA_GRPC_SERVICE_HOST
-            value: metadata-grpc-service.kubeflow.svc.cluster.local
-          - name: METADATA_GRPC_SERVICE_PORT
-            value: "8080"
-          envFrom:
-          - configMapRef:
-              name: metadata-grpc-configmap
-              optional: true
-          image: python:3.8
-          name: user-main
-          resources: {}
+              - --executor_input
+              - '{{$}}'
+              - --function_to_execute
+              - create_katib_experiment_task
+              command:
+              - /tekton/home/launch
+              - --pipeline_name
+              - end-to-end-pipeline
+              - --run_id
+              - $(params.run-id)
+              - --execution_id
+              - $(params.execution-id)
+              - --executor_input
+              - $(params.executor-input)
+              - --component_spec
+              - $(params.component-spec)
+              - --pod_name
+              - $(KFP_POD_NAME)
+              - --pod_uid
+              - $(KFP_POD_UID)
+              - --mlmd_server_address
+              - $(METADATA_GRPC_SERVICE_HOST)
+              - --mlmd_server_port
+              - $(METADATA_GRPC_SERVICE_PORT)
+              - --
+              env:
+              - name: KFP_POD_NAME
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.name
+              - name: KFP_POD_UID
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.uid
+              - name: METADATA_GRPC_SERVICE_HOST
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: METADATA_GRPC_SERVICE_PORT
+                value: "8080"
+              - name: ML_PIPELINE_SERVICE_HOST
+                value: ml-pipeline.kubeflow.svc.cluster.local
+              - name: ML_PIPELINE_SERVICE_PORT_GRPC
+                value: "8887"
+              envFrom:
+              - configMapRef:
+                  name: metadata-grpc-configmap
+                  optional: true
+              image: python:3.8
+              name: user-main
+              resources: {}
       when:
       - input: $(tasks.create-katib-experiment-task-driver.results.cached-decision)
         operator: in
@@ -858,13 +881,17 @@ spec:
           Dict[str, str])])\n    return output(tfjob_chief_spec, tfjob_worker_spec)\n\n"],"image":"python:3.7"}'
       - name: iteration_index
         value: ""
+      - name: kubernetes_config
+        value: ""
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       runAfter:
       - convert-katib-results
-      - root-system-dag-driver
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
     - name: create-tfjob-task
       params:
       - name: executor-input
@@ -875,139 +902,143 @@ spec:
         value: $(context.pipelineRun.uid)
       - name: component-spec
         value: '{"executorLabel":"exec-create-tfjob-task","inputDefinitions":{"parameters":{"best_hps":{"parameterType":"STRING"},"model_volume_name":{"parameterType":"STRING"},"tfjob_name":{"parameterType":"STRING"},"tfjob_namespace":{"parameterType":"STRING"},"training_steps":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"chief_spec":{"parameterType":"STRUCT"},"worker_spec":{"parameterType":"STRUCT"}}}}'
-      runAfter:
-      - create-tfjob-task-driver
+      - name: pod-spec-patch
+        value: $(tasks.create-tfjob-task-driver.results.pod-spec-patch)
       taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: KFPTask
         metadata:
           annotations:
             pipelines.kubeflow.org/v2_pipeline: "true"
           labels:
             pipelines.kubeflow.org/v2_component: "true"
-        params:
-        - name: executor-input
-          type: string
-        - name: execution-id
-          type: string
-        - name: run-id
-          type: string
-        - name: component-spec
-          type: string
-        results:
-        - description: execution id
-          name: execution-id
-        - description: executor input
-          name: executor-input
-        spec: null
-        steps:
-        - command:
-          - launcher-v2
-          - --copy
-          - /tekton/home/launch
-          image: gcr.io/ml-pipeline-test/dev/kfp-launcher-v2:latest
-          imagePullPolicy: Always
-          name: kfp-launcher
-          resources: {}
-        - args:
-          - sh
-          - -c
-          - |2
+        spec:
+          taskSpec:
+            params:
+            - name: executor-input
+              type: string
+            - name: execution-id
+              type: string
+            - name: run-id
+              type: string
+            - name: component-spec
+              type: string
+            steps:
+            - command:
+              - launcher-v2
+              - --copy
+              - /tekton/home/launch
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              imagePullPolicy: Always
+              name: kfp-launcher
+              resources: {}
+            - args:
+              - sh
+              - -c
+              - |2
 
-            if ! [ -x "$(command -v pip)" ]; then
-                python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
-            fi
+                if ! [ -x "$(command -v pip)" ]; then
+                    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
+                fi
 
-            PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
-          - sh
-          - -ec
-          - |
-            program_path=$(mktemp -d)
-            printf "%s" "$0" > "$program_path/ephemeral_component.py"
-            python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
-          - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing
-            import *\n\ndef create_tfjob_task(tfjob_name: str, tfjob_namespace: str,
-            training_steps: str, best_hps: str, model_volume_name: str,\n                     )
-            -> NamedTuple('Outputs', [('chief_spec', Dict[str, str]), ('worker_spec',
-            Dict[str, str])]):\n    # Get parameters from the Katib Experiment.\n
-            \   # Parameters are in the format \"--tf-learning-rate=0.01 --tf-batch-size=100\"\n\n
-            \   # Create the TFJob Chief and Worker specification with the best Hyperparameters.\n
-            \   # TODO (andreyvelich): Use community image for the mnist example.\n
-            \   tfjob_chief_spec = {\n        \"replicas\": 1,\n        \"restartPolicy\":
-            \"OnFailure\",\n        \"template\": {\n            \"metadata\": {\n
-            \               \"annotations\": {\n                    \"sidecar.istio.io/inject\":
-            \"false\"\n                }\n            },\n            \"spec\": {\n
-            \               \"containers\": [\n                    {\n                        \"name\":
-            \"tensorflow\",\n                        \"image\": \"docker.io/liuhougangxa/tf-estimator-mnist\",\n
-            \                       \"command\": [\n                            \"sh\",\n
-            \                           \"-c\"\n                        ],\n                        \"args\":
-            [\n                            \"python /opt/model.py --tf-export-dir=/mnt/export
-            --tf-train-steps={} {}\".format(training_steps, best_hps)\n                        ],\n
-            \                       \"volumeMounts\": [\n                            {\n
-            \                               \"mountPath\": \"/mnt/export\",\n                                \"name\":
-            \"model-volume\"\n                            }\n                        ]\n
-            \                   }\n                ],\n                \"volumes\":
-            [\n                    {\n                        \"name\": \"model-volume\",\n
-            \                       \"persistentVolumeClaim\": {\n                            \"claimName\":
-            model_volume_name\n                        }\n                    }\n
-            \               ]\n            }\n        }\n    }\n\n    tfjob_worker_spec
-            = {\n        \"replicas\": 1,\n        \"restartPolicy\": \"OnFailure\",\n
-            \       \"template\": {\n            \"metadata\": {\n                \"annotations\":
-            {\n                    \"sidecar.istio.io/inject\": \"false\"\n                }\n
-            \           },\n            \"spec\": {\n                \"containers\":
-            [\n                    {\n                        \"name\": \"tensorflow\",\n
-            \                       \"image\": \"docker.io/liuhougangxa/tf-estimator-mnist\",\n
-            \                       \"command\": [\n                            \"sh\",\n
-            \                           \"-c\",\n                        ],\n                        \"args\":
-            [\n                          \"python /opt/model.py --tf-export-dir=/mnt/export
-            --tf-train-steps={} {}\".format(training_steps, best_hps) \n                        ],\n
-            \                   }\n                ],\n            }\n        }\n
-            \   }\n\n    output = NamedTuple('Outputs', [('chief_spec', Dict[str,
-            str]), ('worker_spec', Dict[str, str])])\n    return output(tfjob_chief_spec,
-            tfjob_worker_spec)\n\n"
-          - --executor_input
-          - '{{$}}'
-          - --function_to_execute
-          - create_tfjob_task
-          command:
-          - /tekton/home/launch
-          - --pipeline_name
-          - end-to-end-pipeline
-          - --run_id
-          - $(params.run-id)
-          - --execution_id
-          - $(params.execution-id)
-          - --executor_input
-          - $(params.executor-input)
-          - --component_spec
-          - $(params.component-spec)
-          - --pod_name
-          - $(KFP_POD_NAME)
-          - --pod_uid
-          - $(KFP_POD_UID)
-          - --mlmd_server_address
-          - $(METADATA_GRPC_SERVICE_HOST)
-          - --mlmd_server_port
-          - $(METADATA_GRPC_SERVICE_PORT)
-          - --
-          env:
-          - name: KFP_POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: KFP_POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-          - name: METADATA_GRPC_SERVICE_HOST
-            value: metadata-grpc-service.kubeflow.svc.cluster.local
-          - name: METADATA_GRPC_SERVICE_PORT
-            value: "8080"
-          envFrom:
-          - configMapRef:
-              name: metadata-grpc-configmap
-              optional: true
-          image: python:3.7
-          name: user-main
-          resources: {}
+                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
+              - sh
+              - -ec
+              - |
+                program_path=$(mktemp -d)
+                printf "%s" "$0" > "$program_path/ephemeral_component.py"
+                python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+              - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing
+                import *\n\ndef create_tfjob_task(tfjob_name: str, tfjob_namespace:
+                str, training_steps: str, best_hps: str, model_volume_name: str,\n
+                \                    ) -> NamedTuple('Outputs', [('chief_spec', Dict[str,
+                str]), ('worker_spec', Dict[str, str])]):\n    # Get parameters from
+                the Katib Experiment.\n    # Parameters are in the format \"--tf-learning-rate=0.01
+                --tf-batch-size=100\"\n\n    # Create the TFJob Chief and Worker specification
+                with the best Hyperparameters.\n    # TODO (andreyvelich): Use community
+                image for the mnist example.\n    tfjob_chief_spec = {\n        \"replicas\":
+                1,\n        \"restartPolicy\": \"OnFailure\",\n        \"template\":
+                {\n            \"metadata\": {\n                \"annotations\": {\n
+                \                   \"sidecar.istio.io/inject\": \"false\"\n                }\n
+                \           },\n            \"spec\": {\n                \"containers\":
+                [\n                    {\n                        \"name\": \"tensorflow\",\n
+                \                       \"image\": \"docker.io/liuhougangxa/tf-estimator-mnist\",\n
+                \                       \"command\": [\n                            \"sh\",\n
+                \                           \"-c\"\n                        ],\n                        \"args\":
+                [\n                            \"python /opt/model.py --tf-export-dir=/mnt/export
+                --tf-train-steps={} {}\".format(training_steps, best_hps)\n                        ],\n
+                \                       \"volumeMounts\": [\n                            {\n
+                \                               \"mountPath\": \"/mnt/export\",\n
+                \                               \"name\": \"model-volume\"\n                            }\n
+                \                       ]\n                    }\n                ],\n
+                \               \"volumes\": [\n                    {\n                        \"name\":
+                \"model-volume\",\n                        \"persistentVolumeClaim\":
+                {\n                            \"claimName\": model_volume_name\n
+                \                       }\n                    }\n                ]\n
+                \           }\n        }\n    }\n\n    tfjob_worker_spec = {\n        \"replicas\":
+                1,\n        \"restartPolicy\": \"OnFailure\",\n        \"template\":
+                {\n            \"metadata\": {\n                \"annotations\": {\n
+                \                   \"sidecar.istio.io/inject\": \"false\"\n                }\n
+                \           },\n            \"spec\": {\n                \"containers\":
+                [\n                    {\n                        \"name\": \"tensorflow\",\n
+                \                       \"image\": \"docker.io/liuhougangxa/tf-estimator-mnist\",\n
+                \                       \"command\": [\n                            \"sh\",\n
+                \                           \"-c\",\n                        ],\n
+                \                       \"args\": [\n                          \"python
+                /opt/model.py --tf-export-dir=/mnt/export --tf-train-steps={} {}\".format(training_steps,
+                best_hps) \n                        ],\n                    }\n                ],\n
+                \           }\n        }\n    }\n\n    output = NamedTuple('Outputs',
+                [('chief_spec', Dict[str, str]), ('worker_spec', Dict[str, str])])\n
+                \   return output(tfjob_chief_spec, tfjob_worker_spec)\n\n"
+              - --executor_input
+              - '{{$}}'
+              - --function_to_execute
+              - create_tfjob_task
+              command:
+              - /tekton/home/launch
+              - --pipeline_name
+              - end-to-end-pipeline
+              - --run_id
+              - $(params.run-id)
+              - --execution_id
+              - $(params.execution-id)
+              - --executor_input
+              - $(params.executor-input)
+              - --component_spec
+              - $(params.component-spec)
+              - --pod_name
+              - $(KFP_POD_NAME)
+              - --pod_uid
+              - $(KFP_POD_UID)
+              - --mlmd_server_address
+              - $(METADATA_GRPC_SERVICE_HOST)
+              - --mlmd_server_port
+              - $(METADATA_GRPC_SERVICE_PORT)
+              - --
+              env:
+              - name: KFP_POD_NAME
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.name
+              - name: KFP_POD_UID
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.uid
+              - name: METADATA_GRPC_SERVICE_HOST
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: METADATA_GRPC_SERVICE_PORT
+                value: "8080"
+              - name: ML_PIPELINE_SERVICE_HOST
+                value: ml-pipeline.kubeflow.svc.cluster.local
+              - name: ML_PIPELINE_SERVICE_PORT_GRPC
+                value: "8887"
+              envFrom:
+              - configMapRef:
+                  name: metadata-grpc-configmap
+                  optional: true
+              image: python:3.7
+              name: user-main
+              resources: {}
       when:
       - input: $(tasks.create-tfjob-task-driver.results.cached-decision)
         operator: in
@@ -1041,13 +1072,17 @@ spec:
           open(inferenceservice_artifact.path, ''w'') as f:\n        f.write(json.dumps(inferenceservice_yaml))\n\n"],"image":"python:3.7"}'
       - name: iteration_index
         value: ""
+      - name: kubernetes_config
+        value: ""
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       runAfter:
       - create-serving-task
-      - serving-pipeline-dag-driver
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
     - name: convert-inference-service-to-artifact
       params:
       - name: executor-input
@@ -1058,111 +1093,113 @@ spec:
         value: $(context.pipelineRun.uid)
       - name: component-spec
         value: '{"executorLabel":"exec-convert-inference-service-to-artifact","inputDefinitions":{"parameters":{"inferenceservice_yaml":{"parameterType":"STRUCT"}}},"outputDefinitions":{"artifacts":{"inferenceservice_artifact":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}'
-      runAfter:
-      - convert-inference-service-to-artifact-driver
+      - name: pod-spec-patch
+        value: $(tasks.convert-inference-service-to-artifact-driver.results.pod-spec-patch)
       taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: KFPTask
         metadata:
           annotations:
             pipelines.kubeflow.org/v2_pipeline: "true"
           labels:
             pipelines.kubeflow.org/v2_component: "true"
-        params:
-        - name: executor-input
-          type: string
-        - name: execution-id
-          type: string
-        - name: run-id
-          type: string
-        - name: component-spec
-          type: string
-        results:
-        - description: execution id
-          name: execution-id
-        - description: executor input
-          name: executor-input
-        spec: null
-        steps:
-        - command:
-          - launcher-v2
-          - --copy
-          - /tekton/home/launch
-          image: gcr.io/ml-pipeline-test/dev/kfp-launcher-v2:latest
-          imagePullPolicy: Always
-          name: kfp-launcher
-          resources: {}
-        - args:
-          - sh
-          - -c
-          - |2
+        spec:
+          taskSpec:
+            params:
+            - name: executor-input
+              type: string
+            - name: execution-id
+              type: string
+            - name: run-id
+              type: string
+            - name: component-spec
+              type: string
+            steps:
+            - command:
+              - launcher-v2
+              - --copy
+              - /tekton/home/launch
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              imagePullPolicy: Always
+              name: kfp-launcher
+              resources: {}
+            - args:
+              - sh
+              - -c
+              - |2
 
-            if ! [ -x "$(command -v pip)" ]; then
-                python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
-            fi
+                if ! [ -x "$(command -v pip)" ]; then
+                    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
+                fi
 
-            PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
-          - sh
-          - -ec
-          - |
-            program_path=$(mktemp -d)
-            printf "%s" "$0" > "$program_path/ephemeral_component.py"
-            python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
-          - |2+
+                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
+              - sh
+              - -ec
+              - |
+                program_path=$(mktemp -d)
+                printf "%s" "$0" > "$program_path/ephemeral_component.py"
+                python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+              - |2+
 
-            import kfp
-            from kfp import dsl
-            from kfp.dsl import *
-            from typing import *
+                import kfp
+                from kfp import dsl
+                from kfp.dsl import *
+                from typing import *
 
-            def convert_inference_service_to_artifact(inferenceservice_yaml: Dict[str, str], inferenceservice_artifact: Output[Artifact]):
-                import json
-                with open(inferenceservice_artifact.path, 'w') as f:
-                    f.write(json.dumps(inferenceservice_yaml))
+                def convert_inference_service_to_artifact(inferenceservice_yaml: Dict[str, str], inferenceservice_artifact: Output[Artifact]):
+                    import json
+                    with open(inferenceservice_artifact.path, 'w') as f:
+                        f.write(json.dumps(inferenceservice_yaml))
 
-          - --executor_input
-          - '{{$}}'
-          - --function_to_execute
-          - convert_inference_service_to_artifact
-          command:
-          - /tekton/home/launch
-          - --pipeline_name
-          - end-to-end-pipeline
-          - --run_id
-          - $(params.run-id)
-          - --execution_id
-          - $(params.execution-id)
-          - --executor_input
-          - $(params.executor-input)
-          - --component_spec
-          - $(params.component-spec)
-          - --pod_name
-          - $(KFP_POD_NAME)
-          - --pod_uid
-          - $(KFP_POD_UID)
-          - --mlmd_server_address
-          - $(METADATA_GRPC_SERVICE_HOST)
-          - --mlmd_server_port
-          - $(METADATA_GRPC_SERVICE_PORT)
-          - --
-          env:
-          - name: KFP_POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: KFP_POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-          - name: METADATA_GRPC_SERVICE_HOST
-            value: metadata-grpc-service.kubeflow.svc.cluster.local
-          - name: METADATA_GRPC_SERVICE_PORT
-            value: "8080"
-          envFrom:
-          - configMapRef:
-              name: metadata-grpc-configmap
-              optional: true
-          image: python:3.7
-          name: user-main
-          resources: {}
+              - --executor_input
+              - '{{$}}'
+              - --function_to_execute
+              - convert_inference_service_to_artifact
+              command:
+              - /tekton/home/launch
+              - --pipeline_name
+              - end-to-end-pipeline
+              - --run_id
+              - $(params.run-id)
+              - --execution_id
+              - $(params.execution-id)
+              - --executor_input
+              - $(params.executor-input)
+              - --component_spec
+              - $(params.component-spec)
+              - --pod_name
+              - $(KFP_POD_NAME)
+              - --pod_uid
+              - $(KFP_POD_UID)
+              - --mlmd_server_address
+              - $(METADATA_GRPC_SERVICE_HOST)
+              - --mlmd_server_port
+              - $(METADATA_GRPC_SERVICE_PORT)
+              - --
+              env:
+              - name: KFP_POD_NAME
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.name
+              - name: KFP_POD_UID
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.uid
+              - name: METADATA_GRPC_SERVICE_HOST
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: METADATA_GRPC_SERVICE_PORT
+                value: "8080"
+              - name: ML_PIPELINE_SERVICE_HOST
+                value: ml-pipeline.kubeflow.svc.cluster.local
+              - name: ML_PIPELINE_SERVICE_PORT_GRPC
+                value: "8887"
+              envFrom:
+              - configMapRef:
+                  name: metadata-grpc-configmap
+                  optional: true
+              image: python:3.7
+              name: user-main
+              resources: {}
       when:
       - input: $(tasks.convert-inference-service-to-artifact-driver.results.cached-decision)
         operator: in
@@ -1202,12 +1239,15 @@ spec:
           output(inference_service)\n\n"],"image":"python:3.7"}'
       - name: iteration_index
         value: ""
-      runAfter:
-      - serving-pipeline-dag-driver
+      - name: kubernetes_config
+        value: ""
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
     - name: create-serving-task
       params:
       - name: executor-input
@@ -1218,132 +1258,134 @@ spec:
         value: $(context.pipelineRun.uid)
       - name: component-spec
         value: '{"executorLabel":"exec-create-serving-task","inputDefinitions":{"parameters":{"model_name":{"parameterType":"STRING"},"model_namespace":{"parameterType":"STRING"},"model_volume_name":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"inferenceservice_yaml":{"parameterType":"STRUCT"}}}}'
-      runAfter:
-      - create-serving-task-driver
+      - name: pod-spec-patch
+        value: $(tasks.create-serving-task-driver.results.pod-spec-patch)
       taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: KFPTask
         metadata:
           annotations:
             pipelines.kubeflow.org/v2_pipeline: "true"
           labels:
             pipelines.kubeflow.org/v2_component: "true"
-        params:
-        - name: executor-input
-          type: string
-        - name: execution-id
-          type: string
-        - name: run-id
-          type: string
-        - name: component-spec
-          type: string
-        results:
-        - description: execution id
-          name: execution-id
-        - description: executor input
-          name: executor-input
-        spec: null
-        steps:
-        - command:
-          - launcher-v2
-          - --copy
-          - /tekton/home/launch
-          image: gcr.io/ml-pipeline-test/dev/kfp-launcher-v2:latest
-          imagePullPolicy: Always
-          name: kfp-launcher
-          resources: {}
-        - args:
-          - sh
-          - -c
-          - |2
+        spec:
+          taskSpec:
+            params:
+            - name: executor-input
+              type: string
+            - name: execution-id
+              type: string
+            - name: run-id
+              type: string
+            - name: component-spec
+              type: string
+            steps:
+            - command:
+              - launcher-v2
+              - --copy
+              - /tekton/home/launch
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              imagePullPolicy: Always
+              name: kfp-launcher
+              resources: {}
+            - args:
+              - sh
+              - -c
+              - |2
 
-            if ! [ -x "$(command -v pip)" ]; then
-                python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
-            fi
+                if ! [ -x "$(command -v pip)" ]; then
+                    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
+                fi
 
-            PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
-          - sh
-          - -ec
-          - |
-            program_path=$(mktemp -d)
-            printf "%s" "$0" > "$program_path/ephemeral_component.py"
-            python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
-          - |2+
+                PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-beta.6' && "$0" "$@"
+              - sh
+              - -ec
+              - |
+                program_path=$(mktemp -d)
+                printf "%s" "$0" > "$program_path/ephemeral_component.py"
+                python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+              - |2+
 
-            import kfp
-            from kfp import dsl
-            from kfp.dsl import *
-            from typing import *
+                import kfp
+                from kfp import dsl
+                from kfp.dsl import *
+                from typing import *
 
-            def create_serving_task(model_name: str, model_namespace: str, model_volume_name: str
-                                   ) -> NamedTuple('Outputs', [('inferenceservice_yaml', Dict[str, str])]):
+                def create_serving_task(model_name: str, model_namespace: str, model_volume_name: str
+                                       ) -> NamedTuple('Outputs', [('inferenceservice_yaml', Dict[str, str])]):
 
-                api_version = 'serving.kserve.io/v1beta1'
-                inference_service = {
-                    "apiVersion": api_version,
-                    "kind": "InferenceService",
-                    "metadata": {
-                      "name": model_name,
-                      "namespace": model_namespace,
-                      "annotations": {
-                        "sidecar.istio.io/inject": "false"
-                      }
-                    },
-                    "spec":{
-                      "predictor":{
-                        "tensorflow": {
-                          "storageUri": "pvc://{}/".format(model_volume_name)
+                    api_version = 'serving.kserve.io/v1beta1'
+                    inference_service = {
+                        "apiVersion": api_version,
+                        "kind": "InferenceService",
+                        "metadata": {
+                          "name": model_name,
+                          "namespace": model_namespace,
+                          "annotations": {
+                            "sidecar.istio.io/inject": "false"
+                          }
+                        },
+                        "spec":{
+                          "predictor":{
+                            "tensorflow": {
+                              "storageUri": "pvc://{}/".format(model_volume_name)
+                            }
+                          }
                         }
-                      }
                     }
-                }
 
-                output = NamedTuple('Outputs', [('inferenceservice_yaml', Dict[str, str])])
-                return output(inference_service)
+                    output = NamedTuple('Outputs', [('inferenceservice_yaml', Dict[str, str])])
+                    return output(inference_service)
 
-          - --executor_input
-          - '{{$}}'
-          - --function_to_execute
-          - create_serving_task
-          command:
-          - /tekton/home/launch
-          - --pipeline_name
-          - end-to-end-pipeline
-          - --run_id
-          - $(params.run-id)
-          - --execution_id
-          - $(params.execution-id)
-          - --executor_input
-          - $(params.executor-input)
-          - --component_spec
-          - $(params.component-spec)
-          - --pod_name
-          - $(KFP_POD_NAME)
-          - --pod_uid
-          - $(KFP_POD_UID)
-          - --mlmd_server_address
-          - $(METADATA_GRPC_SERVICE_HOST)
-          - --mlmd_server_port
-          - $(METADATA_GRPC_SERVICE_PORT)
-          - --
-          env:
-          - name: KFP_POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: KFP_POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-          - name: METADATA_GRPC_SERVICE_HOST
-            value: metadata-grpc-service.kubeflow.svc.cluster.local
-          - name: METADATA_GRPC_SERVICE_PORT
-            value: "8080"
-          envFrom:
-          - configMapRef:
-              name: metadata-grpc-configmap
-              optional: true
-          image: python:3.7
-          name: user-main
-          resources: {}
+              - --executor_input
+              - '{{$}}'
+              - --function_to_execute
+              - create_serving_task
+              command:
+              - /tekton/home/launch
+              - --pipeline_name
+              - end-to-end-pipeline
+              - --run_id
+              - $(params.run-id)
+              - --execution_id
+              - $(params.execution-id)
+              - --executor_input
+              - $(params.executor-input)
+              - --component_spec
+              - $(params.component-spec)
+              - --pod_name
+              - $(KFP_POD_NAME)
+              - --pod_uid
+              - $(KFP_POD_UID)
+              - --mlmd_server_address
+              - $(METADATA_GRPC_SERVICE_HOST)
+              - --mlmd_server_port
+              - $(METADATA_GRPC_SERVICE_PORT)
+              - --
+              env:
+              - name: KFP_POD_NAME
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.name
+              - name: KFP_POD_UID
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.uid
+              - name: METADATA_GRPC_SERVICE_HOST
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: METADATA_GRPC_SERVICE_PORT
+                value: "8080"
+              - name: ML_PIPELINE_SERVICE_HOST
+                value: ml-pipeline.kubeflow.svc.cluster.local
+              - name: ML_PIPELINE_SERVICE_PORT_GRPC
+                value: "8887"
+              envFrom:
+              - configMapRef:
+                  name: metadata-grpc-configmap
+                  optional: true
+              image: python:3.7
+              name: user-main
+              resources: {}
       when:
       - input: $(tasks.create-serving-task-driver.results.cached-decision)
         operator: in
@@ -1367,14 +1409,18 @@ spec:
         value: '{"args":["--action","{{$.inputs.parameters[''action'']}}","--inferenceservice-yaml","{{$.inputs.parameters[''inferenceservice_yaml'']}}"],"command":["python","kservedeployer.py"],"image":"quay.io/aipipeline/kserve-component:v0.7.0"}'
       - name: iteration_index
         value: ""
+      - name: kubernetes_config
+        value: ""
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       runAfter:
       - convert-inference-service-to-artifact
       - create-serving-task
-      - serving-pipeline-dag-driver
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
     - name: serving-launcher
       params:
       - name: executor-input
@@ -1385,86 +1431,88 @@ spec:
         value: $(context.pipelineRun.uid)
       - name: component-spec
         value: '{"executorLabel":"exec-serving-launcher","inputDefinitions":{"parameters":{"action":{"parameterType":"STRING"},"inferenceservice_yaml":{"parameterType":"STRUCT"}}}}'
-      runAfter:
-      - serving-launcher-driver
+      - name: pod-spec-patch
+        value: $(tasks.serving-launcher-driver.results.pod-spec-patch)
       taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: KFPTask
         metadata:
           annotations:
             pipelines.kubeflow.org/v2_pipeline: "true"
           labels:
             pipelines.kubeflow.org/v2_component: "true"
-        params:
-        - name: executor-input
-          type: string
-        - name: execution-id
-          type: string
-        - name: run-id
-          type: string
-        - name: component-spec
-          type: string
-        results:
-        - description: execution id
-          name: execution-id
-        - description: executor input
-          name: executor-input
-        spec: null
-        steps:
-        - command:
-          - launcher-v2
-          - --copy
-          - /tekton/home/launch
-          image: gcr.io/ml-pipeline-test/dev/kfp-launcher-v2:latest
-          imagePullPolicy: Always
-          name: kfp-launcher
-          resources: {}
-        - args:
-          - python
-          - kservedeployer.py
-          - --action
-          - '{{$.inputs.parameters[''action'']}}'
-          - --inferenceservice-yaml
-          - '{{$.inputs.parameters[''inferenceservice_yaml'']}}'
-          command:
-          - /tekton/home/launch
-          - --pipeline_name
-          - end-to-end-pipeline
-          - --run_id
-          - $(params.run-id)
-          - --execution_id
-          - $(params.execution-id)
-          - --executor_input
-          - $(params.executor-input)
-          - --component_spec
-          - $(params.component-spec)
-          - --pod_name
-          - $(KFP_POD_NAME)
-          - --pod_uid
-          - $(KFP_POD_UID)
-          - --mlmd_server_address
-          - $(METADATA_GRPC_SERVICE_HOST)
-          - --mlmd_server_port
-          - $(METADATA_GRPC_SERVICE_PORT)
-          - --
-          env:
-          - name: KFP_POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: KFP_POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-          - name: METADATA_GRPC_SERVICE_HOST
-            value: metadata-grpc-service.kubeflow.svc.cluster.local
-          - name: METADATA_GRPC_SERVICE_PORT
-            value: "8080"
-          envFrom:
-          - configMapRef:
-              name: metadata-grpc-configmap
-              optional: true
-          image: quay.io/aipipeline/kserve-component:v0.7.0
-          name: user-main
-          resources: {}
+        spec:
+          taskSpec:
+            params:
+            - name: executor-input
+              type: string
+            - name: execution-id
+              type: string
+            - name: run-id
+              type: string
+            - name: component-spec
+              type: string
+            steps:
+            - command:
+              - launcher-v2
+              - --copy
+              - /tekton/home/launch
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              imagePullPolicy: Always
+              name: kfp-launcher
+              resources: {}
+            - args:
+              - python
+              - kservedeployer.py
+              - --action
+              - '{{$.inputs.parameters[''action'']}}'
+              - --inferenceservice-yaml
+              - '{{$.inputs.parameters[''inferenceservice_yaml'']}}'
+              command:
+              - /tekton/home/launch
+              - --pipeline_name
+              - end-to-end-pipeline
+              - --run_id
+              - $(params.run-id)
+              - --execution_id
+              - $(params.execution-id)
+              - --executor_input
+              - $(params.executor-input)
+              - --component_spec
+              - $(params.component-spec)
+              - --pod_name
+              - $(KFP_POD_NAME)
+              - --pod_uid
+              - $(KFP_POD_UID)
+              - --mlmd_server_address
+              - $(METADATA_GRPC_SERVICE_HOST)
+              - --mlmd_server_port
+              - $(METADATA_GRPC_SERVICE_PORT)
+              - --
+              env:
+              - name: KFP_POD_NAME
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.name
+              - name: KFP_POD_UID
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.uid
+              - name: METADATA_GRPC_SERVICE_HOST
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: METADATA_GRPC_SERVICE_PORT
+                value: "8080"
+              - name: ML_PIPELINE_SERVICE_HOST
+                value: ml-pipeline.kubeflow.svc.cluster.local
+              - name: ML_PIPELINE_SERVICE_PORT_GRPC
+                value: "8887"
+              envFrom:
+              - configMapRef:
+                  name: metadata-grpc-configmap
+                  optional: true
+              image: quay.io/aipipeline/kserve-component:v0.7.0
+              name: user-main
+              resources: {}
       when:
       - input: $(tasks.serving-launcher-driver.results.cached-decision)
         operator: in
@@ -1488,13 +1536,34 @@ spec:
         value: ""
       - name: iteration_index
         value: "-1"
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       runAfter:
       - tfjob-launcher
-      - root-system-dag-driver
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
+    - name: serving-pipeline-dag-pub-driver
+      params:
+      - name: type
+        value: DAG-PUB
+      - name: pipeline_name
+        value: end-to-end-pipeline
+      - name: run_id
+        value: $(context.pipelineRun.uid)
+      - name: dag_execution_id
+        value: $(tasks.serving-pipeline-dag-driver.results.execution-id)
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
+      runAfter:
+      - serving-launcher
+      taskRef:
+        apiVersion: kfp-driver.tekton.dev/v1alpha1
+        kind: KFPDriver
     - name: tfjob-launcher-driver
       params:
       - name: type
@@ -1513,14 +1582,18 @@ spec:
         value: '{"args":["--name","{{$.inputs.parameters[''tfjob_name'']}}","--namespace","{{$.inputs.parameters[''tfjob_namespace'']}}","--workerSpec","{{$.inputs.parameters[''worker_spec'']}}","--chiefSpec","{{$.inputs.parameters[''chief_spec'']}}","--tfjobTimeoutMinutes","{{$.inputs.parameters[''tfjob_timeout_minutes'']}}","--deleteAfterDone","False"],"command":["python","/ml/launch_tfjob.py"],"image":"nikenano/launchernew:latest"}'
       - name: iteration_index
         value: ""
+      - name: kubernetes_config
+        value: ""
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
       runAfter:
       - convert-katib-results
       - create-tfjob-task
-      - root-system-dag-driver
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
     - name: tfjob-launcher
       params:
       - name: executor-input
@@ -1531,94 +1604,96 @@ spec:
         value: $(context.pipelineRun.uid)
       - name: component-spec
         value: '{"executorLabel":"exec-tfjob-launcher","inputDefinitions":{"parameters":{"chief_spec":{"parameterType":"STRUCT"},"tfjob_name":{"parameterType":"STRING"},"tfjob_namespace":{"parameterType":"STRING"},"tfjob_timeout_minutes":{"parameterType":"NUMBER_INTEGER"},"worker_spec":{"parameterType":"STRUCT"}}}}'
-      runAfter:
-      - tfjob-launcher-driver
+      - name: pod-spec-patch
+        value: $(tasks.tfjob-launcher-driver.results.pod-spec-patch)
       taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: KFPTask
         metadata:
           annotations:
             pipelines.kubeflow.org/v2_pipeline: "true"
           labels:
             pipelines.kubeflow.org/v2_component: "true"
-        params:
-        - name: executor-input
-          type: string
-        - name: execution-id
-          type: string
-        - name: run-id
-          type: string
-        - name: component-spec
-          type: string
-        results:
-        - description: execution id
-          name: execution-id
-        - description: executor input
-          name: executor-input
-        spec: null
-        steps:
-        - command:
-          - launcher-v2
-          - --copy
-          - /tekton/home/launch
-          image: gcr.io/ml-pipeline-test/dev/kfp-launcher-v2:latest
-          imagePullPolicy: Always
-          name: kfp-launcher
-          resources: {}
-        - args:
-          - python
-          - /ml/launch_tfjob.py
-          - --name
-          - '{{$.inputs.parameters[''tfjob_name'']}}'
-          - --namespace
-          - '{{$.inputs.parameters[''tfjob_namespace'']}}'
-          - --workerSpec
-          - '{{$.inputs.parameters[''worker_spec'']}}'
-          - --chiefSpec
-          - '{{$.inputs.parameters[''chief_spec'']}}'
-          - --tfjobTimeoutMinutes
-          - '{{$.inputs.parameters[''tfjob_timeout_minutes'']}}'
-          - --deleteAfterDone
-          - "False"
-          command:
-          - /tekton/home/launch
-          - --pipeline_name
-          - end-to-end-pipeline
-          - --run_id
-          - $(params.run-id)
-          - --execution_id
-          - $(params.execution-id)
-          - --executor_input
-          - $(params.executor-input)
-          - --component_spec
-          - $(params.component-spec)
-          - --pod_name
-          - $(KFP_POD_NAME)
-          - --pod_uid
-          - $(KFP_POD_UID)
-          - --mlmd_server_address
-          - $(METADATA_GRPC_SERVICE_HOST)
-          - --mlmd_server_port
-          - $(METADATA_GRPC_SERVICE_PORT)
-          - --
-          env:
-          - name: KFP_POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: KFP_POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-          - name: METADATA_GRPC_SERVICE_HOST
-            value: metadata-grpc-service.kubeflow.svc.cluster.local
-          - name: METADATA_GRPC_SERVICE_PORT
-            value: "8080"
-          envFrom:
-          - configMapRef:
-              name: metadata-grpc-configmap
-              optional: true
-          image: nikenano/launchernew:latest
-          name: user-main
-          resources: {}
+        spec:
+          taskSpec:
+            params:
+            - name: executor-input
+              type: string
+            - name: execution-id
+              type: string
+            - name: run-id
+              type: string
+            - name: component-spec
+              type: string
+            steps:
+            - command:
+              - launcher-v2
+              - --copy
+              - /tekton/home/launch
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              imagePullPolicy: Always
+              name: kfp-launcher
+              resources: {}
+            - args:
+              - python
+              - /ml/launch_tfjob.py
+              - --name
+              - '{{$.inputs.parameters[''tfjob_name'']}}'
+              - --namespace
+              - '{{$.inputs.parameters[''tfjob_namespace'']}}'
+              - --workerSpec
+              - '{{$.inputs.parameters[''worker_spec'']}}'
+              - --chiefSpec
+              - '{{$.inputs.parameters[''chief_spec'']}}'
+              - --tfjobTimeoutMinutes
+              - '{{$.inputs.parameters[''tfjob_timeout_minutes'']}}'
+              - --deleteAfterDone
+              - "False"
+              command:
+              - /tekton/home/launch
+              - --pipeline_name
+              - end-to-end-pipeline
+              - --run_id
+              - $(params.run-id)
+              - --execution_id
+              - $(params.execution-id)
+              - --executor_input
+              - $(params.executor-input)
+              - --component_spec
+              - $(params.component-spec)
+              - --pod_name
+              - $(KFP_POD_NAME)
+              - --pod_uid
+              - $(KFP_POD_UID)
+              - --mlmd_server_address
+              - $(METADATA_GRPC_SERVICE_HOST)
+              - --mlmd_server_port
+              - $(METADATA_GRPC_SERVICE_PORT)
+              - --
+              env:
+              - name: KFP_POD_NAME
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.name
+              - name: KFP_POD_UID
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.uid
+              - name: METADATA_GRPC_SERVICE_HOST
+                value: metadata-grpc-service.kubeflow.svc.cluster.local
+              - name: METADATA_GRPC_SERVICE_PORT
+                value: "8080"
+              - name: ML_PIPELINE_SERVICE_HOST
+                value: ml-pipeline.kubeflow.svc.cluster.local
+              - name: ML_PIPELINE_SERVICE_PORT_GRPC
+                value: "8887"
+              envFrom:
+              - configMapRef:
+                  name: metadata-grpc-configmap
+                  optional: true
+              image: nikenano/launchernew:latest
+              name: user-main
+              resources: {}
       when:
       - input: $(tasks.tfjob-launcher-driver.results.cached-decision)
         operator: in
@@ -1635,15 +1710,37 @@ spec:
       - name: dag_execution_id
         value: "0"
       - name: component
-        value: '{"dag":{"tasks":{"convert-experiment-spec-to-str":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-convert-experiment-spec-to-str"},"dependentTasks":["create-katib-experiment-task"],"inputs":{"parameters":{"experiment_spec_json":{"taskOutputParameter":{"outputParameterKey":"experiment_spec_json","producerTask":"create-katib-experiment-task"}}}},"taskInfo":{"name":"convert-experiment-spec-to-str"}},"convert-katib-results":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-convert-katib-results"},"dependentTasks":["create-dataset"],"inputs":{"artifacts":{"katib_results":{"taskOutputArtifact":{"outputArtifactKey":"parameter_set","producerTask":"create-dataset"}}}},"taskInfo":{"name":"convert-katib-results"}},"create-dataset":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-dataset"},"dependentTasks":["convert-experiment-spec-to-str"],"inputs":{"parameters":{"experiment_name":{"componentInputParameter":"name"},"experiment_namespace":{"componentInputParameter":"namespace"},"experiment_spec_json":{"taskOutputParameter":{"outputParameterKey":"experiment_spec_str_output","producerTask":"convert-experiment-spec-to-str"}},"experiment_timeout_minutes":{"runtimeValue":{"constant":60}}}},"taskInfo":{"name":"create-dataset"}},"create-katib-experiment-task":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-katib-experiment-task"},"inputs":{"parameters":{"experiment_name":{"componentInputParameter":"name"},"experiment_namespace":{"componentInputParameter":"namespace"},"training_steps":{"componentInputParameter":"training_steps"}}},"taskInfo":{"name":"create-katib-experiment-task"}},"create-tfjob-task":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-tfjob-task"},"dependentTasks":["convert-katib-results"],"inputs":{"parameters":{"best_hps":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"convert-katib-results"}},"model_volume_name":{"componentInputParameter":"model_volume_name"},"tfjob_name":{"componentInputParameter":"name"},"tfjob_namespace":{"componentInputParameter":"namespace"},"training_steps":{"componentInputParameter":"training_steps"}}},"taskInfo":{"name":"create-tfjob-task"}},"serving-pipeline":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-serving-pipeline"},"dependentTasks":["tfjob-launcher","root-system-dag-driver"],"inputs":{"parameters":{"model_name":{"componentInputParameter":"name"},"model_namespace":{"componentInputParameter":"namespace"},"model_volume_name":{"componentInputParameter":"model_volume_name"}}},"taskInfo":{"name":"serving-pipeline"}},"tfjob-launcher":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-tfjob-launcher"},"dependentTasks":["convert-katib-results","create-tfjob-task"],"inputs":{"parameters":{"chief_spec":{"taskOutputParameter":{"outputParameterKey":"chief_spec","producerTask":"create-tfjob-task"}},"tfjob_name":{"componentInputParameter":"name"},"tfjob_namespace":{"componentInputParameter":"namespace"},"tfjob_timeout_minutes":{"runtimeValue":{"constant":60}},"worker_spec":{"taskOutputParameter":{"outputParameterKey":"worker_spec","producerTask":"create-tfjob-task"}}}},"taskInfo":{"name":"tfjob-launcher"}}}},"inputDefinitions":{"parameters":{"model_volume_name":{"defaultValue":"workflow1-model-volume","parameterType":"STRING"},"name":{"defaultValue":"mnist-e2e","parameterType":"STRING"},"namespace":{"defaultValue":"kubeflow-user-project","parameterType":"STRING"},"training_steps":{"defaultValue":"200","parameterType":"STRING"}}}}'
+        value: '{"dag":{"tasks":{"convert-experiment-spec-to-str":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-convert-experiment-spec-to-str"},"dependentTasks":["create-katib-experiment-task"],"inputs":{"parameters":{"experiment_spec_json":{"taskOutputParameter":{"outputParameterKey":"experiment_spec_json","producerTask":"create-katib-experiment-task"}}}},"taskInfo":{"name":"convert-experiment-spec-to-str"}},"convert-katib-results":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-convert-katib-results"},"dependentTasks":["create-dataset"],"inputs":{"artifacts":{"katib_results":{"taskOutputArtifact":{"outputArtifactKey":"parameter_set","producerTask":"create-dataset"}}}},"taskInfo":{"name":"convert-katib-results"}},"create-dataset":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-dataset"},"dependentTasks":["convert-experiment-spec-to-str"],"inputs":{"parameters":{"experiment_name":{"componentInputParameter":"name"},"experiment_namespace":{"componentInputParameter":"namespace"},"experiment_spec_json":{"taskOutputParameter":{"outputParameterKey":"experiment_spec_str_output","producerTask":"convert-experiment-spec-to-str"}},"experiment_timeout_minutes":{"runtimeValue":{"constant":60}}}},"taskInfo":{"name":"create-dataset"}},"create-katib-experiment-task":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-katib-experiment-task"},"inputs":{"parameters":{"experiment_name":{"componentInputParameter":"name"},"experiment_namespace":{"componentInputParameter":"namespace"},"training_steps":{"componentInputParameter":"training_steps"}}},"taskInfo":{"name":"create-katib-experiment-task"}},"create-tfjob-task":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-tfjob-task"},"dependentTasks":["convert-katib-results"],"inputs":{"parameters":{"best_hps":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"convert-katib-results"}},"model_volume_name":{"componentInputParameter":"model_volume_name"},"tfjob_name":{"componentInputParameter":"name"},"tfjob_namespace":{"componentInputParameter":"namespace"},"training_steps":{"componentInputParameter":"training_steps"}}},"taskInfo":{"name":"create-tfjob-task"}},"serving-pipeline":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-serving-pipeline"},"dependentTasks":["tfjob-launcher"],"inputs":{"parameters":{"model_name":{"componentInputParameter":"name"},"model_namespace":{"componentInputParameter":"namespace"},"model_volume_name":{"componentInputParameter":"model_volume_name"}}},"taskInfo":{"name":"serving-pipeline"}},"tfjob-launcher":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-tfjob-launcher"},"dependentTasks":["convert-katib-results","create-tfjob-task"],"inputs":{"parameters":{"chief_spec":{"taskOutputParameter":{"outputParameterKey":"chief_spec","producerTask":"create-tfjob-task"}},"tfjob_name":{"componentInputParameter":"name"},"tfjob_namespace":{"componentInputParameter":"namespace"},"tfjob_timeout_minutes":{"runtimeValue":{"constant":60}},"worker_spec":{"taskOutputParameter":{"outputParameterKey":"worker_spec","producerTask":"create-tfjob-task"}}}},"taskInfo":{"name":"tfjob-launcher"}}}},"inputDefinitions":{"parameters":{"model_volume_name":{"defaultValue":"workflow1-model-volume","parameterType":"STRING"},"name":{"defaultValue":"mnist-e2e","parameterType":"STRING"},"namespace":{"defaultValue":"kubeflow-user-project","parameterType":"STRING"},"training_steps":{"defaultValue":"200","parameterType":"STRING"}}}}'
       - name: task
         value: ""
       - name: runtime_config
         value: '{"parameterValues":{"model_volume_name":"workflow1-model-volume","name":"mnist-e2e","namespace":"kubeflow-user-project","training_steps":"200"}}'
       - name: iteration_index
         value: "-1"
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
+      taskRef:
+        apiVersion: kfp-driver.tekton.dev/v1alpha1
+        kind: KFPDriver
+    - name: root-system-dag-pub-driver
+      params:
+      - name: type
+        value: DAG-PUB
+      - name: pipeline_name
+        value: end-to-end-pipeline
+      - name: run_id
+        value: $(context.pipelineRun.uid)
+      - name: dag_execution_id
+        value: $(tasks.root-system-dag-driver.results.execution-id)
+      - name: mlmd_server_address
+        value: metadata-grpc-service.kubeflow.svc.cluster.local
+      - name: mlmd_server_port
+        value: "8080"
+      runAfter:
+      - serving-launcher
       taskRef:
         apiVersion: kfp-driver.tekton.dev/v1alpha1
         kind: KFPDriver
-        name: kfp-driver
 status: {}