Repository: kubeflow/pipelines

Issue:
[backend] v1 caching broken in KFP 1.8 release branch
The root cause is https://github.com/argoproj/argo-workflows/pull/6022 introduced a behavior change that the content of Argo workflow template is no longer logged under `pod.metadata.annotations`, instead it puts the template in container env. 

The change was released in Argo workflow [v3.2.0-rc1](https://github.com/argoproj/argo-workflows/blob/master/CHANGELOG.md#v320-rc1-2021-08-19). And https://github.com/kubeflow/pipelines/pull/6920 upgraded Argo workflow used in KFP from `3.1.14` to `3.2.3`. The change broke v1 caching implementation because it relies on Argo workflow template present in `pod.metadata.annotations`.


Discussion:

Resolving patch:
diff --git a/backend/metadata_writer/src/metadata_writer.py b/backend/metadata_writer/src/metadata_writer.py
--- a/backend/metadata_writer/src/metadata_writer.py
+++ b/backend/metadata_writer/src/metadata_writer.py
@@ -65,7 +65,7 @@ def patch_pod_metadata(
 
 
 ARGO_OUTPUTS_ANNOTATION_KEY = 'workflows.argoproj.io/outputs'
-ARGO_TEMPLATE_ANNOTATION_KEY = 'workflows.argoproj.io/template'
+ARGO_TEMPLATE_ENV_KEY = 'ARGO_TEMPLATE'
 KFP_COMPONENT_SPEC_ANNOTATION_KEY = 'pipelines.kubeflow.org/component_spec'
 KFP_PARAMETER_ARGUMENTS_ANNOTATION_KEY = 'pipelines.kubeflow.org/arguments.parameters'
 METADATA_EXECUTION_ID_LABEL_KEY = 'pipelines.kubeflow.org/metadata_execution_id'
@@ -103,7 +103,7 @@ def argo_artifact_to_uri(artifact: dict) -> str:
     if 's3' in artifact:
         s3_artifact = artifact['s3']
         return '{provider}://{bucket}/{key}'.format(
-            provider=get_object_store_provider(s3_artifact['endpoint']),
+            provider=get_object_store_provider(s3_artifact.get('endpoint', '')),
             bucket=s3_artifact.get('bucket', ''),
             key=s3_artifact.get('key', ''),
         )
@@ -181,8 +181,14 @@ def is_kfp_v2_pod(pod) -> bool:
                 continue
 
             argo_workflow_name = obj.metadata.labels[ARGO_WORKFLOW_LABEL_KEY] # Should exist due to initial filtering
-            argo_template = json.loads(obj.metadata.annotations[ARGO_TEMPLATE_ANNOTATION_KEY])
-            argo_template_name = argo_template['name']
+            argo_template = {}
+            for env in obj.spec.containers[0].env:
+                if env.name == ARGO_TEMPLATE_ENV_KEY:
+                    argo_template = json.loads(env.value)
+                    break
+
+            # Should we throw error instead if argo template not found?
+            argo_template_name = argo_template.get('name', '')
 
             component_name = argo_template_name
             component_version = component_name


Test patch:
diff --git a/backend/src/cache/server/mutation_test.go b/backend/src/cache/server/mutation_test.go
--- a/backend/src/cache/server/mutation_test.go
+++ b/backend/src/cache/server/mutation_test.go
@@ -38,7 +38,6 @@ var (
 		ObjectMeta: metav1.ObjectMeta{
 			Annotations: map[string]string{
 				ArgoWorkflowNodeName: "test_node",
-				ArgoWorkflowTemplate: `{"name": "Does not matter","container":{"command":["echo", "Hello"],"image":"python:3.7"}}`,
 			},
 			Labels: map[string]string{
 				ArgoCompleteLabelKey:    "true",
@@ -51,6 +50,10 @@ var (
 					Name:    "main",
 					Image:   "test_image",
 					Command: []string{"python"},
+					Env: []corev1.EnvVar{{
+						Name:  ArgoWorkflowTemplateEnvKey,
+						Value: `{"name": "Does not matter","container":{"command":["echo", "Hello"],"image":"python:3.7"}}`,
+					}},
 				},
 			},
 			NodeSelector: map[string]string{"disktype": "ssd"},
@@ -244,16 +247,19 @@ func TestMutatePodIfCachedWithTeamplateCleanup(t *testing.T) {
 	fakeClientManager.CacheStore().CreateExecutionCache(executionCache)
 
 	pod := *fakePod.DeepCopy()
-	pod.ObjectMeta.Annotations[ArgoWorkflowTemplate] = `{
-		"name": "Does not matter",
-		"metadata": "anything",
-		"container": {
-			"image": "python:3.7",
-			"command": ["echo", "Hello"]
-		},
-		"outputs": "anything",
-		"foo": "bar"
-	}`
+	pod.Spec.Containers[0].Env = []corev1.EnvVar{{
+		Name: ArgoWorkflowTemplateEnvKey,
+		Value: `{
+			"name": "Does not matter",
+			"metadata": "anything",
+			"container": {
+				"image": "python:3.7",
+				"command": ["echo", "Hello"]
+			},
+			"outputs": "anything",
+			"foo": "bar"
+		}`,
+	}}
 	request := GetFakeRequestFromPod(&pod)
 
 	patchOperation, err := MutatePodIfCached(request, fakeClientManager)