Repository: kubeflow/pipelines
Pull request #11782: feat(backend): Add a mutating webhook for the PipelineVersion kind
Author: login: mprahl
Labels: name: lgtm
name: approved
name: size/L
Created: 2025-03-31T19:45:25Z | Merged: 2025-04-04T16:15:35Z

**Description of your changes:**

This mutating webhook adds labels to query pipeline versions by their pipeline ID and pipeline name, and adds an owner's reference to the pipeline it's a part of.


**Checklist:**
- [x] You have [signed off your commits](https://www.kubeflow.org/docs/about/contributing/#sign-off-your-commits)
- [x] The title for your pull request (PR) should follow our title convention. [Learn more about the pull request title convention used in this repository](https://github.com/kubeflow/pipelines/blob/master/CONTRIBUTING.md#pull-request-title-convention). 


Diff:
diff --git a/.github/resources/manifests/kubernetes-native/kustomization.yaml b/.github/resources/manifests/kubernetes-native/kustomization.yaml
new file mode 100644
index 00000000000..70b451d78bf
--- /dev/null
+++ b/.github/resources/manifests/kubernetes-native/kustomization.yaml
@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../../../manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native
+
+images:
+- name: ghcr.io/kubeflow/kfp-api-server
+  newName: kind-registry:5000/apiserver
+  newTag: latest
+- name: ghcr.io/kubeflow/kfp-persistence-agent
+  newName: kind-registry:5000/persistenceagent
+  newTag: latest
+- name: ghcr.io/kubeflow/kfp-scheduled-workflow-controller
+  newName: kind-registry:5000/scheduledworkflow
+  newTag: latest
+
+patchesStrategicMerge:
+- overlays/apiserver-env.yaml
diff --git a/.github/resources/manifests/kubernetes-native/overlays/apiserver-env.yaml b/.github/resources/manifests/kubernetes-native/overlays/apiserver-env.yaml
new file mode 100644
index 00000000000..1aefd4aa43d
--- /dev/null
+++ b/.github/resources/manifests/kubernetes-native/overlays/apiserver-env.yaml
@@ -0,0 +1,14 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-api-server
+          env:
+            - name: V2_DRIVER_IMAGE
+              value: kind-registry:5000/driver
+            - name: V2_LAUNCHER_IMAGE
+              value: kind-registry:5000/launcher
diff --git a/.github/resources/scripts/deploy-kfp.sh b/.github/resources/scripts/deploy-kfp.sh
index 68252d453c2..15e9eb73308 100755
--- a/.github/resources/scripts/deploy-kfp.sh
+++ b/.github/resources/scripts/deploy-kfp.sh
@@ -43,6 +43,11 @@ fi
 
 # Deploy manifest
 TEST_MANIFESTS=".github/resources/manifests/argo"
+
+if [[ "$PIPELINE_STORE" == "kubernetes" ]]; then
+  TEST_MANIFESTS=".github/resources/manifests/kubernetes-native"
+fi
+
 kubectl apply -k "${TEST_MANIFESTS}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
diff --git a/.github/workflows/kfp-webhooks.yml b/.github/workflows/kfp-webhooks.yml
new file mode 100644
index 00000000000..c70baa0c198
--- /dev/null
+++ b/.github/workflows/kfp-webhooks.yml
@@ -0,0 +1,52 @@
+name: KFP Webhook Integration
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - '.github/resources/**'
+      - '.github/workflows/kfp-webhooks.yml'
+      - 'backend/**'
+
+jobs:
+  webhook-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        k8s_version: [ "v1.29.2", "v1.30.2", "v1.31.0" ]
+    name: KFP Webhooks - K8s ${{ matrix.k8s_version }}
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Create KFP cluster
+      id: create-kfp-cluster
+      uses: ./.github/actions/kfp-cluster
+      with:
+        k8s_version: ${{ matrix.k8s_version }}
+      env:
+        PIPELINE_STORE: kubernetes
+      continue-on-error: true
+
+    - name: Run Webhook Integration Tests
+      id: tests
+      if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
+      run: |
+        make -C backend/test/integration test-webhook
+      continue-on-error: true
+
+    - name: Collect failed logs
+      if: ${{ (steps.create-kfp-cluster.outcome != 'success' ) || ( steps.tests.outcome != 'success' )}}
+      run: |
+        ./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt
+        exit 1
+
+    - name: Collect test results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: kfp-samples-tests-artifacts-k8s-${{ matrix.k8s_version }}
+        path: /tmp/tmp*/*
diff --git a/backend/src/apiserver/main.go b/backend/src/apiserver/main.go
index 5a5bc6dd138..c4a74d24692 100644
--- a/backend/src/apiserver/main.go
+++ b/backend/src/apiserver/main.go
@@ -300,12 +300,13 @@ func startWebhook(controllerClient ctrlclient.Client, wg *sync.WaitGroup) (*http
 
 	topMux := mux.NewRouter()
 
-	pvValidateWebhook, err := webhook.NewPipelineVersionWebhook(controllerClient)
+	pvValidateWebhook, pvMutateWebhook, err := webhook.NewPipelineVersionWebhook(controllerClient)
 	if err != nil {
 		return nil, fmt.Errorf("failed to instantiate the Kubernetes webhook: %v", err)
 	}
 
 	topMux.Handle("/webhooks/validate-pipelineversion", pvValidateWebhook)
+	topMux.Handle("/webhooks/mutate-pipelineversion", pvMutateWebhook)
 
 	webhookServer := &http.Server{
 		Addr:    *webhookPortFlag,
diff --git a/backend/src/apiserver/webhook/pipelineversion_webhook.go b/backend/src/apiserver/webhook/pipelineversion_webhook.go
index f877baac161..c5d6fd1f7c1 100644
--- a/backend/src/apiserver/webhook/pipelineversion_webhook.go
+++ b/backend/src/apiserver/webhook/pipelineversion_webhook.go
@@ -119,14 +119,81 @@ func (p *PipelineVersionsWebhook) ValidateDelete(_ context.Context, _ runtime.Ob
 	return nil, nil
 }
 
-func NewPipelineVersionWebhook(client ctrlclient.Client) (http.Handler, error) {
+func (p *PipelineVersionsWebhook) Default(ctx context.Context, obj runtime.Object) error {
+	pipelineVersion, ok := obj.(*k8sapi.PipelineVersion)
+	if !ok {
+		return &apierrors.StatusError{
+			ErrStatus: metav1.Status{
+				Code:    http.StatusBadRequest,
+				Reason:  metav1.StatusReasonBadRequest,
+				Message: fmt.Sprintf("expected a PipelineVersion object but got %T", pipelineVersion),
+			},
+		}
+	}
+
+	pipeline := &k8sapi.Pipeline{}
+	nsName := types.NamespacedName{Namespace: pipelineVersion.Namespace, Name: pipelineVersion.Spec.PipelineName}
+
+	err := p.Client.Get(ctx, nsName, pipeline)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return newBadRequestError("The spec.pipelineName doesn't map to an existing Pipeline object")
+		}
+
+		return err
+	}
+
+	if pipelineVersion.Labels == nil {
+		pipelineVersion.Labels = map[string]string{}
+	}
+
+	// Labels for efficient querying
+	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline-id"] = string(pipeline.UID)
+	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline"] = pipeline.Name
+
+	trueVal := true
+
+	for i := range pipelineVersion.OwnerReferences {
+		ownerRef := &pipelineVersion.OwnerReferences[i]
+		if ownerRef.APIVersion != k8sapi.GroupVersion.String() || ownerRef.Kind != "Pipeline" {
+			continue
+		}
+
+		ownerRef.Name = pipeline.Name
+		ownerRef.BlockOwnerDeletion = &trueVal
+		ownerRef.UID = pipeline.UID
+
+		return nil
+	}
+
+	pipelineVersion.OwnerReferences = append(pipelineVersion.OwnerReferences, metav1.OwnerReference{
+		APIVersion:         k8sapi.GroupVersion.String(),
+		Kind:               "Pipeline",
+		Name:               pipeline.Name,
+		BlockOwnerDeletion: &trueVal,
+		UID:                pipeline.UID,
+	})
+
+	return nil
+}
+
+// NewPipelineVersionWebhook returns the validating webhook and mutating webhook HTTP handlers
+func NewPipelineVersionWebhook(client ctrlclient.Client) (http.Handler, http.Handler, error) {
 	validating, err := ctrladmission.StandaloneWebhook(
 		ctrladmission.WithCustomValidator(scheme, &k8sapi.PipelineVersion{}, &PipelineVersionsWebhook{Client: client}),
 		ctrladmission.StandaloneOptions{},
 	)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
+	}
+
+	mutating, err := ctrladmission.StandaloneWebhook(
+		ctrladmission.WithCustomDefaulter(scheme, &k8sapi.PipelineVersion{}, &PipelineVersionsWebhook{Client: client}),
+		ctrladmission.StandaloneOptions{},
+	)
+	if err != nil {
+		return nil, nil, err
 	}
 
-	return validating, nil
+	return validating, mutating, nil
 }
diff --git a/backend/src/apiserver/webhook/pipelineversion_webhook_test.go b/backend/src/apiserver/webhook/pipelineversion_webhook_test.go
index 675f68b4faf..86f4c811155 100644
--- a/backend/src/apiserver/webhook/pipelineversion_webhook_test.go
+++ b/backend/src/apiserver/webhook/pipelineversion_webhook_test.go
@@ -18,11 +18,13 @@ import (
 	"encoding/json"
 	"testing"
 
+	"github.com/kubeflow/pipelines/backend/src/crd/kubernetes/v2beta1"
 	k8sapi "github.com/kubeflow/pipelines/backend/src/crd/kubernetes/v2beta1"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/uuid"
 	k8sfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
@@ -36,6 +38,7 @@ func setupPipelineWebhookTest(t *testing.T) (*PipelineVersionsWebhook, string) {
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "test-pipeline",
 				Namespace: "default",
+				UID:       uuid.NewUUID(),
 			},
 		}).Build()
 
@@ -207,3 +210,38 @@ func TestPipelineVersionWebhook_ValidateUpdate_MetadataChangeAllowed(t *testing.
 	_, err := pipelineWebhook.ValidateUpdate(context.TODO(), oldPipelineVersion, newPipelineVersion)
 	assert.NoError(t, err, "Expected no error for metadata-only change")
 }
+
+func TestPipelineVersionWebhook_MutatingUpdate_FixesOwnersRef(t *testing.T) {
+	pipelineWebhook, validPipelineSpecJSON := setupPipelineWebhookTest(t)
+	badUID := uuid.NewUUID()
+
+	pipelineVersion := &k8sapi.PipelineVersion{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test-pipeline-v1",
+			Namespace:  "default",
+			Generation: 1,
+			Labels:     map[string]string{"version": "v2"},
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion: v2beta1.GroupVersion.String(),
+					Kind:       "Pipeline",
+					Name:       "test-pipeline2",
+					UID:        badUID,
+				},
+			},
+		},
+		Spec: k8sapi.PipelineVersionSpec{
+			PipelineName: "test-pipeline",
+			PipelineSpec: k8sapi.PipelineIRSpec{
+				Value: json.RawMessage(validPipelineSpecJSON),
+			},
+		},
+	}
+
+	err := pipelineWebhook.Default(context.TODO(), pipelineVersion)
+	require.NoError(t, err, "Expected no error for fixing the owner's reference")
+	require.Len(t, pipelineVersion.OwnerReferences, 1)
+	require.NotEqual(t, pipelineVersion.OwnerReferences[0].UID, badUID)
+	require.Equal(t, pipelineVersion.OwnerReferences[0].Name, "test-pipeline")
+	require.True(t, *pipelineVersion.OwnerReferences[0].BlockOwnerDeletion)
+}
diff --git a/backend/test/integration/Makefile b/backend/test/integration/Makefile
new file mode 100644
index 00000000000..c4fec777bfb
--- /dev/null
+++ b/backend/test/integration/Makefile
@@ -0,0 +1,3 @@
+.PHONY: test-webhook
+test-webhook:
+	WEBHOOK_INTEGRATION=true go test -v "$(CURDIR)/webhook_test.go"
diff --git a/backend/test/integration/README.md b/backend/test/integration/README.md
index d87a3303105..9fbde2e0b8f 100644
--- a/backend/test/integration/README.md
+++ b/backend/test/integration/README.md
@@ -12,6 +12,13 @@ The default integration test will test the default Database, MySQL.
 3. Or run the following to select certain tests: `NAMESPACE=<kfp-namespace> ./run_tests_locally.sh -testify.m Job`.
    Reference: https://stackoverflow.com/a/43312451
 
+### Webhook Integration Tests
+
+The Kubernetes webhooks require Kubeflow Pipelines to be deployed using the
+`manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native` manifests which is why they are gated by
+the `WEBHOOK_INTEGRATION=true` environment variable value. If the correct environment is deployed, you may run
+the tests with `make -C backend/test/integration test-webhook` from the root of the repository.
+
 ### Run database tests with PostgreSQL
 
 To run this test, you need to first deploy the PostgreSQL images on your Kubernetes cluster. For how to deploy, 
diff --git a/backend/test/integration/webhook_test.go b/backend/test/integration/webhook_test.go
new file mode 100644
index 00000000000..ceecb5de46b
--- /dev/null
+++ b/backend/test/integration/webhook_test.go
@@ -0,0 +1,190 @@
+// Copyright 2025 The Kubeflow Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kubeflow/pipelines/backend/src/common/util"
+	"github.com/kubeflow/pipelines/backend/src/crd/kubernetes/v2beta1"
+	"github.com/stretchr/testify/require"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/client-go/dynamic"
+)
+
+func getSimplePipelineVersion() (*unstructured.Unstructured, *unstructured.Unstructured) {
+	p := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "pipelines.kubeflow.org/v2beta1",
+			"kind":       "Pipeline",
+			"metadata": map[string]interface{}{
+				"name": "simple-pipeline",
+			},
+		},
+	}
+
+	pv := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "pipelines.kubeflow.org/v2beta1",
+			"kind":       "PipelineVersion",
+			"metadata": map[string]interface{}{
+				"name": "simple-pipeline-v1",
+			},
+			"spec": map[string]interface{}{
+				"pipelineName": p.GetName(),
+				"pipelineSpec": map[string]interface{}{
+					"components": map[string]interface{}{
+						"comp-hello": map[string]interface{}{
+							"executorLabel": "exec-hello",
+						},
+					},
+					"deploymentSpec": map[string]interface{}{
+						"executors": map[string]interface{}{
+							"exec-hello": map[string]interface{}{
+								"container": map[string]interface{}{
+									"args":    []interface{}{"hello"},
+									"command": []interface{}{"echo"},
+									"image":   "python3.12",
+								},
+							},
+						},
+					},
+					"pipelineInfo": map[string]interface{}{
+						"description": "An simple pipeline",
+						"name":        "my-pipeline",
+					},
+					"root": map[string]interface{}{
+						"dag": map[string]interface{}{
+							"tasks": map[string]interface{}{
+								"hello": map[string]interface{}{
+									"componentRef": map[string]interface{}{
+										"name": "comp-hello",
+									},
+									"taskInfo": map[string]interface{}{
+										"name": "hello",
+									},
+								},
+							},
+						},
+					},
+					"schemaVersion": "2.1.0",
+					"sdkVersion":    "kfp-2.12.1",
+				},
+			},
+		},
+	}
+
+	return p, pv
+}
+
+// createPipelineVersion will create a PipelineVersion and retry for up to 5 seconds if it fails to be created.
+func createPipelineVersion(
+	t *testing.T, client dynamic.ResourceInterface, pv *unstructured.Unstructured,
+) *unstructured.Unstructured {
+	t.Helper()
+
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*5))
+	defer cancel()
+
+	var prevErr error
+
+	for {
+		newPv, err := client.Create(ctx, pv, metav1.CreateOptions{})
+		if err == nil {
+			return newPv
+		}
+
+		// Workaround because it seems that the error isn't wrapped, so errors.Is doesn't work.
+		if strings.Contains(err.Error(), "context deadline exceeded") {
+			errToAssert := err
+			if prevErr != nil {
+				errToAssert = prevErr
+			}
+
+			require.NoError(t, errToAssert, "Timed out creating the PipelineVersion")
+		}
+
+		prevErr = err
+		time.Sleep(500 * time.Millisecond)
+	}
+}
+
+func TestPipelineVersionWebhookIntegration(t *testing.T) {
+	if os.Getenv("WEBHOOK_INTEGRATION") != "true" {
+		t.Skip("The WEBHOOK_INTEGRATION environment variable is not set to true")
+	}
+
+	config, err := util.GetKubernetesConfig()
+	require.NoError(t, err, "Expected no error to get the Kubernetes configuration")
+
+	client, err := dynamic.NewForConfig(config)
+	require.NoError(t, err, "Expected no error when creating the dynamic client")
+
+	pClient := client.Resource(v2beta1.GroupVersion.WithResource("pipelines")).Namespace("kubeflow")
+	pvClient := client.Resource(v2beta1.GroupVersion.WithResource("pipelineversions")).Namespace("kubeflow")
+
+	p, pv := getSimplePipelineVersion()
+
+	defer func(p *unstructured.Unstructured, pv *unstructured.Unstructured) {
+		err = pvClient.Delete(context.TODO(), pv.GetName(), metav1.DeleteOptions{})
+		if err != nil && !k8serrors.IsNotFound(err) {
+			t.Logf("Failed to delete the pipeline version: %v", err)
+		}
+
+		err := pClient.Delete(context.TODO(), p.GetName(), metav1.DeleteOptions{})
+		if err != nil && !k8serrors.IsNotFound(err) {
+			t.Logf("Failed to delete the pipeline: %v", err)
+		}
+	}(p, pv)
+
+	p, err = pClient.Create(context.TODO(), p, metav1.CreateOptions{})
+	require.NoError(t, err, "Expected no error when creating the pipeline")
+
+	pv = createPipelineVersion(t, pvClient, pv)
+
+	t.Log("Checking that the mutating webhook set the pipeline-id label")
+	require.Equal(t, pv.GetLabels()["pipelines.kubeflow.org/pipeline-id"], string(p.GetUID()))
+
+	t.Log("Checking that the mutating webhook set the pipeline label")
+	require.Equal(t, pv.GetLabels()["pipelines.kubeflow.org/pipeline"], p.GetName())
+
+	t.Log("Checking that the mutating webhook added the owners reference of the Pipeline object")
+	ownerRefFound := false
+
+	for _, ownerRef := range pv.GetOwnerReferences() {
+		if ownerRef.APIVersion == v2beta1.GroupVersion.String() && ownerRef.Kind == "Pipeline" {
+			ownerRefFound = true
+			require.Equal(t, ownerRef.UID, p.GetUID())
+
+			break
+		}
+	}
+
+	require.True(t, ownerRefFound, "Expected to find an owner ref of the Pipeline on the PipelineVersion")
+
+	// This test is to ensure the validating webhook is available. Most of the test coverage occurs in unit tests.
+	t.Log("Checking that the validating webhook ensures the PipelineVersion is immutable")
+	err = unstructured.SetNestedField(pv.Object, "new-name", "spec", "pipelineSpec", "pipelineInfo", "name")
+	require.NoError(t, err, "Failed to set a field on the PipelineVersion using unstructured.SetNestedField")
+
+	_, err = pvClient.Update(context.TODO(), pv, metav1.UpdateOptions{})
+	require.ErrorContains(t, err, "Pipeline spec is immutable")
+}
diff --git a/manifests/kustomize/base/webhook/kustomization.yaml b/manifests/kustomize/base/webhook/kustomization.yaml
index 68bd739915e..c21c7ca5e6a 100644
--- a/manifests/kustomize/base/webhook/kustomization.yaml
+++ b/manifests/kustomize/base/webhook/kustomization.yaml
@@ -4,6 +4,7 @@ namespace: kubeflow
 
 resources:
   - pipelineversion-validating-webhook-config.yaml
+  - pipelineversion-mutating-webhook-config.yaml
 
 configurations:
 - params.yaml
diff --git a/manifests/kustomize/base/webhook/params.yaml b/manifests/kustomize/base/webhook/params.yaml
index 9dff26f3a72..911553f62ab 100644
--- a/manifests/kustomize/base/webhook/params.yaml
+++ b/manifests/kustomize/base/webhook/params.yaml
@@ -5,3 +5,9 @@ varReference:
     kind: ValidatingWebhookConfiguration
   - path: webhooks/name
     kind: ValidatingWebhookConfiguration
+  - path: metadata/annotations
+    kind: MutatingWebhookConfiguration
+  - path: webhooks/clientConfig/service/namespace
+    kind: MutatingWebhookConfiguration
+  - path: webhooks/name
+    kind: MutatingWebhookConfiguration
diff --git a/manifests/kustomize/base/webhook/pipelineversion-mutating-webhook-config.yaml b/manifests/kustomize/base/webhook/pipelineversion-mutating-webhook-config.yaml
new file mode 100644
index 00000000000..949e2afdfda
--- /dev/null
+++ b/manifests/kustomize/base/webhook/pipelineversion-mutating-webhook-config.yaml
@@ -0,0 +1,27 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: pipelineversions.pipelines.kubeflow.org
+webhooks:
+  - name: pipelineversions.pipelines.kubeflow.org
+    rules:
+      - operations:
+          - CREATE
+          - UPDATE
+        apiGroups:
+          - pipelines.kubeflow.org
+        apiVersions:
+          - v2beta1
+        resources:
+          - pipelineversions
+        scope: Namespaced
+    admissionReviewVersions:
+      - v1
+    sideEffects: None
+    failurePolicy: Fail
+    clientConfig:
+      service:
+        name: ml-pipeline
+        namespace: $(kfp-namespace)
+        path: /webhooks/mutate-pipelineversion
+        port: 8443
diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native/kustomization.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native/kustomization.yaml
index 3f4d2be678b..a23edbd0d24 100644
--- a/manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native/kustomization.yaml
+++ b/manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native/kustomization.yaml
@@ -23,4 +23,8 @@ patches:
   - path: patches/validating-webhook.yaml
     target:
       kind: ValidatingWebhookConfiguration
-      name: pipelineversions.pipelines.kubeflow.org
\ No newline at end of file
+      name: pipelineversions.pipelines.kubeflow.org
+  - path: patches/mutating-webhook.yaml
+    target:
+      kind: MutatingWebhookConfiguration
+      name: pipelineversions.pipelines.kubeflow.org
diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native/patches/deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native/patches/deployment.yaml
index e73ff46ae02..9895c154769 100644
--- a/manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native/patches/deployment.yaml
+++ b/manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native/patches/deployment.yaml
@@ -18,6 +18,7 @@ spec:
             - "-logtostderr=true"
             - "--webhookTLSCertPath=/etc/webhook/certs/tls.crt"
             - "--webhookTLSKeyPath=/etc/webhook/certs/tls.key"
+            - "--pipelinesStoreKubernetes=true"
           volumeMounts:
             - name: webhook-certs
               mountPath: /etc/webhook/certs
diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native/patches/mutating-webhook.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native/patches/mutating-webhook.yaml
new file mode 100644
index 00000000000..35a074db7f2
--- /dev/null
+++ b/manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native/patches/mutating-webhook.yaml
@@ -0,0 +1,6 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: pipelineversions.pipelines.kubeflow.org
+  annotations:
+    cert-manager.io/inject-ca-from: $(kfp-namespace)/kfp-api-webhook-cert
diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-multi-user-k8s-native/kustomization.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-multi-user-k8s-native/kustomization.yaml
index 878a03dd274..37e42f4ba8d 100644
--- a/manifests/kustomize/env/cert-manager/platform-agnostic-multi-user-k8s-native/kustomization.yaml
+++ b/manifests/kustomize/env/cert-manager/platform-agnostic-multi-user-k8s-native/kustomization.yaml
@@ -24,3 +24,7 @@ patches:
     target:
       kind: ValidatingWebhookConfiguration
       name: pipelineversions.pipelines.kubeflow.org
+  - path: patches/mutating-webhook.yaml
+    target:
+      kind: MutatingWebhookConfiguration
+      name: pipelineversions.pipelines.kubeflow.org
diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-multi-user-k8s-native/patches/deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-multi-user-k8s-native/patches/deployment.yaml
index 7fbf2253180..94c086eba6b 100644
--- a/manifests/kustomize/env/cert-manager/platform-agnostic-multi-user-k8s-native/patches/deployment.yaml
+++ b/manifests/kustomize/env/cert-manager/platform-agnostic-multi-user-k8s-native/patches/deployment.yaml
@@ -19,6 +19,7 @@ spec:
             - "-logtostderr=true"
             - "--webhookTLSCertPath=/etc/webhook/certs/tls.crt"
             - "--webhookTLSKeyPath=/etc/webhook/certs/tls.key"
+            - "--pipelinesStoreKubernetes=true"
           volumeMounts:
             - name: webhook-certs
               mountPath: /etc/webhook/certs
diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-multi-user-k8s-native/patches/mutating-webhook.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-multi-user-k8s-native/patches/mutating-webhook.yaml
new file mode 100644
index 00000000000..35a074db7f2
--- /dev/null
+++ b/manifests/kustomize/env/cert-manager/platform-agnostic-multi-user-k8s-native/patches/mutating-webhook.yaml
@@ -0,0 +1,6 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: pipelineversions.pipelines.kubeflow.org
+  annotations:
+    cert-manager.io/inject-ca-from: $(kfp-namespace)/kfp-api-webhook-cert
diff --git a/manifests/kustomize/env/dev-kind/kustomization.yaml b/manifests/kustomize/env/dev-kind/kustomization.yaml
index 4f5757c958e..685cc69e9b3 100644
--- a/manifests/kustomize/env/dev-kind/kustomization.yaml
+++ b/manifests/kustomize/env/dev-kind/kustomization.yaml
@@ -141,6 +141,16 @@ patches:
         clientConfig:
           service:
             name: ml-pipeline-reverse-proxy
+- patch: |-
+    apiVersion: admissionregistration.k8s.io/v1
+    kind: MutatingWebhookConfiguration
+    metadata:
+      name: pipelineversions.pipelines.kubeflow.org
+    webhooks:
+      - name: pipelineversions.pipelines.kubeflow.org
+        clientConfig:
+          service:
+            name: ml-pipeline-reverse-proxy
 - patch: |-
     apiVersion: cert-manager.io/v1
     kind: Certificate
diff --git a/test/sdk-execution-tests/sdk_execution_tests.py b/test/sdk-execution-tests/sdk_execution_tests.py
index 6c36d2fa1de..42bf655a2d5 100644
--- a/test/sdk-execution-tests/sdk_execution_tests.py
+++ b/test/sdk-execution-tests/sdk_execution_tests.py
@@ -20,10 +20,14 @@
 from kfp import client
 from kfp import dsl  # noqa
 import kfp_server_api
+import kubernetes.client
+import kubernetes.client.rest
+import kubernetes.config
 import pytest
 import yaml
 
 KFP_ENDPOINT = os.environ['KFP_ENDPOINT']
+KFP_NAMESPACE = os.getenv("KFP_NAMESPACE", "kubeflow")
 TIMEOUT_SECONDS = os.environ['TIMEOUT_SECONDS']
 CURRENT_DIR = os.path.abspath(os.path.dirname(__file__))
 PROJECT_ROOT = os.path.abspath(
@@ -32,6 +36,7 @@
                            'test_data_config.yaml')
 
 kfp_client = client.Client(host=KFP_ENDPOINT)
+kubernetes.config.load_kube_config()
 
 
 @dataclasses.dataclass
@@ -123,6 +128,14 @@ def test(test_case: TestCase) -> None:
     api_run = wait(run_result)
     assert api_run.state == test_case.expected_state, f'Pipeline {test_case.name} ended with incorrect status: {api_run.state}. More info: {run_url}'
 
+    try:
+        print(f'Deleting the Argo Workflow for run {api_run.run_id}')
+        kubernetes.client.CustomObjectsApi().delete_collection_namespaced_custom_object(
+            "argoproj.io", "v1alpha1", KFP_NAMESPACE, "workflows", label_selector=f'pipeline/runid={api_run.run_id}'
+        )
+    except kubernetes.client.rest.ApiException as e:
+        print(f'Failed to delete the Argo Workflow for run {api_run.run_id}: {e}', file=sys.stderr)
+
 
 if __name__ == '__main__':
     pytest.main()


Reviews:
user: login: HumairAK
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-02T19:46:55Z
user: login: mprahl
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-03T16:37:05Z
user: login: mprahl
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-03T16:46:04Z
user: login: HumairAK
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-03T18:59:15Z
user: login: mprahl
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-03T20:29:48Z

Review comments:
diff_hunk: @@ -93,3 +93,7 @@ kind-load-driver-debug:
 
 .PHONY: kind-build-and-load-driver-debug
 kind-build-and-load-driver-debug: image_driver_debug kind-load-driver-debug
+
+.PHONY: test-webhook
+test-webhook:
+	WEBHOOK_INTEGRATION=true go test -v -run '.+WebhookIntegration$$' "$(CURDIR)/src/apiserver/webhook/..."
path: backend/Makefile
user: login: HumairAK
body: this and the integration seems oddly (maybe arguably) placed when we have a `backend/test/integration` location
created_at: 2025-04-02T19:36:48Z
author_association: COLLABORATOR
diff_hunk: @@ -119,14 +119,77 @@ func (p *PipelineVersionsWebhook) ValidateDelete(_ context.Context, _ runtime.Ob
 	return nil, nil
 }
 
-func NewPipelineVersionWebhook(client ctrlclient.Client) (http.Handler, error) {
+func (p *PipelineVersionsWebhook) Default(ctx context.Context, obj runtime.Object) error {
+	pipelineVersion, ok := obj.(*k8sapi.PipelineVersion)
+	if !ok {
+		return &apierrors.StatusError{
+			ErrStatus: metav1.Status{
+				Code:    http.StatusBadRequest,
+				Reason:  metav1.StatusReasonBadRequest,
+				Message: fmt.Sprintf("expected a PipelineVersion object but got %T", pipelineVersion),
+			},
+		}
+	}
+
+	pipeline := &k8sapi.Pipeline{}
+	nsName := types.NamespacedName{Namespace: pipelineVersion.Namespace, Name: pipelineVersion.Spec.PipelineName}
+
+	err := p.Client.Get(ctx, nsName, pipeline)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return newBadRequestError("The spec.pipelineName doesn't map to an existing Pipeline object")
+		}
+
+		return err
+	}
+
+	if pipelineVersion.Labels == nil {
+		pipelineVersion.Labels = map[string]string{}
+	}
+
+	// Labels for efficient querying
+	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline-id"] = string(pipeline.UID)
+	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline"] = string(pipeline.Name)
path: backend/src/apiserver/webhook/pipelineversion_webhook.go
user: login: HumairAK
body: ```suggestion
	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline"] = pipeline.Name
```
created_at: 2025-04-02T19:38:50Z
author_association: COLLABORATOR
diff_hunk: @@ -207,3 +217,196 @@ func TestPipelineVersionWebhook_ValidateUpdate_MetadataChangeAllowed(t *testing.
 	_, err := pipelineWebhook.ValidateUpdate(context.TODO(), oldPipelineVersion, newPipelineVersion)
 	assert.NoError(t, err, "Expected no error for metadata-only change")
 }
+
+func TestPipelineVersionWebhook_MutatingUpdate_FixesOwnersRef(t *testing.T) {
+	pipelineWebhook, validPipelineSpecJSON := setupPipelineWebhookTest(t)
+	badUID := uuid.NewUUID()
+
+	pipelineVersion := &k8sapi.PipelineVersion{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test-pipeline-v1",
+			Namespace:  "default",
+			Generation: 1,
+			Labels:     map[string]string{"version": "v2"},
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion: v2beta1.GroupVersion.String(),
+					Kind:       "Pipeline",
+					Name:       "test-pipeline2",
+					UID:        badUID,
+				},
+			},
+		},
+		Spec: k8sapi.PipelineVersionSpec{
+			PipelineName: "test-pipeline",
+			PipelineSpec: k8sapi.PipelineIRSpec{
+				Value: json.RawMessage(validPipelineSpecJSON),
+			},
+		},
+	}
+
+	err := pipelineWebhook.Default(context.TODO(), pipelineVersion)
+	require.NoError(t, err, "Expected no error for fixing the owner's reference")
+	require.Len(t, pipelineVersion.OwnerReferences, 1)
+	require.NotEqual(t, pipelineVersion.OwnerReferences[0].UID, badUID)
+	require.Equal(t, pipelineVersion.OwnerReferences[0].Name, "test-pipeline")
+}
+
+func getSimplePipelineVersion() (*unstructured.Unstructured, *unstructured.Unstructured) {
+	p := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "pipelines.kubeflow.org/v2beta1",
+			"kind":       "Pipeline",
+			"metadata": map[string]interface{}{
+				"name": "simple-pipeline",
+			},
+		},
+	}
+
+	pv := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "pipelines.kubeflow.org/v2beta1",
+			"kind":       "PipelineVersion",
+			"metadata": map[string]interface{}{
+				"name": "simple-pipeline-v1",
+			},
+			"spec": map[string]interface{}{
+				"pipelineName": p.GetName(),
+				"pipelineSpec": map[string]interface{}{
+					"components": map[string]interface{}{
+						"comp-hello": map[string]interface{}{
+							"executorLabel": "exec-hello",
+						},
+					},
+					"deploymentSpec": map[string]interface{}{
+						"executors": map[string]interface{}{
+							"exec-hello": map[string]interface{}{
+								"container": map[string]interface{}{
+									"args":    []interface{}{"hello"},
+									"command": []interface{}{"echo"},
+									"image":   "python3.12",
+								},
+							},
+						},
+					},
+					"pipelineInfo": map[string]interface{}{
+						"description": "An simple pipeline",
+						"name":        "my-pipeline",
+					},
+					"root": map[string]interface{}{
+						"dag": map[string]interface{}{
+							"tasks": map[string]interface{}{
+								"hello": map[string]interface{}{
+									"componentRef": map[string]interface{}{
+										"name": "comp-hello",
+									},
+									"taskInfo": map[string]interface{}{
+										"name": "hello",
+									},
+								},
+							},
+						},
+					},
+					"schemaVersion": "2.1.0",
+					"sdkVersion":    "kfp-2.12.1",
+				},
+			},
+		},
+	}
+
+	return p, pv
+}
+
+// createPipelineVersion will create a PipelineVersion and retry for up to 5 seconds if it fails to be created.
+func createPipelineVersion(
+	t *testing.T, client dynamic.ResourceInterface, pv *unstructured.Unstructured,
+) *unstructured.Unstructured {
+	t.Helper()
+
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*5))
+	defer cancel()
+
+	var prevErr error
+
+	for {
+		newPv, err := client.Create(ctx, pv, metav1.CreateOptions{})
+		if err == nil {
+			return newPv
+		}
+
+		// Workaround because it seems that the error isn't wrapped, so errors.Is doesn't work.
+		if strings.Contains(err.Error(), "context deadline exceeded") {
+			errToAssert := err
+			if prevErr != nil {
+				errToAssert = prevErr
+			}
+
+			require.NoError(t, errToAssert, "Timed out creating the PipelineVersion")
+		}
+
+		prevErr = err
+		time.Sleep(500 * time.Millisecond)
+	}
+}
+
+func TestPipelineVersionWebhookIntegration(t *testing.T) {
+	if os.Getenv("WEBHOOK_INTEGRATION") != "true" {
+		t.Skip("The WEBHOOK_INTEGRATION environment variable is not set to true")
+	}
+
+	config, err := util.GetKubernetesConfig()
+	require.NoError(t, err, "Expected no error to get the Kubernetes configuration")
+
+	client, err := dynamic.NewForConfig(config)
+	require.NoError(t, err, "Expected no error when creating the dynamic client")
+
+	pClient := client.Resource(v2beta1.GroupVersion.WithResource("pipelines")).Namespace("kubeflow")
+	pvClient := client.Resource(v2beta1.GroupVersion.WithResource("pipelineversions")).Namespace("kubeflow")
+
+	p, pv := getSimplePipelineVersion()
+
+	defer func(p *unstructured.Unstructured, pv *unstructured.Unstructured) {
+		err = pvClient.Delete(context.TODO(), pv.GetName(), metav1.DeleteOptions{})
+		if err != nil && !k8serrors.IsNotFound(err) {
+			t.Logf("Failed to delete the pipeline version: %v", err)
+		}
+
+		err := pClient.Delete(context.TODO(), p.GetName(), metav1.DeleteOptions{})
+		if err != nil && !k8serrors.IsNotFound(err) {
+			t.Logf("Failed to delete the pipeline: %v", err)
+		}
+	}(p, pv)
+
+	p, err = pClient.Create(context.TODO(), p, metav1.CreateOptions{})
+	require.NoError(t, err, "Expected no error when creating the pipeline")
+
+	pv = createPipelineVersion(t, pvClient, pv)
+
+	t.Log("Checking that the mutating webhook set the pipeline-id label")
+	require.Equal(t, pv.GetLabels()["pipelines.kubeflow.org/pipeline-id"], string(p.GetUID()))
+
+	t.Log("Checking that the mutating webhook set the pipeline label")
+	require.Equal(t, pv.GetLabels()["pipelines.kubeflow.org/pipeline"], string(p.GetName()))
path: backend/src/apiserver/webhook/pipelineversion_webhook_test.go
user: login: HumairAK
body: ```suggestion
	require.Equal(t, pv.GetLabels()["pipelines.kubeflow.org/pipeline"], p.GetName())
```
created_at: 2025-04-02T19:42:53Z
author_association: COLLABORATOR
diff_hunk: @@ -119,14 +119,77 @@ func (p *PipelineVersionsWebhook) ValidateDelete(_ context.Context, _ runtime.Ob
 	return nil, nil
 }
 
-func NewPipelineVersionWebhook(client ctrlclient.Client) (http.Handler, error) {
+func (p *PipelineVersionsWebhook) Default(ctx context.Context, obj runtime.Object) error {
+	pipelineVersion, ok := obj.(*k8sapi.PipelineVersion)
+	if !ok {
+		return &apierrors.StatusError{
+			ErrStatus: metav1.Status{
+				Code:    http.StatusBadRequest,
+				Reason:  metav1.StatusReasonBadRequest,
+				Message: fmt.Sprintf("expected a PipelineVersion object but got %T", pipelineVersion),
+			},
+		}
+	}
+
+	pipeline := &k8sapi.Pipeline{}
+	nsName := types.NamespacedName{Namespace: pipelineVersion.Namespace, Name: pipelineVersion.Spec.PipelineName}
+
+	err := p.Client.Get(ctx, nsName, pipeline)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return newBadRequestError("The spec.pipelineName doesn't map to an existing Pipeline object")
+		}
+
+		return err
+	}
+
+	if pipelineVersion.Labels == nil {
+		pipelineVersion.Labels = map[string]string{}
+	}
+
+	// Labels for efficient querying
+	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline-id"] = string(pipeline.UID)
+	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline"] = string(pipeline.Name)
+
+	for i := range pipelineVersion.OwnerReferences {
+		ownerRef := &pipelineVersion.OwnerReferences[i]
+		if ownerRef.APIVersion != k8sapi.GroupVersion.String() || ownerRef.Kind != "Pipeline" {
+			continue
+		}
+
+		ownerRef.Name = pipeline.Name
+		ownerRef.UID = pipeline.UID
+
+		return nil
+	}
+
+	pipelineVersion.OwnerReferences = append(pipelineVersion.OwnerReferences, metav1.OwnerReference{
+		APIVersion: k8sapi.GroupVersion.String(),
+		Kind:       "Pipeline",
+		Name:       pipeline.Name,
+		UID:        pipeline.UID,
+	})
path: backend/src/apiserver/webhook/pipelineversion_webhook.go
user: login: HumairAK
body: an observation here, I'm wondering how this affects pipeline deletions, currently the db store implementation you can't delete the Pipeline without first having to delete the PipelineVersions, the use of ownerreferences here makes it a non issue for the k8s native store implementation, something to note for @rimolive maybe when doing pipeline deletions

created_at: 2025-04-02T19:46:14Z
author_association: COLLABORATOR
diff_hunk: @@ -93,3 +93,7 @@ kind-load-driver-debug:
 
 .PHONY: kind-build-and-load-driver-debug
 kind-build-and-load-driver-debug: image_driver_debug kind-load-driver-debug
+
+.PHONY: test-webhook
+test-webhook:
+	WEBHOOK_INTEGRATION=true go test -v -run '.+WebhookIntegration$$' "$(CURDIR)/src/apiserver/webhook/..."
path: backend/Makefile
user: login: mprahl
body: Whoops, thanks for pointing that out.
created_at: 2025-04-03T16:37:05Z
author_association: COLLABORATOR
diff_hunk: @@ -119,14 +119,77 @@ func (p *PipelineVersionsWebhook) ValidateDelete(_ context.Context, _ runtime.Ob
 	return nil, nil
 }
 
-func NewPipelineVersionWebhook(client ctrlclient.Client) (http.Handler, error) {
+func (p *PipelineVersionsWebhook) Default(ctx context.Context, obj runtime.Object) error {
+	pipelineVersion, ok := obj.(*k8sapi.PipelineVersion)
+	if !ok {
+		return &apierrors.StatusError{
+			ErrStatus: metav1.Status{
+				Code:    http.StatusBadRequest,
+				Reason:  metav1.StatusReasonBadRequest,
+				Message: fmt.Sprintf("expected a PipelineVersion object but got %T", pipelineVersion),
+			},
+		}
+	}
+
+	pipeline := &k8sapi.Pipeline{}
+	nsName := types.NamespacedName{Namespace: pipelineVersion.Namespace, Name: pipelineVersion.Spec.PipelineName}
+
+	err := p.Client.Get(ctx, nsName, pipeline)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return newBadRequestError("The spec.pipelineName doesn't map to an existing Pipeline object")
+		}
+
+		return err
+	}
+
+	if pipelineVersion.Labels == nil {
+		pipelineVersion.Labels = map[string]string{}
+	}
+
+	// Labels for efficient querying
+	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline-id"] = string(pipeline.UID)
+	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline"] = string(pipeline.Name)
+
+	for i := range pipelineVersion.OwnerReferences {
+		ownerRef := &pipelineVersion.OwnerReferences[i]
+		if ownerRef.APIVersion != k8sapi.GroupVersion.String() || ownerRef.Kind != "Pipeline" {
+			continue
+		}
+
+		ownerRef.Name = pipeline.Name
+		ownerRef.UID = pipeline.UID
+
+		return nil
+	}
+
+	pipelineVersion.OwnerReferences = append(pipelineVersion.OwnerReferences, metav1.OwnerReference{
+		APIVersion: k8sapi.GroupVersion.String(),
+		Kind:       "Pipeline",
+		Name:       pipeline.Name,
+		UID:        pipeline.UID,
+	})
path: backend/src/apiserver/webhook/pipelineversion_webhook.go
user: login: mprahl
body: @HumairAK and @rimolive do you think we should set `blockOwnerDeletion` to prevent the pipeline from being deleted if the pipeline versions are still around? I lean towards no because I like the idea of being able to fully delete a pipeline with one API call but wanted your take.
created_at: 2025-04-03T16:46:04Z
author_association: COLLABORATOR
diff_hunk: @@ -119,14 +119,77 @@ func (p *PipelineVersionsWebhook) ValidateDelete(_ context.Context, _ runtime.Ob
 	return nil, nil
 }
 
-func NewPipelineVersionWebhook(client ctrlclient.Client) (http.Handler, error) {
+func (p *PipelineVersionsWebhook) Default(ctx context.Context, obj runtime.Object) error {
+	pipelineVersion, ok := obj.(*k8sapi.PipelineVersion)
+	if !ok {
+		return &apierrors.StatusError{
+			ErrStatus: metav1.Status{
+				Code:    http.StatusBadRequest,
+				Reason:  metav1.StatusReasonBadRequest,
+				Message: fmt.Sprintf("expected a PipelineVersion object but got %T", pipelineVersion),
+			},
+		}
+	}
+
+	pipeline := &k8sapi.Pipeline{}
+	nsName := types.NamespacedName{Namespace: pipelineVersion.Namespace, Name: pipelineVersion.Spec.PipelineName}
+
+	err := p.Client.Get(ctx, nsName, pipeline)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return newBadRequestError("The spec.pipelineName doesn't map to an existing Pipeline object")
+		}
+
+		return err
+	}
+
+	if pipelineVersion.Labels == nil {
+		pipelineVersion.Labels = map[string]string{}
+	}
+
+	// Labels for efficient querying
+	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline-id"] = string(pipeline.UID)
+	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline"] = string(pipeline.Name)
+
+	for i := range pipelineVersion.OwnerReferences {
+		ownerRef := &pipelineVersion.OwnerReferences[i]
+		if ownerRef.APIVersion != k8sapi.GroupVersion.String() || ownerRef.Kind != "Pipeline" {
+			continue
+		}
+
+		ownerRef.Name = pipeline.Name
+		ownerRef.UID = pipeline.UID
+
+		return nil
+	}
+
+	pipelineVersion.OwnerReferences = append(pipelineVersion.OwnerReferences, metav1.OwnerReference{
+		APIVersion: k8sapi.GroupVersion.String(),
+		Kind:       "Pipeline",
+		Name:       pipeline.Name,
+		UID:        pipeline.UID,
+	})
path: backend/src/apiserver/webhook/pipelineversion_webhook.go
user: login: HumairAK
body: sigh, I agree with you but I think there has been discussion in the past about adding a separate api or toggle in the pipeline deletion call something like a parameter "cascade=true" to delete all at once for the db case, so in this case I'm leaning towards consistency and blocking it, but open to thoughts 
created_at: 2025-04-03T18:59:14Z
author_association: COLLABORATOR
diff_hunk: @@ -119,14 +119,77 @@ func (p *PipelineVersionsWebhook) ValidateDelete(_ context.Context, _ runtime.Ob
 	return nil, nil
 }
 
-func NewPipelineVersionWebhook(client ctrlclient.Client) (http.Handler, error) {
+func (p *PipelineVersionsWebhook) Default(ctx context.Context, obj runtime.Object) error {
+	pipelineVersion, ok := obj.(*k8sapi.PipelineVersion)
+	if !ok {
+		return &apierrors.StatusError{
+			ErrStatus: metav1.Status{
+				Code:    http.StatusBadRequest,
+				Reason:  metav1.StatusReasonBadRequest,
+				Message: fmt.Sprintf("expected a PipelineVersion object but got %T", pipelineVersion),
+			},
+		}
+	}
+
+	pipeline := &k8sapi.Pipeline{}
+	nsName := types.NamespacedName{Namespace: pipelineVersion.Namespace, Name: pipelineVersion.Spec.PipelineName}
+
+	err := p.Client.Get(ctx, nsName, pipeline)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return newBadRequestError("The spec.pipelineName doesn't map to an existing Pipeline object")
+		}
+
+		return err
+	}
+
+	if pipelineVersion.Labels == nil {
+		pipelineVersion.Labels = map[string]string{}
+	}
+
+	// Labels for efficient querying
+	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline-id"] = string(pipeline.UID)
+	pipelineVersion.Labels["pipelines.kubeflow.org/pipeline"] = string(pipeline.Name)
+
+	for i := range pipelineVersion.OwnerReferences {
+		ownerRef := &pipelineVersion.OwnerReferences[i]
+		if ownerRef.APIVersion != k8sapi.GroupVersion.String() || ownerRef.Kind != "Pipeline" {
+			continue
+		}
+
+		ownerRef.Name = pipeline.Name
+		ownerRef.UID = pipeline.UID
+
+		return nil
+	}
+
+	pipelineVersion.OwnerReferences = append(pipelineVersion.OwnerReferences, metav1.OwnerReference{
+		APIVersion: k8sapi.GroupVersion.String(),
+		Kind:       "Pipeline",
+		Name:       pipeline.Name,
+		UID:        pipeline.UID,
+	})
path: backend/src/apiserver/webhook/pipelineversion_webhook.go
user: login: mprahl
body: Okay, I added `blockOwnerDeletion`!
created_at: 2025-04-03T20:29:48Z
author_association: COLLABORATOR

PR comments:
user: login: google-oss-prow[bot]
created_at: 2025-03-31T19:45:27Z
body: Skipping CI for Draft Pull Request.
If you want CI signal for your change, please convert it to an actual PR.
You can still manually trigger a test run with `/test all`
author_association: NONE
user: login: mprahl
created_at: 2025-04-03T18:03:54Z
body: /rerun-workflow "KFP SDK execution tests"
author_association: COLLABORATOR
user: login: mprahl
created_at: 2025-04-03T18:07:14Z
body: /rerun-workflow "CI Check"
author_association: COLLABORATOR
user: login: HumairAK
created_at: 2025-04-03T19:01:19Z
body: /rerun-workflow "KFP SDK execution tests"
author_association: COLLABORATOR
user: login: rimolive
created_at: 2025-04-04T12:31:24Z
body: /rerun-workflow "KFP SDK execution tests"
author_association: MEMBER
user: login: HumairAK
created_at: 2025-04-04T16:14:54Z
body: /lgtm
/approve

thanks for the update to help resolve the test flakiness! 🥳 
author_association: COLLABORATOR
user: login: google-oss-prow[bot]
created_at: 2025-04-04T16:15:00Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubeflow/pipelines/pull/11782#issuecomment-2779207869" title="Approved">HumairAK</a>*

The full list of commands accepted by this bot can be found [here](https://oss.gprow.dev/command-help?repo=kubeflow%2Fpipelines).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[.github/OWNERS](https://github.com/kubeflow/pipelines/blob/master/.github/OWNERS)~~ [HumairAK]
- ~~[backend/OWNERS](https://github.com/kubeflow/pipelines/blob/master/backend/OWNERS)~~ [HumairAK]
- ~~[manifests/kustomize/OWNERS](https://github.com/kubeflow/pipelines/blob/master/manifests/kustomize/OWNERS)~~ [HumairAK]
- ~~[test/OWNERS](https://github.com/kubeflow/pipelines/blob/master/test/OWNERS)~~ [HumairAK]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: NONE