Repository: akuity/kargo
Pull request #1650 (2024-03-19 15:22:35)

feat(verification): improve transient error handling
Addresses (part of) #1640

This improves the handling of transient Kubernetes API errors which may occur during the verification step of a Stage's current Freight.

It does this by selectively returning errors which should be retried, causing the reconciler to not leave the `Verifying` phase and allowing it to retry the action it performed on the next reconciliation attempt.

To allow the reconciler to distinguish a creation error from a failed attempt to retrieve results, closer inspection is performed on the existence of an `AnalysisRun` reference in the `VerificationInfo`.

netlify[bot]: ### <span aria-hidden="true">👷</span> Deploy Preview for *docs-kargo-akuity-io* processing.


|  Name | Link |
|:-:|------------------------|
|<span aria-hidden="true">🔨</span> Latest commit | 01c4bbf5f1312ac0cbcafb06abe25ba270ddc426 |
|<span aria-hidden="true">🔍</span> Latest deploy log | https://app.netlify.com/sites/docs-kargo-akuity-io/deploys/65f9adbec160de0008d20742 |
codecov[bot]: ## [Codecov](https://app.codecov.io/gh/akuity/kargo/pull/1650?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=akuity) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 44.11%. Comparing base [(`ffaad65`)](https://app.codecov.io/gh/akuity/kargo/commit/ffaad653ec36ae7299f567d62407fde14fcf0140?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=akuity) to head [(`01c4bbf`)](https://app.codecov.io/gh/akuity/kargo/pull/1650?dropdown=coverage&src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=akuity).


<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #1650      +/-   ##
==========================================
+ Coverage   43.96%   44.11%   +0.15%     
==========================================
  Files         199      200       +1     
  Lines       12846    12881      +35     
==========================================
+ Hits         5648     5683      +35     
  Misses       6956     6956              
  Partials      242      242              
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/akuity/kargo/pull/1650?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=akuity).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=akuity).

krancour: This LGTM after merge conflicts are resolved.

I may be PR'ing something similar for Promotions shortly.
hiddeco: Has been rebased.

Patch:
diff --git a/api/v1alpha1/stage_types.go b/api/v1alpha1/stage_types.go
index aefdf0a9..0ea7d59d 100644
--- a/api/v1alpha1/stage_types.go
+++ b/api/v1alpha1/stage_types.go
@@ -738,6 +738,12 @@ type VerificationInfo struct {
 	AnalysisRun *AnalysisRunReference `json:"analysisRun,omitempty" protobuf:"bytes,3,opt,name=analysisRun"`
 }
 
+// HasAnalysisRun returns a bool indicating whether the VerificationInfo has an
+// associated AnalysisRun.
+func (v *VerificationInfo) HasAnalysisRun() bool {
+	return v != nil && v.AnalysisRun != nil
+}
+
 type VerificationInfoStack []VerificationInfo
 
 // UpdateOrPush updates the VerificationInfo with the same ID as the provided
diff --git a/cmd/controlplane/api.go b/cmd/controlplane/api.go
index a03ca388..08de30f6 100644
--- a/cmd/controlplane/api.go
+++ b/cmd/controlplane/api.go
@@ -39,7 +39,7 @@ func newAPICommand() *cobra.Command {
 		Logger: log.StandardLogger(),
 	}
 
-	return &cobra.Command{
+	cmd := &cobra.Command{
 		Use:               "api",
 		DisableAutoGenTag: true,
 		SilenceErrors:     true,
@@ -50,6 +50,8 @@ func newAPICommand() *cobra.Command {
 			return cmdOpts.run(cmd.Context())
 		},
 	}
+
+	return cmd
 }
 
 func (o *apiOptions) complete() {
@@ -72,11 +74,11 @@ func (o *apiOptions) run(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("error setting up internal Kubernetes API client: %w", err)
 	}
+
 	kubeClient, err := newWrappedKubernetesClient(ctx, clientCfg, internalClient, cfg)
 	if err != nil {
 		return fmt.Errorf("error creating Kubernetes client for Kargo API server: %w", err)
 	}
-
 	switch {
 	case !cfg.RolloutsIntegrationEnabled:
 		o.Logger.Info("Argo Rollouts integration is disabled")
@@ -91,10 +93,10 @@ func (o *apiOptions) run(ctx context.Context) error {
 	}
 
 	if cfg.AdminConfig != nil {
-		log.Info("admin account is enabled")
+		o.Logger.Info("admin account is enabled")
 	}
 	if cfg.OIDCConfig != nil {
-		log.WithFields(log.Fields{
+		o.Logger.WithFields(log.Fields{
 			"issuerURL":   cfg.OIDCConfig.IssuerURL,
 			"clientID":    cfg.OIDCConfig.ClientID,
 			"cliClientID": cfg.OIDCConfig.CLIClientID,
diff --git a/cmd/controlplane/controller.go b/cmd/controlplane/controller.go
index 0c5d82ef..e5b51f43 100644
--- a/cmd/controlplane/controller.go
+++ b/cmd/controlplane/controller.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"sync"
 
@@ -32,238 +33,294 @@ import (
 	_ "github.com/akuity/kargo/internal/gitprovider/github"
 )
 
+type controllerOptions struct {
+	ShardName  string
+	KubeConfig string
+
+	ArgoCDEnabled       bool
+	ArgoCDKubeConfig    string
+	ArgoCDNamespaceOnly bool
+
+	Logger *log.Logger
+}
+
 func newControllerCommand() *cobra.Command {
-	return &cobra.Command{
+	cmdOpts := &controllerOptions{
+		Logger: log.StandardLogger(),
+	}
+
+	cmd := &cobra.Command{
 		Use:               "controller",
 		DisableAutoGenTag: true,
 		SilenceErrors:     true,
 		SilenceUsage:      true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			ctx := cmd.Context()
+			cmdOpts.complete()
 
-			version := versionpkg.GetVersion()
-			shardName := os.GetEnv("SHARD_NAME", "")
+			return cmdOpts.run(cmd.Context())
+		},
+	}
 
-			startupLogEntry := log.WithFields(log.Fields{
-				"version": version.Version,
-				"commit":  version.GitCommit,
-			})
-			if shardName != "" {
-				startupLogEntry = startupLogEntry.WithField("shard", shardName)
-			}
-			startupLogEntry.Info("Starting Kargo Controller")
-
-			stagesReconcilerCfg := stages.ReconcilerConfigFromEnv()
-			var kargoMgr manager.Manager
-			{
-				// If the env var is undefined, this will resolve to kubeconfig for the
-				// cluster the controller is running in.
-				//
-				// It is typically defined if this controller is running somewhere other
-				// than where the Kargo resources live. One example of this would be a
-				// sharded topology wherein Kargo controllers run on application
-				// clusters, with Kargo resources hosted in a centralized management
-				// cluster.
-				restCfg, err :=
-					kubernetes.GetRestConfig(ctx, os.GetEnv("KUBECONFIG", ""))
-				if err != nil {
-					return fmt.Errorf("error loading REST config for Kargo controller manager: %w", err)
-				}
-				restCfg.ContentType = runtime.ContentTypeJSON
-
-				scheme := runtime.NewScheme()
-				if err = corev1.AddToScheme(scheme); err != nil {
-					return fmt.Errorf(
-						"error adding Kubernetes core API to Kargo controller manager scheme: %w",
-						err,
-					)
-				}
-				if err = kargoapi.AddToScheme(scheme); err != nil {
-					return fmt.Errorf(
-						"error adding Kargo API to Kargo controller manager scheme: %w",
-						err,
-					)
-				}
-				if stagesReconcilerCfg.RolloutsIntegrationEnabled {
-					if argoRolloutsExists(ctx, restCfg) {
-						log.Info("Argo Rollouts integration is enabled")
-						if err = rollouts.AddToScheme(scheme); err != nil {
-							return fmt.Errorf(
-								"error adding Argo Rollouts API to Kargo controller manager scheme: %w",
-								err,
-							)
-						}
-					} else {
-						log.Warn(
-							"Argo Rollouts integration was enabled, but no Argo Rollouts " +
-								"CRDs were found. Proceeding without Argo Rollouts integration.",
-						)
-					}
-				}
-
-				secretReq, err := controller.GetCredentialsRequirement()
-				if err != nil {
-					return fmt.Errorf("error getting label requirement for credentials Secrets: %w", err)
-				}
-
-				cacheOpts := cache.Options{
-					ByObject: map[client.Object]cache.ByObject{
-						// Only watch Secrets matching the label requirements
-						// for credentials.
-						&corev1.Secret{}: {
-							Label: labels.NewSelector().Add(*secretReq),
-						},
-					},
-				}
-
-				if kargoMgr, err = ctrl.NewManager(
-					restCfg,
-					ctrl.Options{
-						Scheme: scheme,
-						Metrics: server.Options{
-							BindAddress: "0",
-						},
-						Cache: cacheOpts,
-					},
-				); err != nil {
-					return fmt.Errorf("error initializing Kargo controller manager: %w", err)
-				}
-			}
+	return cmd
+}
 
-			var argocdMgr manager.Manager
-			if types.MustParseBool(os.GetEnv("ARGOCD_INTEGRATION_ENABLED", "true")) {
-				// If the env var is undefined, this will resolve to kubeconfig for the
-				// cluster the controller is running in.
-				//
-				// It is typically defined if this controller is running somewhere other
-				// than where the Argo CD resources live. Two examples of this would
-				// involve topologies wherein Kargo controllers run EITHER sharded
-				// across application clusters OR in a centralized management cluster,
-				// but with Argo CD deployed to a different management cluster.
-				restCfg, err :=
-					kubernetes.GetRestConfig(ctx, os.GetEnv("ARGOCD_KUBECONFIG", ""))
-				if err != nil {
-					return fmt.Errorf("error loading REST config for Argo CD controller manager: %w", err)
-				}
-				restCfg.ContentType = runtime.ContentTypeJSON
-
-				argocdNamespace := libargocd.Namespace()
-
-				// There's a chance there is only permission to interact with Argo CD
-				// Application resources in a single namespace, so we will use that
-				// namespace when attempting to determine if Argo CD CRDs are installed.
-				if argoCDExists(ctx, restCfg, argocdNamespace) {
-					log.Info("Argo CD integration is enabled")
-					scheme := runtime.NewScheme()
-					if err = corev1.AddToScheme(scheme); err != nil {
-						return fmt.Errorf(
-							"error adding Kubernetes core API to Argo CD controller manager scheme: %w",
-							err,
-						)
-					}
-					if err = argocd.AddToScheme(scheme); err != nil {
-						return fmt.Errorf(
-							"error adding Argo CD API to Argo CD controller manager scheme: %w",
-							err,
-						)
-					}
-					cacheOpts := cache.Options{} // Watches all namespaces by default
-					if types.MustParseBool(
-						os.GetEnv("ARGOCD_WATCH_ARGOCD_NAMESPACE_ONLY", "false"),
-					) {
-						cacheOpts.DefaultNamespaces = map[string]cache.Config{
-							argocdNamespace: {},
-						}
-					}
-					if argocdMgr, err = ctrl.NewManager(
-						restCfg,
-						ctrl.Options{
-							Scheme: scheme,
-							Metrics: server.Options{
-								BindAddress: "0",
-							},
-							Cache: cacheOpts,
-						},
-					); err != nil {
-						return fmt.Errorf("error initializing Argo CD Application controller manager: %w", err)
-					}
-				} else {
-					log.Warn(
-						"ARGO CD integration was enabled, but no Argo CD CRDs were " +
-							"found. Proceeding without Argo CD integration.",
-					)
-				}
-			} else {
-				log.Info("Argo CD integration is disabled")
-			}
+func (o *controllerOptions) complete() {
+	o.ShardName = os.GetEnv("SHARD_NAME", "")
+	o.KubeConfig = os.GetEnv("KUBECONFIG", "")
+	o.ArgoCDEnabled = types.MustParseBool(os.GetEnv("ARGOCD_INTEGRATION_ENABLED", "true"))
+	o.ArgoCDKubeConfig = os.GetEnv("ARGOCD_KUBECONFIG", "")
+	o.ArgoCDNamespaceOnly = types.MustParseBool(os.GetEnv("ARGOCD_WATCH_ARGOCD_NAMESPACE_ONLY", "false"))
+}
 
-			credentialsDB := credentials.NewKubernetesDatabase(
-				kargoMgr.GetClient(),
-				credentials.KubernetesDatabaseConfigFromEnv(),
-			)
+func (o *controllerOptions) run(ctx context.Context) error {
+	version := versionpkg.GetVersion()
 
-			if err := promotions.SetupReconcilerWithManager(
-				ctx,
-				kargoMgr,
-				argocdMgr,
-				credentialsDB,
-				shardName,
-			); err != nil {
-				return fmt.Errorf("error setting up Promotions reconciler: %w", err)
-			}
+	startupLogEntry := o.Logger.WithFields(log.Fields{
+		"version": version.Version,
+		"commit":  version.GitCommit,
+	})
+	if o.ShardName != "" {
+		startupLogEntry = startupLogEntry.WithField("shard", o.ShardName)
+	}
+	startupLogEntry.Info("Starting Kargo Controller")
 
-			if err := stages.SetupReconcilerWithManager(
-				ctx,
-				kargoMgr,
-				argocdMgr,
-				stagesReconcilerCfg,
-			); err != nil {
-				return fmt.Errorf("error setting up Stages reconciler: %w", err)
-			}
+	stagesReconcilerCfg := stages.ReconcilerConfigFromEnv()
 
-			if err := warehouses.SetupReconcilerWithManager(
-				kargoMgr,
-				credentialsDB,
-				shardName,
-			); err != nil {
-				return fmt.Errorf("error setting up Warehouses reconciler: %w", err)
-			}
+	kargoMgr, err := o.setupKargoManager(ctx, stagesReconcilerCfg)
+	if err != nil {
+		return fmt.Errorf("error initializing Kargo controller manager: %w", err)
+	}
 
-			var errChan = make(chan error)
+	argocdMgr, err := o.setupArgoCDManager(ctx)
+	if err != nil {
+		return fmt.Errorf("error initializing Argo CD Application controller manager: %w", err)
+	}
 
-			wg := sync.WaitGroup{}
+	credentialsDB := credentials.NewKubernetesDatabase(
+		kargoMgr.GetClient(),
+		credentials.KubernetesDatabaseConfigFromEnv(),
+	)
 
-			if argocdMgr != nil {
-				wg.Add(1)
-				go func() {
-					defer wg.Done()
-					if err := argocdMgr.Start(ctx); err != nil {
-						errChan <- fmt.Errorf("error starting argo cd manager: %w", err)
-					}
-				}()
-			}
+	if err := o.setupReconcilers(ctx, kargoMgr, argocdMgr, credentialsDB, stagesReconcilerCfg); err != nil {
+		return fmt.Errorf("error setting up reconcilers: %w", err)
+	}
+
+	return o.startManagers(ctx, kargoMgr, argocdMgr)
+}
+
+func (o *controllerOptions) setupKargoManager(
+	ctx context.Context,
+	stagesReconcilerCfg stages.ReconcilerConfig,
+) (manager.Manager, error) {
+	// If the env var is undefined, this will resolve to kubeconfig for the
+	// cluster the controller is running in.
+	//
+	// It is typically defined if this controller is running somewhere other
+	// than where the Kargo resources live. One example of this would be a
+	// sharded topology wherein Kargo controllers run on application
+	// clusters, with Kargo resources hosted in a centralized management
+	// cluster.
+	restCfg, err := kubernetes.GetRestConfig(ctx, o.KubeConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error loading REST config for Kargo controller manager: %w", err)
+	}
+	restCfg.ContentType = runtime.ContentTypeJSON
 
-			wg.Add(1)
-			go func() {
-				defer wg.Done()
-				if err := kargoMgr.Start(ctx); err != nil {
-					errChan <- fmt.Errorf("error starting kargo manager: %w", err)
-				}
-			}()
-
-			// Adapt wg to a channel that can be used in a select
-			doneCh := make(chan struct{})
-			go func() {
-				wg.Wait()
-				close(doneCh)
-			}()
-
-			select {
-			case err := <-errChan:
-				return err
-			case <-doneCh:
-				return nil
+	scheme := runtime.NewScheme()
+	if err = corev1.AddToScheme(scheme); err != nil {
+		return nil, fmt.Errorf(
+			"error adding Kubernetes core API to Kargo controller manager scheme: %w",
+			err,
+		)
+	}
+	if err = kargoapi.AddToScheme(scheme); err != nil {
+		return nil, fmt.Errorf(
+			"error adding Kargo API to Kargo controller manager scheme: %w",
+			err,
+		)
+	}
+	if stagesReconcilerCfg.RolloutsIntegrationEnabled {
+		if argoRolloutsExists(ctx, restCfg) {
+			log.Info("Argo Rollouts integration is enabled")
+			if err = rollouts.AddToScheme(scheme); err != nil {
+				return nil, fmt.Errorf(
+					"error adding Argo Rollouts API to Kargo controller manager scheme: %w",
+					err,
+				)
 			}
+		} else {
+			log.Warn(
+				"Argo Rollouts integration was enabled, but no Argo Rollouts " +
+					"CRDs were found. Proceeding without Argo Rollouts integration.",
+			)
+		}
+	}
+
+	secretReq, err := controller.GetCredentialsRequirement()
+	if err != nil {
+		return nil, fmt.Errorf("error getting label requirement for credentials Secrets: %w", err)
+	}
+
+	cacheOpts := cache.Options{
+		ByObject: map[client.Object]cache.ByObject{
+			// Only watch Secrets matching the label requirements
+			// for credentials.
+			&corev1.Secret{}: {
+				Label: labels.NewSelector().Add(*secretReq),
+			},
 		},
 	}
+
+	return ctrl.NewManager(
+		restCfg,
+		ctrl.Options{
+			Scheme: scheme,
+			Metrics: server.Options{
+				BindAddress: "0",
+			},
+			Cache: cacheOpts,
+		},
+	)
+}
+
+func (o *controllerOptions) setupArgoCDManager(ctx context.Context) (manager.Manager, error) {
+	if !o.ArgoCDEnabled {
+		o.Logger.Info("Argo CD integration is disabled")
+		return nil, nil
+	}
+
+	// If the env var is undefined, this will resolve to kubeconfig for the
+	// cluster the controller is running in.
+	//
+	// It is typically defined if this controller is running somewhere other
+	// than where the Argo CD resources live. Two examples of this would
+	// involve topologies wherein Kargo controllers run EITHER sharded
+	// across application clusters OR in a centralized management cluster,
+	// but with Argo CD deployed to a different management cluster.
+	restCfg, err := kubernetes.GetRestConfig(ctx, o.ArgoCDKubeConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error loading REST config for Argo CD controller manager: %w", err)
+	}
+	restCfg.ContentType = runtime.ContentTypeJSON
+
+	argocdNamespace := libargocd.Namespace()
+
+	// There's a chance there is only permission to interact with Argo CD
+	// Application resources in a single namespace, so we will use that
+	// namespace when attempting to determine if Argo CD CRDs are installed.
+	if !argoCDExists(ctx, restCfg, argocdNamespace) {
+		o.Logger.Warn(
+			"Argo CD integration was enabled, but no Argo CD CRDs were found. " +
+				"Proceeding without Argo CD integration.",
+		)
+		return nil, nil
+	}
+
+	o.Logger.Info("Argo CD integration is enabled")
+
+	scheme := runtime.NewScheme()
+	if err = corev1.AddToScheme(scheme); err != nil {
+		return nil, fmt.Errorf(
+			"error adding Kubernetes core API to Argo CD controller manager scheme: %w",
+			err,
+		)
+	}
+	if err = argocd.AddToScheme(scheme); err != nil {
+		return nil, fmt.Errorf(
+			"error adding Argo CD API to Argo CD controller manager scheme: %w",
+			err,
+		)
+	}
+	cacheOpts := cache.Options{} // Watches all namespaces by default
+	if o.ArgoCDNamespaceOnly {
+		cacheOpts.DefaultNamespaces = map[string]cache.Config{
+			argocdNamespace: {},
+		}
+	}
+
+	return ctrl.NewManager(
+		restCfg,
+		ctrl.Options{
+			Scheme: scheme,
+			Metrics: server.Options{
+				BindAddress: "0",
+			},
+			Cache: cacheOpts,
+		},
+	)
+}
+
+func (o *controllerOptions) setupReconcilers(
+	ctx context.Context,
+	kargoMgr, argocdMgr manager.Manager,
+	credentialsDB credentials.Database,
+	stagesReconcilerCfg stages.ReconcilerConfig,
+) error {
+	if err := promotions.SetupReconcilerWithManager(
+		ctx,
+		kargoMgr,
+		argocdMgr,
+		credentialsDB,
+		o.ShardName,
+	); err != nil {
+		return fmt.Errorf("error setting up Promotions reconciler: %w", err)
+	}
+
+	if err := stages.SetupReconcilerWithManager(
+		ctx,
+		kargoMgr,
+		argocdMgr,
+		stagesReconcilerCfg,
+	); err != nil {
+		return fmt.Errorf("error setting up Stages reconciler: %w", err)
+	}
+
+	if err := warehouses.SetupReconcilerWithManager(
+		kargoMgr,
+		credentialsDB,
+		o.ShardName,
+	); err != nil {
+		return fmt.Errorf("error setting up Warehouses reconciler: %w", err)
+	}
+
+	return nil
+}
+
+func (o *controllerOptions) startManagers(ctx context.Context, kargoMgr, argocdMgr manager.Manager) error {
+	var (
+		errChan = make(chan error)
+		wg      = sync.WaitGroup{}
+	)
+
+	if argocdMgr != nil {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := argocdMgr.Start(ctx); err != nil {
+				errChan <- fmt.Errorf("error starting argo cd manager: %w", err)
+			}
+		}()
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if err := kargoMgr.Start(ctx); err != nil {
+			errChan <- fmt.Errorf("error starting kargo manager: %w", err)
+		}
+	}()
+
+	// Adapt wg to a channel that can be used in a select
+	doneCh := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(doneCh)
+	}()
+
+	select {
+	case err := <-errChan:
+		return err
+	case <-doneCh:
+		return nil
+	}
 }
diff --git a/cmd/controlplane/garbage_collector.go b/cmd/controlplane/garbage_collector.go
index a515b4fa..d7625d3c 100644
--- a/cmd/controlplane/garbage_collector.go
+++ b/cmd/controlplane/garbage_collector.go
@@ -21,76 +21,104 @@ import (
 	versionpkg "github.com/akuity/kargo/internal/version"
 )
 
+type garbageCollectorOptions struct {
+	KubeConfig string
+
+	Logger *log.Logger
+}
+
 func newGarbageCollectorCommand() *cobra.Command {
-	return &cobra.Command{
+	cmdOpts := &garbageCollectorOptions{
+		Logger: log.StandardLogger(),
+	}
+
+	cmd := &cobra.Command{
 		Use:               "garbage-collector",
 		DisableAutoGenTag: true,
 		SilenceErrors:     true,
 		SilenceUsage:      true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			ctx := cmd.Context()
-
-			version := versionpkg.GetVersion()
-			log.WithFields(log.Fields{
-				"version": version.Version,
-				"commit":  version.GitCommit,
-			}).Info("Starting Kargo Garbage Collector")
-
-			cfg := garbage.CollectorConfigFromEnv()
-
-			var mgr manager.Manager
-			{
-				restCfg, err :=
-					kubernetes.GetRestConfig(ctx, os.GetEnv("KUBECONFIG", ""))
-				if err != nil {
-					return fmt.Errorf("error loading REST config: %w", err)
-				}
-				scheme := runtime.NewScheme()
-				if err = corev1.AddToScheme(scheme); err != nil {
-					return fmt.Errorf("error adding Kubernetes core API to scheme: %w", err)
-				}
-				if err = kargoapi.AddToScheme(scheme); err != nil {
-					return fmt.Errorf("error adding Kargo API to scheme: %w", err)
-				}
-				if mgr, err = ctrl.NewManager(
-					restCfg,
-					ctrl.Options{
-						Scheme: scheme,
-						Metrics: server.Options{
-							BindAddress: "0",
-						},
-					},
-				); err != nil {
-					return fmt.Errorf("error initializing controller manager: %w", err)
-				}
-				// Index Promotions by Stage
-				if err = kubeclient.IndexPromotionsByStage(ctx, mgr); err != nil {
-					return fmt.Errorf("error indexing Promotions by Stage: %w", err)
-				}
-				// Index Freight by Warehouse
-				if err = kubeclient.IndexFreightByWarehouse(ctx, mgr); err != nil {
-					return fmt.Errorf("error indexing Freight by Warehouse: %w", err)
-				}
-				// Index Stages by Freight
-				if err = kubeclient.IndexStagesByFreight(ctx, mgr); err != nil {
-					return fmt.Errorf("error indexing Stages by Freight: %w", err)
-				}
-			}
-
-			ctx, cancel := context.WithCancel(ctx)
-			defer cancel()
-
-			go func() {
-				if err := mgr.Start(ctx); err != nil {
-					panic(fmt.Errorf("start manager: %w", err))
-				}
-			}()
-
-			if !mgr.GetCache().WaitForCacheSync(ctx) {
-				return errors.New("error waiting for cache sync")
-			}
-
-			return garbage.NewCollector(mgr.GetClient(), cfg).Run(ctx)
+			cmdOpts.complete()
+
+			return cmdOpts.run(cmd.Context())
+		},
+	}
+
+	return cmd
+}
+
+func (o *garbageCollectorOptions) complete() {
+	o.KubeConfig = os.GetEnv("KUBECONFIG", "")
+}
+
+func (o *garbageCollectorOptions) run(ctx context.Context) error {
+	version := versionpkg.GetVersion()
+
+	o.Logger.WithFields(log.Fields{
+		"version": version.Version,
+		"commit":  version.GitCommit,
+	}).Info("Starting Kargo Garbage Collector")
+
+	mgr, err := o.setupManager(ctx)
+	if err != nil {
+		return fmt.Errorf("error setting up controller manager: %w", err)
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	go func() {
+		if err := mgr.Start(ctx); err != nil {
+			panic(fmt.Errorf("start manager: %w", err))
+		}
+	}()
+
+	if !mgr.GetCache().WaitForCacheSync(ctx) {
+		return errors.New("error waiting for cache sync")
+	}
+
+	cfg := garbage.CollectorConfigFromEnv()
+	return garbage.NewCollector(mgr.GetClient(), cfg).Run(ctx)
+}
+
+func (o *garbageCollectorOptions) setupManager(ctx context.Context) (manager.Manager, error) {
+	restCfg, err := kubernetes.GetRestConfig(ctx, o.KubeConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error loading REST config: %w", err)
+	}
+
+	scheme := runtime.NewScheme()
+	if err = corev1.AddToScheme(scheme); err != nil {
+		return nil, fmt.Errorf("error adding Kubernetes core API to scheme: %w", err)
+	}
+	if err = kargoapi.AddToScheme(scheme); err != nil {
+		return nil, fmt.Errorf("error adding Kargo API to scheme: %w", err)
+	}
+
+	mgr, err := ctrl.NewManager(
+		restCfg,
+		ctrl.Options{
+			Scheme: scheme,
+			Metrics: server.Options{
+				BindAddress: "0",
+			},
 		},
+	)
+	if err != nil {
+		return nil, fmt.Errorf("error initializing controller manager: %w", err)
+	}
+
+	// Index Promotions by Stage
+	if err = kubeclient.IndexPromotionsByStage(ctx, mgr); err != nil {
+		return nil, fmt.Errorf("error indexing Promotions by Stage: %w", err)
+	}
+	// Index Freight by Warehouse
+	if err = kubeclient.IndexFreightByWarehouse(ctx, mgr); err != nil {
+		return nil, fmt.Errorf("error indexing Freight by Warehouse: %w", err)
+	}
+	// Index Stages by Freight
+	if err = kubeclient.IndexStagesByFreight(ctx, mgr); err != nil {
+		return nil, fmt.Errorf("error indexing Stages by Freight: %w", err)
 	}
+	return mgr, nil
 }
diff --git a/cmd/controlplane/management_controller.go b/cmd/controlplane/management_controller.go
index dff9ff71..3137cbd3 100644
--- a/cmd/controlplane/management_controller.go
+++ b/cmd/controlplane/management_controller.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 
 	log "github.com/sirupsen/logrus"
@@ -20,78 +21,100 @@ import (
 	versionpkg "github.com/akuity/kargo/internal/version"
 )
 
+type managementControllerOptions struct {
+	KubeConfig string
+
+	Logger *log.Logger
+}
+
 func newManagementControllerCommand() *cobra.Command {
-	return &cobra.Command{
+	cmdOpts := &managementControllerOptions{
+		Logger: log.StandardLogger(),
+	}
+
+	cmd := &cobra.Command{
 		Use:               "management-controller",
 		DisableAutoGenTag: true,
 		SilenceErrors:     true,
 		SilenceUsage:      true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			ctx := cmd.Context()
-
-			version := versionpkg.GetVersion()
-
-			log.WithFields(log.Fields{
-				"version": version.Version,
-				"commit":  version.GitCommit,
-			}).Info("Starting Kargo Management Controller")
-
-			var kargoMgr manager.Manager
-			{
-				restCfg, err :=
-					kubernetes.GetRestConfig(ctx, os.GetEnv("KUBECONFIG", ""))
-				if err != nil {
-					return fmt.Errorf("error loading REST config for Kargo controller manager: %w", err)
-				}
-				restCfg.ContentType = runtime.ContentTypeJSON
-
-				scheme := runtime.NewScheme()
-				if err = corev1.AddToScheme(scheme); err != nil {
-					return fmt.Errorf(
-						"error adding Kubernetes core API to Kargo controller manager scheme: %w",
-						err,
-					)
-				}
-				if err = rbacv1.AddToScheme(scheme); err != nil {
-					return fmt.Errorf(
-						"error adding Kubernetes RBAC API to Kargo controller manager scheme: %w",
-						err,
-					)
-				}
-				if err = kargoapi.AddToScheme(scheme); err != nil {
-					return fmt.Errorf(
-						"error adding Kargo API to Kargo controller manager scheme: %w",
-						err,
-					)
-				}
-				if kargoMgr, err = ctrl.NewManager(
-					restCfg,
-					ctrl.Options{
-						Scheme: scheme,
-						Metrics: server.Options{
-							BindAddress: "0",
-						},
-					},
-				); err != nil {
-					return fmt.Errorf("error initializing Kargo controller manager: %w", err)
-				}
-			}
-
-			if err := namespaces.SetupReconcilerWithManager(kargoMgr); err != nil {
-				return fmt.Errorf("error setting up Namespaces reconciler: %w", err)
-			}
-
-			if err := projects.SetupReconcilerWithManager(
-				kargoMgr,
-				projects.ReconcilerConfigFromEnv(),
-			); err != nil {
-				return fmt.Errorf("error setting up Projects reconciler: %w", err)
-			}
-
-			if err := kargoMgr.Start(ctx); err != nil {
-				return fmt.Errorf("error starting kargo manager: %w", err)
-			}
-			return nil
+			cmdOpts.complete()
+
+			return cmdOpts.run(cmd.Context())
 		},
 	}
+
+	return cmd
+}
+
+func (o *managementControllerOptions) complete() {
+	o.KubeConfig = os.GetEnv("KUBECONFIG", "")
+}
+
+func (o *managementControllerOptions) run(ctx context.Context) error {
+	version := versionpkg.GetVersion()
+
+	o.Logger.WithFields(log.Fields{
+		"version": version.Version,
+		"commit":  version.GitCommit,
+	}).Info("Starting Kargo Management Controller")
+
+	kargoMgr, err := o.setupManager(ctx)
+	if err != nil {
+		return fmt.Errorf("error initializing Kargo controller manager: %w", err)
+	}
+
+	if err := namespaces.SetupReconcilerWithManager(kargoMgr); err != nil {
+		return fmt.Errorf("error setting up Namespaces reconciler: %w", err)
+	}
+
+	if err := projects.SetupReconcilerWithManager(
+		kargoMgr,
+		projects.ReconcilerConfigFromEnv(),
+	); err != nil {
+		return fmt.Errorf("error setting up Projects reconciler: %w", err)
+	}
+
+	if err := kargoMgr.Start(ctx); err != nil {
+		return fmt.Errorf("error starting kargo manager: %w", err)
+	}
+	return nil
+}
+
+func (o *managementControllerOptions) setupManager(ctx context.Context) (manager.Manager, error) {
+	restCfg, err := kubernetes.GetRestConfig(ctx, o.KubeConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error loading REST config for Kargo controller manager: %w", err)
+	}
+	restCfg.ContentType = runtime.ContentTypeJSON
+
+	scheme := runtime.NewScheme()
+	if err = corev1.AddToScheme(scheme); err != nil {
+		return nil, fmt.Errorf(
+			"error adding Kubernetes core API to Kargo controller manager scheme: %w",
+			err,
+		)
+	}
+	if err = rbacv1.AddToScheme(scheme); err != nil {
+		return nil, fmt.Errorf(
+			"error adding Kubernetes RBAC API to Kargo controller manager scheme: %w",
+			err,
+		)
+	}
+	if err = kargoapi.AddToScheme(scheme); err != nil {
+		return nil, fmt.Errorf(
+			"error adding Kargo API to Kargo controller manager scheme: %w",
+			err,
+		)
+	}
+
+	return ctrl.NewManager(
+		restCfg,
+		ctrl.Options{
+			Scheme: scheme,
+			Metrics: server.Options{
+				BindAddress: "0",
+			},
+		},
+	)
 }
diff --git a/cmd/controlplane/version.go b/cmd/controlplane/version.go
index 9375d5f0..98f5d877 100644
--- a/cmd/controlplane/version.go
+++ b/cmd/controlplane/version.go
@@ -3,25 +3,36 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 
 	"github.com/spf13/cobra"
 
 	versionpkg "github.com/akuity/kargo/internal/version"
 )
 
+type versionOptions struct{}
+
 func newVersionCommand() *cobra.Command {
-	return &cobra.Command{
+	cmdOpts := &versionOptions{}
+
+	cmd := &cobra.Command{
 		Use:               "version",
 		DisableAutoGenTag: true,
 		SilenceErrors:     true,
 		SilenceUsage:      true,
-		RunE: func(*cobra.Command, []string) error {
-			version, err := json.MarshalIndent(versionpkg.GetVersion(), "", "  ")
-			if err != nil {
-				return fmt.Errorf("marshal version: %w", err)
-			}
-			fmt.Println(string(version))
-			return nil
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			return cmdOpts.run(cmd.OutOrStdout())
 		},
 	}
+
+	return cmd
+}
+
+func (o *versionOptions) run(out io.Writer) error {
+	version, err := json.MarshalIndent(versionpkg.GetVersion(), "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal version: %w", err)
+	}
+	_, _ = fmt.Fprintln(out, string(version))
+	return nil
 }
diff --git a/cmd/controlplane/webhooks.go b/cmd/controlplane/webhooks.go
index ae597fcd..206ee52f 100644
--- a/cmd/controlplane/webhooks.go
+++ b/cmd/controlplane/webhooks.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 
 	log "github.com/sirupsen/logrus"
@@ -25,86 +26,106 @@ import (
 	"github.com/akuity/kargo/internal/webhook/warehouse"
 )
 
+type webhooksServerOptions struct {
+	KubeConfig string
+
+	Logger *log.Logger
+}
+
 func newWebhooksServerCommand() *cobra.Command {
-	return &cobra.Command{
+	cmdOpts := &webhooksServerOptions{
+		Logger: log.StandardLogger(),
+	}
+
+	cmd := &cobra.Command{
 		Use:               "webhooks-server",
 		DisableAutoGenTag: true,
 		SilenceErrors:     true,
 		SilenceUsage:      true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			ctx := cmd.Context()
-
-			version := versionpkg.GetVersion()
-			log.WithFields(log.Fields{
-				"version": version.Version,
-				"commit":  version.GitCommit,
-			}).Info("Starting Kargo Webhooks Server")
-
-			restCfg, err := kubernetes.GetRestConfig(ctx, os.GetEnv("KUBECONFIG", ""))
-			if err != nil {
-				return fmt.Errorf("error getting REST config: %w", err)
-			}
-
-			scheme := runtime.NewScheme()
-			if err = corev1.AddToScheme(scheme); err != nil {
-				return fmt.Errorf("add corev1 to scheme: %w", err)
-			}
-			if err = rbacv1.AddToScheme(scheme); err != nil {
-				return fmt.Errorf("add rbacv1 to scheme: %w", err)
-			}
-			if err = authzv1.AddToScheme(scheme); err != nil {
-				return fmt.Errorf("add authzv1 to scheme: %w", err)
-			}
-			if err = kargoapi.AddToScheme(scheme); err != nil {
-				return fmt.Errorf("add kargo api to scheme: %w", err)
-			}
-
-			mgr, err := ctrl.NewManager(
-				restCfg,
-				ctrl.Options{
-					Scheme: scheme,
-					WebhookServer: webhook.NewServer(
-						webhook.Options{
-							Port: 9443,
-						},
-					),
-					Metrics: server.Options{
-						BindAddress: "0",
-					},
+			cmdOpts.complete()
+
+			return cmdOpts.run(cmd.Context())
+		},
+	}
+
+	return cmd
+}
+
+func (o *webhooksServerOptions) complete() {
+	o.KubeConfig = os.GetEnv("KUBECONFIG", "")
+}
+
+func (o *webhooksServerOptions) run(ctx context.Context) error {
+	version := versionpkg.GetVersion()
+	o.Logger.WithFields(log.Fields{
+		"version": version.Version,
+		"commit":  version.GitCommit,
+	}).Info("Starting Kargo Webhooks Server")
+
+	restCfg, err := kubernetes.GetRestConfig(ctx, o.KubeConfig)
+	if err != nil {
+		return fmt.Errorf("error getting REST config: %w", err)
+	}
+
+	scheme := runtime.NewScheme()
+	if err = corev1.AddToScheme(scheme); err != nil {
+		return fmt.Errorf("add corev1 to scheme: %w", err)
+	}
+	if err = rbacv1.AddToScheme(scheme); err != nil {
+		return fmt.Errorf("add rbacv1 to scheme: %w", err)
+	}
+	if err = authzv1.AddToScheme(scheme); err != nil {
+		return fmt.Errorf("add authzv1 to scheme: %w", err)
+	}
+	if err = kargoapi.AddToScheme(scheme); err != nil {
+		return fmt.Errorf("add kargo api to scheme: %w", err)
+	}
+
+	mgr, err := ctrl.NewManager(
+		restCfg,
+		ctrl.Options{
+			Scheme: scheme,
+			WebhookServer: webhook.NewServer(
+				webhook.Options{
+					Port: 9443,
 				},
-			)
-			if err != nil {
-				return fmt.Errorf("new manager: %w", err)
-			}
-
-			// Index Stages by Freight
-			if err = kubeclient.IndexStagesByFreight(ctx, mgr); err != nil {
-				return fmt.Errorf("index Stages by Freight: %w", err)
-			}
-
-			if err = freight.SetupWebhookWithManager(mgr); err != nil {
-				return fmt.Errorf("setup Freight webhook: %w", err)
-			}
-			if err = project.SetupWebhookWithManager(
-				mgr,
-				project.WebhookConfigFromEnv(),
-			); err != nil {
-				return fmt.Errorf("setup Project webhook: %w", err)
-			}
-			if err = promotion.SetupWebhookWithManager(mgr); err != nil {
-				return fmt.Errorf("setup Promotion webhook: %w", err)
-			}
-			if err = stage.SetupWebhookWithManager(mgr); err != nil {
-				return fmt.Errorf("setup Stage webhook: %w", err)
-			}
-			if err = warehouse.SetupWebhookWithManager(mgr); err != nil {
-				return fmt.Errorf("setup Warehouse webhook: %w", err)
-			}
-
-			if err := mgr.Start(ctx); err != nil {
-				return fmt.Errorf("start Kargo webhook manager: %w", err)
-			}
-			return nil
+			),
+			Metrics: server.Options{
+				BindAddress: "0",
+			},
 		},
+	)
+	if err != nil {
+		return fmt.Errorf("new manager: %w", err)
+	}
+
+	// Index Stages by Freight
+	if err = kubeclient.IndexStagesByFreight(ctx, mgr); err != nil {
+		return fmt.Errorf("index Stages by Freight: %w", err)
+	}
+
+	if err = freight.SetupWebhookWithManager(mgr); err != nil {
+		return fmt.Errorf("setup Freight webhook: %w", err)
+	}
+	if err = project.SetupWebhookWithManager(
+		mgr,
+		project.WebhookConfigFromEnv(),
+	); err != nil {
+		return fmt.Errorf("setup Project webhook: %w", err)
+	}
+	if err = promotion.SetupWebhookWithManager(mgr); err != nil {
+		return fmt.Errorf("setup Promotion webhook: %w", err)
+	}
+	if err = stage.SetupWebhookWithManager(mgr); err != nil {
+		return fmt.Errorf("setup Stage webhook: %w", err)
+	}
+	if err = warehouse.SetupWebhookWithManager(mgr); err != nil {
+		return fmt.Errorf("setup Warehouse webhook: %w", err)
+	}
+
+	if err := mgr.Start(ctx); err != nil {
+		return fmt.Errorf("start Kargo webhook manager: %w", err)
 	}
+	return nil
 }
diff --git a/internal/controller/stages/stages.go b/internal/controller/stages/stages.go
index 2496ff32..ab3e0387 100644
--- a/internal/controller/stages/stages.go
+++ b/internal/controller/stages/stages.go
@@ -86,7 +86,7 @@ type reconciler struct {
 	startVerificationFn func(
 		context.Context,
 		*kargoapi.Stage,
-	) *kargoapi.VerificationInfo
+	) (*kargoapi.VerificationInfo, error)
 
 	abortVerificationFn func(
 		context.Context,
@@ -96,7 +96,7 @@ type reconciler struct {
 	getVerificationInfoFn func(
 		context.Context,
 		*kargoapi.Stage,
-	) *kargoapi.VerificationInfo
+	) (*kargoapi.VerificationInfo, error)
 
 	getAnalysisTemplateFn func(
 		context.Context,
@@ -759,14 +759,32 @@ func (r *reconciler) syncNormalStage(
 			//       check if verification step is necessary and if yes execute
 			//       step irrespective of phase
 			if status.Phase == kargoapi.StagePhaseVerifying || status.Phase == kargoapi.StagePhaseNotApplicable {
-				if status.CurrentFreight.VerificationInfo == nil {
+				if !status.CurrentFreight.VerificationInfo.HasAnalysisRun() {
 					if status.Health == nil || status.Health.Status == kargoapi.HealthStateHealthy {
 						log.Debug("starting verification")
-						status.CurrentFreight.VerificationInfo = r.startVerificationFn(ctx, stage)
+						var err error
+						if status.CurrentFreight.VerificationInfo, err = r.startVerificationFn(
+							ctx,
+							stage,
+						); err != nil && !status.CurrentFreight.VerificationInfo.HasAnalysisRun() {
+							status.CurrentFreight.VerificationHistory.UpdateOrPush(
+								*status.CurrentFreight.VerificationInfo,
+							)
+							return status, fmt.Errorf("error starting verification: %w", err)
+						}
 					}
 				} else {
 					log.Debug("checking verification results")
-					status.CurrentFreight.VerificationInfo = r.getVerificationInfoFn(ctx, stage)
+					var err error
+					if status.CurrentFreight.VerificationInfo, err = r.getVerificationInfoFn(
+						ctx,
+						stage,
+					); err != nil && status.CurrentFreight.VerificationInfo.HasAnalysisRun() {
+						status.CurrentFreight.VerificationHistory.UpdateOrPush(
+							*status.CurrentFreight.VerificationInfo,
+						)
+						return status, fmt.Errorf("error getting verification info: %w", err)
+					}
 
 					// Abort the verification if it's still running and the Stage has
 					// been marked to do so.
diff --git a/internal/controller/stages/verification.go b/internal/controller/stages/verification.go
index be0ba651..69dad915 100644
--- a/internal/controller/stages/verification.go
+++ b/internal/controller/stages/verification.go
@@ -16,13 +16,23 @@ import (
 
 	kargoapi "github.com/akuity/kargo/api/v1alpha1"
 	rollouts "github.com/akuity/kargo/internal/controller/rollouts/api/v1alpha1"
+	"github.com/akuity/kargo/internal/kubeclient"
 	"github.com/akuity/kargo/internal/logging"
 )
 
+// startVerification starts a verification for the given Stage. If the Stage
+// does not have a reverification annotation, it checks if there is an existing
+// AnalysisRun for the Stage and Freight. If there is, it returns the status of
+// this AnalysisRun. If there is not, it creates a new AnalysisRun for the Stage
+// and Freight.
+//
+// In case of an error, it returns a VerificationInfo with the error message and
+// the phase set to Error. If the error may be due to a transient issue, it is
+// returned, so that the caller can retry the operation.
 func (r *reconciler) startVerification(
 	ctx context.Context,
 	stage *kargoapi.Stage,
-) *kargoapi.VerificationInfo {
+) (*kargoapi.VerificationInfo, error) {
 	if !r.cfg.RolloutsIntegrationEnabled {
 		return &kargoapi.VerificationInfo{
 			ID:        uuid.NewString(),
@@ -30,7 +40,7 @@ func (r *reconciler) startVerification(
 			Phase:     kargoapi.VerificationPhaseError,
 			Message: "Rollouts integration is disabled on this controller; " +
 				"cannot start verification",
-		}
+		}, nil
 	}
 
 	logger := logging.LoggerFromContext(ctx)
@@ -64,7 +74,7 @@ func (r *reconciler) startVerification(
 					stage.Namespace,
 					err,
 				).Error(),
-			}
+			}, err
 		}
 		if len(analysisRuns.Items) > 0 {
 			// Sort the AnalysisRuns by creation timestamp, so that the most recent
@@ -84,7 +94,7 @@ func (r *reconciler) startVerification(
 					Namespace: latestAnalysisRun.Namespace,
 					Phase:     string(latestAnalysisRun.Status.Phase),
 				},
-			}
+			}, nil
 		}
 	}
 
@@ -111,7 +121,7 @@ func (r *reconciler) startVerification(
 					stage.Namespace,
 					err,
 				).Error(),
-			}
+			}, err
 		}
 		if template == nil {
 			return &kargoapi.VerificationInfo{
@@ -123,7 +133,7 @@ func (r *reconciler) startVerification(
 					templateRef.Name,
 					stage.Namespace,
 				).Error(),
-			}
+			}, nil
 		}
 		templates[i] = template
 	}
@@ -147,7 +157,7 @@ func (r *reconciler) startVerification(
 				stage.Namespace,
 				err,
 			).Error(),
-		}
+		}, err
 	}
 	if freight == nil {
 		return &kargoapi.VerificationInfo{
@@ -159,7 +169,7 @@ func (r *reconciler) startVerification(
 				stage.Status.CurrentFreight.Name,
 				stage.Namespace,
 			).Error(),
-		}
+		}, nil
 	}
 
 	run, err := r.buildAnalysisRunFn(stage, freight, templates)
@@ -175,7 +185,7 @@ func (r *reconciler) startVerification(
 				stage.Namespace,
 				err,
 			).Error(),
-		}
+		}, nil
 	}
 
 	if err := r.createAnalysisRunFn(ctx, run); err != nil {
@@ -189,7 +199,7 @@ func (r *reconciler) startVerification(
 				run.Namespace,
 				err,
 			).Error(),
-		}
+		}, kubeclient.IgnoreInvalid(err) // Ignore errors which are due to validation issues
 	}
 
 	return &kargoapi.VerificationInfo{
@@ -200,13 +210,22 @@ func (r *reconciler) startVerification(
 			Name:      run.Name,
 			Namespace: run.Namespace,
 		},
-	}
+	}, nil
 }
 
+// getVerificationInfo returns the status of the AnalysisRun for the given Stage.
+//
+// In case of an error, it returns a VerificationInfo with the error message and
+// the phase set to Error. If the error may be due to a transient issue, it is
+// returned, so that the caller can retry the operation.
+//
+// If an error is returned, the AnalysisRun reference in the VerificationInfo
+// will always be set to the AnalysisRun that was being checked. This is to
+// ensure the caller can continue to track the status of the AnalysisRun.
 func (r *reconciler) getVerificationInfo(
 	ctx context.Context,
 	stage *kargoapi.Stage,
-) *kargoapi.VerificationInfo {
+) (*kargoapi.VerificationInfo, error) {
 	if !r.cfg.RolloutsIntegrationEnabled {
 		return &kargoapi.VerificationInfo{
 			ID:        stage.Status.CurrentFreight.VerificationInfo.ID,
@@ -214,7 +233,7 @@ func (r *reconciler) getVerificationInfo(
 			Phase:     kargoapi.VerificationPhaseError,
 			Message: "Rollouts integration is disabled on this controller; cannot " +
 				"get verification info",
-		}
+		}, nil
 	}
 
 	analysisRunName := stage.Status.CurrentFreight.VerificationInfo.AnalysisRun.Name
@@ -237,7 +256,8 @@ func (r *reconciler) getVerificationInfo(
 				stage.Namespace,
 				err,
 			).Error(),
-		}
+			AnalysisRun: stage.Status.CurrentFreight.VerificationInfo.AnalysisRun.DeepCopy(),
+		}, err
 	}
 	if analysisRun == nil {
 		return &kargoapi.VerificationInfo{
@@ -249,7 +269,7 @@ func (r *reconciler) getVerificationInfo(
 				analysisRunName,
 				stage.Namespace,
 			).Error(),
-		}
+		}, nil
 	}
 	return &kargoapi.VerificationInfo{
 		ID:        stage.Status.CurrentFreight.VerificationInfo.ID,
@@ -260,7 +280,7 @@ func (r *reconciler) getVerificationInfo(
 			Namespace: analysisRun.Namespace,
 			Phase:     string(analysisRun.Status.Phase),
 		},
-	}
+	}, nil
 }
 
 func (r *reconciler) abortVerification(
@@ -298,6 +318,7 @@ func (r *reconciler) abortVerification(
 				ar.Namespace,
 				err,
 			).Error(),
+			AnalysisRun: stage.Status.CurrentFreight.VerificationInfo.AnalysisRun.DeepCopy(),
 		}
 	}
 
@@ -307,14 +328,11 @@ func (r *reconciler) abortVerification(
 	// We do not use the further information from the AnalysisRun, as this
 	// will indicate a "Succeeded" phase due to Argo Rollouts behavior.
 	return &kargoapi.VerificationInfo{
-		ID:        stage.Status.CurrentFreight.VerificationInfo.ID,
-		StartTime: ptr.To(ar.CreationTimestamp),
-		Phase:     kargoapi.VerificationPhaseAborted,
-		Message:   "Verification aborted by user",
-		AnalysisRun: &kargoapi.AnalysisRunReference{
-			Name:      ar.Name,
-			Namespace: ar.Namespace,
-		},
+		ID:          stage.Status.CurrentFreight.VerificationInfo.ID,
+		StartTime:   ptr.To(ar.CreationTimestamp),
+		Phase:       kargoapi.VerificationPhaseAborted,
+		Message:     "Verification aborted by user",
+		AnalysisRun: stage.Status.CurrentFreight.VerificationInfo.AnalysisRun.DeepCopy(),
 	}
 }
 
diff --git a/internal/kubeclient/errors.go b/internal/kubeclient/errors.go
new file mode 100644
index 00000000..d792cbe9
--- /dev/null
+++ b/internal/kubeclient/errors.go
@@ -0,0 +1,14 @@
+package kubeclient
+
+import (
+	"k8s.io/apimachinery/pkg/api/errors"
+)
+
+// IgnoreInvalid returns nil on Invalid errors.
+// All other values that are not Invalid errors or nil are returned unmodified.
+func IgnoreInvalid(err error) error {
+	if errors.IsInvalid(err) {
+		return nil
+	}
+	return err
+}


Test patch:
diff --git a/api/v1alpha1/stage_types_test.go b/api/v1alpha1/stage_types_test.go
index 07e5c024..15fbfc2a 100644
--- a/api/v1alpha1/stage_types_test.go
+++ b/api/v1alpha1/stage_types_test.go
@@ -6,6 +6,37 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestVerificationInfo_HasAnalysisRun(t *testing.T) {
+	testCases := []struct {
+		name           string
+		info           *VerificationInfo
+		expectedResult bool
+	}{
+		{
+			name:           "VerificationInfo is nil",
+			info:           nil,
+			expectedResult: false,
+		},
+		{
+			name:           "AnalysisRun is nil",
+			info:           &VerificationInfo{},
+			expectedResult: false,
+		},
+		{
+			name: "AnalysisRun is not nil",
+			info: &VerificationInfo{
+				AnalysisRun: &AnalysisRunReference{},
+			},
+			expectedResult: true,
+		},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			require.Equal(t, testCase.expectedResult, testCase.info.HasAnalysisRun())
+		})
+	}
+}
+
 func TestFreightReferenceStackUpdateOrPush(t *testing.T) {
 	testCases := []struct {
 		name          string
diff --git a/internal/controller/stages/stages_test.go b/internal/controller/stages/stages_test.go
index 74874ae2..d70c46a4 100644
--- a/internal/controller/stages/stages_test.go
+++ b/internal/controller/stages/stages_test.go
@@ -364,7 +364,7 @@ func TestSyncNormalStage(t *testing.T) {
 				startVerificationFn: func(
 					context.Context,
 					*kargoapi.Stage,
-				) *kargoapi.VerificationInfo {
+				) (*kargoapi.VerificationInfo, error) {
 					return &kargoapi.VerificationInfo{
 						ID:      "new-fake-id",
 						Phase:   kargoapi.VerificationPhasePending,
@@ -372,7 +372,7 @@ func TestSyncNormalStage(t *testing.T) {
 						AnalysisRun: &kargoapi.AnalysisRunReference{
 							Name: "new-fake-analysis-run",
 						},
-					}
+					}, nil
 				},
 			},
 			assertions: func(
@@ -477,11 +477,11 @@ func TestSyncNormalStage(t *testing.T) {
 				startVerificationFn: func(
 					context.Context,
 					*kargoapi.Stage,
-				) *kargoapi.VerificationInfo {
+				) (*kargoapi.VerificationInfo, error) {
 					return &kargoapi.VerificationInfo{
 						Phase:   kargoapi.VerificationPhaseError,
 						Message: "something went wrong",
-					}
+					}, nil
 				},
 			},
 			assertions: func(
@@ -517,6 +517,71 @@ func TestSyncNormalStage(t *testing.T) {
 			},
 		},
 
+		{
+			name: "retryable error starting verification",
+			stage: &kargoapi.Stage{
+				Spec: &kargoapi.StageSpec{
+					PromotionMechanisms: &kargoapi.PromotionMechanisms{},
+					Verification:        &kargoapi.Verification{},
+				},
+				Status: kargoapi.StageStatus{
+					Phase:          kargoapi.StagePhaseVerifying,
+					CurrentFreight: &kargoapi.FreightReference{},
+				},
+			},
+			reconciler: &reconciler{
+				hasNonTerminalPromotionsFn: noNonTerminalPromotionsFn,
+				checkHealthFn: func(
+					context.Context,
+					kargoapi.FreightReference,
+					[]kargoapi.ArgoCDAppUpdate,
+				) *kargoapi.Health {
+					return nil
+				},
+				startVerificationFn: func(
+					context.Context,
+					*kargoapi.Stage,
+				) (*kargoapi.VerificationInfo, error) {
+					return &kargoapi.VerificationInfo{
+						Phase:   kargoapi.VerificationPhaseError,
+						Message: "something went wrong",
+					}, errors.New("retryable error")
+				},
+			},
+			assertions: func(
+				t *testing.T,
+				initialStatus kargoapi.StageStatus,
+				newStatus kargoapi.StageStatus,
+				err error,
+			) {
+				require.ErrorContains(t, err, "retryable error")
+				require.NotNil(t, newStatus.CurrentFreight)
+				require.Equal(t, kargoapi.StagePhaseVerifying, newStatus.Phase)
+
+				expectInfo := kargoapi.VerificationInfo{
+					Phase:   kargoapi.VerificationPhaseError,
+					Message: "something went wrong",
+				}
+
+				require.Equal(
+					t,
+					&expectInfo,
+					newStatus.CurrentFreight.VerificationInfo,
+				)
+				require.Equal(
+					t,
+					kargoapi.VerificationInfoStack{expectInfo},
+					newStatus.CurrentFreight.VerificationHistory,
+				)
+
+				// Everything else should be returned unchanged
+				newStatus.CurrentFreight.VerificationInfo = nil
+				newStatus.CurrentFreight.VerificationHistory = nil
+				newStatus.Phase = initialStatus.Phase
+				require.Equal(t, initialStatus, newStatus)
+			},
+		},
+
 		{
 			name: "error checking verification result",
 			stage: &kargoapi.Stage{
@@ -527,7 +592,13 @@ func TestSyncNormalStage(t *testing.T) {
 				Status: kargoapi.StageStatus{
 					Phase: kargoapi.StagePhaseVerifying,
 					CurrentFreight: &kargoapi.FreightReference{
-						VerificationInfo: &kargoapi.VerificationInfo{},
+						VerificationInfo: &kargoapi.VerificationInfo{
+							Phase: kargoapi.VerificationPhasePending,
+							AnalysisRun: &kargoapi.AnalysisRunReference{
+								Name:      "fake-analysis-run",
+								Namespace: "fake-namespace",
+							},
+						},
 					},
 				},
 			},
@@ -540,11 +611,11 @@ func TestSyncNormalStage(t *testing.T) {
 				) *kargoapi.Health {
 					return nil
 				},
-				getVerificationInfoFn: func(_ context.Context, _ *kargoapi.Stage) *kargoapi.VerificationInfo {
+				getVerificationInfoFn: func(_ context.Context, _ *kargoapi.Stage) (*kargoapi.VerificationInfo, error) {
 					return &kargoapi.VerificationInfo{
 						Phase:   kargoapi.VerificationPhaseError,
 						Message: "something went wrong",
-					}
+					}, nil
 				},
 			},
 			assertions: func(
@@ -572,6 +643,77 @@ func TestSyncNormalStage(t *testing.T) {
 			},
 		},
 
+		{
+			name: "retryable error checking verification result",
+			stage: &kargoapi.Stage{
+				Spec: &kargoapi.StageSpec{
+					PromotionMechanisms: &kargoapi.PromotionMechanisms{},
+					Verification:        &kargoapi.Verification{},
+				},
+				Status: kargoapi.StageStatus{
+					Phase: kargoapi.StagePhaseVerifying,
+					CurrentFreight: &kargoapi.FreightReference{
+						VerificationInfo: &kargoapi.VerificationInfo{
+							Phase: kargoapi.VerificationPhasePending,
+							AnalysisRun: &kargoapi.AnalysisRunReference{
+								Name:      "fake-analysis-run",
+								Namespace: "fake-namespace",
+							},
+						},
+					},
+				},
+			},
+			reconciler: &reconciler{
+				hasNonTerminalPromotionsFn: noNonTerminalPromotionsFn,
+				checkHealthFn: func(
+					context.Context,
+					kargoapi.FreightReference,
+					[]kargoapi.ArgoCDAppUpdate,
+				) *kargoapi.Health {
+					return nil
+				},
+				getVerificationInfoFn: func(_ context.Context, _ *kargoapi.Stage) (*kargoapi.VerificationInfo, error) {
+					return &kargoapi.VerificationInfo{
+						Phase:   kargoapi.VerificationPhaseError,
+						Message: "something went wrong",
+						AnalysisRun: &kargoapi.AnalysisRunReference{
+							Name:      "fake-analysis-run",
+							Namespace: "fake-namespace",
+						},
+					}, errors.New("retryable error")
+				},
+			},
+			assertions: func(
+				t *testing.T,
+				initialStatus kargoapi.StageStatus,
+				newStatus kargoapi.StageStatus,
+				err error,
+			) {
+				require.ErrorContains(t, err, "retryable error")
+				require.NotNil(t, newStatus.CurrentFreight)
+				require.Equal(
+					t,
+					&kargoapi.VerificationInfo{
+						Phase:   kargoapi.VerificationPhaseError,
+						Message: "something went wrong",
+						AnalysisRun: &kargoapi.AnalysisRunReference{
+							Name:      "fake-analysis-run",
+							Namespace: "fake-namespace",
+						},
+					},
+					newStatus.CurrentFreight.VerificationInfo,
+				)
+				require.Len(t, newStatus.CurrentFreight.VerificationHistory, 1)
+
+				// Phase should not be changed to Steady
+				require.Equal(t, kargoapi.StagePhaseVerifying, newStatus.Phase)
+				// Everything else should be unchanged
+				newStatus.Phase = initialStatus.Phase
+				newStatus.CurrentFreight = initialStatus.CurrentFreight
+				require.Equal(t, initialStatus, newStatus)
+			},
+		},
+
 		{
 			name: "verification aborted",
 			stage: &kargoapi.Stage{
@@ -609,8 +751,8 @@ func TestSyncNormalStage(t *testing.T) {
 				getVerificationInfoFn: func(
 					_ context.Context,
 					s *kargoapi.Stage,
-				) *kargoapi.VerificationInfo {
-					return s.Status.CurrentFreight.VerificationInfo
+				) (*kargoapi.VerificationInfo, error) {
+					return s.Status.CurrentFreight.VerificationInfo, nil
 				},
 				abortVerificationFn: func(
 					context.Context,
@@ -681,10 +823,10 @@ func TestSyncNormalStage(t *testing.T) {
 				getVerificationInfoFn: func(
 					_ context.Context,
 					s *kargoapi.Stage,
-				) *kargoapi.VerificationInfo {
+				) (*kargoapi.VerificationInfo, error) {
 					i := s.Status.CurrentFreight.VerificationInfo.DeepCopy()
 					i.Phase = kargoapi.VerificationPhaseError
-					return i
+					return i, nil
 				},
 				abortVerificationFn: func(
 					context.Context,
@@ -1172,7 +1314,13 @@ func TestSyncNormalStage(t *testing.T) {
 					Phase:            kargoapi.StagePhaseVerifying,
 					CurrentPromotion: &kargoapi.PromotionInfo{},
 					CurrentFreight: &kargoapi.FreightReference{
-						VerificationInfo: &kargoapi.VerificationInfo{},
+						VerificationInfo: &kargoapi.VerificationInfo{
+							Phase: kargoapi.VerificationPhasePending,
+							AnalysisRun: &kargoapi.AnalysisRunReference{
+								Name:      "fake-analysis-run",
+								Namespace: "fake-namespace",
+							},
+						},
 					},
 				},
 			},
@@ -1190,7 +1338,7 @@ func TestSyncNormalStage(t *testing.T) {
 				getVerificationInfoFn: func(
 					context.Context,
 					*kargoapi.Stage,
-				) *kargoapi.VerificationInfo {
+				) (*kargoapi.VerificationInfo, error) {
 					return &kargoapi.VerificationInfo{
 						Phase: kargoapi.VerificationPhaseSuccessful,
 						AnalysisRun: &kargoapi.AnalysisRunReference{
@@ -1198,7 +1346,7 @@ func TestSyncNormalStage(t *testing.T) {
 							Namespace: "fake-namespace",
 							Phase:     string(rollouts.AnalysisPhaseSuccessful),
 						},
-					}
+					}, nil
 				},
 				verifyFreightInStageFn: func(context.Context, string, string, string) error {
 					return nil
diff --git a/internal/controller/stages/verification_test.go b/internal/controller/stages/verification_test.go
index e44d6019..2f3e3adb 100644
--- a/internal/controller/stages/verification_test.go
+++ b/internal/controller/stages/verification_test.go
@@ -7,7 +7,9 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -22,12 +24,13 @@ func TestStartVerification(t *testing.T) {
 		name       string
 		stage      *kargoapi.Stage
 		reconciler *reconciler
-		assertions func(*testing.T, *kargoapi.VerificationInfo)
+		assertions func(*testing.T, *kargoapi.VerificationInfo, error)
 	}{
 		{
 			name:       "rollouts integration not enabled",
 			reconciler: &reconciler{},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.NoError(t, err)
 				require.Contains(
 					t,
 					vi.Message,
@@ -57,7 +60,8 @@ func TestStartVerification(t *testing.T) {
 					return errors.New("something went wrong")
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.ErrorContains(t, err, "something went wrong")
 				require.Contains(t, vi.Message, "something went wrong")
 				require.Contains(t, vi.Message, "error listing AnalysisRuns for Stage")
 			},
@@ -87,7 +91,8 @@ func TestStartVerification(t *testing.T) {
 					return nil
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.NoError(t, err)
 				require.NotNil(t, vi)
 				require.Empty(t, vi.Message)
 			},
@@ -160,7 +165,8 @@ func TestStartVerification(t *testing.T) {
 					return nil
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.NoError(t, err)
 				require.NotNilf(t, vi, "expected non-nil VerificationInfo")
 				require.NotEmptyf(t, vi.ID, "expected non-empty VerificationInfo.ID")
 				require.Equal(t, kargoapi.VerificationPhasePending, vi.Phase)
@@ -204,7 +210,8 @@ func TestStartVerification(t *testing.T) {
 					return nil, errors.New("something went wrong")
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.ErrorContains(t, err, "something went wrong")
 				require.NotNil(t, vi)
 				require.Contains(t, vi.Message, "something went wrong")
 				require.Contains(t, vi.Message, "error getting AnalysisTemplate")
@@ -244,7 +251,8 @@ func TestStartVerification(t *testing.T) {
 					return nil, nil
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.NoError(t, err)
 				require.NotNil(t, vi)
 				require.Contains(t, vi.Message, "AnalysisTemplate")
 				require.Contains(t, vi.Message, "not found")
@@ -291,7 +299,8 @@ func TestStartVerification(t *testing.T) {
 					return nil, fmt.Errorf("something went wrong")
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.ErrorContains(t, err, "something went wrong")
 				require.NotNil(t, vi)
 				require.Contains(t, vi.Message, "something went wrong")
 				require.Contains(t, vi.Message, "error getting Freight")
@@ -338,7 +347,8 @@ func TestStartVerification(t *testing.T) {
 					return nil, nil
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.NoError(t, err)
 				require.NotNil(t, vi)
 				require.Contains(t, vi.Message, "Freight")
 				require.Contains(t, vi.Message, "not found")
@@ -391,7 +401,8 @@ func TestStartVerification(t *testing.T) {
 					return nil, errors.New("something went wrong")
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.NoError(t, err)
 				require.NotNil(t, vi)
 				require.Contains(t, vi.Message, "something went wrong")
 				require.Contains(t, vi.Message, "error building AnalysisRun for Stage")
@@ -452,12 +463,76 @@ func TestStartVerification(t *testing.T) {
 					return errors.New("something went wrong")
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.ErrorContains(t, err, "something went wrong")
 				require.NotNil(t, vi)
 				require.Contains(t, vi.Message, "something went wrong")
 				require.Contains(t, vi.Message, "error creating AnalysisRun")
 			},
 		},
+		{
+			name: "validation error creating AnalysisRun",
+			stage: &kargoapi.Stage{
+				Spec: &kargoapi.StageSpec{
+					Verification: &kargoapi.Verification{
+						AnalysisTemplates: []kargoapi.AnalysisTemplateReference{{}},
+					},
+				},
+				Status: kargoapi.StageStatus{
+					CurrentFreight: &kargoapi.FreightReference{
+						Name: "fake-id",
+					},
+				},
+			},
+			reconciler: &reconciler{
+				cfg: ReconcilerConfig{
+					RolloutsIntegrationEnabled: true,
+				},
+				listAnalysisRunsFn: func(
+					context.Context,
+					client.ObjectList,
+					...client.ListOption,
+				) error {
+					return nil
+				},
+				getAnalysisTemplateFn: func(
+					context.Context,
+					client.Client,
+					types.NamespacedName,
+				) (*rollouts.AnalysisTemplate, error) {
+					return &rollouts.AnalysisTemplate{}, nil
+				},
+				getFreightFn: func(
+					context.Context,
+					client.Client,
+					types.NamespacedName,
+				) (*kargoapi.Freight, error) {
+					return &kargoapi.Freight{}, nil
+				},
+				buildAnalysisRunFn: func(
+					*kargoapi.Stage,
+					*kargoapi.Freight,
+					[]*rollouts.AnalysisTemplate,
+				) (*rollouts.AnalysisRun, error) {
+					return &rollouts.AnalysisRun{}, nil
+				},
+				createAnalysisRunFn: func(
+					context.Context,
+					client.Object,
+					...client.CreateOption,
+				) error {
+					return apierrors.NewInvalid(schema.GroupKind{
+						Group: kargoapi.GroupVersion.Group,
+						Kind:  "AnalysisRun",
+					}, "fake-name", nil)
+				},
+			},
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.NoError(t, err)
+				require.NotNil(t, vi)
+				require.Contains(t, vi.Message, "error creating AnalysisRun")
+			},
+		},
 		{
 			name: "success",
 			stage: &kargoapi.Stage{
@@ -518,7 +593,8 @@ func TestStartVerification(t *testing.T) {
 					return nil
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.NoError(t, err)
 				require.NotNilf(t, vi, "expected non-nil VerificationInfo")
 				require.NotEmptyf(t, vi.ID, "expected non-empty VerificationInfo.ID")
 				require.Equal(t, kargoapi.VerificationPhasePending, vi.Phase)
@@ -531,12 +607,14 @@ func TestStartVerification(t *testing.T) {
 	}
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
+			info, err := testCase.reconciler.startVerification(
+				context.Background(),
+				testCase.stage,
+			)
 			testCase.assertions(
 				t,
-				testCase.reconciler.startVerification(
-					context.Background(),
-					testCase.stage,
-				),
+				info,
+				err,
 			)
 		})
 	}
@@ -547,7 +625,7 @@ func TestGetVerificationInfo(t *testing.T) {
 		name       string
 		stage      *kargoapi.Stage
 		reconciler *reconciler
-		assertions func(*testing.T, *kargoapi.VerificationInfo)
+		assertions func(*testing.T, *kargoapi.VerificationInfo, error)
 	}{
 		{
 			name:       "rollouts integration not enabled",
@@ -559,7 +637,8 @@ func TestGetVerificationInfo(t *testing.T) {
 					},
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.NoError(t, err)
 				require.NotNil(t, vi)
 				require.Contains(
 					t,
@@ -595,7 +674,8 @@ func TestGetVerificationInfo(t *testing.T) {
 					return nil, errors.New("something went wrong")
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.ErrorContains(t, err, "something went wrong")
 				require.NotNil(t, vi)
 				require.Contains(t, vi.Message, "something went wrong")
 				require.Contains(t, vi.Message, "error getting AnalysisRun")
@@ -628,7 +708,8 @@ func TestGetVerificationInfo(t *testing.T) {
 					return nil, nil
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.NoError(t, err)
 				require.NotNil(t, vi)
 				require.Contains(t, vi.Message, "AnalysisRun")
 				require.Contains(t, vi.Message, "not found")
@@ -669,8 +750,8 @@ func TestGetVerificationInfo(t *testing.T) {
 					}, nil
 				},
 			},
-			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo) {
-				require.NotNil(t, vi.StartTime)
+			assertions: func(t *testing.T, vi *kargoapi.VerificationInfo, err error) {
+				require.NoError(t, err)
 				require.Equal(
 					t,
 					&kargoapi.VerificationInfo{
@@ -689,12 +770,14 @@ func TestGetVerificationInfo(t *testing.T) {
 	}
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
+			info, err := testCase.reconciler.getVerificationInfo(
+				context.Background(),
+				testCase.stage,
+			)
 			testCase.assertions(
 				t,
-				testCase.reconciler.getVerificationInfo(
-					context.Background(),
-					testCase.stage,
-				),
+				info,
+				err,
 			)
 		})
 	}
diff --git a/internal/kubeclient/errors_test.go b/internal/kubeclient/errors_test.go
new file mode 100644
index 00000000..2e1b3a14
--- /dev/null
+++ b/internal/kubeclient/errors_test.go
@@ -0,0 +1,49 @@
+package kubeclient
+
+import (
+	"errors"
+	"testing"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	kargoapi "github.com/akuity/kargo/api/v1alpha1"
+)
+
+func TestIgnoreInvalid(t *testing.T) {
+	testCases := []struct {
+		name    string
+		err     error
+		wantErr bool
+	}{
+		{
+			name:    "nil",
+			err:     nil,
+			wantErr: false,
+		},
+		{
+			name:    "arbitrary error",
+			err:     errors.New("not invalid"),
+			wantErr: true,
+		},
+		{
+			name: "invalid",
+			err: apierrors.NewInvalid(
+				schema.GroupKind{
+					Group: kargoapi.GroupVersion.Group,
+					Kind:  "Freight",
+				},
+				"freight",
+				nil,
+			),
+			wantErr: false,
+		},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			if err := IgnoreInvalid(testCase.err); (err != nil) != testCase.wantErr {
+				t.Errorf("IgnoreInvalid() error = %v, wantErr %v", err, testCase.wantErr)
+			}
+		})
+	}
+}