Repository: rancher/fleet
Pull request #2665 (2024-07-23 09:46:51)

Make Helm release garbage collection interval configurable
Refers to #2169.

This also adds end-to-end tests validating that old releases are cleaned up when updating a bundle's namespace or release name.



Patch:
diff --git a/.github/scripts/deploy-fleet.sh b/.github/scripts/deploy-fleet.sh
index 1acd08ce..00a8a247 100755
--- a/.github/scripts/deploy-fleet.sh
+++ b/.github/scripts/deploy-fleet.sh
@@ -66,6 +66,7 @@ eventually helm upgrade --install fleet charts/fleet \
   $shards_settings \
   --set-string extraEnv[0].name=EXPERIMENTAL_OCI_STORAGE \
   --set-string extraEnv[0].value=true \
+  --set garbageCollectionInterval=1s \
   --set debug=true --set debugLevel=1
 
 # wait for controller and agent rollout
diff --git a/charts/fleet-agent/templates/configmap.yaml b/charts/fleet-agent/templates/configmap.yaml
index f3e83a89..5b052e87 100644
--- a/charts/fleet-agent/templates/configmap.yaml
+++ b/charts/fleet-agent/templates/configmap.yaml
@@ -10,4 +10,7 @@ data:
       {{ end }}
       "clientID":"{{.Values.clientID}}",
       "agentTLSMode": "{{.Values.agentTLSMode}}"
+      {{ if .Values.garbageCollectionInterval }}
+      "garbageCollectionInterval": "{{.Values.garbageCollectionInterval}}"
+      {{ end }}
     }
diff --git a/charts/fleet-crd/templates/crds.yaml b/charts/fleet-crd/templates/crds.yaml
index f20173ca..235b4892 100644
--- a/charts/fleet-crd/templates/crds.yaml
+++ b/charts/fleet-crd/templates/crds.yaml
@@ -5656,6 +5656,10 @@ spec:
                       nullable: true
                       type: string
                   type: object
+                garbageCollectionInterval:
+                  description: GarbageCollectionInterval determines how often agents
+                    clean up obsolete Helm releases.
+                  type: string
                 namespace:
                   description: 'Namespace is the cluster namespace, it contains the
                     clusters service
diff --git a/charts/fleet/templates/configmap.yaml b/charts/fleet/templates/configmap.yaml
index 44774021..e2af6ca7 100644
--- a/charts/fleet/templates/configmap.yaml
+++ b/charts/fleet/templates/configmap.yaml
@@ -12,6 +12,9 @@ data:
       "apiServerCA": "{{b64enc .Values.apiServerCA}}",
       "agentCheckinInterval": "{{.Values.agentCheckinInterval}}",
       "agentTLSMode": "{{.Values.agentTLSMode}}",
+      {{ if .Values.garbageCollectionInterval }}
+      "garbageCollectionInterval": "{{.Values.garbageCollectionInterval}}",
+      {{ end }}
       "ignoreClusterRegistrationLabels": {{.Values.ignoreClusterRegistrationLabels}},
       "bootstrap": {
         "paths": "{{.Values.bootstrap.paths}}",
diff --git a/dev/setup-fleet b/dev/setup-fleet
index c0b5fa88..60895e83 100755
--- a/dev/setup-fleet
+++ b/dev/setup-fleet
@@ -42,6 +42,7 @@ helm -n cattle-fleet-system upgrade --install --create-namespace --wait --reset-
   $shards_settings \
   --set-string extraEnv[0].name=EXPERIMENTAL_OCI_STORAGE \
   --set-string extraEnv[0].value=true \
+  --set garbageCollectionInterval=1s \
   --set debug=true --set debugLevel=1 fleet charts/fleet
 
 # wait for controller and agent rollout
diff --git a/internal/cmd/agent/clusterstatus.go b/internal/cmd/agent/clusterstatus.go
index d8e5795c..6609b5cb 100644
--- a/internal/cmd/agent/clusterstatus.go
+++ b/internal/cmd/agent/clusterstatus.go
@@ -89,7 +89,7 @@ func (cs *ClusterStatus) Run(cmd *cobra.Command, args []string) error {
 
 	// cannot start without kubeconfig for upstream cluster
 	setupLog.Info("Fetching kubeconfig for upstream cluster from registration", "namespace", cs.Namespace)
-	agentInfo, err := register.Get(ctx, cs.Namespace, localConfig)
+	agentInfo, _, err := register.Get(ctx, cs.Namespace, localConfig)
 	if err != nil {
 		setupLog.Error(err, "failed to get kubeconfig from upstream cluster")
 		return err
diff --git a/internal/cmd/agent/deployer/cleanup/cleanup.go b/internal/cmd/agent/deployer/cleanup/cleanup.go
index 28ec23ce..d0df8564 100644
--- a/internal/cmd/agent/deployer/cleanup/cleanup.go
+++ b/internal/cmd/agent/deployer/cleanup/cleanup.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
+	"helm.sh/helm/v3/pkg/action"
 
 	"github.com/rancher/fleet/internal/helmdeployer"
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
@@ -26,26 +27,48 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
+type HelmDeployer interface {
+	NewListAction() *action.List
+	ListDeployments(list helmdeployer.ListAction) ([]helmdeployer.DeployedBundle, error)
+	DeleteRelease(ctx context.Context, deployed helmdeployer.DeployedBundle) error
+	Delete(ctx context.Context, name string) error
+}
+
 type Cleanup struct {
 	client           client.Client
 	fleetNamespace   string
 	defaultNamespace string
-	helmDeployer     *helmdeployer.Helm
+	helmDeployer     HelmDeployer
 	cleanupOnce      sync.Once
 
 	mapper meta.RESTMapper
 	// localDynamicClient is a dynamic client for the cluster the agent is running on (local cluster).
 	localDynamicClient *dynamic.DynamicClient
+
+	garbageCollectionInterval time.Duration
 }
 
-func New(upstream client.Client, mapper meta.RESTMapper, localDynamicClient *dynamic.DynamicClient, deployer *helmdeployer.Helm, fleetNamespace string, defaultNamespace string) *Cleanup {
+func New(
+	upstream client.Client,
+	mapper meta.RESTMapper,
+	localDynamicClient *dynamic.DynamicClient,
+	deployer HelmDeployer,
+	fleetNamespace string,
+	defaultNamespace string,
+	garbageCollectionInterval time.Duration,
+) *Cleanup {
+	if garbageCollectionInterval == 0 {
+		garbageCollectionInterval = durations.GarbageCollect
+	}
+
 	return &Cleanup{
-		client:             upstream,
-		mapper:             mapper,
-		localDynamicClient: localDynamicClient,
-		helmDeployer:       deployer,
-		fleetNamespace:     fleetNamespace,
-		defaultNamespace:   defaultNamespace,
+		client:                    upstream,
+		mapper:                    mapper,
+		localDynamicClient:        localDynamicClient,
+		helmDeployer:              deployer,
+		fleetNamespace:            fleetNamespace,
+		defaultNamespace:          defaultNamespace,
+		garbageCollectionInterval: garbageCollectionInterval,
 	}
 }
 
@@ -92,7 +115,7 @@ func (c *Cleanup) garbageCollect(ctx context.Context) {
 		select {
 		case <-ctx.Done():
 			return
-		case <-time.After(wait.Jitter(durations.GarbageCollect, 1.0)):
+		case <-time.After(wait.Jitter(c.garbageCollectionInterval, 1.0)):
 		}
 	}
 }
diff --git a/internal/cmd/agent/operator.go b/internal/cmd/agent/operator.go
index c2288f98..6348e425 100644
--- a/internal/cmd/agent/operator.go
+++ b/internal/cmd/agent/operator.go
@@ -13,6 +13,7 @@ import (
 	"github.com/rancher/fleet/internal/cmd/agent/deployer/monitor"
 	"github.com/rancher/fleet/internal/cmd/agent/register"
 	"github.com/rancher/fleet/internal/cmd/agent/trigger"
+	"github.com/rancher/fleet/internal/config"
 	"github.com/rancher/fleet/internal/helmdeployer"
 	"github.com/rancher/fleet/internal/manifest"
 	"github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
@@ -54,7 +55,7 @@ func start(ctx context.Context, localConfig *rest.Config, systemNamespace, agent
 	// Registration is done in an init container. If we are here, we are already registered.
 	// Retrieve the existing config from the registration.
 	// Cannot start without kubeconfig for upstream cluster:
-	upstreamConfig, fleetNamespace, clusterName, err := loadRegistration(ctx, systemNamespace, localConfig)
+	upstreamConfig, agentConfig, fleetNamespace, clusterName, err := loadRegistration(ctx, systemNamespace, localConfig)
 	if err != nil {
 		setupLog.Error(err, "unable to load registration and start manager")
 		return err
@@ -84,7 +85,16 @@ func start(ctx context.Context, localConfig *rest.Config, systemNamespace, agent
 	localCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	reconciler, err := newReconciler(ctx, localCtx, mgr, localConfig, systemNamespace, fleetNamespace, agentScope)
+	reconciler, err := newReconciler(
+		ctx,
+		localCtx,
+		mgr,
+		localConfig,
+		systemNamespace,
+		fleetNamespace,
+		agentScope,
+		agentConfig,
+	)
 	if err != nil {
 		setupLog.Error(err, "unable to set up bundledeployment reconciler")
 		return err
@@ -116,29 +126,48 @@ func start(ctx context.Context, localConfig *rest.Config, systemNamespace, agent
 	return nil
 }
 
-func loadRegistration(ctx context.Context, systemNamespace string, localConfig *rest.Config) (*rest.Config, string, string, error) {
-	agentInfo, err := register.Get(ctx, systemNamespace, localConfig)
+func loadRegistration(
+	ctx context.Context,
+	systemNamespace string,
+	localConfig *rest.Config,
+) (*rest.Config, config.Config, string, string, error) {
+	agentInfo, agentConfig, err := register.Get(ctx, systemNamespace, localConfig)
 	if err != nil {
 		setupLog.Error(err, "cannot get registration info for upstream cluster")
-		return nil, "", "", err
+		return nil, config.Config{}, "", "", err
 	}
 
 	// fleetNamespace is the upstream cluster namespace from AgentInfo, e.g. cluster-fleet-ID
 	fleetNamespace, _, err := agentInfo.ClientConfig.Namespace()
 	if err != nil {
 		setupLog.Error(err, "cannot get upstream namespace from registration")
-		return nil, "", "", err
+		return nil, config.Config{}, "", "", err
 	}
 
 	upstreamConfig, err := agentInfo.ClientConfig.ClientConfig()
 	if err != nil {
 		setupLog.Error(err, "cannot get upstream kubeconfig from registration")
-		return nil, "", "", err
+		return nil, config.Config{}, "", "", err
 	}
-	return upstreamConfig, fleetNamespace, agentInfo.ClusterName, nil
+
+	if agentConfig == nil {
+		setupLog.Error(err, "unexpected nil agent config")
+		return nil, config.Config{}, "", "", err
+	}
+
+	return upstreamConfig, *agentConfig, fleetNamespace, agentInfo.ClusterName, nil
 }
 
-func newReconciler(ctx, localCtx context.Context, mgr manager.Manager, localConfig *rest.Config, systemNamespace, fleetNamespace, agentScope string) (*controller.BundleDeploymentReconciler, error) {
+func newReconciler(
+	ctx context.Context,
+	localCtx context.Context,
+	mgr manager.Manager,
+	localConfig *rest.Config,
+	systemNamespace string,
+	fleetNamespace string,
+	agentScope string,
+	agentConfig config.Config,
+) (*controller.BundleDeploymentReconciler, error) {
 	upstreamClient := mgr.GetClient()
 
 	// Build client for local cluster
@@ -215,6 +244,7 @@ func newReconciler(ctx, localCtx context.Context, mgr manager.Manager, localConf
 		helmDeployer,
 		fleetNamespace,
 		defaultNamespace,
+		agentConfig.GarbageCollectionInterval.Duration,
 	)
 
 	return &controller.BundleDeploymentReconciler{
diff --git a/internal/cmd/agent/register/register.go b/internal/cmd/agent/register/register.go
index 48f7a4a5..9eac5626 100644
--- a/internal/cmd/agent/register/register.go
+++ b/internal/cmd/agent/register/register.go
@@ -50,18 +50,18 @@ type AgentInfo struct {
 // Get a registration token from the local cluster and fail if it does not exist.
 // This does not create a new registration process and only works after
 // registration has been completed.
-func Get(ctx context.Context, namespace string, cfg *rest.Config) (*AgentInfo, error) {
+func Get(ctx context.Context, namespace string, cfg *rest.Config) (*AgentInfo, *config.Config, error) {
 	cfg = rest.CopyConfig(cfg)
 	// disable the rate limiter
 	cfg.RateLimiter = ratelimit.None
 	k8s, err := core.NewFactoryFromConfig(cfg)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	agentConfig, err := config.Lookup(ctx, namespace, config.AgentConfigName, k8s.Core().V1().ConfigMap())
 	if err != nil {
-		return nil, fmt.Errorf(
+		return nil, nil, fmt.Errorf(
 			"failed to look up client config %s/%s: %w",
 			namespace,
 			config.AgentConfigName,
@@ -76,22 +76,22 @@ func Get(ctx context.Context, namespace string, cfg *rest.Config) (*AgentInfo, e
 	secret, err := k8s.Core().V1().Secret().Get(namespace, CredName, metav1.GetOptions{})
 	if apierrors.IsNotFound(err) {
 		logrus.Warn("Cannot find fleet-agent secret")
-		return nil, err
+		return nil, nil, err
 	} else if err != nil {
 		logrus.Error("Cannot get fleet-agent secret")
-		return nil, err
+		return nil, nil, err
 	}
 
 	clientConfig, err := clientcmd.NewClientConfigFromBytes(secret.Data[Kubeconfig])
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	return &AgentInfo{
 		ClusterNamespace: string(secret.Data[ClusterNamespace]),
 		ClusterName:      string(secret.Data[ClusterName]),
 		ClientConfig:     clientConfig,
-	}, nil
+	}, agentConfig, nil
 }
 
 // Register creates a fleet-agent secret with the upstream kubeconfig, by
diff --git a/internal/cmd/controller/agentmanagement/agent/config.go b/internal/cmd/controller/agentmanagement/agent/config.go
index c88983a8..a944d261 100644
--- a/internal/cmd/controller/agentmanagement/agent/config.go
+++ b/internal/cmd/controller/agentmanagement/agent/config.go
@@ -12,9 +12,10 @@ import (
 )
 
 type ConfigOptions struct {
-	Labels       map[string]string
-	ClientID     string
-	AgentTLSMode string
+	Labels                    map[string]string
+	ClientID                  string
+	AgentTLSMode              string
+	GarbageCollectionInterval metav1.Duration
 }
 
 func agentConfig(ctx context.Context, agentNamespace, controllerNamespace string, cg *client.Getter, opts *ConfigOptions) ([]runtime.Object, error) {
@@ -38,9 +39,10 @@ func agentConfig(ctx context.Context, agentNamespace, controllerNamespace string
 
 func configObjects(controllerNamespace string, co *ConfigOptions) ([]runtime.Object, error) {
 	cm, err := config.ToConfigMap(controllerNamespace, config.AgentConfigName, &config.Config{
-		Labels:       co.Labels,
-		ClientID:     co.ClientID,
-		AgentTLSMode: co.AgentTLSMode,
+		Labels:                    co.Labels,
+		ClientID:                  co.ClientID,
+		AgentTLSMode:              co.AgentTLSMode,
+		GarbageCollectionInterval: co.GarbageCollectionInterval,
 	})
 	if err != nil {
 		return nil, err
diff --git a/internal/cmd/controller/agentmanagement/controllers/cluster/import.go b/internal/cmd/controller/agentmanagement/controllers/cluster/import.go
index 4297fbb2..1df456a6 100644
--- a/internal/cmd/controller/agentmanagement/controllers/cluster/import.go
+++ b/internal/cmd/controller/agentmanagement/controllers/cluster/import.go
@@ -97,7 +97,8 @@ func (i *importHandler) onConfig(config *config.Config) error {
 
 		hasConfigChanged := config.APIServerURL != cluster.Status.APIServerURL ||
 			hashStatusField(config.APIServerCA) != cluster.Status.APIServerCAHash ||
-			config.AgentTLSMode != cluster.Status.AgentTLSMode
+			config.AgentTLSMode != cluster.Status.AgentTLSMode ||
+			hasGarbageCollectionIntervalChanged(config, cluster)
 
 		if hasConfigChanged {
 			logrus.Infof("API server config changed, trigger cluster import for cluster %s/%s", cluster.Namespace, cluster.Name)
@@ -314,9 +315,10 @@ func (i *importHandler) importCluster(cluster *fleet.Cluster, status fleet.Clust
 			APIServerCA:  apiServerCA,
 			APIServerURL: apiServerURL,
 			ConfigOptions: agent.ConfigOptions{
-				ClientID:     cluster.Spec.ClientID,
-				Labels:       clusterLabels,
-				AgentTLSMode: cfg.AgentTLSMode,
+				ClientID:                  cluster.Spec.ClientID,
+				Labels:                    clusterLabels,
+				AgentTLSMode:              cfg.AgentTLSMode,
+				GarbageCollectionInterval: cfg.GarbageCollectionInterval,
 			},
 			ManifestOptions: agent.ManifestOptions{
 				AgentEnvVars:     cluster.Spec.AgentEnvVars,
@@ -385,6 +387,7 @@ func (i *importHandler) importCluster(cluster *fleet.Cluster, status fleet.Clust
 	status.APIServerURL = apiServerURL
 	status.APIServerCAHash = hashStatusField(apiServerCA)
 	status.AgentTLSMode = cfg.AgentTLSMode
+	status.GarbageCollectionInterval = &cfg.GarbageCollectionInterval
 
 	return status, nil
 }
@@ -432,3 +435,9 @@ func (i *importHandler) restConfigFromKubeConfig(data []byte, agentTLSMode strin
 
 	return clientcmd.NewDefaultClientConfig(raw, &clientcmd.ConfigOverrides{}).ClientConfig()
 }
+
+func hasGarbageCollectionIntervalChanged(config *config.Config, cluster fleet.Cluster) bool {
+	return (config.GarbageCollectionInterval.Duration != 0 && cluster.Status.GarbageCollectionInterval == nil) ||
+		(cluster.Status.GarbageCollectionInterval != nil &&
+			config.GarbageCollectionInterval.Duration != cluster.Status.GarbageCollectionInterval.Duration)
+}
diff --git a/internal/config/config.go b/internal/config/config.go
index d7f92fe9..98fd05e5 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -113,6 +113,9 @@ type Config struct {
 	// canceling the request.  Used to retrieve the latest commit of configured
 	// git repositories.
 	GitClientTimeout metav1.Duration `json:"gitClientTimeout,omitempty"`
+
+	// GarbageCollectionInterval determines how often agents clean up obsolete Helm releases.
+	GarbageCollectionInterval metav1.Duration `json:"garbageCollectionInterval,omitempty"`
 }
 
 type Bootstrap struct {
diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go
index f7ee1c32..e7c58886 100644
--- a/internal/manifest/manifest.go
+++ b/internal/manifest/manifest.go
@@ -109,10 +109,10 @@ func (m *Manifest) ID() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return toSHA256ID(shasum), nil
+	return ToSHA256ID(shasum), nil
 }
 
-// toSHA256ID generates a valid Kubernetes name (max length of 64) from a provided SHA256 sum
-func toSHA256ID(shasum string) string {
+// ToSHA256ID generates a valid Kubernetes name (max length of 64) from a provided SHA256 sum
+func ToSHA256ID(shasum string) string {
 	return ("s-" + shasum)[:63]
 }
diff --git a/internal/manifest/store.go b/internal/manifest/store.go
index f5a29996..77f36ef9 100644
--- a/internal/manifest/store.go
+++ b/internal/manifest/store.go
@@ -14,12 +14,12 @@ import (
 
 func NewStore(client client.Client) *ContentStore {
 	return &ContentStore{
-		client: client,
+		Client: client,
 	}
 }
 
 type ContentStore struct {
-	client client.Client
+	Client client.Client
 }
 
 // Store stores the manifest as a content resource.
@@ -30,7 +30,7 @@ func (c *ContentStore) Store(ctx context.Context, manifest *Manifest) error {
 		return err
 	}
 
-	if err := c.client.Get(ctx, types.NamespacedName{Name: id}, &fleet.Content{}); err != nil && !apierrors.IsNotFound(err) {
+	if err := c.Client.Get(ctx, types.NamespacedName{Name: id}, &fleet.Content{}); err != nil && !apierrors.IsNotFound(err) {
 		return err
 	} else if err == nil {
 		return nil
@@ -55,7 +55,7 @@ func (c *ContentStore) createContents(ctx context.Context, id string, manifest *
 		return err
 	}
 
-	err = c.client.Create(ctx, &fleet.Content{
+	err = c.Client.Create(ctx, &fleet.Content{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: id,
 		},
diff --git a/internal/mocks/helm_deployer_mock.go b/internal/mocks/helm_deployer_mock.go
new file mode 100644
index 00000000..d27da382
--- /dev/null
+++ b/internal/mocks/helm_deployer_mock.go
@@ -0,0 +1,99 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/rancher/fleet/internal/cmd/agent/deployer/cleanup (interfaces: HelmDeployer)
+//
+// Generated by this command:
+//
+//	mockgen --build_flags=--mod=mod -destination=../../../../mocks/helm_deployer_mock.go -package=mocks github.com/rancher/fleet/internal/cmd/agent/deployer/cleanup HelmDeployer
+//
+
+// Package mocks is a generated GoMock package.
+package mocks
+
+import (
+	context "context"
+	reflect "reflect"
+
+	helmdeployer "github.com/rancher/fleet/internal/helmdeployer"
+	gomock "go.uber.org/mock/gomock"
+	action "helm.sh/helm/v3/pkg/action"
+)
+
+// MockHelmDeployer is a mock of HelmDeployer interface.
+type MockHelmDeployer struct {
+	ctrl     *gomock.Controller
+	recorder *MockHelmDeployerMockRecorder
+}
+
+// MockHelmDeployerMockRecorder is the mock recorder for MockHelmDeployer.
+type MockHelmDeployerMockRecorder struct {
+	mock *MockHelmDeployer
+}
+
+// NewMockHelmDeployer creates a new mock instance.
+func NewMockHelmDeployer(ctrl *gomock.Controller) *MockHelmDeployer {
+	mock := &MockHelmDeployer{ctrl: ctrl}
+	mock.recorder = &MockHelmDeployerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockHelmDeployer) EXPECT() *MockHelmDeployerMockRecorder {
+	return m.recorder
+}
+
+// Delete mocks base method.
+func (m *MockHelmDeployer) Delete(arg0 context.Context, arg1 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Delete", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Delete indicates an expected call of Delete.
+func (mr *MockHelmDeployerMockRecorder) Delete(arg0, arg1 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockHelmDeployer)(nil).Delete), arg0, arg1)
+}
+
+// DeleteRelease mocks base method.
+func (m *MockHelmDeployer) DeleteRelease(arg0 context.Context, arg1 helmdeployer.DeployedBundle) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteRelease", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DeleteRelease indicates an expected call of DeleteRelease.
+func (mr *MockHelmDeployerMockRecorder) DeleteRelease(arg0, arg1 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRelease", reflect.TypeOf((*MockHelmDeployer)(nil).DeleteRelease), arg0, arg1)
+}
+
+// ListDeployments mocks base method.
+func (m *MockHelmDeployer) ListDeployments(arg0 helmdeployer.ListAction) ([]helmdeployer.DeployedBundle, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListDeployments", arg0)
+	ret0, _ := ret[0].([]helmdeployer.DeployedBundle)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListDeployments indicates an expected call of ListDeployments.
+func (mr *MockHelmDeployerMockRecorder) ListDeployments(arg0 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDeployments", reflect.TypeOf((*MockHelmDeployer)(nil).ListDeployments), arg0)
+}
+
+// NewListAction mocks base method.
+func (m *MockHelmDeployer) NewListAction() *action.List {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NewListAction")
+	ret0, _ := ret[0].(*action.List)
+	return ret0
+}
+
+// NewListAction indicates an expected call of NewListAction.
+func (mr *MockHelmDeployerMockRecorder) NewListAction() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewListAction", reflect.TypeOf((*MockHelmDeployer)(nil).NewListAction))
+}
diff --git a/pkg/apis/fleet.cattle.io/v1alpha1/cluster_types.go b/pkg/apis/fleet.cattle.io/v1alpha1/cluster_types.go
index dc1ed40c..aa428459 100644
--- a/pkg/apis/fleet.cattle.io/v1alpha1/cluster_types.go
+++ b/pkg/apis/fleet.cattle.io/v1alpha1/cluster_types.go
@@ -214,6 +214,9 @@ type ClusterStatus struct {
 	Display ClusterDisplay `json:"display,omitempty"`
 	// AgentStatus contains information about the agent.
 	Agent AgentStatus `json:"agent,omitempty"`
+
+	// GarbageCollectionInterval determines how often agents clean up obsolete Helm releases.
+	GarbageCollectionInterval *metav1.Duration `json:"garbageCollectionInterval,omitempty"`
 }
 
 type ClusterDisplay struct {
diff --git a/pkg/apis/fleet.cattle.io/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/fleet.cattle.io/v1alpha1/zz_generated.deepcopy.go
index e64aa508..3ca16b6f 100644
--- a/pkg/apis/fleet.cattle.io/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/fleet.cattle.io/v1alpha1/zz_generated.deepcopy.go
@@ -1093,6 +1093,11 @@ func (in *ClusterStatus) DeepCopyInto(out *ClusterStatus) {
 	}
 	out.Display = in.Display
 	in.Agent.DeepCopyInto(&out.Agent)
+	if in.GarbageCollectionInterval != nil {
+		in, out := &in.GarbageCollectionInterval, &out.GarbageCollectionInterval
+		*out = new(v1.Duration)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterStatus.


Test patch:
diff --git a/e2e/assets/gitrepo/gitrepo.yaml b/e2e/assets/gitrepo/gitrepo.yaml
index 8d07f5b7..00f74ed4 100644
--- a/e2e/assets/gitrepo/gitrepo.yaml
+++ b/e2e/assets/gitrepo/gitrepo.yaml
@@ -6,6 +6,8 @@ spec:
   repo: {{.Repo}}
   branch: {{.Branch}}
   pollingInterval: {{.PollingInterval}}
+  {{- if .TargetNamespace }}
   targetNamespace: {{.TargetNamespace}}
+  {{- end }}
   paths:
   - examples
diff --git a/e2e/assets/helm/repo/with-release-name/config-chart/Chart.yaml b/e2e/assets/helm/repo/with-release-name/config-chart/Chart.yaml
new file mode 100644
index 00000000..7ba744f9
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/config-chart/Chart.yaml
@@ -0,0 +1,6 @@
+apiVersion: v2
+name: config-chart
+description: A test chart that verifies its config
+type: application
+version: 0.1.0
+appVersion: "1.16.0"
diff --git a/e2e/assets/helm/repo/with-release-name/config-chart/templates/configmap.yaml b/e2e/assets/helm/repo/with-release-name/config-chart/templates/configmap.yaml
new file mode 100644
index 00000000..340a61bf
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/config-chart/templates/configmap.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: test-simple-chart-config
+data:
+  test: "value123"
+  name: {{ .Values.name }}
diff --git a/e2e/assets/helm/repo/with-release-name/config-chart/values.yaml b/e2e/assets/helm/repo/with-release-name/config-chart/values.yaml
new file mode 100644
index 00000000..3266b4da
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/config-chart/values.yaml
@@ -0,0 +1,1 @@
+name: default-name
diff --git a/e2e/assets/helm/repo/with-release-name/crds-only/crds-chart/Chart.yaml b/e2e/assets/helm/repo/with-release-name/crds-only/crds-chart/Chart.yaml
new file mode 100644
index 00000000..b6f09293
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/crds-only/crds-chart/Chart.yaml
@@ -0,0 +1,6 @@
+apiVersion: v2
+name: crds-chart
+description: A test chart that only contains CRDs
+type: application
+version: 0.1.0
+appVersion: "1.16.0"
diff --git a/e2e/assets/helm/repo/with-release-name/crds-only/crds-chart/templates/crds.yaml b/e2e/assets/helm/repo/with-release-name/crds-only/crds-chart/templates/crds.yaml
new file mode 100644
index 00000000..4c87bb28
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/crds-only/crds-chart/templates/crds.yaml
@@ -0,0 +1,41 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  name: foobars.crd.test
+spec:
+  group: crd.test
+  names:
+    kind: Foobar
+    listKind: FoobarList
+    plural: foobars
+    singular: foobar
+  scope: Namespaced
+  versions:
+    - name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: 'Foobar is a dummy resource.'
+          properties:
+            apiVersion:
+              description: 'APIVersion is always necessary.'
+              type: string
+            foo:
+              description: 'Foo is important.'
+              nullable: true
+              type: string
+            bar:
+              description: 'Bar does not walk into a Foo. A Foo walks into a bar.'
+              nullable: true
+              type: string
+            kind:
+              description: 'Kindness is everything.'
+              type: string
+            metadata:
+              type: object
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
diff --git a/e2e/assets/helm/repo/with-release-name/crds-only/fleet.yaml b/e2e/assets/helm/repo/with-release-name/crds-only/fleet.yaml
new file mode 100644
index 00000000..7de353d4
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/crds-only/fleet.yaml
@@ -0,0 +1,5 @@
+helm:
+  chart: crds-chart
+  values:
+    name: example-value
+  releaseName: test-config-release
diff --git a/e2e/assets/helm/repo/with-release-name/fleet.yaml b/e2e/assets/helm/repo/with-release-name/fleet.yaml
new file mode 100644
index 00000000..b1464bd8
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/fleet.yaml
@@ -0,0 +1,5 @@
+helm:
+  chart: config-chart
+  values:
+    name: example-value
+  releaseName: test-config-release
diff --git a/e2e/assets/helm/repo/with-release-name/no-crds/config-chart/Chart.yaml b/e2e/assets/helm/repo/with-release-name/no-crds/config-chart/Chart.yaml
new file mode 100644
index 00000000..7ba744f9
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/no-crds/config-chart/Chart.yaml
@@ -0,0 +1,6 @@
+apiVersion: v2
+name: config-chart
+description: A test chart that verifies its config
+type: application
+version: 0.1.0
+appVersion: "1.16.0"
diff --git a/e2e/assets/helm/repo/with-release-name/no-crds/config-chart/templates/configmap.yaml b/e2e/assets/helm/repo/with-release-name/no-crds/config-chart/templates/configmap.yaml
new file mode 100644
index 00000000..340a61bf
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/no-crds/config-chart/templates/configmap.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: test-simple-chart-config
+data:
+  test: "value123"
+  name: {{ .Values.name }}
diff --git a/e2e/assets/helm/repo/with-release-name/no-crds/config-chart/values.yaml b/e2e/assets/helm/repo/with-release-name/no-crds/config-chart/values.yaml
new file mode 100644
index 00000000..3266b4da
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/no-crds/config-chart/values.yaml
@@ -0,0 +1,1 @@
+name: default-name
diff --git a/e2e/assets/helm/repo/with-release-name/no-crds/fleet.yaml b/e2e/assets/helm/repo/with-release-name/no-crds/fleet.yaml
new file mode 100644
index 00000000..b1464bd8
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/no-crds/fleet.yaml
@@ -0,0 +1,5 @@
+helm:
+  chart: config-chart
+  values:
+    name: example-value
+  releaseName: test-config-release
diff --git a/e2e/assets/helm/repo/with-release-name/no-crds/values.yaml b/e2e/assets/helm/repo/with-release-name/no-crds/values.yaml
new file mode 100644
index 00000000..b0a9f2b3
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/no-crds/values.yaml
@@ -0,0 +1,1 @@
+url: global.fleet.clusterLabels.name
diff --git a/e2e/assets/helm/repo/with-release-name/values.yaml b/e2e/assets/helm/repo/with-release-name/values.yaml
new file mode 100644
index 00000000..b0a9f2b3
--- /dev/null
+++ b/e2e/assets/helm/repo/with-release-name/values.yaml
@@ -0,0 +1,1 @@
+url: global.fleet.clusterLabels.name
diff --git a/e2e/single-cluster/release_cleanup_test.go b/e2e/single-cluster/release_cleanup_test.go
new file mode 100644
index 00000000..e7b6e228
--- /dev/null
+++ b/e2e/single-cluster/release_cleanup_test.go
@@ -0,0 +1,223 @@
+package singlecluster_test
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"github.com/go-git/go-git/v5"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/rancher/fleet/e2e/testenv"
+	"github.com/rancher/fleet/e2e/testenv/githelper"
+	"github.com/rancher/fleet/e2e/testenv/kubectl"
+)
+
+var _ = Describe("Monitoring Helm releases along GitRepo/bundle namespace updates", Ordered, func() {
+	var (
+		k            kubectl.Command
+		r            = rand.New(rand.NewSource(GinkgoRandomSeed()))
+		oldNamespace string
+		newNamespace string
+		gitrepoName  string
+		path         string
+	)
+
+	BeforeAll(func() {
+		k = env.Kubectl.Namespace(env.Namespace)
+		oldNamespace = testenv.NewNamespaceName("target", r)
+		newNamespace = testenv.NewNamespaceName("target", r)
+		gitrepoName = "releases-cleanup"
+		path = "simple-chart"
+
+		err := testenv.CreateGitRepo(
+			k,
+			oldNamespace,
+			gitrepoName,
+			"master",
+			"",
+			path,
+		)
+		Expect(err).ToNot(HaveOccurred())
+
+		// Check for existence of Helm release as the only release in the target namespace
+		checkRelease(oldNamespace, fmt.Sprintf("%s-%s", gitrepoName, path))
+	})
+
+	AfterAll(func() {
+		out, err := k.Delete("gitrepo", gitrepoName)
+		Expect(err).ToNot(HaveOccurred(), out)
+
+		out, err = k.Delete("ns", oldNamespace, "--wait=false")
+		Expect(err).ToNot(HaveOccurred(), out)
+
+		out, err = k.Delete("ns", newNamespace, "--wait=false")
+		Expect(err).ToNot(HaveOccurred(), out)
+	})
+
+	When("updating a bundle's namespace", func() {
+		It("creates a new release in the new namespace", func() {
+			out, err := k.Patch(
+				"gitrepo",
+				gitrepoName,
+				"--type=merge",
+				"-p",
+				fmt.Sprintf(`{"spec":{"targetNamespace":"%s"}}`, newNamespace),
+			)
+			Expect(err).ToNot(HaveOccurred(), out)
+
+			checkRelease(newNamespace, fmt.Sprintf("%s-%s", gitrepoName, path))
+		})
+
+		It("deletes the old release in the previous namespace", func() {
+			Eventually(func(g Gomega) {
+				cmd := exec.Command("helm", "list", "-q", "-n", oldNamespace)
+				out, err := cmd.CombinedOutput()
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(string(out)).To(BeEmpty())
+			}).Should(Succeed())
+		})
+	})
+})
+
+var _ = Describe("Monitoring Helm releases along GitRepo/bundle release name updates", Ordered, Label("infra-setup"), func() {
+	var (
+		tmpDir           string
+		clonedir         string
+		k                kubectl.Command
+		gh               *githelper.Git
+		clone            *git.Repository
+		repoName         string
+		inClusterRepoURL string
+		gitrepoName      string
+		r                = rand.New(rand.NewSource(GinkgoRandomSeed()))
+		targetNamespace  string
+		oldReleaseName   string
+		newReleaseName   string
+		assetPath        string
+	)
+
+	BeforeEach(func() {
+		k = env.Kubectl.Namespace(env.Namespace)
+		oldReleaseName = "test-config-release"
+		newReleaseName = "new-test-config-release"
+		repoName = "repo"
+
+		DeferCleanup(func() {
+			out, err := k.Delete("gitrepo", gitrepoName)
+			Expect(err).ToNot(HaveOccurred(), out)
+
+			if targetNamespace != "" {
+				out, err = k.Delete("ns", targetNamespace, "--wait=false")
+				Expect(err).ToNot(HaveOccurred(), out)
+			}
+
+			_ = os.RemoveAll(tmpDir)
+		})
+	})
+
+	JustBeforeEach(func() {
+		// Build git repo URL reachable _within_ the cluster, for the GitRepo
+		host, err := githelper.BuildGitHostname(env.Namespace)
+		Expect(err).ToNot(HaveOccurred())
+
+		addr, err := githelper.GetExternalRepoAddr(env, port, repoName)
+		Expect(err).ToNot(HaveOccurred())
+		gh = githelper.NewHTTP(addr)
+
+		inClusterRepoURL = gh.GetInClusterURL(host, port, repoName)
+
+		tmpDir, _ = os.MkdirTemp("", "fleet-")
+		clonedir = path.Join(tmpDir, repoName)
+
+		gitrepoName = testenv.RandomFilename("releases-test", r)
+
+		err = testenv.ApplyTemplate(k, testenv.AssetPath("gitrepo/gitrepo.yaml"), struct {
+			Name            string
+			Repo            string
+			Branch          string
+			PollingInterval string
+			TargetNamespace string
+		}{
+			gitrepoName,
+			inClusterRepoURL,
+			gh.Branch,
+			"15s",           // default
+			targetNamespace, // to avoid conflicts with other tests
+		})
+		Expect(err).ToNot(HaveOccurred())
+
+		clone, err = gh.Create(clonedir, testenv.AssetPath(assetPath), "examples")
+		Expect(err).ToNot(HaveOccurred())
+
+		// Check for existence of Helm release with old name in the target namespace
+		checkRelease(targetNamespace, oldReleaseName)
+	})
+
+	When("updating the release name of a bundle containing upstream Kubernetes resources", func() {
+		BeforeEach(func() {
+			assetPath = "helm/repo/with-release-name/no-crds"
+			targetNamespace = testenv.NewNamespaceName("target", r)
+		})
+		It("replaces the release", func() {
+			By("updating the release name in fleet.yaml")
+			// Not adding `takeOwnership: true` leads to errors from Helm checking its own annotations and failing to
+			// deploy the new release because the config map already exists and belongs to the old one.
+			replace(
+				path.Join(clonedir, "examples", "fleet.yaml"),
+				oldReleaseName,
+				fmt.Sprintf("%s\n  takeOwnership: true", newReleaseName),
+			)
+
+			_, err := gh.Update(clone)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("creating a release with the new name")
+			checkRelease(targetNamespace, newReleaseName)
+		})
+	})
+
+	When("updating the release name of a bundle containing only CRDs", func() {
+		BeforeEach(func() {
+			assetPath = "helm/repo/with-release-name/crds-only"
+			targetNamespace = ""
+			DeferCleanup(func() {
+				out, err := k.Delete("crd", "foobars.crd.test")
+				Expect(err).ToNot(HaveOccurred(), out)
+			})
+		})
+		It("replaces the release", func() {
+			By("updating the release name in fleet.yaml")
+			replace(
+				path.Join(clonedir, "examples", "fleet.yaml"),
+				oldReleaseName,
+				fmt.Sprintf("%s\n  takeOwnership: true", newReleaseName),
+			)
+
+			_, err := gh.Update(clone)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("creating a release with the new name")
+			checkRelease("default", newReleaseName)
+		})
+	})
+})
+
+// checkRelease validates that namespace eventually contains a single release named releaseName.
+func checkRelease(namespace, releaseName string) {
+	var releases []string
+	Eventually(func(g Gomega) {
+		cmd := exec.Command("helm", "list", "-q", "-n", namespace)
+		out, err := cmd.CombinedOutput()
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(string(out)).ToNot(BeEmpty())
+
+		releases = strings.Split(strings.TrimSpace(string(out)), "\n")
+
+		g.Expect(len(releases)).To(Equal(1), strings.Join(releases, ","))
+		g.Expect(releases[0]).To(Equal(releaseName))
+	}).Should(Succeed())
+}
diff --git a/integrationtests/agent/suite_test.go b/integrationtests/agent/suite_test.go
index f7e5195a..e4256941 100644
--- a/integrationtests/agent/suite_test.go
+++ b/integrationtests/agent/suite_test.go
@@ -185,7 +185,9 @@ func newReconciler(ctx context.Context, mgr manager.Manager, lookup *lookup) *co
 		localDynamic,
 		helmDeployer,
 		fleetNamespace,
-		defaultNamespace)
+		defaultNamespace,
+		0,
+	)
 
 	return &controller.BundleDeploymentReconciler{
 		Client: upstreamClient,
diff --git a/internal/cmd/agent/deployer/cleanup/cleanup_test.go b/internal/cmd/agent/deployer/cleanup/cleanup_test.go
new file mode 100644
index 00000000..ba561394
--- /dev/null
+++ b/internal/cmd/agent/deployer/cleanup/cleanup_test.go
@@ -0,0 +1,89 @@
+//go:generate mockgen --build_flags=--mod=mod -destination=../../../../mocks/helm_deployer_mock.go -package=mocks github.com/rancher/fleet/internal/cmd/agent/deployer/cleanup HelmDeployer
+
+package cleanup
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/rancher/fleet/internal/helmdeployer"
+	"github.com/rancher/fleet/internal/mocks"
+	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
+	"go.uber.org/mock/gomock"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+func TestCleanupReleases(t *testing.T) {
+	fleetNS := "foo"   // Used to get bundle deployments by bundle ID
+	defaultNS := "bar" // Used to compute the expected release key
+
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	deployedBundles := []helmdeployer.DeployedBundle{
+		{
+			BundleID:    "ID1",
+			ReleaseName: fmt.Sprintf("%s/TestRelease1", defaultNS),
+		},
+		{
+			BundleID:    "ID2",
+			ReleaseName: fmt.Sprintf("%s/TestRelease2", defaultNS),
+		},
+		{
+			BundleID:    "ID3",
+			ReleaseName: fmt.Sprintf("%s/TestRelease3", defaultNS),
+		},
+	}
+
+	mockClient := mocks.NewMockClient(mockCtrl)
+	bd := &fleet.BundleDeployment{}
+	mockClient.EXPECT().Get(gomock.Any(), types.NamespacedName{Namespace: fleetNS, Name: "ID1"}, bd).DoAndReturn(
+		func(_ context.Context, _ types.NamespacedName, bd *fleet.BundleDeployment, _ ...interface{}) error {
+			bd.Spec.Options.TargetNamespace = defaultNS
+			bd.Spec.Options.Helm = &fleet.HelmOptions{
+				ReleaseName: "TestRelease1", // will be kept
+			}
+
+			return nil
+		},
+	)
+
+	mockClient.EXPECT().Get(gomock.Any(), types.NamespacedName{Namespace: fleetNS, Name: "ID2"}, bd).DoAndReturn(
+		func(_ context.Context, _ types.NamespacedName, bd *fleet.BundleDeployment, _ ...interface{}) error {
+			bd.Spec.Options.TargetNamespace = defaultNS
+			bd.Spec.Options.Helm = &fleet.HelmOptions{
+				ReleaseName: "TestRelease2-old", // will be deleted
+			}
+
+			return nil
+		},
+	)
+
+	mockClient.EXPECT().Get(gomock.Any(), types.NamespacedName{Namespace: fleetNS, Name: "ID3"}, bd).DoAndReturn(
+		func(_ context.Context, _ types.NamespacedName, bd *fleet.BundleDeployment, _ ...interface{}) error {
+			bd.Spec.Options.TargetNamespace = defaultNS + "-old" // will be deleted
+			bd.Spec.Options.Helm = &fleet.HelmOptions{
+				ReleaseName: "TestRelease3",
+			}
+
+			return nil
+		},
+	)
+
+	mockHelmDeployer := mocks.NewMockHelmDeployer(mockCtrl)
+	mockHelmDeployer.EXPECT().NewListAction()
+	mockHelmDeployer.EXPECT().ListDeployments(gomock.Any()).Return(deployedBundles, nil)
+	mockHelmDeployer.EXPECT().DeleteRelease(gomock.Any(), deployedBundles[1]).Return(nil)
+	mockHelmDeployer.EXPECT().DeleteRelease(gomock.Any(), deployedBundles[2]).Return(nil)
+
+	cleanup := New(mockClient, nil, nil, mockHelmDeployer, fleetNS, defaultNS, 1*time.Second)
+
+	err := cleanup.cleanup(context.Background(), log.FromContext(context.Background()).WithName("test"))
+
+	if err != nil {
+		t.Errorf("cleanup failed: %v", err)
+	}
+}
diff --git a/internal/manifest/store_test.go b/internal/manifest/store_test.go
index e012bf54..94097206 100644
--- a/internal/manifest/store_test.go
+++ b/internal/manifest/store_test.go
@@ -1,4 +1,4 @@
-package manifest
+package manifest_test
 
 import (
 	"bytes"
@@ -11,14 +11,15 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 
+	"github.com/rancher/fleet/internal/manifest"
 	"github.com/rancher/fleet/internal/mocks"
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 )
 
-func testManifest(t *testing.T, data string) *Manifest {
+func testManifest(t *testing.T, data string) *manifest.Manifest {
 	t.Helper()
 
-	m, err := FromJSON([]byte(data), "")
+	m, err := manifest.FromJSON([]byte(data), "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -35,7 +36,7 @@ func Test_contentStore_Store(t *testing.T) {
 	resources := `{"resources": [{"name": "foo", "content": "bar"}]}`
 	checksum := "752ebbb975f52eea5e87950ef2ca5de4055de3c68a17f54d94527d7fd79c21fd"
 	type args struct {
-		manifest *Manifest
+		manifest *manifest.Manifest
 		cached   bool
 	}
 	tests := []struct {
@@ -48,7 +49,7 @@ func Test_contentStore_Store(t *testing.T) {
 			args: args{
 				manifest: testManifest(t, resources),
 			},
-			want: toSHA256ID(checksum),
+			want: manifest.ToSHA256ID(checksum),
 		},
 		{
 			name: "existing manifest",
@@ -56,7 +57,7 @@ func Test_contentStore_Store(t *testing.T) {
 				manifest: testManifest(t, resources),
 				cached:   true,
 			},
-			want: toSHA256ID(checksum),
+			want: manifest.ToSHA256ID(checksum),
 		},
 	}
 	for _, tt := range tests {
@@ -64,7 +65,7 @@ func Test_contentStore_Store(t *testing.T) {
 			ctrl := gomock.NewController(t)
 			client := mocks.NewMockClient(ctrl)
 
-			store := &ContentStore{client}
+			store := &manifest.ContentStore{client}
 			ctx := context.TODO()
 			nsn := types.NamespacedName{Name: tt.want}