Repository: argoproj-labs/applicationset (go)

Issue:
Applicationset-controller overloads api-server and have memory leak
We have only 1 ApplicationSet and it kills our k8s control-plane.

ArgoCD v1.8.7+eb3d1fb

applicationset-controller memory usage
<img width="2283" alt="applicationset-controller" src="https://user-images.githubusercontent.com/11005895/110884353-a90d1900-82ed-11eb-83f8-fce8c9115d8c.png">

api-server go-routines
<img width="2287" alt="api-server" src="https://user-images.githubusercontent.com/11005895/110884648-15881800-82ee-11eb-8ed1-83ccfe8fdc4d.png">

applicationset-controller logs
```
time="2021-03-12T01:07:25Z" level=info msg="Starting configmap/secret informers"
time="2021-03-12T01:07:25Z" level=info msg="Configmap/secret informer synced"
time="2021-03-12T01:07:25Z" level=info msg="matched cluster secret" cluster=argocd-eu-west-1
time="2021-03-12T01:07:25Z" level=info msg="matched cluster secret" cluster=argocd-us-east-1
time="2021-03-12T01:07:25Z" level=info msg="generated 2 applications" generator="&{0xc000848f00 0xc00012c000 0xc0007abce0 devops}"
time="2021-03-12T01:07:25Z" level=info msg="unchanged Application" app=eu-west-1-web-autotests appSet=web-autotests
2021-03-12T01:07:25.915Z	DEBUG	controller-runtime.manager.events	Normal	{"object": {"kind":"ApplicationSet","namespace":"devops","name":"web-autotests","uid":"a0e6349b-1920-4b8c-8a1d-dc713b04af3f","apiVersion":"argoproj.io/v1alpha1","resourceVersion":"66469235"}, "reason": "unchanged", "message": "unchanged Application \"eu-west-1-web-autotests\""}
time="2021-03-12T01:07:25Z" level=info msg="unchanged Application" app=us-east-1-web-autotests appSet=web-autotests
time="2021-03-12T01:07:25Z" level=info msg="end reconcile" requeueAfter=0s
time="2021-03-12T01:07:25Z" level=info msg="Starting configmap/secret informers"
2021-03-12T01:07:25.916Z	DEBUG	controller-runtime.manager.events	Normal	{"object": {"kind":"ApplicationSet","namespace":"devops","name":"web-autotests","uid":"a0e6349b-1920-4b8c-8a1d-dc713b04af3f","apiVersion":"argoproj.io/v1alpha1","resourceVersion":"66469235"}, "reason": "unchanged", "message": "unchanged Application \"us-east-1-web-autotests\""}
time="2021-03-12T01:07:26Z" level=info msg="Configmap/secret informer synced"
time="2021-03-12T01:07:26Z" level=info msg="matched cluster secret" cluster=argocd-us-east-1
time="2021-03-12T01:07:26Z" level=info msg="matched cluster secret" cluster=argocd-eu-west-1
time="2021-03-12T01:07:26Z" level=info msg="generated 2 applications" generator="&{0xc000848f00 0xc00012c000 0xc0007abce0 devops}"
time="2021-03-12T01:07:26Z" level=info msg="unchanged Application" app=us-east-1-web-autotests appSet=web-autotests
2021-03-12T01:07:26.124Z	DEBUG	controller-runtime.manager.events	Normal	{"object": {"kind":"ApplicationSet","namespace":"devops","name":"web-autotests","uid":"a0e6349b-1920-4b8c-8a1d-dc713b04af3f","apiVersion":"argoproj.io/v1alpha1","resourceVersion":"66469235"}, "reason": "unchanged", "message": "unchanged Application \"us-east-1-web-autotests\""}
time="2021-03-12T01:07:26Z" level=info msg="unchanged Application" app=eu-west-1-web-autotests appSet=web-autotests
time="2021-03-12T01:07:26Z" level=info msg="end reconcile" requeueAfter=0s
2021-03-12T01:07:26.125Z	DEBUG	controller-runtime.manager.events	Normal	{"object": {"kind":"ApplicationSet","namespace":"devops","name":"web-autotests","uid":"a0e6349b-1920-4b8c-8a1d-dc713b04af3f","apiVersion":"argoproj.io/v1alpha1","resourceVersion":"66469235"}, "reason": "unchanged", "message": "unchanged Application \"eu-west-1-web-autotests\""}


time="2021-03-12T01:10:25Z" level=info msg="Starting configmap/secret informers"
time="2021-03-12T01:10:25Z" level=info msg="Configmap/secret informer synced"
time="2021-03-12T01:10:25Z" level=info msg="matched cluster secret" cluster=argocd-eu-west-1
time="2021-03-12T01:10:25Z" level=info msg="matched cluster secret" cluster=argocd-us-east-1
time="2021-03-12T01:10:25Z" level=info msg="generated 2 applications" generator="&{0xc000848f00 0xc00012c000 0xc0007abce0 devops}"
time="2021-03-12T01:10:25Z" level=info msg="unchanged Application" app=eu-west-1-web-autotests appSet=web-autotests
2021-03-12T01:10:25.907Z	DEBUG	controller-runtime.manager.events	Normal	{"object": {"kind":"ApplicationSet","namespace":"devops","name":"web-autotests","uid":"a0e6349b-1920-4b8c-8a1d-dc713b04af3f","apiVersion":"argoproj.io/v1alpha1","resourceVersion":"66469235"}, "reason": "unchanged", "message": "unchanged Application \"eu-west-1-web-autotests\""}
time="2021-03-12T01:10:25Z" level=info msg="unchanged Application" app=us-east-1-web-autotests appSet=web-autotests
2021-03-12T01:10:25.908Z	DEBUG	controller-runtime.manager.events	Normal	{"object": {"kind":"ApplicationSet","namespace":"devops","name":"web-autotests","uid":"a0e6349b-1920-4b8c-8a1d-dc713b04af3f","apiVersion":"argoproj.io/v1alpha1","resourceVersion":"66469235"}, "reason": "unchanged", "message": "unchanged Application \"us-east-1-web-autotests\""}
time="2021-03-12T01:10:25Z" level=info msg="end reconcile" requeueAfter=0s
time="2021-03-12T01:10:25Z" level=info msg="Starting configmap/secret informers"
time="2021-03-12T01:10:26Z" level=info msg="Configmap/secret informer synced"
time="2021-03-12T01:10:26Z" level=info msg="matched cluster secret" cluster=argocd-eu-west-1
time="2021-03-12T01:10:26Z" level=info msg="matched cluster secret" cluster=argocd-us-east-1
time="2021-03-12T01:10:26Z" level=info msg="generated 2 applications" generator="&{0xc000848f00 0xc00012c000 0xc0007abce0 devops}"
time="2021-03-12T01:10:26Z" level=info msg="unchanged Application" app=eu-west-1-web-autotests appSet=web-autotests
2021-03-12T01:10:26.016Z	DEBUG	controller-runtime.manager.events	Normal	{"object": {"kind":"ApplicationSet","namespace":"devops","name":"web-autotests","uid":"a0e6349b-1920-4b8c-8a1d-dc713b04af3f","apiVersion":"argoproj.io/v1alpha1","resourceVersion":"66469235"}, "reason": "unchanged", "message": "unchanged Application \"eu-west-1-web-autotests\""}
time="2021-03-12T01:10:26Z" level=info msg="unchanged Application" app=us-east-1-web-autotests appSet=web-autotests
2021-03-12T01:10:26.017Z	DEBUG	controller-runtime.manager.events	Normal	{"object": {"kind":"ApplicationSet","namespace":"devops","name":"web-autotests","uid":"a0e6349b-1920-4b8c-8a1d-dc713b04af3f","apiVersion":"argoproj.io/v1alpha1","resourceVersion":"66469235"}, "reason": "unchanged", "message": "unchanged Application \"us-east-1-web-autotests\""}
time="2021-03-12T01:10:26Z" level=info msg="end reconcile" requeueAfter=0s
```


Pull request:
Applicationset-controller overloads api-server and have memory leak
Fixes #153 

This was caused by our creating a SettingsManager (SettingsManager in settings.go in Argo CD) with every call of the ClusterGenerator. Since the settings manager contains a client and 2 listers, each instance was opening sockets and multiple goroutines to handle resource caching with the k8s server. The socket and these connections were what were occupying the leaked memory:
```
# From SettingsManager
	clientset  kubernetes.Interface
	secrets    v1listers.SecretLister
	configmaps v1listers.ConfigMapLister
```

The fix is:
- Create a single instance of SettingsManager (which is what Argo CD does), and query that instance on each cluster generator call. 
- I've also added Argo's  stat logger, which outputs the memory usage and # of goroutines every 10 minutes, which should allow us to catch similar issues in the future.


Patch:
diff --git a/main.go b/main.go
index 33fa32e..6adda40 100644
--- a/main.go
+++ b/main.go
@@ -21,6 +21,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 
@@ -39,6 +40,7 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 
 	appclientset "github.com/argoproj/argo-cd/pkg/client/clientset/versioned"
+	"github.com/argoproj/pkg/stats"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -150,6 +152,9 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "ApplicationSet")
 		os.Exit(1)
 	}
+
+	stats.StartStatsTicker(10 * time.Minute)
+
 	// +kubebuilder:scaffold:builder
 
 	setupLog.Info("Starting manager")
diff --git a/pkg/controllers/applicationset_controller.go b/pkg/controllers/applicationset_controller.go
index 01fdcc5..a9549fb 100644
--- a/pkg/controllers/applicationset_controller.go
+++ b/pkg/controllers/applicationset_controller.go
@@ -81,6 +81,11 @@ func (r *ApplicationSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	// Do not attempt to further reconcile the ApplicationSet if it is being deleted.
+	if applicationSetInfo.ObjectMeta.DeletionTimestamp != nil {
+		return ctrl.Result{}, nil
+	}
+
 	// Log a warning if there are unrecognized generators
 	checkInvalidGenerators(&applicationSetInfo)
 
@@ -407,21 +412,36 @@ func (r *ApplicationSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
 }
 
 // createOrUpdateInCluster will create / update application resources in the cluster.
-// For new application it will call create
-// For application that need to update it will call update
-// The function also adds owner reference to all applications, and uses it for delete them.
+// - For new applications, it will call create
+// - For existing application, it will call update
+// The function also adds owner reference to all applications, and uses it to delete them.
 func (r *ApplicationSetReconciler) createOrUpdateInCluster(ctx context.Context, applicationSet argoprojiov1alpha1.ApplicationSet, desiredApplications []argov1alpha1.Application) error {
 
 	var firstError error
-	//create or updates the application in appList
-	for _, app := range desiredApplications {
-		appLog := log.WithFields(log.Fields{"app": app.Name, "appSet": applicationSet.Name})
-		app.Namespace = applicationSet.Namespace
+	// Creates or updates the application in appList
+	for _, generatedApp := range desiredApplications {
+
+		appLog := log.WithFields(log.Fields{"app": generatedApp.Name, "appSet": applicationSet.Name})
+		generatedApp.Namespace = applicationSet.Namespace
+
+		found := &argov1alpha1.Application{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      generatedApp.Name,
+				Namespace: generatedApp.Namespace,
+			},
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "Application",
+				APIVersion: "argoproj.io/v1alpha1",
+			},
+		}
 
-		found := app
-		action, err := utils.CreateOrUpdate(ctx, r.Client, &found, func() error {
-			found.Spec = app.Spec
-			return controllerutil.SetControllerReference(&applicationSet, &found, r.Scheme)
+		action, err := utils.CreateOrUpdate(ctx, r.Client, found, func() error {
+			// Copy only the Application/ObjectMeta fields that are significant, from the generatedApp
+			found.Spec = generatedApp.Spec
+			found.ObjectMeta.Annotations = generatedApp.Annotations
+			found.ObjectMeta.Finalizers = generatedApp.Finalizers
+			found.ObjectMeta.Labels = generatedApp.Labels
+			return controllerutil.SetControllerReference(&applicationSet, found, r.Scheme)
 		})
 
 		if err != nil {
@@ -432,7 +452,7 @@ func (r *ApplicationSetReconciler) createOrUpdateInCluster(ctx context.Context,
 			continue
 		}
 
-		r.Recorder.Eventf(&applicationSet, core.EventTypeNormal, fmt.Sprint(action), "%s Application %q", action, app.Name)
+		r.Recorder.Eventf(&applicationSet, core.EventTypeNormal, fmt.Sprint(action), "%s Application %q", action, generatedApp.Name)
 		appLog.Logf(log.InfoLevel, "%s Application", action)
 	}
 	return firstError
diff --git a/pkg/generators/cluster.go b/pkg/generators/cluster.go
index 01aec27..be89a57 100644
--- a/pkg/generators/cluster.go
+++ b/pkg/generators/cluster.go
@@ -30,15 +30,22 @@ type ClusterGenerator struct {
 	ctx       context.Context
 	clientset kubernetes.Interface
 	// namespace is the Argo CD namespace
-	namespace string
+	namespace       string
+	settingsManager *settings.SettingsManager
+	argoDB          db.ArgoDB
 }
 
 func NewClusterGenerator(c client.Client, ctx context.Context, clientset kubernetes.Interface, namespace string) Generator {
+
+	settingsManager := settings.NewSettingsManager(ctx, clientset, namespace)
+
 	g := &ClusterGenerator{
-		Client:    c,
-		ctx:       ctx,
-		clientset: clientset,
-		namespace: namespace,
+		Client:          c,
+		ctx:             ctx,
+		clientset:       clientset,
+		namespace:       namespace,
+		settingsManager: settingsManager,
+		argoDB:          db.NewDB(namespace, settingsManager, clientset),
 	}
 	return g
 }
@@ -67,9 +74,7 @@ func (g *ClusterGenerator) GenerateParams(
 	ignoreLocalClusters := len(appSetGenerator.Clusters.Selector.MatchExpressions) > 0 || len(appSetGenerator.Clusters.Selector.MatchLabels) > 0
 
 	// ListCluster from Argo CD's util/db package will include the local cluster in the list of clusters
-	settingsMgr := settings.NewSettingsManager(g.ctx, g.clientset, g.namespace)
-	newDB := db.NewDB(g.namespace, settingsMgr, g.clientset)
-	clustersFromArgoCD, err := newDB.ListClusters(g.ctx)
+	clustersFromArgoCD, err := g.argoDB.ListClusters(g.ctx)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/utils/createOrUpdate.go b/pkg/utils/createOrUpdate.go
index 7d60281..57c94ff 100644
--- a/pkg/utils/createOrUpdate.go
+++ b/pkg/utils/createOrUpdate.go
@@ -15,9 +15,21 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
-// CreateOrUpdate overrides "sigs.k8s.io/controller-runtime" function to add equality for argov1alpha1.ApplicationDestination
-// argov1alpha1.ApplicationDestination has a private variable, so the default implementation fails to compare it
+// CreateOrUpdate overrides "sigs.k8s.io/controller-runtime" function
+// in sigs.k8s.io/controller-runtime/pkg/controller/controllerutil/controllerutil.go
+// to add equality for argov1alpha1.ApplicationDestination
+// argov1alpha1.ApplicationDestination has a private variable, so the default
+// implementation fails to compare it.
+//
+// CreateOrUpdate creates or updates the given object in the Kubernetes
+// cluster. The object's desired state must be reconciled with the existing
+// state inside the passed in callback MutateFn.
+//
+// The MutateFn is called regardless of creating or updating an object.
+//
+// It returns the executed operation and an error.
 func CreateOrUpdate(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) {
+
 	key := client.ObjectKeyFromObject(obj)
 	if err := c.Get(ctx, key, obj); err != nil {
 		if !errors.IsNotFound(err) {


Test patch:
diff --git a/pkg/controllers/applicationset_controller_test.go b/pkg/controllers/applicationset_controller_test.go
index 8ad98d1..21c399d 100644
--- a/pkg/controllers/applicationset_controller_test.go
+++ b/pkg/controllers/applicationset_controller_test.go
@@ -302,20 +302,27 @@ func TestCreateOrUpdateInCluster(t *testing.T) {
 	assert.Nil(t, err)
 
 	for _, c := range []struct {
-		appSet     argoprojiov1alpha1.ApplicationSet
-		existsApps []argov1alpha1.Application
-		apps       []argov1alpha1.Application
-		expected   []argov1alpha1.Application
+		// name is human-readable test name
+		name string
+		// appSet is the ApplicationSet we are generating resources for
+		appSet argoprojiov1alpha1.ApplicationSet
+		// existingApps are the apps that already exist on the cluster
+		existingApps []argov1alpha1.Application
+		// desiredApps are the generated apps to create/update
+		desiredApps []argov1alpha1.Application
+		// expected is what we expect the cluster Applications to look like, after createOrUpdateInCluster
+		expected []argov1alpha1.Application
 	}{
 		{
+			name: "Create an app that doesn't exist",
 			appSet: argoprojiov1alpha1.ApplicationSet{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "name",
 					Namespace: "namespace",
 				},
 			},
-			existsApps: nil,
-			apps: []argov1alpha1.Application{
+			existingApps: nil,
+			desiredApps: []argov1alpha1.Application{
 				{
 					ObjectMeta: metav1.ObjectMeta{
 						Name: "app1",
@@ -337,6 +344,7 @@ func TestCreateOrUpdateInCluster(t *testing.T) {
 			},
 		},
 		{
+			name: "Update an existing app with a different project name",
 			appSet: argoprojiov1alpha1.ApplicationSet{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "name",
@@ -350,7 +358,7 @@ func TestCreateOrUpdateInCluster(t *testing.T) {
 					},
 				},
 			},
-			existsApps: []argov1alpha1.Application{
+			existingApps: []argov1alpha1.Application{
 				{
 					TypeMeta: metav1.TypeMeta{
 						Kind:       "Application",
@@ -366,7 +374,7 @@ func TestCreateOrUpdateInCluster(t *testing.T) {
 					},
 				},
 			},
-			apps: []argov1alpha1.Application{
+			desiredApps: []argov1alpha1.Application{
 				{
 					ObjectMeta: metav1.ObjectMeta{
 						Name: "app1",
@@ -394,6 +402,7 @@ func TestCreateOrUpdateInCluster(t *testing.T) {
 			},
 		},
 		{
+			name: "Create a new app and check it doesn't replace the existing app",
 			appSet: argoprojiov1alpha1.ApplicationSet{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "name",
@@ -407,7 +416,7 @@ func TestCreateOrUpdateInCluster(t *testing.T) {
 					},
 				},
 			},
-			existsApps: []argov1alpha1.Application{
+			existingApps: []argov1alpha1.Application{
 				{
 					TypeMeta: metav1.TypeMeta{
 						Kind:       "Application",
@@ -423,7 +432,7 @@ func TestCreateOrUpdateInCluster(t *testing.T) {
 					},
 				},
 			},
-			apps: []argov1alpha1.Application{
+			desiredApps: []argov1alpha1.Application{
 				{
 					ObjectMeta: metav1.ObjectMeta{
 						Name: "app2",
@@ -450,39 +459,316 @@ func TestCreateOrUpdateInCluster(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "Ensure that labels and annotations are added (via update) into an exiting application",
+			appSet: argoprojiov1alpha1.ApplicationSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "name",
+					Namespace: "namespace",
+				},
+				Spec: argoprojiov1alpha1.ApplicationSetSpec{
+					Template: argoprojiov1alpha1.ApplicationSetTemplate{
+						Spec: argov1alpha1.ApplicationSpec{
+							Project: "project",
+						},
+					},
+				},
+			},
+			existingApps: []argov1alpha1.Application{
+				{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "Application",
+						APIVersion: "argoproj.io/v1alpha1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name:            "app1",
+						Namespace:       "namespace",
+						ResourceVersion: "2",
+					},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project: "project",
+					},
+				},
+			},
+			desiredApps: []argov1alpha1.Application{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:        "app1",
+						Labels:      map[string]string{"label-key": "label-value"},
+						Annotations: map[string]string{"annot-key": "annot-value"},
+					},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project: "project",
+					},
+				},
+			},
+			expected: []argov1alpha1.Application{
+				{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "Application",
+						APIVersion: "argoproj.io/v1alpha1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name:            "app1",
+						Namespace:       "namespace",
+						Labels:          map[string]string{"label-key": "label-value"},
+						Annotations:     map[string]string{"annot-key": "annot-value"},
+						ResourceVersion: "3",
+					},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project: "project",
+					},
+				},
+			},
+		},
+		{
+			name: "Ensure that labels and annotations are removed from an existing app",
+			appSet: argoprojiov1alpha1.ApplicationSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "name",
+					Namespace: "namespace",
+				},
+				Spec: argoprojiov1alpha1.ApplicationSetSpec{
+					Template: argoprojiov1alpha1.ApplicationSetTemplate{
+						Spec: argov1alpha1.ApplicationSpec{
+							Project: "project",
+						},
+					},
+				},
+			},
+			existingApps: []argov1alpha1.Application{
+				{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "Application",
+						APIVersion: "argoproj.io/v1alpha1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name:            "app1",
+						Namespace:       "namespace",
+						ResourceVersion: "2",
+						Labels:          map[string]string{"label-key": "label-value"},
+						Annotations:     map[string]string{"annot-key": "annot-value"},
+					},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project: "project",
+					},
+				},
+			},
+			desiredApps: []argov1alpha1.Application{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "app1",
+					},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project: "project",
+					},
+				},
+			},
+			expected: []argov1alpha1.Application{
+				{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "Application",
+						APIVersion: "argoproj.io/v1alpha1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name:            "app1",
+						Namespace:       "namespace",
+						ResourceVersion: "3",
+					},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project: "project",
+					},
+				},
+			},
+		},
+		{
+			name: "Ensure that status and operation fields are not overriden by an update, when removing labels/annotations",
+			appSet: argoprojiov1alpha1.ApplicationSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "name",
+					Namespace: "namespace",
+				},
+				Spec: argoprojiov1alpha1.ApplicationSetSpec{
+					Template: argoprojiov1alpha1.ApplicationSetTemplate{
+						Spec: argov1alpha1.ApplicationSpec{
+							Project: "project",
+						},
+					},
+				},
+			},
+			existingApps: []argov1alpha1.Application{
+				{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "Application",
+						APIVersion: "argoproj.io/v1alpha1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name:            "app1",
+						Namespace:       "namespace",
+						ResourceVersion: "2",
+						Labels:          map[string]string{"label-key": "label-value"},
+						Annotations:     map[string]string{"annot-key": "annot-value"},
+					},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project: "project",
+					},
+					Status: argov1alpha1.ApplicationStatus{
+						Resources: []argov1alpha1.ResourceStatus{{Name: "sample-name"}},
+					},
+					Operation: &argov1alpha1.Operation{
+						Sync: &argov1alpha1.SyncOperation{Revision: "sample-revision"},
+					},
+				},
+			},
+			desiredApps: []argov1alpha1.Application{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "app1",
+					},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project: "project",
+					},
+				},
+			},
+			expected: []argov1alpha1.Application{
+				{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "Application",
+						APIVersion: "argoproj.io/v1alpha1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name:            "app1",
+						Namespace:       "namespace",
+						ResourceVersion: "3",
+					},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project: "project",
+					},
+					Status: argov1alpha1.ApplicationStatus{
+						Resources: []argov1alpha1.ResourceStatus{{Name: "sample-name"}},
+					},
+					Operation: &argov1alpha1.Operation{
+						Sync: &argov1alpha1.SyncOperation{Revision: "sample-revision"},
+					},
+				},
+			},
+		},
+		{
+			name: "Ensure that status and operation fields are not overriden by an update, when removing labels/annotations and adding other fields",
+			appSet: argoprojiov1alpha1.ApplicationSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "name",
+					Namespace: "namespace",
+				},
+				Spec: argoprojiov1alpha1.ApplicationSetSpec{
+					Template: argoprojiov1alpha1.ApplicationSetTemplate{
+						Spec: argov1alpha1.ApplicationSpec{
+							Project:     "project",
+							Source:      argov1alpha1.ApplicationSource{Path: "path", TargetRevision: "revision", RepoURL: "repoURL"},
+							Destination: argov1alpha1.ApplicationDestination{Server: "server", Namespace: "namespace"},
+						},
+					},
+				},
+			},
+			existingApps: []argov1alpha1.Application{
+				{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "Application",
+						APIVersion: "argoproj.io/v1alpha1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name:            "app1",
+						Namespace:       "namespace",
+						ResourceVersion: "2",
+					},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project: "project",
+					},
+					Status: argov1alpha1.ApplicationStatus{
+						Resources: []argov1alpha1.ResourceStatus{{Name: "sample-name"}},
+					},
+					Operation: &argov1alpha1.Operation{
+						Sync: &argov1alpha1.SyncOperation{Revision: "sample-revision"},
+					},
+				},
+			},
+			desiredApps: []argov1alpha1.Application{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:        "app1",
+						Labels:      map[string]string{"label-key": "label-value"},
+						Annotations: map[string]string{"annot-key": "annot-value"},
+					},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project:     "project",
+						Source:      argov1alpha1.ApplicationSource{Path: "path", TargetRevision: "revision", RepoURL: "repoURL"},
+						Destination: argov1alpha1.ApplicationDestination{Server: "server", Namespace: "namespace"},
+					},
+				},
+			},
+			expected: []argov1alpha1.Application{
+				{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "Application",
+						APIVersion: "argoproj.io/v1alpha1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name:            "app1",
+						Namespace:       "namespace",
+						Labels:          map[string]string{"label-key": "label-value"},
+						Annotations:     map[string]string{"annot-key": "annot-value"},
+						ResourceVersion: "3",
+					},
+					Spec: argov1alpha1.ApplicationSpec{
+						Project:     "project",
+						Source:      argov1alpha1.ApplicationSource{Path: "path", TargetRevision: "revision", RepoURL: "repoURL"},
+						Destination: argov1alpha1.ApplicationDestination{Server: "server", Namespace: "namespace"},
+					},
+					Status: argov1alpha1.ApplicationStatus{
+						Resources: []argov1alpha1.ResourceStatus{{Name: "sample-name"}},
+					},
+					Operation: &argov1alpha1.Operation{
+						Sync: &argov1alpha1.SyncOperation{Revision: "sample-revision"},
+					},
+				},
+			},
+		},
 	} {
-		initObjs := []client.Object{&c.appSet}
-		for _, a := range c.existsApps {
-			err = controllerutil.SetControllerReference(&c.appSet, &a, scheme)
-			assert.Nil(t, err)
-			initObjs = append(initObjs, &a)
-		}
 
-		client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build()
+		t.Run(c.name, func(t *testing.T) {
 
-		r := ApplicationSetReconciler{
-			Client:   client,
-			Scheme:   scheme,
-			Recorder: record.NewFakeRecorder(len(initObjs) + len(c.expected)),
-		}
+			initObjs := []client.Object{&c.appSet}
 
-		err = r.createOrUpdateInCluster(context.TODO(), c.appSet, c.apps)
-		assert.Nil(t, err)
+			for _, a := range c.existingApps {
+				err = controllerutil.SetControllerReference(&c.appSet, &a, scheme)
+				assert.Nil(t, err)
+				initObjs = append(initObjs, &a)
+			}
 
-		for _, obj := range c.expected {
-			got := &argov1alpha1.Application{}
-			_ = client.Get(context.Background(), crtclient.ObjectKey{
-				Namespace: obj.Namespace,
-				Name:      obj.Name,
-			}, got)
+			client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build()
 
-			err = controllerutil.SetControllerReference(&c.appSet, &obj, r.Scheme)
+			r := ApplicationSetReconciler{
+				Client:   client,
+				Scheme:   scheme,
+				Recorder: record.NewFakeRecorder(len(initObjs) + len(c.expected)),
+			}
+
+			err = r.createOrUpdateInCluster(context.TODO(), c.appSet, c.desiredApps)
 			assert.Nil(t, err)
 
-			assert.Equal(t, obj, *got)
-		}
-	}
+			for _, obj := range c.expected {
+				got := &argov1alpha1.Application{}
+				_ = client.Get(context.Background(), crtclient.ObjectKey{
+					Namespace: obj.Namespace,
+					Name:      obj.Name,
+				}, got)
 
+				err = controllerutil.SetControllerReference(&c.appSet, &obj, r.Scheme)
+				assert.Nil(t, err)
+				assert.Equal(t, obj, *got)
+			}
+		})
+	}
 }
 
 func TestCreateApplications(t *testing.T) {
diff --git a/test/e2e/applicationset/applicationset_test.go b/test/e2e/applicationset/applicationset_test.go
index 0319981..51adf54 100644
--- a/test/e2e/applicationset/applicationset_test.go
+++ b/test/e2e/applicationset/applicationset_test.go
@@ -18,8 +18,9 @@ func TestSimpleListGenerator(t *testing.T) {
 			APIVersion: "argoproj.io/v1alpha1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "my-cluster-guestbook",
-			Namespace: utils.ArgoCDNamespace,
+			Name:       "my-cluster-guestbook",
+			Namespace:  utils.ArgoCDNamespace,
+			Finalizers: []string{"resources-finalizer.argocd.argoproj.io"},
 		},
 		Spec: argov1alpha1.ApplicationSpec{
 			Project: "default",
@@ -35,6 +36,7 @@ func TestSimpleListGenerator(t *testing.T) {
 		},
 	}
 	var expectedAppNewNamespace *argov1alpha1.Application
+	var expectedAppNewMetadata *argov1alpha1.Application
 
 	Given(t).
 		// Create a ListGenerator-based ApplicationSet
@@ -79,9 +81,21 @@ func TestSimpleListGenerator(t *testing.T) {
 			appset.Spec.Template.Spec.Destination.Namespace = "guestbook2"
 		}).Then().Expect(ApplicationsExist([]argov1alpha1.Application{*expectedAppNewNamespace})).
 
+		// Update the metadata fields in the appset template, and make sure it propagates to the apps
+		When().
+		And(func() {
+			expectedAppNewMetadata = expectedAppNewNamespace.DeepCopy()
+			expectedAppNewMetadata.ObjectMeta.Annotations = map[string]string{"annotation-key": "annotation-value"}
+			expectedAppNewMetadata.ObjectMeta.Labels = map[string]string{"label-key": "label-value"}
+		}).
+		Update(func(appset *v1alpha1.ApplicationSet) {
+			appset.Spec.Template.Annotations = map[string]string{"annotation-key": "annotation-value"}
+			appset.Spec.Template.Labels = map[string]string{"label-key": "label-value"}
+		}).Then().Expect(ApplicationsExist([]argov1alpha1.Application{*expectedAppNewMetadata})).
+
 		// Delete the ApplicationSet, and verify it deletes the Applications
 		When().
-		Delete().Then().Expect(ApplicationsDoNotExist([]argov1alpha1.Application{*expectedAppNewNamespace}))
+		Delete().Then().Expect(ApplicationsDoNotExist([]argov1alpha1.Application{*expectedAppNewMetadata}))
 
 }
 
@@ -93,8 +107,9 @@ func TestSimpleGitDirectoryGenerator(t *testing.T) {
 				APIVersion: "argoproj.io/v1alpha1",
 			},
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      name,
-				Namespace: utils.ArgoCDNamespace,
+				Name:       name,
+				Namespace:  utils.ArgoCDNamespace,
+				Finalizers: []string{"resources-finalizer.argocd.argoproj.io"},
 			},
 			Spec: argov1alpha1.ApplicationSpec{
 				Project: "default",
@@ -118,6 +133,7 @@ func TestSimpleGitDirectoryGenerator(t *testing.T) {
 	}
 
 	var expectedAppsNewNamespace []argov1alpha1.Application
+	var expectedAppsNewMetadata []argov1alpha1.Application
 
 	Given(t).
 		When().
@@ -169,6 +185,21 @@ func TestSimpleGitDirectoryGenerator(t *testing.T) {
 			appset.Spec.Template.Spec.Destination.Namespace = "guestbook2"
 		}).Then().Expect(ApplicationsExist(expectedAppsNewNamespace)).
 
+		// Update the metadata fields in the appset template, and make sure it propagates to the apps
+		When().
+		And(func() {
+			for _, expectedApp := range expectedAppsNewNamespace {
+				expectedAppNewMetadata := expectedApp.DeepCopy()
+				expectedAppNewMetadata.ObjectMeta.Annotations = map[string]string{"annotation-key": "annotation-value"}
+				expectedAppNewMetadata.ObjectMeta.Labels = map[string]string{"label-key": "label-value"}
+				expectedAppsNewMetadata = append(expectedAppsNewMetadata, *expectedAppNewMetadata)
+			}
+		}).
+		Update(func(appset *v1alpha1.ApplicationSet) {
+			appset.Spec.Template.Annotations = map[string]string{"annotation-key": "annotation-value"}
+			appset.Spec.Template.Labels = map[string]string{"label-key": "label-value"}
+		}).Then().Expect(ApplicationsExist(expectedAppsNewMetadata)).
+
 		// Delete the ApplicationSet, and verify it deletes the Applications
 		When().
 		Delete().Then().Expect(ApplicationsDoNotExist(expectedAppsNewNamespace))
@@ -183,8 +214,9 @@ func TestSimpleGitFilesGenerator(t *testing.T) {
 				APIVersion: "argoproj.io/v1alpha1",
 			},
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      name,
-				Namespace: utils.ArgoCDNamespace,
+				Name:       name,
+				Namespace:  utils.ArgoCDNamespace,
+				Finalizers: []string{"resources-finalizer.argocd.argoproj.io"},
 			},
 			Spec: argov1alpha1.ApplicationSpec{
 				Project: "default",
@@ -207,6 +239,7 @@ func TestSimpleGitFilesGenerator(t *testing.T) {
 	}
 
 	var expectedAppsNewNamespace []argov1alpha1.Application
+	var expectedAppsNewMetadata []argov1alpha1.Application
 
 	Given(t).
 		When().
@@ -258,6 +291,21 @@ func TestSimpleGitFilesGenerator(t *testing.T) {
 			appset.Spec.Template.Spec.Destination.Namespace = "guestbook2"
 		}).Then().Expect(ApplicationsExist(expectedAppsNewNamespace)).
 
+		// Update the metadata fields in the appset template, and make sure it propagates to the apps
+		When().
+		And(func() {
+			for _, expectedApp := range expectedAppsNewNamespace {
+				expectedAppNewMetadata := expectedApp.DeepCopy()
+				expectedAppNewMetadata.ObjectMeta.Annotations = map[string]string{"annotation-key": "annotation-value"}
+				expectedAppNewMetadata.ObjectMeta.Labels = map[string]string{"label-key": "label-value"}
+				expectedAppsNewMetadata = append(expectedAppsNewMetadata, *expectedAppNewMetadata)
+			}
+		}).
+		Update(func(appset *v1alpha1.ApplicationSet) {
+			appset.Spec.Template.Annotations = map[string]string{"annotation-key": "annotation-value"}
+			appset.Spec.Template.Labels = map[string]string{"label-key": "label-value"}
+		}).Then().Expect(ApplicationsExist(expectedAppsNewMetadata)).
+
 		// Delete the ApplicationSet, and verify it deletes the Applications
 		When().
 		Delete().Then().Expect(ApplicationsDoNotExist(expectedAppsNewNamespace))
diff --git a/test/e2e/applicationset/cluster_e2e_test.go b/test/e2e/applicationset/cluster_e2e_test.go
index 20b3d0e..abfad15 100644
--- a/test/e2e/applicationset/cluster_e2e_test.go
+++ b/test/e2e/applicationset/cluster_e2e_test.go
@@ -18,8 +18,9 @@ func TestSimpleClusterGenerator(t *testing.T) {
 			APIVersion: "argoproj.io/v1alpha1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "cluster1-guestbook",
-			Namespace: utils.ArgoCDNamespace,
+			Name:       "cluster1-guestbook",
+			Namespace:  utils.ArgoCDNamespace,
+			Finalizers: []string{"resources-finalizer.argocd.argoproj.io"},
 		},
 		Spec: argov1alpha1.ApplicationSpec{
 			Project: "default",
@@ -36,6 +37,7 @@ func TestSimpleClusterGenerator(t *testing.T) {
 	}
 
 	var expectedAppNewNamespace *argov1alpha1.Application
+	var expectedAppNewMetadata *argov1alpha1.Application
 
 	Given(t).
 		// Create a ClusterGenerator-based ApplicationSet
@@ -85,6 +87,18 @@ func TestSimpleClusterGenerator(t *testing.T) {
 			appset.Spec.Template.Spec.Destination.Namespace = "guestbook2"
 		}).Then().Expect(ApplicationsExist([]argov1alpha1.Application{*expectedAppNewNamespace})).
 
+		// Update the metadata fields in the appset template, and make sure it propagates to the apps
+		When().
+		And(func() {
+			expectedAppNewMetadata = expectedAppNewNamespace.DeepCopy()
+			expectedAppNewMetadata.ObjectMeta.Annotations = map[string]string{"annotation-key": "annotation-value"}
+			expectedAppNewMetadata.ObjectMeta.Labels = map[string]string{"label-key": "label-value"}
+		}).
+		Update(func(appset *v1alpha1.ApplicationSet) {
+			appset.Spec.Template.Annotations = map[string]string{"annotation-key": "annotation-value"}
+			appset.Spec.Template.Labels = map[string]string{"label-key": "label-value"}
+		}).Then().Expect(ApplicationsExist([]argov1alpha1.Application{*expectedAppNewMetadata})).
+
 		// Delete the ApplicationSet, and verify it deletes the Applications
 		When().
 		Delete().Then().Expect(ApplicationsDoNotExist([]argov1alpha1.Application{*expectedAppNewNamespace}))
@@ -98,8 +112,9 @@ func TestClusterGeneratorWithLocalCluster(t *testing.T) {
 			APIVersion: "argoproj.io/v1alpha1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "in-cluster-guestbook",
-			Namespace: utils.ArgoCDNamespace,
+			Name:       "in-cluster-guestbook",
+			Namespace:  utils.ArgoCDNamespace,
+			Finalizers: []string{"resources-finalizer.argocd.argoproj.io"},
 		},
 		Spec: argov1alpha1.ApplicationSpec{
 			Project: "default",
@@ -144,6 +159,7 @@ func TestClusterGeneratorWithLocalCluster(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			var expectedAppNewNamespace *argov1alpha1.Application
+			var expectedAppNewMetadata *argov1alpha1.Application
 
 			// Create the expected application from the template, and copy in the destination from the test case
 			expectedApp := *expectedAppTemplate.DeepCopy()
@@ -186,6 +202,18 @@ func TestClusterGeneratorWithLocalCluster(t *testing.T) {
 					appset.Spec.Template.Spec.Destination.Namespace = "guestbook2"
 				}).Then().Expect(ApplicationsExist([]argov1alpha1.Application{*expectedAppNewNamespace})).
 
+				// Update the metadata fields in the appset template, and make sure it propagates to the apps
+				When().
+				And(func() {
+					expectedAppNewMetadata = expectedAppNewNamespace.DeepCopy()
+					expectedAppNewMetadata.ObjectMeta.Annotations = map[string]string{"annotation-key": "annotation-value"}
+					expectedAppNewMetadata.ObjectMeta.Labels = map[string]string{"label-key": "label-value"}
+				}).
+				Update(func(appset *v1alpha1.ApplicationSet) {
+					appset.Spec.Template.Annotations = map[string]string{"annotation-key": "annotation-value"}
+					appset.Spec.Template.Labels = map[string]string{"label-key": "label-value"}
+				}).Then().Expect(ApplicationsExist([]argov1alpha1.Application{*expectedAppNewMetadata})).
+
 				// Delete the ApplicationSet, and verify it deletes the Applications
 				When().
 				Delete().Then().Expect(ApplicationsDoNotExist([]argov1alpha1.Application{*expectedAppNewNamespace}))
diff --git a/test/e2e/fixture/applicationsets/expectation.go b/test/e2e/fixture/applicationsets/expectation.go
index 1427bf4..e17ec74 100644
--- a/test/e2e/fixture/applicationsets/expectation.go
+++ b/test/e2e/fixture/applicationsets/expectation.go
@@ -7,6 +7,7 @@ import (
 
 	argov1alpha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
 	"github.com/argoproj/gitops-engine/pkg/diff"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 type state = string
@@ -110,7 +111,16 @@ func filterFields(input argov1alpha1.Application) argov1alpha1.Application {
 
 	spec := input.Spec
 
+	metaCopy := input.ObjectMeta.DeepCopy()
+
 	output := argov1alpha1.Application{
+		ObjectMeta: metav1.ObjectMeta{
+			Labels:      metaCopy.Labels,
+			Annotations: metaCopy.Annotations,
+			Name:        metaCopy.Name,
+			Namespace:   metaCopy.Namespace,
+			Finalizers:  metaCopy.Finalizers,
+		},
 		Spec: argov1alpha1.ApplicationSpec{
 			Source: argov1alpha1.ApplicationSource{
 				Path:           spec.Source.Path,