Repository: argoproj-labs/argocd-operator
Pull request #835 (2022-12-22 18:08:15)

feat: Allow user instance monitoring
**What type of PR is this?**
> Uncomment only one ` /kind` line, and delete the rest.
> For example, `> /kind bug` would simply become: `/kind bug`

> /kind bug
> /kind chore
> /kind cleanup
> /kind failing-test
/kind enhancement
> /kind documentation
> /kind code-refactoring


**What does this PR do / why we need it**:
This PR allows users to enable workload status monitoring for a given argo-cd instance. Users can enable monitoring for their instance by setting `.spec.monitoring.enabled` to `true`, which creates a PrometheusRule for alerts (with 7 opinionated alert rules within it - one for each workload). The rules are configured to fire when a workload has not had the required number of ready replicas for a certain duration of time out of the box. Users are free to delete or make changes to the alert rule and the operator would not overwrite the action. 

**Have you updated the necessary documentation?**
Not yet

* [x] Documentation update is required by this PR.
* [ ] Documentation has been updated.

**Which issue(s) this PR fixes**:

Fixes #?

**How to test changes / Special notes to the reviewer**:
1. Launch operator locally against an openshift cluster or any cluster that contains the prometheus API
2. create a namespace and argo-cd instance within that namespace
3. If using openshift - enable user workload monitoring for your cluster by running `oc -n openshift-monitoring edit configmap cluster-monitoring-config` and setting `data.config.yaml.enableUserWorkload = true` (https://docs.openshift.com/container-platform/4.6/monitoring/enabling-monitoring-for-user-defined-projects.html?extIdCarryOver=true&sc_cid=701f2000001Css5AAC#enabling-monitoring-for-user-defined-projects_enabling-monitoring-for-user-defined-projects)
4. Set `.spec.monitoring.enabled=true` in CR
5. Check that prometheusRule is created in the namespace 
6. On OpenShift you can go to Admin > Observe > Alerting > Alerting Rules to ensure that all 7 alerting rules show up in the list (Remove any pre-applied filters0)
7. Set `.spec.applicationSet.image=test-image` to create a situation where # desired replicas for appset controller != # ready replicas 
8. Go back to Admin > Observe > Alerting > Alerts and you should see the `ApplicationSetControllerNotReady` alert in this list (remove all pre-applied filters) with state initially set as "pending" which should change to "firing" after 10m 


drpaneas: Great work @jaideepr97 💪 🚀 🚀  I've added a couple of comments, but I wouldn't consider any of those as blockers.

- Some corner test-cases are missing, but in general **awesome** work for adding unit-tests for the alerts!!!
- This PR adds a new YAML manifest. Is this auto-generated or you wrote it by hand?
- This can be done in separate story as well: a doc listing the alerts we support.
drpaneas: > @drpaneas Thanks for taking a look :) I don't see any comments on here though, did they get saved?

sorry I forgot to submit them. They should be visible now ;)
wtam2018: Please add documentation in the same PR.

Patch:
diff --git a/api/v1alpha1/argocd_types.go b/api/v1alpha1/argocd_types.go
index 71490097..a8823562 100644
--- a/api/v1alpha1/argocd_types.go
+++ b/api/v1alpha1/argocd_types.go
@@ -614,6 +614,15 @@ type KustomizeVersionSpec struct {
 	Path string `json:"path,omitempty"`
 }
 
+// ArgoCDMonitoringSpec is used to configure workload status monitoring for a given Argo CD instance.
+// It triggers creation of serviceMonitor and PrometheusRules that alert users when a given workload
+// status meets a certain criteria. For e.g, it can fire an alert if the application controller is
+// pending for x mins consecutively.
+type ArgoCDMonitoringSpec struct {
+	// Enabled defines whether workload status monitoring is enabled for this instance or not
+	Enabled bool `json:"enabled"`
+}
+
 //ArgoCDNodePlacementSpec is used to specify NodeSelector and Tolerations for Argo CD workloads
 type ArgoCDNodePlacementSpec struct {
 	// NodeSelector is a field of PodSpec, it is a map of key value pairs used for node selection
@@ -701,6 +710,9 @@ type ArgoCDSpec struct {
 	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="OIDC Config'",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:text","urn:alm:descriptor:com.tectonic.ui:advanced"}
 	OIDCConfig string `json:"oidcConfig,omitempty"`
 
+	// Monitoring defines whether workload status monitoring configuration for this instance.
+	Monitoring ArgoCDMonitoringSpec `json:"monitoring,omitempty"`
+
 	// NodePlacement defines NodeSelectors and Taints for Argo CD workloads
 	NodePlacement *ArgoCDNodePlacementSpec `json:"nodePlacement,omitempty"`
 
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index f98cbe95..391711b0 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -489,6 +489,21 @@ func (in *ArgoCDList) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ArgoCDMonitoringSpec) DeepCopyInto(out *ArgoCDMonitoringSpec) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArgoCDMonitoringSpec.
+func (in *ArgoCDMonitoringSpec) DeepCopy() *ArgoCDMonitoringSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ArgoCDMonitoringSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ArgoCDNodePlacementSpec) DeepCopyInto(out *ArgoCDNodePlacementSpec) {
 	*out = *in
@@ -893,6 +908,7 @@ func (in *ArgoCDSpec) DeepCopyInto(out *ArgoCDSpec) {
 		*out = make([]KustomizeVersionSpec, len(*in))
 		copy(*out, *in)
 	}
+	out.Monitoring = in.Monitoring
 	if in.NodePlacement != nil {
 		in, out := &in.NodePlacement, &out.NodePlacement
 		*out = new(ArgoCDNodePlacementSpec)
diff --git a/bundle/manifests/argoproj.io_argocds.yaml b/bundle/manifests/argoproj.io_argocds.yaml
index b3ffacae..992f2ef2 100644
--- a/bundle/manifests/argoproj.io_argocds.yaml
+++ b/bundle/manifests/argoproj.io_argocds.yaml
@@ -782,6 +782,17 @@ spec:
                       type: string
                   type: object
                 type: array
+              monitoring:
+                description: Monitoring defines whether workload status monitoring
+                  configuration for this instance.
+                properties:
+                  enabled:
+                    description: Enabled defines whether workload status monitoring
+                      is enabled for this instance or not
+                    type: boolean
+                required:
+                - enabled
+                type: object
               nodePlacement:
                 description: NodePlacement defines NodeSelectors and Taints for Argo
                   CD workloads
diff --git a/config/crd/bases/argoproj.io_argocds.yaml b/config/crd/bases/argoproj.io_argocds.yaml
index 3cce9a95..9362fc3f 100644
--- a/config/crd/bases/argoproj.io_argocds.yaml
+++ b/config/crd/bases/argoproj.io_argocds.yaml
@@ -784,6 +784,17 @@ spec:
                       type: string
                   type: object
                 type: array
+              monitoring:
+                description: Monitoring defines whether workload status monitoring
+                  configuration for this instance.
+                properties:
+                  enabled:
+                    description: Enabled defines whether workload status monitoring
+                      is enabled for this instance or not
+                    type: boolean
+                required:
+                - enabled
+                type: object
               nodePlacement:
                 description: NodePlacement defines NodeSelectors and Taints for Argo
                   CD workloads
diff --git a/controllers/argocd/prometheus.go b/controllers/argocd/prometheus.go
index 88578a14..23675785 100644
--- a/controllers/argocd/prometheus.go
+++ b/controllers/argocd/prometheus.go
@@ -20,6 +20,7 @@ import (
 
 	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	argoprojv1a1 "github.com/argoproj-labs/argocd-operator/api/v1alpha1"
@@ -247,3 +248,150 @@ func (r *ReconcileArgoCD) reconcileServerMetricsServiceMonitor(cr *argoprojv1a1.
 	}
 	return r.Client.Create(context.TODO(), sm)
 }
+
+// reconcilePrometheusRule reconciles the PrometheusRule that triggers alerts based on workload statuses
+func (r *ReconcileArgoCD) reconcilePrometheusRule(cr *argoprojv1a1.ArgoCD) error {
+
+	promRule := newPrometheusRule(cr.Namespace, "argocd-component-status-alert")
+
+	if argoutil.IsObjectFound(r.Client, cr.Namespace, promRule.Name, promRule) {
+
+		if !cr.Spec.Monitoring.Enabled {
+			// PrometheusRule exists but enabled flag has been set to false, delete the PrometheusRule
+			log.Info("instance monitoring disabled, deleting component status tracking prometheusRule")
+			return r.Client.Delete(context.TODO(), promRule)
+		}
+		return nil // PrometheusRule found, do nothing
+	}
+
+	if !cr.Spec.Monitoring.Enabled {
+		return nil // Monitoring not enabled, do nothing.
+	}
+
+	ruleGroups := []monitoringv1.RuleGroup{
+		{
+			Name: "ArgoCDComponentStatus",
+			Rules: []monitoringv1.Rule{
+				{
+					Alert: "ApplicationControllerNotReady",
+					Annotations: map[string]string{
+						"message": fmt.Sprintf("application controller deployment for Argo CD instance in namespace %s is not running", cr.Namespace),
+					},
+					Expr: intstr.IntOrString{
+						Type:   intstr.String,
+						StrVal: fmt.Sprintf("kube_statefulset_status_replicas{statefulset=\"%s\", namespace=\"%s\"} != kube_statefulset_status_replicas_ready{statefulset=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(cr.Name+"-application-controller"), cr.Namespace, fmt.Sprintf(cr.Name+"-application-controller"), cr.Namespace),
+					},
+					For: "1m",
+					Labels: map[string]string{
+						"severity": "critical",
+					},
+				},
+				{
+					Alert: "ServerNotReady",
+					Annotations: map[string]string{
+						"message": fmt.Sprintf("server deployment for Argo CD instance in namespace %s is not running", cr.Namespace),
+					},
+					Expr: intstr.IntOrString{
+						Type:   intstr.String,
+						StrVal: fmt.Sprintf("kube_deployment_status_replicas{deployment=\"%s\", namespace=\"%s\"} != kube_deployment_status_replicas_ready{deployment=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(cr.Name+"-server"), cr.Namespace, fmt.Sprintf(cr.Name+"-server"), cr.Namespace),
+					},
+					For: "1m",
+					Labels: map[string]string{
+						"severity": "critical",
+					},
+				},
+				{
+					Alert: "RepoServerNotReady",
+					Annotations: map[string]string{
+						"message": fmt.Sprintf("repo server deployment for Argo CD instance in namespace %s is not running", cr.Namespace),
+					},
+					Expr: intstr.IntOrString{
+						Type:   intstr.String,
+						StrVal: fmt.Sprintf("kube_deployment_status_replicas{deployment=\"%s\", namespace=\"%s\"} != kube_deployment_status_replicas_ready{deployment=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(cr.Name+"-repo-server"), cr.Namespace, fmt.Sprintf(cr.Name+"-repo-server"), cr.Namespace),
+					},
+					For: "1m",
+					Labels: map[string]string{
+						"severity": "critical",
+					},
+				},
+				{
+					Alert: "ApplicationSetControllerNotReady",
+					Annotations: map[string]string{
+						"message": fmt.Sprintf("applicationSet controller deployment for Argo CD instance in namespace %s is not running", cr.Namespace),
+					},
+					Expr: intstr.IntOrString{
+						Type:   intstr.String,
+						StrVal: fmt.Sprintf("kube_deployment_status_replicas{deployment=\"%s\", namespace=\"%s\"} != kube_deployment_status_replicas_ready{deployment=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(cr.Name+"-applicationset-controller"), cr.Namespace, fmt.Sprintf(cr.Name+"-applicationset-controller"), cr.Namespace),
+					},
+					For: "5m",
+					Labels: map[string]string{
+						"severity": "warning",
+					},
+				},
+				{
+					Alert: "DexNotReady",
+					Annotations: map[string]string{
+						"message": fmt.Sprintf("dex deployment for Argo CD instance in namespace %s is not running", cr.Namespace),
+					},
+					Expr: intstr.IntOrString{
+						Type:   intstr.String,
+						StrVal: fmt.Sprintf("kube_deployment_status_replicas{deployment=\"%s\", namespace=\"%s\"} != kube_deployment_status_replicas_ready{deployment=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(cr.Name+"-dex-server"), cr.Namespace, fmt.Sprintf(cr.Name+"-dex-server"), cr.Namespace),
+					},
+					For: "5m",
+					Labels: map[string]string{
+						"severity": "warning",
+					},
+				},
+				{
+					Alert: "NotificationsControllerNotReady",
+					Annotations: map[string]string{
+						"message": fmt.Sprintf("notifications controller deployment for Argo CD instance in namespace %s is not running", cr.Namespace),
+					},
+					Expr: intstr.IntOrString{
+						Type:   intstr.String,
+						StrVal: fmt.Sprintf("kube_deployment_status_replicas{deployment=\"%s\", namespace=\"%s\"} != kube_deployment_status_replicas_ready{deployment=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(cr.Name+"-notifications-controller"), cr.Namespace, fmt.Sprintf(cr.Name+"-notifications-controller"), cr.Namespace),
+					},
+					For: "5m",
+					Labels: map[string]string{
+						"severity": "warning",
+					},
+				},
+				{
+					Alert: "RedisNotReady",
+					Annotations: map[string]string{
+						"message": fmt.Sprintf("redis deployment for Argo CD instance in namespace %s is not running", cr.Namespace),
+					},
+					Expr: intstr.IntOrString{
+						Type:   intstr.String,
+						StrVal: fmt.Sprintf("kube_deployment_status_replicas{deployment=\"%s\", namespace=\"%s\"} != kube_deployment_status_replicas_ready{deployment=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(cr.Name+"-redis"), cr.Namespace, fmt.Sprintf(cr.Name+"-redis"), cr.Namespace),
+					},
+					For: "5m",
+					Labels: map[string]string{
+						"severity": "warning",
+					},
+				},
+			},
+		},
+	}
+	promRule.Spec.Groups = ruleGroups
+
+	if err := controllerutil.SetControllerReference(cr, promRule, r.Scheme); err != nil {
+		return err
+	}
+
+	log.Info("instance monitoring enabled, creating component status tracking prometheusRule")
+	return r.Client.Create(context.TODO(), promRule) // Create PrometheusRule
+}
+
+// newPrometheusRule returns an empty PrometheusRule
+func newPrometheusRule(namespace, alertRuleName string) *monitoringv1.PrometheusRule {
+
+	promRule := &monitoringv1.PrometheusRule{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      alertRuleName,
+			Namespace: namespace,
+		},
+		Spec: monitoringv1.PrometheusRuleSpec{},
+	}
+	return promRule
+}
diff --git a/controllers/argocd/util.go b/controllers/argocd/util.go
index fc93596c..a761613f 100644
--- a/controllers/argocd/util.go
+++ b/controllers/argocd/util.go
@@ -797,6 +797,11 @@ func (r *ReconcileArgoCD) reconcileResources(cr *argoprojv1a1.ArgoCD) error {
 			return err
 		}
 
+		// Reconciles prometheusRule created to alert based on argo-cd workload status
+		if err := r.reconcilePrometheusRule(cr); err != nil {
+			return err
+		}
+
 		if err := r.reconcileMetricsServiceMonitor(cr); err != nil {
 			return err
 		}
diff --git a/deploy/olm-catalog/argocd-operator/0.6.0/argoproj.io_argocds.yaml b/deploy/olm-catalog/argocd-operator/0.6.0/argoproj.io_argocds.yaml
index b3ffacae..992f2ef2 100644
--- a/deploy/olm-catalog/argocd-operator/0.6.0/argoproj.io_argocds.yaml
+++ b/deploy/olm-catalog/argocd-operator/0.6.0/argoproj.io_argocds.yaml
@@ -782,6 +782,17 @@ spec:
                       type: string
                   type: object
                 type: array
+              monitoring:
+                description: Monitoring defines whether workload status monitoring
+                  configuration for this instance.
+                properties:
+                  enabled:
+                    description: Enabled defines whether workload status monitoring
+                      is enabled for this instance or not
+                    type: boolean
+                required:
+                - enabled
+                type: object
               nodePlacement:
                 description: NodePlacement defines NodeSelectors and Taints for Argo
                   CD workloads
diff --git a/docs/usage/monitoring.md b/docs/usage/monitoring.md
new file mode 100644
index 00000000..f0342883
--- /dev/null
+++ b/docs/usage/monitoring.md
@@ -0,0 +1,79 @@
+# Instance workload monitoring 
+
+Instance workload monitoring allows you to be better informed of the state of your Argo CD instance by enabling alerts for the instance. Alerts are triggered when the instance's component workload (application-controller, repo-server, server etc.) pods are unable to come up for whatever reason and there is a drift between the number of ready replicas and the number of desired replicas for a certain period of time.  
+
+**Note:** Pre-requisites for this feature:
+- Your cluster is configured with Prometheus, and the Argo CD instance is in a namespace that can be monitored via Prometheus
+- Your cluster has the `kube-state-metrics` service running 
+
+**Note:** This feature is only concerned with availability of Argo CD workloads, and does not focus on monitoring for performance. 
+
+
+Instance workload monitoring can be enabled by setting `.spec.monitoring.enabled` to `true` on a given Argo CD instance.
+For example:
+
+```
+apiVersion: argoproj.io/v1alpha1
+kind: ArgoCD
+metadata:
+  name: example-argocd
+  labels:
+    example: example-argocd
+spec:
+  ...
+  monitoring:
+    enabled: true
+  ...
+```
+
+Instance workload monitoring is set to `false` by default.
+
+Enabling this setting allows the operator to create a `PrometheusRule` containing pre-configured alert rules for all the workloads 9statefulsets/deployments) managed by the instance. Here is a sample alert rule included in the PrometheusRule created by the operator:
+
+```
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: argocd-component-status-alert
+  namespace: $NAMESPACE
+spec:
+  groups:
+    - name: ArgoCDComponentStatus
+      rules:
+        ...
+        - alert: ApplicationSetControllerNotReady
+          annotations:
+            message: >-
+              applicationSet controller deployment for Argo CD instance in
+              namespace "default" is not running
+          expr: >-
+            kube_deployment_status_replicas{deployment="argocd-applicationset-controller",
+            namespace="default"} !=
+            kube_deployment_status_replicas_ready{deployment="argocd-applicationset-controller",
+            namespace="default"} 
+          for: 10m
+          labels:
+            severity: warning
+        ...
+```
+
+Users are free to modify/delete alert rules as wish, changes made to the rules will not be overwritten by the operator. 
+
+Instance workload monitoring can be disabled by setting `.spec.monitoring.enabled` to `false` on a given Argo CD instance.
+For example:
+
+```
+apiVersion: argoproj.io/v1alpha1
+kind: ArgoCD
+metadata:
+  name: example-argocd
+  labels:
+    example: example-argocd
+spec:
+  ...
+  monitoring:
+    enabled: false
+  ...
+```
+
+Disabling workload monitoring will delete the created PrometheusRule. 
\ No newline at end of file


Test patch:
diff --git a/controllers/argocd/prometheus_test.go b/controllers/argocd/prometheus_test.go
new file mode 100644
index 00000000..31a47cc2
--- /dev/null
+++ b/controllers/argocd/prometheus_test.go
@@ -0,0 +1,201 @@
+package argocd
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	argoprojv1alpha1 "github.com/argoproj-labs/argocd-operator/api/v1alpha1"
+	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+)
+
+func TestReconcileWorkloadStatusAlertRule(t *testing.T) {
+	tests := []struct {
+		name              string
+		argocd            *argoprojv1alpha1.ArgoCD
+		wantPromRuleFound bool
+		existingPromRule  bool
+	}{
+		{
+			name: "monitoring enabled, no existing prom rule",
+			argocd: makeTestArgoCD(func(cr *argoprojv1alpha1.ArgoCD) {
+				cr.Spec.Monitoring.Enabled = true
+			}),
+			existingPromRule:  false,
+			wantPromRuleFound: true,
+		},
+		{
+			name: "monitoring disabled, no existing prom rule",
+			argocd: makeTestArgoCD(func(cr *argoprojv1alpha1.ArgoCD) {
+				cr.Spec.Monitoring.Enabled = false
+			}),
+			existingPromRule:  false,
+			wantPromRuleFound: false,
+		},
+		{
+			name: "monitoring enabled, existing prom rule",
+			argocd: makeTestArgoCD(func(cr *argoprojv1alpha1.ArgoCD) {
+				cr.Spec.Monitoring.Enabled = true
+			}),
+			existingPromRule:  true,
+			wantPromRuleFound: true,
+		},
+		{
+			name: "monitoring disabled, existing prom rule",
+			argocd: makeTestArgoCD(func(cr *argoprojv1alpha1.ArgoCD) {
+				cr.Spec.Monitoring.Enabled = false
+			}),
+			existingPromRule:  true,
+			wantPromRuleFound: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			desiredRuleGroup := []monitoringv1.RuleGroup{
+				{
+					Name: "ArgoCDComponentStatus",
+					Rules: []monitoringv1.Rule{
+						{
+							Alert: "ApplicationControllerNotReady",
+							Annotations: map[string]string{
+								"message": fmt.Sprintf("application controller deployment for Argo CD instance in namespace %s is not running", test.argocd.Namespace),
+							},
+							Expr: intstr.IntOrString{
+								Type:   intstr.String,
+								StrVal: fmt.Sprintf("kube_statefulset_status_replicas{statefulset=\"%s\", namespace=\"%s\"} != kube_statefulset_status_replicas_ready{statefulset=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(test.argocd.Name+"-application-controller"), test.argocd.Namespace, fmt.Sprintf(test.argocd.Name+"-application-controller"), test.argocd.Namespace),
+							},
+							For: "1m",
+							Labels: map[string]string{
+								"severity": "critical",
+							},
+						},
+						{
+							Alert: "ServerNotReady",
+							Annotations: map[string]string{
+								"message": fmt.Sprintf("server deployment for Argo CD instance in namespace %s is not running", test.argocd.Namespace),
+							},
+							Expr: intstr.IntOrString{
+								Type:   intstr.String,
+								StrVal: fmt.Sprintf("kube_deployment_status_replicas{deployment=\"%s\", namespace=\"%s\"} != kube_deployment_status_replicas_ready{deployment=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(test.argocd.Name+"-server"), test.argocd.Namespace, fmt.Sprintf(test.argocd.Name+"-server"), test.argocd.Namespace),
+							},
+							For: "1m",
+							Labels: map[string]string{
+								"severity": "critical",
+							},
+						},
+						{
+							Alert: "RepoServerNotReady",
+							Annotations: map[string]string{
+								"message": fmt.Sprintf("repo server deployment for Argo CD instance in namespace %s is not running", test.argocd.Namespace),
+							},
+							Expr: intstr.IntOrString{
+								Type:   intstr.String,
+								StrVal: fmt.Sprintf("kube_deployment_status_replicas{deployment=\"%s\", namespace=\"%s\"} != kube_deployment_status_replicas_ready{deployment=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(test.argocd.Name+"-repo-server"), test.argocd.Namespace, fmt.Sprintf(test.argocd.Name+"-repo-server"), test.argocd.Namespace),
+							},
+							For: "1m",
+							Labels: map[string]string{
+								"severity": "critical",
+							},
+						},
+						{
+							Alert: "ApplicationSetControllerNotReady",
+							Annotations: map[string]string{
+								"message": fmt.Sprintf("applicationSet controller deployment for Argo CD instance in namespace %s is not running", test.argocd.Namespace),
+							},
+							Expr: intstr.IntOrString{
+								Type:   intstr.String,
+								StrVal: fmt.Sprintf("kube_deployment_status_replicas{deployment=\"%s\", namespace=\"%s\"} != kube_deployment_status_replicas_ready{deployment=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(test.argocd.Name+"-applicationset-controller"), test.argocd.Namespace, fmt.Sprintf(test.argocd.Name+"-applicationset-controller"), test.argocd.Namespace),
+							},
+							For: "5m",
+							Labels: map[string]string{
+								"severity": "warning",
+							},
+						},
+						{
+							Alert: "DexNotReady",
+							Annotations: map[string]string{
+								"message": fmt.Sprintf("dex deployment for Argo CD instance in namespace %s is not running", test.argocd.Namespace),
+							},
+							Expr: intstr.IntOrString{
+								Type:   intstr.String,
+								StrVal: fmt.Sprintf("kube_deployment_status_replicas{deployment=\"%s\", namespace=\"%s\"} != kube_deployment_status_replicas_ready{deployment=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(test.argocd.Name+"-dex-server"), test.argocd.Namespace, fmt.Sprintf(test.argocd.Name+"-dex-server"), test.argocd.Namespace),
+							},
+							For: "5m",
+							Labels: map[string]string{
+								"severity": "warning",
+							},
+						},
+						{
+							Alert: "NotificationsControllerNotReady",
+							Annotations: map[string]string{
+								"message": fmt.Sprintf("notifications controller deployment for Argo CD instance in namespace %s is not running", test.argocd.Namespace),
+							},
+							Expr: intstr.IntOrString{
+								Type:   intstr.String,
+								StrVal: fmt.Sprintf("kube_deployment_status_replicas{deployment=\"%s\", namespace=\"%s\"} != kube_deployment_status_replicas_ready{deployment=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(test.argocd.Name+"-notifications-controller"), test.argocd.Namespace, fmt.Sprintf(test.argocd.Name+"-notifications-controller"), test.argocd.Namespace),
+							},
+							For: "5m",
+							Labels: map[string]string{
+								"severity": "warning",
+							},
+						},
+						{
+							Alert: "RedisNotReady",
+							Annotations: map[string]string{
+								"message": fmt.Sprintf("redis deployment for Argo CD instance in namespace %s is not running", test.argocd.Namespace),
+							},
+							Expr: intstr.IntOrString{
+								Type:   intstr.String,
+								StrVal: fmt.Sprintf("kube_deployment_status_replicas{deployment=\"%s\", namespace=\"%s\"} != kube_deployment_status_replicas_ready{deployment=\"%s\", namespace=\"%s\"} ", fmt.Sprintf(test.argocd.Name+"-redis"), test.argocd.Namespace, fmt.Sprintf(test.argocd.Name+"-redis"), test.argocd.Namespace),
+							},
+							For: "5m",
+							Labels: map[string]string{
+								"severity": "warning",
+							},
+						},
+					},
+				},
+			}
+
+			r := makeTestReconciler(t, test.argocd)
+			err := monitoringv1.AddToScheme(r.Scheme)
+			assert.NoError(t, err)
+
+			if test.existingPromRule {
+				r.Client.Create(context.TODO(), newPrometheusRule(test.argocd.Namespace, "argocd-component-status-alert"))
+			}
+
+			err = r.reconcilePrometheusRule(test.argocd)
+
+			// reconciler doesn't need to do anything and should return nil
+			if (test.existingPromRule && test.wantPromRuleFound) || (!test.existingPromRule && !test.wantPromRuleFound) {
+				if err != nil {
+					t.Fatal("expected nil response but got non-nil response")
+				}
+			} else {
+				// reconciler either needs to create rule or delete it
+				testRule := &monitoringv1.PrometheusRule{}
+				err = r.Client.Get(context.TODO(), types.NamespacedName{
+					Name:      "argocd-component-status-alert",
+					Namespace: test.argocd.Namespace,
+				}, testRule)
+
+				if test.wantPromRuleFound && err != nil {
+					t.Fatal("unexpected error - prometheusRule not found")
+				} else if !test.wantPromRuleFound && err == nil {
+					t.Fatal("expected error but did not get one - prometheusRule not deleted")
+				}
+
+				if !test.existingPromRule {
+					assert.Equal(t, desiredRuleGroup, testRule.Spec.Groups)
+				}
+
+			}
+		})
+	}
+}