Repository: cortexproject/cortex (Go)

Issue:
Retry alertmanager delete silence when one instance fails
**Describe the bug**
Currently alertmanager silence handler only retries on GET requests https://github.com/cortexproject/cortex/blob/master/pkg/alertmanager/distributor.go#L250-L260 but it seems like from the discussion in https://github.com/cortexproject/cortex/pull/4422 we should also be able to retry on DELETE. 

**Expected behavior**
Delete silence request should retry to a difference instance if the delete request in one  AM instance fails.



Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f15d8218222..5949b4b20cc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@
 * [ENHANCEMENT] Query Frontend/Scheduler: Time check in query priority now considers overall data select time window (including range selectors, modifiers and lookback delta). #5758
 * [ENHANCEMENT] Querier: Added `querier.store-gateway-query-stats-enabled` to enable or disable store gateway query stats log. #5749
 * [ENHANCEMENT] Upgrade to go 1.21.6. #5765
+* [ENHANCEMENT] AlertManager: Retrying AlertManager Delete Silence on error #5794
 * [BUGFIX] Distributor: Do not use label with empty values for sharding #5717
 * [BUGFIX] Query Frontend: queries with negative offset should check whether it is cacheable or not. #5719
 * [BUGFIX] Redis Cache: pass `cache_size` config correctly. #5734
diff --git a/pkg/alertmanager/distributor.go b/pkg/alertmanager/distributor.go
index 4da13e112f3..378971e843d 100644
--- a/pkg/alertmanager/distributor.go
+++ b/pkg/alertmanager/distributor.go
@@ -247,14 +247,14 @@ func (d *Distributor) doUnary(userID string, w http.ResponseWriter, r *http.Requ
 	// we forward the request to only only of the alertmanagers.
 
 	var instances []ring.InstanceDesc
-	if req.GetMethod() == "GET" {
+	if req.GetMethod() == "GET" || req.GetMethod() == "DELETE" {
 		instances = replicationSet.Instances
 		// Randomize the list of instances to not always query the same one.
 		rand.Shuffle(len(instances), func(i, j int) {
 			instances[i], instances[j] = instances[j], instances[i]
 		})
 	} else {
-		//Picking 1 instance at Random for Non-Get Unary Read requests, as shuffling through large number of instances might increase complexity
+		//Picking 1 instance at Random for Non-Get and Non-Delete Unary Read requests, as shuffling through large number of instances might increase complexity
 		randN := rand.Intn(len(replicationSet.Instances))
 		instances = replicationSet.Instances[randN : randN+1]
 	}


Test patch:
diff --git a/pkg/alertmanager/distributor_test.go b/pkg/alertmanager/distributor_test.go
index 38be937903f..ddda42749f9 100644
--- a/pkg/alertmanager/distributor_test.go
+++ b/pkg/alertmanager/distributor_test.go
@@ -200,6 +200,24 @@ func TestDistributor_DistributeRequest(t *testing.T) {
 			expStatusCode:      http.StatusOK,
 			expectedTotalCalls: 1,
 			route:              "/silence/id",
+		}, {
+			name:               "Delete /silence/id should try all alert managers on error",
+			numAM:              3,
+			numHappyAM:         0,
+			replicationFactor:  3,
+			isDelete:           true,
+			expStatusCode:      http.StatusInternalServerError,
+			expectedTotalCalls: 3,
+			route:              "/silence/id",
+		}, {
+			name:               "Delete /silence/id is sent to 3 AM when 2 are not happy",
+			numAM:              3,
+			numHappyAM:         1,
+			replicationFactor:  3,
+			isDelete:           true,
+			expStatusCode:      http.StatusOK,
+			expectedTotalCalls: 3,
+			route:              "/silence/id",
 		}, {
 			name:               "Read /status is sent to only 1 AM",
 			numAM:              5,