Repository: VictoriaMetrics/VictoriaMetrics
Pull request #4214 (2023-04-27 10:35:22)

app/vmalert: follow-up after 6c322b4a00fd5e85687c03bf27b1b6113f819248
https://github.com/VictoriaMetrics/VictoriaMetrics/commit/6c322b4a00fd5e85687c03bf27b1b6113f819248

codecov[bot]: ## [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4214?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) Report
Patch coverage: **`100.00`**% and project coverage change: **`-0.05`** :warning:
> Comparison is base [(`6c322b4`)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/6c322b4a00fd5e85687c03bf27b1b6113f819248?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) 61.60% compared to head [(`e2915fc`)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4214?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) 61.56%.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #4214      +/-   ##
==========================================
- Coverage   61.60%   61.56%   -0.05%     
==========================================
  Files         325      325              
  Lines       58576    58577       +1     
==========================================
- Hits        36086    36060      -26     
- Misses      20519    20541      +22     
- Partials     1971     1976       +5     
```


| [Impacted Files](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4214?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) | Coverage Δ | |
|---|---|---|
| [app/vmalert/config/config.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4214?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtYWxlcnQvY29uZmlnL2NvbmZpZy5nbw==) | `82.53% <ø> (ø)` | |
| [app/vmalert/group.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4214?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtYWxlcnQvZ3JvdXAuZ28=) | `81.84% <100.00%> (+0.79%)` | :arrow_up: |
| [app/vmalert/notifier/alertmanager.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4214?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtYWxlcnQvbm90aWZpZXIvYWxlcnRtYW5hZ2VyLmdv) | `79.51% <100.00%> (+3.61%)` | :arrow_up: |

... and [7 files with indirect coverage changes](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4214/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics)


</details>

[:umbrella: View full report in Codecov by Sentry](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4214?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).   
:loudspeaker: Do you have feedback about the report comment? [Let us know in this issue](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).


Patch:
diff --git a/app/vmalert/README.md b/app/vmalert/README.md
index 1e9c489e4..749a161d2 100644
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@@ -144,6 +144,15 @@ params:
 headers:
   [ <string>, ...]
 
+# Optional list of HTTP headers in form `header-name: value`
+# applied for all alert notifications sent to notifiers 
+# generated by rules of this group.
+# For example:
+#  notifier_headers:
+#    - "TenantID: foo"
+notifier_headers:
+        [ <string>, ...]
+
 # Optional list of labels added to every rule within a group.
 # It has priority over the external labels.
 # Labels are commonly used for adding environment
diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go
index 16fb5bab1..0fc201ef9 100644
--- a/app/vmalert/config/config.go
+++ b/app/vmalert/config/config.go
@@ -36,7 +36,7 @@ type Group struct {
 	Params url.Values `yaml:"params"`
 	// Headers contains optional HTTP headers added to each rule request
 	Headers []Header `yaml:"headers,omitempty"`
-	// NotifierHeaders contains optional HTTP headers added to each alert request which will send to notifier
+	// NotifierHeaders contains optional HTTP headers sent to notifiers for generated notifications
 	NotifierHeaders []Header `yaml:"notifier_headers,omitempty"`
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
diff --git a/app/vmalert/group.go b/app/vmalert/group.go
index acab27d64..1aae05b5d 100644
--- a/app/vmalert/group.go
+++ b/app/vmalert/group.go
@@ -300,9 +300,8 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 	e := &executor{
 		rw:                       rw,
 		notifiers:                nts,
+		notifierHeaders:          g.NotifierHeaders,
 		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
-
-		notifierHeaders: g.NotifierHeaders,
 	}
 
 	evalTS := time.Now()
@@ -379,6 +378,8 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 			// ensure that staleness is tracked or existing rules only
 			e.purgeStaleSeries(g.Rules)
 
+			e.notifierHeaders = g.NotifierHeaders
+
 			if g.Interval != ng.Interval {
 				g.Interval = ng.Interval
 				t.Stop()
@@ -412,8 +413,10 @@ func getResolveDuration(groupInterval, delta, maxDuration time.Duration) time.Du
 }
 
 type executor struct {
-	notifiers func() []notifier.Notifier
-	rw        *remotewrite.Client
+	notifiers       func() []notifier.Notifier
+	notifierHeaders map[string]string
+
+	rw *remotewrite.Client
 
 	previouslySentSeriesToRWMu sync.Mutex
 	// previouslySentSeriesToRW stores series sent to RW on previous iteration
@@ -421,8 +424,6 @@ type executor struct {
 	// where `ruleID` is ID of the Rule within a Group
 	// and `ruleLabels` is []prompb.Label marshalled to a string
 	previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
-
-	notifierHeaders map[string]string
 }
 
 func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration, limit int) chan error {
diff --git a/app/vmalert/notifier/alertmanager.go b/app/vmalert/notifier/alertmanager.go
index d30d34d7c..527ad17f8 100644
--- a/app/vmalert/notifier/alertmanager.go
+++ b/app/vmalert/notifier/alertmanager.go
@@ -51,16 +51,16 @@ func (am *AlertManager) Close() {
 func (am AlertManager) Addr() string { return am.addr }
 
 // Send an alert or resolve message
-func (am *AlertManager) Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) error {
+func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[string]string) error {
 	am.metrics.alertsSent.Add(len(alerts))
-	err := am.send(ctx, alerts, notifierHeaders)
+	err := am.send(ctx, alerts, headers)
 	if err != nil {
 		am.metrics.alertsSendErrors.Add(len(alerts))
 	}
 	return err
 }
 
-func (am *AlertManager) send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) error {
+func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[string]string) error {
 	b := &bytes.Buffer{}
 	writeamRequest(b, alerts, am.argFunc, am.relabelConfigs)
 
@@ -69,7 +69,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, notifierHeader
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
-	for key, value := range notifierHeaders {
+	for key, value := range headers {
 		req.Header.Set(key, value)
 	}
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3edf8882d..fdc96b107 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -32,6 +32,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add the metric relabel playground feature to the vmui. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3807).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy requests for unauthorized users. See [this doc](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4083).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to specify default route (`default_url`) for processing non-matched requests. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4084). 
+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): support configuring of custom HTTP headers sent to notifiers on the Group level. See  [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3260).
 
 * BUGFIX: reduce the probability of sudden increase in the number of small parts on systems with small number of CPU cores.
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): fix performance issue when migrating data from VictoriaMetrics according to [these docs](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Add the ability to speed up the data migration via `--vm-native-disable-retries` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4092).
diff --git a/docs/vmalert.md b/docs/vmalert.md
index abf2232eb..e354cfbfd 100644
--- a/docs/vmalert.md
+++ b/docs/vmalert.md
@@ -148,6 +148,15 @@ params:
 headers:
   [ <string>, ...]
 
+# Optional list of HTTP headers in form `header-name: value`
+# applied for all alert notifications sent to notifiers 
+# generated by rules of this group.
+# For example:
+#  notifier_headers:
+#    - "TenantID: foo"
+notifier_headers:
+        [ <string>, ...]
+
 # Optional list of labels added to every rule within a group.
 # It has priority over the external labels.
 # Labels are commonly used for adding environment


Test patch:
diff --git a/app/vmalert/config/config_test.go b/app/vmalert/config/config_test.go
index 694e6ff0b..71eee2822 100644
--- a/app/vmalert/config/config_test.go
+++ b/app/vmalert/config/config_test.go
@@ -538,6 +538,24 @@ rules:
 `)
 	})
 
+	t.Run("`notifier_headers` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+notifier_headers:
+  - "TenantID: foo"
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+notifier_headers:
+  - "TenantID: bar"
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+
 	t.Run("`debug` change", func(t *testing.T) {
 		f(t, `
 name: TestGroup
diff --git a/app/vmalert/config/testdata/rules/rules2-good.rules b/app/vmalert/config/testdata/rules/rules2-good.rules
index 7b17282a0..6fb2d4a2c 100644
--- a/app/vmalert/config/testdata/rules/rules2-good.rules
+++ b/app/vmalert/config/testdata/rules/rules2-good.rules
@@ -5,6 +5,8 @@ groups:
     limit: 1000
     headers:
       - "MyHeader: foo"
+    notifier_headers:
+      - "MyHeader: foo"
     params:
       denyPartialResponse: ["true"]
     rules:
diff --git a/app/vmalert/notifier/alertmanager_test.go b/app/vmalert/notifier/alertmanager_test.go
index 9bc79009f..5f3ed8c6e 100644
--- a/app/vmalert/notifier/alertmanager_test.go
+++ b/app/vmalert/notifier/alertmanager_test.go
@@ -25,6 +25,7 @@ func TestAlertManager_Addr(t *testing.T) {
 
 func TestAlertManager_Send(t *testing.T) {
 	const baUser, baPass = "foo", "bar"
+	const headerKey, headerValue = "TenantID", "foo"
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
 		t.Errorf("should not be called")
@@ -73,6 +74,11 @@ func TestAlertManager_Send(t *testing.T) {
 			if a[0].EndAt.IsZero() {
 				t.Errorf("expected non-zero end time")
 			}
+		case 3:
+			if r.Header.Get(headerKey) != headerValue {
+				t.Errorf("expected header %q to be set to %q; got %q instead",
+					headerKey, headerValue, r.Header.Get(headerKey))
+			}
 		}
 	})
 	srv := httptest.NewServer(mux)
@@ -108,4 +114,7 @@ func TestAlertManager_Send(t *testing.T) {
 	if c != 2 {
 		t.Errorf("expected 2 calls(count from zero) to server got %d", c)
 	}
+	if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
 }