Repository: VictoriaMetrics/VictoriaMetrics
Pull request #6762 (2024-08-05 15:56:46)

vmalert: attaches HTTP headers defined in notifier configuration file…
… for each request to notifiers



Patch:
diff --git a/app/vmalert/notifier/alertmanager.go b/app/vmalert/notifier/alertmanager.go
index d183224d5..ff268dd99 100644
--- a/app/vmalert/notifier/alertmanager.go
+++ b/app/vmalert/notifier/alertmanager.go
@@ -76,9 +76,6 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
-	for key, value := range headers {
-		req.Header.Set(key, value)
-	}
 
 	if am.timeout > 0 {
 		var cancel context.CancelFunc
@@ -94,6 +91,11 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 			return err
 		}
 	}
+	// external headers have higher priority
+	for key, value := range headers {
+		req.Header.Set(key, value)
+	}
+
 	resp, err := am.client.Do(req)
 	if err != nil {
 		return err
@@ -145,7 +147,9 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	aCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
 		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
-		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams))
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams),
+		utils.WithHeaders(strings.Join(authCfg.Headers, "^^")),
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
diff --git a/app/vmselect/netstorage/tmp_blocks_file.go b/app/vmselect/netstorage/tmp_blocks_file.go
index 78288d538..f62b8e48e 100644
--- a/app/vmselect/netstorage/tmp_blocks_file.go
+++ b/app/vmselect/netstorage/tmp_blocks_file.go
@@ -138,7 +138,7 @@ func (tbf *tmpBlocksFile) Finalize() error {
 	tbf.buf = tbf.buf[:0]
 	r := fs.NewReaderAt(tbf.f)
 
-	// Hint the OS that the file is read almost sequentiallly.
+	// Hint the OS that the file is read almost sequentially.
 	// This should reduce the number of disk seeks, which is important
 	// for HDDs.
 	r.MustFadviseSequentialRead(true)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index f6cb6ab75..d0c4ec55e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -33,6 +33,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 * FEATURE: add `/influx/health` health-check handler for Influx endpoints. This is needed as some clients use the health endpoint to determine if the server is healthy and ready for data ingestion. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6653) for the details.
 * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl/): add `--vm-backoff-retries`, `--vm-backoff-factor`, `--vm-backoff-min-duration` and `--vm-native-backoff-retries`, `--vm-native-backoff-factor`, `--vm-native-backoff-min-duration` command-line flags. These flags allow to change backoff policy config for import requests to VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6622).
 
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): respect HTTP headers defined in [notifier configuration file](https://docs.victoriametrics.com/vmalert/#notifier-configuration-file) for each request to notifiers. Previously, this param was ignored by mistake.
+
 ## [v1.102.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.1)
 
 Released at 2024-08-01
diff --git a/docs/data-ingestion/Alloy.md b/docs/data-ingestion/Alloy.md
new file mode 100644
index 000000000..02c33019f
--- /dev/null
+++ b/docs/data-ingestion/Alloy.md
@@ -0,0 +1,222 @@
+---
+title: Grafana Alloy
+weight: 3
+sort: 3
+menu:
+  docs:
+    identifier: "alloy"
+    parent: "data-ingestion"
+    weight: 3
+aliases:
+  - /data-ingestion/Grafana-Alloy.html
+  - /data-ingestion/grafana-alloy.html
+  - /data-ingestion/Grafana-Agent.html
+  - /data-ingestion/grafana-agent.html
+
+---
+
+[Grafana Alloy](https://grafana.com/docs/alloy/latest/) supports sending data via the Prometheus remote write protocol and OpenTelemetry Protocol (OTLP).
+Collecting metrics and forwarding them to VictoriaMetrics using Prometheus scraping and remote writing is more straightforward, but using OpenTelemetry enables more complex processing operations to occur before sending data to VictoriaMetrics.
+
+The Alloy configuration file can be found in the following location depending on your platform:
+- Linux: `/etc/alloy/config.alloy`
+- Windows: `%ProgramFiles%\GrafanaLabs\Alloy\config.alloy`
+- MacOS: `$(brew --prefix)/etc/alloy/config.alloy`
+
+To configure Grafana Alloy to push collected metrics to VictoriaMetrics via Prometheus remote write protocol,
+update the `prometheus.remote_write` endpoint:
+```Alloy
+prometheus.remote_write "victoriametrics" {
+  endpoint {
+    url = "https://<victoriametrics-addr>/prometheus/api/v1/write"
+  }
+}
+```
+
+For pushing data to VictoriaMetrics cluster the `url` should point to vminsert and include 
+the [tenantID](https://docs.victoriametrics.com/cluster-victoriametrics/#url-format):
+```sh
+https://<vminsert-addr>/insert/<tenant_id>/prometheus/api/v1/write
+```
+
+> Note: read more about [multitenancy](https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy)
+> or [multitenancy via labels](https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy-via-labels).
+
+After the configuration has been updated, Alloy must be reloaded or restarted for the change to be applied:
+- Linux: `sudo systemctl reload alloy.service`
+- Windows: `Restart-Service Alloy`, this can also be done from the GUI using task manager
+- MacOS: `brew services restart alloy`
+- Helm chart: changing the `alloy.configMap` in the Alloy [helm values](https://raw.githubusercontent.com/grafana/alloy/main/operations/helm/charts/alloy/values.yaml)
+
+## remote write
+
+In the example below we will be using the node exporter component built into Alloy to generate metrics,
+but any Prometheus scrape target can forward data to VictoriaMetrics.
+Metrics are forwarded from the scrape target to VictoriaMetrics by creating a `prometheus.remote_write` component
+and configuring the `promethues.scrape` component to forward metrics to the `prometheus.remote_write` component.
+
+```Alloy
+prometheus.exporter.unix "nodeexporter" {}
+
+prometheus.scrape "nodeexporter" {
+  targets = prometheus.exporter.unix.nodeexporter.targets
+  forward_to = [prometheus.remote_write.victoriametrics.receiver]
+}
+
+prometheus.remote_write "victoriametrics" {
+  endpoint {
+    url = "https://<victoriametrics-addr>/prometheus/api/v1/write"
+  }
+}
+```
+
+## remote write with basic authentication
+
+This is the same as the previous configuration but adds the `basic_auth` parameters:
+
+```Alloy
+prometheus.exporter.unix "nodeexporter" {}
+
+prometheus.scrape "nodeexporter" {
+  targets = prometheus.exporter.unix.nodeexporter.targets
+  forward_to = [prometheus.remote_write.victoriametrics.receiver]
+}
+
+prometheus.remote_write "victoriametrics" {
+  endpoint {
+    url = "https://<victoriametrics-addr>/prometheus/api/v1/write"
+    basic_auth {
+      username = "<victoriametrics_user>"
+      password = "<victoriametrics_password>"
+    }
+  }
+}
+```
+
+## remote write with bearer authentication
+
+This is the same as the first config but adds the `bearer_token` parameter:
+
+```Alloy
+prometheus.exporter.unix "nodeexporter" {}
+
+prometheus.scrape "nodeexporter" {
+  targets = prometheus.exporter.unix.nodeexporter.targets
+  forward_to = [prometheus.remote_write.victoriametrics.receiver]
+}
+
+prometheus.remote_write "victoriametrics" {
+  endpoint {
+    url = "https://<victoriametrics-addr>/prometheus/api/v1/write"
+    bearer_token  = "<token>"
+  }
+}
+```
+
+## OpenTelemetry
+
+```Alloy
+prometheus.exporter.unix "nodeexporter" {}
+
+prometheus.scrape "nodeexporter" {
+  targets = prometheus.exporter.unix.nodeexporter.targets
+  forward_to = [otelcol.receiver.prometheus.victoriametrics.receiver]
+}
+
+otelcol.receiver.prometheus "victoriametrics" {
+  output {
+    metrics = [otelcol.processor.batch.batch.input]
+  }
+}
+
+otelcol.processor.batch "batch" {
+  output {
+    metrics = [otelcol.exporter.otlphttp.victoriametrics.input]
+  }
+}
+
+otelcol.exporter.otlphttp "victoriametrics" {
+  client {
+    endpoint = "http://<victoriametrics-addr>:<victoriametrics_port>/opentelemetry"
+  }
+}
+```
+
+## OpenTelemetry with Basic Authentication
+
+This is the same configuration without authentication but contains the `otelcol.auth.basic` block 
+and references it in `otelcol.exporter.otlphttp`:
+
+```Alloy
+prometheus.exporter.unix "nodeexporter" {}
+
+prometheus.scrape "nodeexporter" {
+  targets = prometheus.exporter.unix.nodeexporter.targets
+  forward_to = [otelcol.receiver.prometheus.victoriametrics.receiver]
+}
+
+otelcol.auth.basic "otel_auth" {
+  username = "<user>"
+  password = "<password>"
+}
+
+otelcol.receiver.prometheus "victoriametrics" {
+  output {
+    metrics = [otelcol.processor.batch.batch.input]
+  }
+}
+
+otelcol.processor.batch "batch" {
+  output {
+    metrics = [otelcol.exporter.otlphttp.victoriametrics.input]
+  }
+}
+
+otelcol.exporter.otlphttp "victoriametrics" {
+  client {
+    endpoint = "https://<victoriametrics-addr:<victoriametrics_port>/opentelemetry"
+    auth = otelcol.auth.basic.otel_auth.handler
+  }
+}
+```
+
+## OpenTelemetry with Bearer Authentication
+
+This is the same as the basic authentication configuration but swaps the `otelcol.auth.basic` for `otelcol.auth.bearer`:
+
+```Alloy
+prometheus.exporter.unix "nodeexporter" {}
+
+prometheus.scrape "nodeexporter" {
+  targets = prometheus.exporter.unix.nodeexporter.targets
+  forward_to = [otelcol.receiver.prometheus.victoriametrics.receiver]
+}
+
+otelcol.auth.bearer "otel_auth" {
+  token = "<token>"  
+}
+
+otelcol.receiver.prometheus "victoriametrics" {
+  output {
+    metrics = [otelcol.processor.batch.batch.input]
+  }
+}
+
+otelcol.processor.batch "batch" {
+  output {
+    metrics = [otelcol.exporter.otlphttp.victoriametrics.input]
+  }
+}
+
+otelcol.exporter.otlphttp "victoriametrics" {
+  client {
+    endpoint = "https://<victoriametrics-addr>:<victoriametrics_port>/opentelemetry"
+    auth = otelcol.auth.bearer.otel_auth.handler
+  }
+}
+```
+
+## References
+
+- [Grafana Alloy Helm Chart](https://github.com/grafana/alloy/tree/main/operations/helm)
+- [Grafana Alloy Documenation](https://grafana.com/docs/alloy/latest)
diff --git a/docs/data-ingestion/Prometheus.md b/docs/data-ingestion/Prometheus.md
new file mode 100644
index 000000000..a0cc1f9bb
--- /dev/null
+++ b/docs/data-ingestion/Prometheus.md
@@ -0,0 +1,22 @@
+---
+title: Prometheus
+weight: 1
+sort: 1
+menu:
+  docs:
+    identifier: "prometheus"
+    parent: "data-ingestion"
+    weight: 1
+aliases:
+  - /data-ingestion/prometheus.html
+  - /data-ingestion/Prometheus.html
+---
+
+VictoriaMetrics supports ingesting data from Prometheus via the Prometheus remote write protocol.
+The documentation for pushing data from Prometheus to VictoriaMetrics is located [here](https://docs.victoriametrics.com/#prometheus-setup)
+
+## References
+
+- [Prometheus configuration documentation](https://prometheus.io/docs/prometheus/latest/configuration/configuration/)
+- [Prometheus Helm chart values](https://github.com/prometheus-community/helm-charts/blob/main/charts/prometheus/values.yaml)
+- [Prometheus Feature Flags](https://prometheus.io/docs/prometheus/latest/feature_flags/#prometheus-agent)
diff --git a/docs/data-ingestion/pbs-form.webp b/docs/data-ingestion/Proxmox-pbs-form.webp
similarity index 100%
rename from docs/data-ingestion/pbs-form.webp
rename to docs/data-ingestion/Proxmox-pbs-form.webp
diff --git a/docs/data-ingestion/pbs-nav.webp b/docs/data-ingestion/Proxmox-pbs-nav.webp
similarity index 100%
rename from docs/data-ingestion/pbs-nav.webp
rename to docs/data-ingestion/Proxmox-pbs-nav.webp
diff --git a/docs/data-ingestion/pve-form.webp b/docs/data-ingestion/Proxmox-pve-form.webp
similarity index 100%
rename from docs/data-ingestion/pve-form.webp
rename to docs/data-ingestion/Proxmox-pve-form.webp
diff --git a/docs/data-ingestion/pve-nav.webp b/docs/data-ingestion/Proxmox-pve-nav.webp
similarity index 100%
rename from docs/data-ingestion/pve-nav.webp
rename to docs/data-ingestion/Proxmox-pve-nav.webp
diff --git a/docs/data-ingestion/Proxmox.md b/docs/data-ingestion/Proxmox.md
index f5eb91936..44bff2dff 100644
--- a/docs/data-ingestion/Proxmox.md
+++ b/docs/data-ingestion/Proxmox.md
@@ -1,68 +1,68 @@
 ---
 title: Proxmox
-weight: 1
-sort: 1
+weight: 6
+sort: 6
 menu:
   docs:
     identifier: "proxmox"
     parent: "data-ingestion"
-    weight: 1
-    # sort: 1
+    weight: 6
 aliases:
   - /data-ingestion/proxmox.html
   - /data-ingestion/Proxmox.html
 ---
-Since Proxmox Virtual Environment(PVE) and Proxmox Backup Server(PBS) support sending data using the InfluxDB We can use the InfluxDB write support built into VictoriaMetrics
-Currently PVE and PBS only support using an Authorization Token for authentication and does not support basic auth or a username and password.
+
+Since Proxmox Virtual Environment(PVE) and Proxmox Backup Server(PBS) support sending data using the InfluxDB 
+we can use the InfluxDB write support built into VictoriaMetrics.
+Currently, PVE and PBS only support using an Authorization Token for authentication and does not support basic auth 
+or a username and password.
 
 ## Proxmox Virtual Environment (PVE)
-If want help Sending your data to VictoriaMetrics Cloud check out [our blog](https://victoriametrics.com/blog/proxmox-monitoring-with-dbaas/).
+
+> If you want to send your data to VictoriaMetrics Cloud check out [our blog](https://victoriametrics.com/blog/proxmox-monitoring-with-dbaas/).
 
 1. Login to PVE as an administrator
 2. Go to DataCenter > MetricServer > Add > InfluxDB
 
-![PVE Metric Navigation](pve-nav.webp)
+![PVE Metric Navigation](Proxmox-pve-nav.webp)
 
 3. Set the parameters as follows:
   - Name: VictoriaMetrics (can be changed to any string)
   - Server: the hostname or IP of your VictoriaMetrics Instance
-  - Port: This will vary depending how you are sending data to VictoriaMetrics, but the defaults for all components are listed in the [data ingestion documentation](https://docs.victoriametrics.com/data-ingestion.html)
+  - Port: This will vary depending on how you are sending data to VictoriaMetrics, but the defaults for all components are listed in the [data ingestion documentation](https://docs.victoriametrics.com/data-ingestion.html)
   - Protocol: use HTTPS if you have TLS/SSL configured otherwise use HTTP
-  - Organization: leave empty since it doesn't get used
-  - Bucket: leave empty since it doesn't get used
+  - Organization: leave it empty since it doesn't get used
+  - Bucket: leave it empty since it doesn't get used
   - Token: your token from vmauth or leave blank if you don't have authentication enabled
   - If you need to ignore TLS/SSL errors check the advanced box and uncheck the verify certificate box
 4. Click the `Create` button
 
-![PVE Metric Form](pve-form.webp)
+![PVE Metric Form](Proxmox-pve-form.webp)
 
 5. Run `system_uptime{object="nodes"}` in vmui or in the explore view in Grafana to verify metrics from PVE are being sent to VictoriaMetrics.
 You should see 1 time series per node in your PVE cluster.
 
 ## Proxmox Backup Server (PBS)
-1. Login to PBS as an administrator
-2. Go to Configuration > Metrics Server > Add > InfluxDB
 
+1. Log in to PBS as an administrator
+2. Go to Configuration > Metrics Server > Add > InfluxDB
 
-![PBS Metric Navigation](pbs-nav.webp)
-
+![PBS Metric Navigation](Proxmox-pbs-nav.webp)
 
 3.  Set the parameters as follows:
   - Name: VictoriaMetrics (can be set to any string)
   - URL: http(s)://<ip_or_host>:<port>
-    - set the URL to https if you have TLS enabled and http if you do not
-    - Port: This will vary depending how you are sending data to VictoriaMetrics, but the defaults for all components are listed in the [data ingestion documentation](https://docs.victoriametrics.com/data-ingestion.html)
-  - Organization: leave empty since it doesn't get used
-  - Bucket: leave empty since it doesn't get used
+    - set the URL to HTTPS if you have TLS enabled and HTTP if you do not
+    - Port: This will vary depending on how you are sending data to VictoriaMetrics, but the defaults for all components are listed in the [data ingestion documentation](https://docs.victoriametrics.com/data-ingestion.html)
+  - Organization: leave it empty since it doesn't get used
+  - Bucket: leave it empty since it doesn't get used
   - Token: your token from vmauth or leave blank if you don't have authentication enabled
 4. Click the `Create` button
 
-
-![PBS Metric Form](pbs-form.webp)
-
+![PBS Metric Form](Proxmox-pbs-form.webp)
 
 5. Run `cpustat_idle{object="host"}` in vmui or in the explore view in Grafana to verify metrics from PBS are being to VictoriaMetrics.
 
-
 # References
+
 - [Blog Post for configuring VictoriaMetrics Cloud and Proxmox VE](https://victoriametrics.com/blog/proxmox-monitoring-with-dbaas/)
diff --git a/docs/data-ingestion/README.md b/docs/data-ingestion/README.md
index 58b9ed793..442b8b80e 100644
--- a/docs/data-ingestion/README.md
+++ b/docs/data-ingestion/README.md
@@ -6,14 +6,22 @@ If you are unsure what port number to use when pushing data to VictoriaMetrics s
 - VictoriaMetrics Single: 8428
 - vmagent: 8429
 - vmauth: 8427
-- vminsert: 8482
+- vminsert: 8480
 
 In the rest of the documentation we will assume you have configured your push endpoint to use TLS/SSL on port 443 so the urls in the rest of the documentation will look like `https://<victoriametrics_url>` instead of `http://<victoriametrics_url>:8428` for VictoriaMetrics single.
 
 ## Documented Collectors/Agents
-* [Telegraf]({{< relref "Telegraf.md" >}})
-* [Vector]({{< relref "Vector.md" >}})
+
+
+- [Telegraf](https://docs.victoriametrics.com/data-ingestion/telegraf/)
+- [Vector](https://docs.victoriametrics.com/data-ingestion/vector/)
+- [vmagent](https://docs.victoriametrics.com/data-ingestion/vmagent)
+- [Grafana Alloy](https://docs.victoriametrics.com/data-ingestion/alloy/)
+- [Prometheus](https://docs.victoriametrics.com/data-ingestion/prometheus/)
+
 
 ## Supported Platforms
-* [Proxmox Virtual Environment and Proxmox Backup Server]({{< relref "Proxmox.md" >}})
+
+
+- [Proxmox Virtual Environment and Proxmox Backup Server](https://docs.victoriametrics.com/data-ingestion/proxmox/)
 
diff --git a/docs/data-ingestion/Telegraf.md b/docs/data-ingestion/Telegraf.md
index a74e588f5..a716d0dc2 100644
--- a/docs/data-ingestion/Telegraf.md
+++ b/docs/data-ingestion/Telegraf.md
@@ -1,13 +1,12 @@
 ---
 title: Telegraf
-weight: 1
-sort: 1
+weight: 5
+sort: 5
 menu:
   docs:
     identifier: "telegraf"
     parent: "data-ingestion"
-    weight: 1
-    # sort: 1
+    weight: 5
 aliases:
   - /data-ingestion/telegraf.html
   - /data-ingestion/Telegraf.html
@@ -22,10 +21,10 @@ victoriametrics_user="telegraf"
 victoriametrics_password="password"
 victoriametrics_token="my_token"
 ```
-and they can be referenced in a Telegraf configuration file by prepending the variable name with `$` ex. `$victoriametrics_url` will be translated to `https://metrics.example.com` if it referenced in a Telegraf configuration using the values from `/etc/default/telegraf` in the values seen above.
-Otherwise please replace the variables below to fit your setup.
+and they can be referenced in a Telegraf configuration file by prepending the variable name with `$` ex. `$victoriametrics_url` will be translated to `https://metrics.example.com` if it is referenced in a Telegraf configuration using the values from `/etc/default/telegraf` in the values seen above.
+Otherwise, please replace the variables below to fit your setup.
 
-If you want to mimic this behavior on windows please read [Influx Data's blog on storing variables in the registry](https://www.influxdata.com/blog/using-telegraf-on-windows/)
+If you want to mimic this behavior on Windows please read [Influx Data's blog on storing variables in the registry](https://www.influxdata.com/blog/using-telegraf-on-windows/)
 
 ## Minimum Configuration with no Authentication
 ```toml
@@ -67,7 +66,7 @@ This is the same as the minimum configuration but adds the authorization header
 ```
 
 ## Route certain metrics
-If you only want to route certain metrics to VictoriaMetrics use the `namepass` option with a comma separated listed of the measurements you wish to send to VictoriaMetrics.
+If you only want to route certain metrics to VictoriaMetrics use the `namepass` option with a comma separated list of the measurements you wish to send to VictoriaMetrics.
 
 ```
 [[outputs.influxdb]]
@@ -100,5 +99,5 @@ This is not recommended since it can allow sending metrics to a compromised site
 # References 
 - [Install Telegraf](https://docs.influxdata.com/telegraf/v1/install/)
 - [InfluxDBv1 output for Telegraf](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/influxdb)
-- [Storing Telegraf variables in the windows registry](https://www.influxdata.com/blog/using-telegraf-on-windows/)
+- [Storing Telegraf variables in the Windows registry](https://www.influxdata.com/blog/using-telegraf-on-windows/)
 - [Telegraf variables](https://docs.influxdata.com/telegraf/v1/configuration/#example-telegraf-environment-variables)
diff --git a/docs/data-ingestion/Vector.md b/docs/data-ingestion/Vector.md
index 47489111e..e2c283d69 100644
--- a/docs/data-ingestion/Vector.md
+++ b/docs/data-ingestion/Vector.md
@@ -1,19 +1,18 @@
 ---
 title: Vector
-weight: 1
-sort: 1
+weight: 4
+sort: 4
 menu:
   docs:
     identifier: "Vector"
     parent: "data-ingestion"
-    weight: 1
-    # sort: 1
+    weight: 4
 aliases:
   - /data-ingestion/Vector.html
   - /data-ingestion/vector.html
 ---
 To Send data to Vector you need to configure with a Prometheus remote write sink and forward metrics to that sink from at least 1 source.
-You will need to replace the values in `<>` with your to match your setup.
+You will need to replace the values in `<>` to match your setup.
 
 ## Minimum Config
 ```yaml
@@ -117,4 +116,4 @@ sinks:
 
 # References
 - [Vector documentation](https://vector.dev/docs/)
-- [VictoriaLogs documentation for using vector]({{< ref "/victorialogs/data-ingestion/vector" >}})
+- [VictoriaLogs documentation for using vector](https://docs.victoriametrics.com/victorialogs/data-ingestion/vector)
diff --git a/docs/data-ingestion/vmagent.md b/docs/data-ingestion/vmagent.md
new file mode 100644
index 000000000..8be84f119
--- /dev/null
+++ b/docs/data-ingestion/vmagent.md
@@ -0,0 +1,89 @@
+---
+title: vmagent
+weight: 2
+sort: 2
+menu:
+  docs:
+    identifier: "vmagent"
+    parent: "data-ingestion"
+    weight: 2
+aliases:
+  - /data-ingestion/vmagent.html
+  - /data-ingestion/VMAgent.html
+---
+
+vmagent can receive data via the same protocols as VictoriaMetrics Single or Cluster versions,
+as well as scrape Prometheus endpoints. In other words, 
+it supports both [Push](https://docs.victoriametrics.com/keyconcepts/#push-model) and [Pull](https://docs.victoriametrics.com/keyconcepts/#pull-model) models.
+
+This section of the documentation only covers forwarding data from vmagent to another destination.
+For extra information about vmagent as well as quickstart guide please refer to the [vmagent documentation](https://docs.victoriametrics.com/vmagent/).
+
+To configure vmagent to push metrics to VictoriaMetrics via Prometheus remote write protocol,
+configure the `-remoteWrite.url` cmd-line flag:
+
+```sh
+/path/to/vmagent -remoteWrite.url=https://<victoriametrics_url>/api/v1/write
+```
+
+For pushing data to VictoriaMetrics cluster the `-remoteWrite.url` should point to vminsert and include
+the [tenantID](https://docs.victoriametrics.com/cluster-victoriametrics/#url-format):
+```sh
+/path/to/vmagent -remoteWrite.url=https://<vminsert-addr>/insert/<tenant_id>/prometheus/api/v1/write
+```
+
+> Note: read more about [multitenancy](https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy)
+> or [multitenancy via labels](https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy-via-labels).
+
+Please note, `-remoteWrite.url` cmd-line flag can be specified multiple times with different values. In this case,
+vmagent will [replicate](https://docs.victoriametrics.com/vmagent/#replication-and-high-availability) data to each 
+specified destination. In addition, it is possible to configure [metrics sharding](https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages)
+across `-remoteWrite.url` destinations.
+
+## Remote write with basic authentication
+
+This requires setting the `-remoteWrite.basicAuth.username` and `-remoteWrite.basicAuth.password` command line flags:
+
+```sh
+/path/to/vmagent -remoteWrite.url=https://<victoriametrics_url>/api/v1/write \
+ -remoteWrite.basicAuth.username=<username> \
+ -remoteWrite.basicAuth.password=<password>
+```
+
+## Remote write with bearer Authentication
+
+```sh
+/path/to/vmagent -remoteWrite.url=https://<victoriametrics_url>/api/v1/write \
+ -remoteWrite.bearerToken=<token>
+```
+
+The token can be placed in a file and accessed via the `-remoteWrite.bearerTokenFile` command line argument.
+The file needs to be readable by the user vmagent is running as. The token file should only contain the token as seen below:
+
+```
+<token>
+```
+
+The command to run vmagent with a token file will be the following:
+
+```sh
+/path/to/vmagent -remoteWrite.url=https://<victoriametrics_url>/api/v1/write \
+ -remoteWrite.bearerTokenFile=/path/to/tokenfile
+
+```
+
+## Ignore TLS/SSL errors
+
+If you are using self-signed certificates you can either ignore certificates issues using 
+the `-remoteWrite.tlsInsecureSkipVerify`, which is a security risk, or use `-remoteWrite.tlsCAFile` to point
+to a file containing the self-signed CA certificate:
+
+```sh
+/path/to/vmagent -remoteWrite.url=https://<victoriametrics_url>/api/v1/write \
+ -remoteWrite.tlsInsecureSkipVerify
+```
+
+## References
+
+- [vmagent docs](https://docs.victoriametrics.com/vmagent/)
+- [vmagent commandline flags](https://docs.victoriametrics.com/vmagent/#advanced-usage)
diff --git a/lib/fs/fs.go b/lib/fs/fs.go
index 980066b10..9a451fb24 100644
--- a/lib/fs/fs.go
+++ b/lib/fs/fs.go
@@ -52,7 +52,7 @@ func MustWriteSync(path string, data []byte) {
 // in the middle of the write.
 //
 // If the file at path already exists, then the file is overwritten atomically if canOverwrite is true.
-// Otherwise error is returned.
+// Otherwise, error is returned.
 func MustWriteAtomic(path string, data []byte, canOverwrite bool) {
 	// Check for the existing file. It is expected that
 	// the MustWriteAtomic function cannot be called concurrently


Test patch:
diff --git a/app/vmalert/notifier/alertmanager_test.go b/app/vmalert/notifier/alertmanager_test.go
index 94868b8bf..e792090c4 100644
--- a/app/vmalert/notifier/alertmanager_test.go
+++ b/app/vmalert/notifier/alertmanager_test.go
@@ -3,6 +3,7 @@ package notifier
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
@@ -48,6 +49,9 @@ func TestAlertManager_Send(t *testing.T) {
 			conn, _, _ := w.(http.Hijacker).Hijack()
 			_ = conn.Close()
 		case 1:
+			if r.Header.Get(headerKey) != headerValue {
+				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
+			}
 			w.WriteHeader(500)
 		case 2:
 			var a []struct {
@@ -72,6 +76,9 @@ func TestAlertManager_Send(t *testing.T) {
 			if a[0].EndAt.IsZero() {
 				t.Fatalf("expected non-zero end time")
 			}
+			if r.Header.Get(headerKey) != "bar" {
+				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
+			}
 		case 3:
 			if r.Header.Get(headerKey) != headerValue {
 				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
@@ -86,6 +93,7 @@ func TestAlertManager_Send(t *testing.T) {
 			Username: baUser,
 			Password: promauth.NewSecret(baPass),
 		},
+		Headers: []string{fmt.Sprintf("%s:%s", headerKey, headerValue)},
 	}
 	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
@@ -105,7 +113,7 @@ func TestAlertManager_Send(t *testing.T) {
 		Start:       time.Now().UTC(),
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
-	}}, nil); err != nil {
+	}}, map[string]string{headerKey: "bar"}); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 	if c != 2 {