Repository: VictoriaMetrics/VictoriaMetrics
Pull request #5608 (2024-01-12 08:08:57)

app/vmselect/promql: properly handle possible negative results caused…
… by float operations precision error in rollup functions like rate() or increase()

related https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5571

hagen1778: Tests are failing(
Haleygo: > Tests are failing(

Sorry, fixed, please check)

Patch:
diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go
index 5279f9d9c..94e26a1b6 100644
--- a/app/vmselect/netstorage/netstorage.go
+++ b/app/vmselect/netstorage/netstorage.go
@@ -1044,7 +1044,7 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 			for xw := range workCh {
 				if err := f(&xw.mn, &xw.b, tr, workerID); err != nil {
 					errGlobalLock.Lock()
-					if errGlobal != nil {
+					if errGlobal == nil {
 						errGlobal = err
 						atomic.StoreUint32(&mustStop, 1)
 					}
diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go
index cd5f2bbff..14d9c5c08 100644
--- a/app/vmselect/promql/rollup.go
+++ b/app/vmselect/promql/rollup.go
@@ -859,6 +859,11 @@ func removeCounterResets(values []float64) {
 		}
 		prevValue = v
 		values[i] = v + correction
+		// Check again, there could be precision error in float operations,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5571
+		if i > 0 && values[i] < values[i-1] {
+			values[i] = values[i-1]
+		}
 	}
 }
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 20e713147..29f56196c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -30,7 +30,7 @@ The sandbox cluster installation is running under the constant load generated by
 
 * SECURITY: upgrade Go builder from Go1.21.5 to Go1.21.6. See [the list of issues addressed in Go1.21.6](https://github.com/golang/go/issues?q=milestone%3AGo1.21.6+label%3ACherryPickApproved).
 
-* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for Service Discovery of the Hetzner Cloud and Hetzner Robot API targets. [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3154).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for discovering [Hetzner Cloud](https://www.hetzner.com/cloud) and [Hetzner Robot](https://docs.hetzner.com/robot) scrape targets. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3154) and [these docs](https://docs.victoriametrics.com/sd_configs.html#hetzner_sd_configs).
 * FEATURE: [graphite](https://docs.victoriametrics.com/#graphite-render-api-usage): add support for negative index in `groupByNode` and `aliasByNode` functions. Thanks to @rbizos for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5581).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [DataDog v2 data ingestion protocol](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics). See [these docs](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4451).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose ability to set OAuth2 endpoint parameters per each `-remoteWrite.url` via the command-line flag `-remoteWrite.oauth2.endpointParams`. See [these docs](https://docs.victoriametrics.com/vmagent.html#advanced-usage). Thanks to @mhill-holoplot for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5427).
@@ -40,15 +40,17 @@ The sandbox cluster installation is running under the constant load generated by
   - `-remoteRead.oauth2.endpointParams` for `-remoteRead.url`
   - `-remoteWrite.oauth2.endpointParams` for `-remoteWrite.url`
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy incoming requests to different backends based on the requested host via `src_hosts` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth.html#generic-http-proxy-for-different-backends).
-* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): rename cmd-line flag `vm-native-disable-retries` to `vm-native-disable-per-metric-migration` to better reflect its meaning.
+* FEATURE: all VictoriaMetrics components: break HTTP client connection if an error occurs after the server at `-httpListenAddr` already sent response status code. Previously such an error couldn't be detected at client side. Now the client will get an error about invalid chunked response. The error message is simultaneously written to the server log and in the last line of the response. This should help detecting errors when migrating data between VictoriaMetrics instances by [vmctl](https://docs.victoriametrics.com/vmctl.html). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5645).
 * FEATURE: all VictoriaMetrics components: add ability to specify arbitrary HTTP headers to send with every request to `-pushmetrics.url`. See [`push metrics` docs](https://docs.victoriametrics.com/#push-metrics).
 * FEATURE: all VictoriaMetrics components: add `-metrics.exposeMetadata` command-line flag, which allows displaying `TYPE` and `HELP` metadata at `/metrics` page exposed at `-httpListenAddr`. This may be needed when the `/metrics` page is scraped by collector, which requires the `TYPE` and `HELP` metadata such as [Google Cloud Managed Prometheus](https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type).
 * FEATURE: dashboards/cluster: add panels for detailed visualization of traffic usage between vmstorage, vminsert, vmselect components and their clients. New panels are available in the rows dedicated to specific components.
 * FEATURE: dashboards/cluster: update "Slow Queries" panel to show percentage of the slow queries to the total number of read queries served by vmselect. The percentage value should make it more clear for users whether there is a service degradation.
 * FEATURE: dashboards/single: change dashboard title from `VictoriaMetrics` to `VictoriaMetrics - single-node`. The new title should provide better understanding of this dashboard purpose.
-* FEATURE [vmctl](https://docs.victoriametrics.com/vmctl.html): add `-vm-native-src-insecure-skip-verify` and `-vm-native-dst-insecure-skip-verify` command-line flags for native protocol. It can be used for skipping TLS certificate verification when connecting to the source or destination addresses.
+* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): rename cmd-line flag `vm-native-disable-retries` to `vm-native-disable-per-metric-migration` to better reflect its meaning.
+* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `-vm-native-src-insecure-skip-verify` and `-vm-native-dst-insecure-skip-verify` command-line flags for native protocol. It can be used for skipping TLS certificate verification when connecting to the source or destination addresses.
 * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): add `job` label to `DiskRunsOutOfSpace` alerting rule, so it is easier to understand to which installation the triggered instance belongs.
 
+* BUGFIX: properly return errors from [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). Previously these errors were silently suppressed. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5649).
 * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly return full results when `-search.skipSlowReplicas` command-line flag is passed to `vmselect` and when [vmstorage groups](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#vmstorage-groups-at-vmselect) are in use. Previously partial results could be returned in this case.
 * BUGFIX: `vminsert`: properly accept samples via [OpenTelemetry data ingestion protocol](https://docs.victoriametrics.com/#sending-data-via-opentelemetry) when these samples have no [resource attributes](https://opentelemetry.io/docs/instrumentation/go/resources/). Previously such samples were silently skipped.
 * BUGFIX: `vmstorage`: added missing `-inmemoryDataFlushInterval` command-line flag, which was missing in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) after implementing [this feature](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3337) in [v1.85.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.0).
@@ -61,11 +63,12 @@ The sandbox cluster installation is running under the constant load generated by
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): retry on import errors in `vm-native` mode. Before, retries happened only on writes into a network connection between source and destination. But errors returned by server after all the data was transmitted were logged, but not retried.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly assume role with [AWS IRSA authorization](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). Previously role chaining was not supported. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3822) for details.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): exit if there is config syntax error in [`scrape_config_files`](https://docs.victoriametrics.com/vmagent.html#loading-scrape-configs-from-multiple-files) when `-promscrape.config.strictParse=true`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5508).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not store scrape response for target in memory when staleness markers are disabled. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5577) for details.
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix a link for the statistic inaccuracy explanation in the cardinality explorer tool. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5460).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): send `step` param for instant queries. The change reverts [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3896) due to reasons explained in [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3896#issuecomment-1896704401).
 * BUGFIX: all: fix potential panic during components shutdown when [metrics push](https://docs.victoriametrics.com/#push-metrics) is configured. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5548). Thanks to @zhdd99 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5549).
 * BUGFIX: [vmselect](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly determine time range search for instant queries with too big look-behind window like `foo[100y]`. Previously, such queries could return empty responses even if `foo` is present in database.
-* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): send `step` param for instant queries. The change reverts [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3896) due to reasons explained in [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3896#issuecomment-1896704401).
-
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly handle possible negative results caused by float operations precision error in rollup functions like rate() or increase(). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5571).
 
 ## [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0)
 
diff --git a/docs/anomaly-detection/Overview.md b/docs/anomaly-detection/Overview.md
index ed196b2e0..824ea01d2 100644
--- a/docs/anomaly-detection/Overview.md
+++ b/docs/anomaly-detection/Overview.md
@@ -1,5 +1,5 @@
 ---
-title: vmanomaly overview
+title: Overview
 weight: 1
 sort: 1
 menu:
@@ -7,13 +7,12 @@ menu:
     identifier: "vmanomaly-overview"
     parent: 'anomaly-detection'
     weight: 1
-    title: Overview
 aliases:
 - /anomaly-detection.html
 ---
 
 
-# vmanomaly overview
+# Overview
 
 **_vmanomaly_ is a part of [enterprise package](https://docs.victoriametrics.com/enterprise.html). You need to request a [free trial license](https://victoriametrics.com/products/enterprise/trial/) for evaluation.**
 
diff --git a/docs/sd_configs.md b/docs/sd_configs.md
index 0a5c6044d..6b4a2c2ce 100644
--- a/docs/sd_configs.md
+++ b/docs/sd_configs.md
@@ -27,6 +27,7 @@ aliases:
 * `eureka_sd_configs` is for discovering and scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka). See [these docs](#eureka_sd_configs).
 * `file_sd_configs` is for scraping targets defined in external files (aka file-based service discovery). See [these docs](#file_sd_configs).
 * `gce_sd_configs` is for discovering and scraping [Google Compute Engine](https://cloud.google.com/compute) targets. See [these docs](#gce_sd_configs).
+* `hetzner_sd_configs` is for discovering and scraping [Hetzner Cloud](https://www.hetzner.com/cloud) and [Hetzner Robot](https://docs.hetzner.com/robot) targets. See [these docs](#hetzner_sd_configs).
 * `http_sd_configs` is for discovering and scraping targets provided by external http-based service discovery. See [these docs](#http_sd_configs).
 * `kubernetes_sd_configs` is for discovering and scraping [Kubernetes](https://kubernetes.io/) targets. See [these docs](#kubernetes_sd_configs).
 * `kuma_sd_configs` is for discovering and scraping [Kuma](https://kuma.io) targets. See [these docs](#kuma_sd_configs).
@@ -34,7 +35,6 @@ aliases:
 * `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [these docs](#openstack_sd_configs).
 * `static_configs` is for scraping statically defined targets. See [these docs](#static_configs).
 * `yandexcloud_sd_configs` is for discovering and scraping [Yandex Cloud](https://cloud.yandex.com/en/) targets. See [these docs](#yandexcloud_sd_configs).
-* `hetzner_sd_configs` is for discovering and scraping [Hetzner Cloud](https://www.hetzner.com/cloud) and [Hetzner Robot](https://robot.hetzner.com/) targets. See [these docs](#hetzner_sd_configs).
 
 Note that the `refresh_interval` option isn't supported for these scrape configs. Use the corresponding `-promscrape.*CheckInterval`
 command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs`
@@ -797,6 +797,80 @@ The following meta labels are available on discovered targets during [relabeling
 
 The list of discovered GCE targets is refreshed at the interval, which can be configured via `-promscrape.gceSDCheckInterval` command-line flag.
 
+## hetzner_sd_configs
+
+Hetzner SD configuration allows retrieving scrape targets from [Hetzner Cloud](https://www.hetzner.com/cloud) and [Hetzner Robot](https://docs.hetzner.com/robot).
+
+Configuration example:
+
+```yaml
+scrape_configs:
+- job_name: hetzner
+  hetzner_sd_configs:
+    # The mandatory Hetzner role for entity discovery.
+    # Must be either 'robot' or 'hcloud'.
+    role: "hcloud"
+
+    # Required credentials for API server authentication for 'hcloud' role.
+    authorization:
+      credentials: "..."
+      # type: "..."  # default: Bearer
+      # credentials_file: "..."  # is mutually-exclusive with credentials
+
+    # Required credentials for API server authentication for 'robot' role.
+    #
+    # basic_auth:
+    #  username: "..."
+    #  password: "..."
+    #  password_file: "..."  # is mutually-exclusive with password
+
+    # port is an optional port to scrape metrics from.
+    # By default, port 80 is used.
+    # port: ...
+
+    # Additional HTTP API client options can be specified here.
+    # See https://docs.victoriametrics.com/sd_configs.html#http-api-client-options
+```
+
+Each discovered target has an [`__address__`](https://docs.victoriametrics.com/relabeling.html#how-to-modify-scrape-urls-in-targets) label set
+to `<FQDN>:<port>`, where FQDN is discovered instance address and `<port>` is the port from the `hetzner_sd_configs` (default port is `80`).
+
+The following meta labels are available on discovered targets during [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling):
+
+Common labels for both `hcloud` and `robot` roles:
+
+* `__meta_hetzner_datacenter`: the datacenter of the server
+* `__meta_hetzner_public_ipv4`: the public IPv4 address of the server
+* `__meta_hetzner_public_ipv6_network`: the public IPv6 network (/64) of the server
+* `__meta_hetzner_role`: the current role `hcloud` or `robot`
+* `__meta_hetzner_server_id`: the ID of the server
+* `__meta_hetzner_server_name`: the name of the server
+* `__meta_hetzner_server_status`: the status of the server
+
+Additional labels for `role: hcloud`:
+
+* `__meta_hetzner_hcloud_datacenter_location`: the location of the server
+* `__meta_hetzner_hcloud_datacenter_location_network_zone`: the network zone of the server
+* `__meta_hetzner_hcloud_cpu_cores`: the CPU cores count of the server
+* `__meta_hetzner_hcloud_cpu_type`: the CPU type of the server (shared or dedicated)
+* `__meta_hetzner_hcloud_disk_size_gb`: the disk size of the server (in GB)
+* `__meta_hetzner_hcloud_image_description`: the description of the server image
+* `__meta_hetzner_hcloud_image_name`: the image name of the server
+* `__meta_hetzner_hcloud_image_os_flavor`: the OS flavor of the server image
+* `__meta_hetzner_hcloud_image_os_version`: the OS version of the server image
+* `__meta_hetzner_hcloud_label_<labelname>`: each label of the server
+* `__meta_hetzner_hcloud_labelpresent_<labelname>`: true for each label of the server
+* `__meta_hetzner_hcloud_memory_size_gb`: the amount of memory of the server (in GB)
+* `__meta_hetzner_hcloud_private_ipv4_<networkname>`: the private IPv4 address of the server within a given network
+* `__meta_hetzner_hcloud_server_type`: the type of the server
+
+Additional labels for `role: robot`:
+
+* `__meta_hetzner_robot_cancelled`: the server cancellation status
+* `__meta_hetzner_robot_product`: the product of the server
+
+The list of discovered Hetzner targets is refreshed at the interval, which can be configured via `-promscrape.hetznerSDCheckInterval` command-line flag.
+
 ## http_sd_configs
 
 HTTP-based service discovery fetches targets from the specified `url`.
@@ -1375,86 +1449,6 @@ The following meta labels are available on discovered targets during [relabeling
 
 The list of discovered Yandex Cloud targets is refreshed at the interval, which can be configured via `-promscrape.yandexcloudSDCheckInterval` command-line flag.
 
-## hetzner_sd_configs
-
-Hetzner SD configuration allows to retrieving scrape targets from [Hetzner Cloud](https://www.hetzner.com/cloud) and [Hetzner Robot](https://robot.hetzner.com/).
-
-Configuration example:
-
-```yaml
-scrape_configs:
-- job_name: hetzner
-  hetzner_sd_configs:
-    # Define the mandatory Hetzner role for entity discovery.
-    # Must be either 'robot' or 'hcloud'.
-    role: <string>
-
-    # Credentials for API server authentication.
-    # Note: `basic_auth` is required for 'robot' role.
-    # `authorization` is required for 'hcloud' role.
-    # `basic_auth` and `authorization` are mutually exclusive options.
-    # Similarly, `password` and `password_file` cannot be used together.
-    #   ...
-
-    # port is an optional port to scrape metrics from.
-    # By default, port 80 is used.
-    # port: ...
-```
-
-```yaml
-scrape_configs:
-- job_name: hcloud
-  hetzner_sd_configs:
-    - role: hcloud
-      authorization:
-        credentials: ZGI12cup........
-
-- job_name: robot
-  hetzner_sd_configs:
-    - role: robot
-      basic_auth:
-        username: hello
-        password: password-example
-```
-
-Each discovered target has an [`__address__`](https://docs.victoriametrics.com/relabeling.html#how-to-modify-scrape-urls-in-targets) label set
-to the FQDN of the discovered instance.
-
-The following meta labels are available on discovered targets during [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling):
-
-Hetzner Labels (Avalibaly for both `hcloud` and `robot` Roles.)
-
-* `__meta_hetzner_server_id`: the ID of the server
-* `__meta_hetzner_server_name`: the name of the server
-* `__meta_hetzner_server_status`: the status of the server
-* `__meta_hetzner_public_ipv4`: the public IPv4 address of the server
-* `__meta_hetzner_public_ipv6_network`: the public IPv6 network (/64) of the server
-* `__meta_hetzner_datacenter`: the datacenter of the server
-
-Hetzner Labels (Only whetn `hcloud` Role is set)
-
-* `__meta_hetzner_hcloud_image_name`: the image name of the server
-* `__meta_hetzner_hcloud_image_description`: the description of the server image
-* `__meta_hetzner_hcloud_image_os_flavor`: the OS flavor of the server image
-* `__meta_hetzner_hcloud_image_os_version`: the OS version of the server image
-* `__meta_hetzner_hcloud_datacenter_location`: the location of the server
-* `__meta_hetzner_hcloud_datacenter_location_network_zone`: the network zone of the server
-* `__meta_hetzner_hcloud_server_type`: the type of the server
-* `__meta_hetzner_hcloud_cpu_cores`: the CPU cores count of the server
-* `__meta_hetzner_hcloud_cpu_type`: the CPU type of the server (shared or dedicated)
-* `__meta_hetzner_hcloud_memory_size_gb`: the amount of memory of the server (in GB)
-* `__meta_hetzner_hcloud_disk_size_gb`: the disk size of the server (in GB)
-* `__meta_hetzner_hcloud_private_ipv4_<networkname>`: the private IPv4 address of the server within a given network
-* `__meta_hetzner_hcloud_label_<labelname>`: each label of the server
-* `__meta_hetzner_hcloud_labelpresent_<labelname>`: true for each label of the server
-
-Hetzner Labels (Only whetn `robot` Role is set)
-
-* `__meta_hetzner_robot_product`: the product of the server
-* `__meta_hetzner_robot_cancelled`: the server cancellation status
-
-The list of discovered Yandex Cloud targets is refreshed at the interval, which can be configured via `-promscrape.hetznerSDCheckInterval` command-line flag.
-
 ## scrape_configs
 
 The `scrape_configs` section at file pointed by `-promscrape.config` command-line flag can contain [supported service discovery options](#supported-service-discovery-configs).
diff --git a/docs/vmagent.md b/docs/vmagent.md
index 1e35e9aa7..7e39f5ede 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -1787,6 +1787,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Interval for checking for changes in 'file_sd_config'. See https://docs.victoriametrics.com/sd_configs.html#file_sd_configs for details (default 1m0s)
   -promscrape.gceSDCheckInterval duration
      Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#gce_sd_configs for details (default 1m0s)
+  -promscrape.hetznerSDCheckInterval duration
+     Interval for checking for changes in Hetnzer API. This works only if hetzner_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#hetzner_sd_configs for details (default 30s)
   -promscrape.httpSDCheckInterval duration
      Interval for checking for changes in http endpoint service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#http_sd_configs for details (default 1m0s)
   -promscrape.kubernetes.apiServerTimeout duration
@@ -1826,8 +1828,6 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors
   -promscrape.yandexcloudSDCheckInterval duration
      Interval for checking for changes in Yandex Cloud API. This works only if yandexcloud_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#yandexcloud_sd_configs for details (default 30s)
-  -promscrape.hetznerSDCheckInterval duration
-     Interval for checking for changes in Hetnzer API. This works only if hetzner_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#hetzner_sd_configs for details (default 30s)
   -pushmetrics.disableCompression
      Whether to disable request body compression when pushing metrics to every -pushmetrics.url
   -pushmetrics.extraLabel array
diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go
index 27ba674bc..b847843c1 100644
--- a/lib/httpserver/httpserver.go
+++ b/lib/httpserver/httpserver.go
@@ -214,8 +214,10 @@ var gzipHandlerWrapper = func() func(http.Handler) http.HandlerFunc {
 	return hw
 }()
 
-var metricsHandlerDuration = metrics.NewHistogram(`vm_http_request_duration_seconds{path="/metrics"}`)
-var connTimeoutClosedConns = metrics.NewCounter(`vm_http_conn_timeout_closed_conns_total`)
+var (
+	metricsHandlerDuration = metrics.NewHistogram(`vm_http_request_duration_seconds{path="/metrics"}`)
+	connTimeoutClosedConns = metrics.NewCounter(`vm_http_conn_timeout_closed_conns_total`)
+)
 
 var hostname = func() string {
 	h, err := os.Hostname()
@@ -358,6 +360,10 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 		if !CheckBasicAuth(w, r) {
 			return
 		}
+
+		w = &responseWriterWithAbort{
+			ResponseWriter: w,
+		}
 		if rh(w, r) {
 			return
 		}
@@ -471,6 +477,69 @@ func GetQuotedRemoteAddr(r *http.Request) string {
 	return strconv.Quote(remoteAddr)
 }
 
+type responseWriterWithAbort struct {
+	http.ResponseWriter
+
+	sentHeaders bool
+	aborted     bool
+}
+
+func (rwa *responseWriterWithAbort) Write(data []byte) (int, error) {
+	if rwa.aborted {
+		return 0, fmt.Errorf("response connection is aborted")
+	}
+	if !rwa.sentHeaders {
+		rwa.sentHeaders = true
+	}
+	return rwa.ResponseWriter.Write(data)
+}
+
+func (rwa *responseWriterWithAbort) WriteHeader(statusCode int) {
+	if rwa.aborted {
+		logger.WarnfSkipframes(1, "cannot write response headers with statusCode=%d, since the response connection has been aborted", statusCode)
+		return
+	}
+	if rwa.sentHeaders {
+		logger.WarnfSkipframes(1, "cannot write response headers with statusCode=%d, since they were already sent", statusCode)
+		return
+	}
+	rwa.ResponseWriter.WriteHeader(statusCode)
+	rwa.sentHeaders = true
+}
+
+// abort aborts the client connection associated with rwa.
+//
+// The last http chunk in the response stream is intentionally written incorrectly,
+// so the client, which reads the response, could notice this error.
+func (rwa *responseWriterWithAbort) abort() {
+	if !rwa.sentHeaders {
+		logger.Panicf("BUG: abort can be called only after http response headers are sent")
+	}
+	if rwa.aborted {
+		logger.WarnfSkipframes(2, "cannot abort the connection, since it has been already aborted")
+		return
+	}
+	hj, ok := rwa.ResponseWriter.(http.Hijacker)
+	if !ok {
+		logger.Panicf("BUG: ResponseWriter must implement http.Hijacker interface")
+	}
+	conn, bw, err := hj.Hijack()
+	if err != nil {
+		logger.WarnfSkipframes(2, "cannot hijack response connection: %s", err)
+		return
+	}
+
+	// Just write an error message into the client connection as is without http chunked encoding.
+	// This is needed in order to notify the client about the aborted connection.
+	_, _ = bw.WriteString("\nthe connection has been aborted; see the last line in the response and/or in the server log for the reason\n")
+	_ = bw.Flush()
+
+	// Forcibly close the client connection in order to break http keep-alive at client side.
+	_ = conn.Close()
+
+	rwa.aborted = true
+}
+
 // Errorf writes formatted error message to w and to logger.
 func Errorf(w http.ResponseWriter, r *http.Request, format string, args ...interface{}) {
 	errStr := fmt.Sprintf(format, args...)
@@ -488,6 +557,13 @@ func Errorf(w http.ResponseWriter, r *http.Request, format string, args ...inter
 			break
 		}
 	}
+	if rwa, ok := w.(*responseWriterWithAbort); ok && rwa.sentHeaders {
+		// HTTP status code has been already sent to client, so it cannot be sent again.
+		// Just write errStr to the response and abort the client connection, so the client could notice the error.
+		fmt.Fprintf(w, "\n%s\n", errStr)
+		rwa.abort()
+		return
+	}
 	http.Error(w, errStr, statusCode)
 }
 
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index f93961875..6da5fc85c 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -307,6 +307,7 @@ type ScrapeConfig struct {
 	EurekaSDConfigs       []eureka.SDConfig       `yaml:"eureka_sd_configs,omitempty"`
 	FileSDConfigs         []FileSDConfig          `yaml:"file_sd_configs,omitempty"`
 	GCESDConfigs          []gce.SDConfig          `yaml:"gce_sd_configs,omitempty"`
+	HetznerSDConfigs      []hetzner.SDConfig      `yaml:"hetzner_sd_configs,omitempty"`
 	HTTPSDConfigs         []http.SDConfig         `yaml:"http_sd_configs,omitempty"`
 	KubernetesSDConfigs   []kubernetes.SDConfig   `yaml:"kubernetes_sd_configs,omitempty"`
 	KumaSDConfigs         []kuma.SDConfig         `yaml:"kuma_sd_configs,omitempty"`
@@ -314,7 +315,6 @@ type ScrapeConfig struct {
 	OpenStackSDConfigs    []openstack.SDConfig    `yaml:"openstack_sd_configs,omitempty"`
 	StaticConfigs         []StaticConfig          `yaml:"static_configs,omitempty"`
 	YandexCloudSDConfigs  []yandexcloud.SDConfig  `yaml:"yandexcloud_sd_configs,omitempty"`
-	HetznerSDConfigs      []hetzner.SDConfig      `yaml:"hetzner_sd_configs,omitempty"`
 
 	// These options are supported only by lib/promscrape.
 	DisableCompression  bool                       `yaml:"disable_compression,omitempty"`
@@ -376,6 +376,9 @@ func (sc *ScrapeConfig) mustStop() {
 	for i := range sc.GCESDConfigs {
 		sc.GCESDConfigs[i].MustStop()
 	}
+	for i := range sc.HetznerSDConfigs {
+		sc.HetznerSDConfigs[i].MustStop()
+	}
 	for i := range sc.HTTPSDConfigs {
 		sc.HTTPSDConfigs[i].MustStop()
 	}
@@ -670,6 +673,16 @@ func (cfg *Config) getGCESDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 	return cfg.getScrapeWorkGeneric(visitConfigs, "gce_sd_config", prev)
 }
 
+// getHetznerSDScrapeWork returns `hetzner_sd_configs` ScrapeWork from cfg.
+func (cfg *Config) getHetznerSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
+	visitConfigs := func(sc *ScrapeConfig, visitor func(sdc targetLabelsGetter)) {
+		for i := range sc.HetznerSDConfigs {
+			visitor(&sc.HetznerSDConfigs[i])
+		}
+	}
+	return cfg.getScrapeWorkGeneric(visitConfigs, "hetzner_sd_config", prev)
+}
+
 // getHTTPDScrapeWork returns `http_sd_configs` ScrapeWork from cfg.
 func (cfg *Config) getHTTPDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 	visitConfigs := func(sc *ScrapeConfig, visitor func(sdc targetLabelsGetter)) {
@@ -748,16 +761,6 @@ func (cfg *Config) getYandexCloudSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork
 	return cfg.getScrapeWorkGeneric(visitConfigs, "yandexcloud_sd_config", prev)
 }
 
-// getHetznerSDScrapeWork returns `hetzner_sd_configs` ScrapeWork from cfg.
-func (cfg *Config) getHetznerSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
-	visitConfigs := func(sc *ScrapeConfig, visitor func(sdc targetLabelsGetter)) {
-		for i := range sc.HetznerSDConfigs {
-			visitor(&sc.HetznerSDConfigs[i])
-		}
-	}
-	return cfg.getScrapeWorkGeneric(visitConfigs, "hetzner_sd_config", prev)
-}
-
 type targetLabelsGetter interface {
 	GetLabels(baseDir string) ([]*promutils.Labels, error)
 }
diff --git a/lib/promscrape/discovery/hetzner/api.go b/lib/promscrape/discovery/hetzner/api.go
index 8f1bd2b0d..e34b29cfb 100644
--- a/lib/promscrape/discovery/hetzner/api.go
+++ b/lib/promscrape/discovery/hetzner/api.go
@@ -28,17 +28,19 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	var apiServer string
 	switch sdc.Role {
 	case "robot":
+		// See https://robot.hetzner.com/doc/webservice/en.html
 		apiServer = "https://robot-ws.your-server.de"
 		if hcc.BasicAuth == nil {
-			return nil, fmt.Errorf("basic_auth must be set when role is `%q`", sdc.Role)
+			return nil, fmt.Errorf("basic_auth must be set when role is `robot`")
 		}
 	case "hcloud":
-		apiServer = "https://api.hetzner.cloud/v1"
+		// See https://docs.hetzner.cloud/
+		apiServer = "https://api.hetzner.cloud"
 		if hcc.Authorization == nil {
-			return nil, fmt.Errorf("authorization must be set when role is `%q`", sdc.Role)
+			return nil, fmt.Errorf("authorization must be set when role is `hcloud`")
 		}
 	default:
-		return nil, fmt.Errorf("skipping unexpected role=%q; must be one of `robot` or `hcloud`", sdc.Role)
+		return nil, fmt.Errorf("unexpected role=%q; must be one of `robot` or `hcloud`", sdc.Role)
 	}
 
 	ac, err := hcc.NewConfig(baseDir)
diff --git a/lib/promscrape/discovery/hetzner/hcloud.go b/lib/promscrape/discovery/hetzner/hcloud.go
index f7fde4070..9ec688068 100644
--- a/lib/promscrape/discovery/hetzner/hcloud.go
+++ b/lib/promscrape/discovery/hetzner/hcloud.go
@@ -3,73 +3,189 @@ package hetzner
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
-// HcloudServerList represents a list of servers from Hetzner Cloud API.
-type HcloudServerList struct {
-	Servers []HcloudServer `json:"servers"`
+// getHCloudServerLabels returns labels for hcloud servers obtained from the given cfg
+func getHCloudServerLabels(cfg *apiConfig) ([]*promutils.Labels, error) {
+	networks, err := getHCloudNetworks(cfg)
+	if err != nil {
+		return nil, err
+	}
+	servers, err := getHCloudServers(cfg)
+	if err != nil {
+		return nil, err
+	}
+	var ms []*promutils.Labels
+	for i := range servers {
+		ms = appendHCloudTargetLabels(ms, &servers[i], networks, cfg.port)
+	}
+	return ms, nil
 }
 
-// HcloudServer represents the structure of server data.
-type HcloudServer struct {
-	ID         int               `json:"id"`
-	Name       string            `json:"name"`
-	Status     string            `json:"status"`
-	PublicNet  PublicNet         `json:"public_net,omitempty"`
-	PrivateNet []PrivateNet      `json:"private_net,omitempty"`
-	ServerType ServerType        `json:"server_type"`
-	Datacenter Datacenter        `json:"datacenter"`
-	Image      Image             `json:"image"`
-	Labels     map[string]string `json:"labels"`
+func appendHCloudTargetLabels(ms []*promutils.Labels, server *HCloudServer, networks []HCloudNetwork, port int) []*promutils.Labels {
+	m := promutils.NewLabels(24)
+
+	addr := discoveryutils.JoinHostPort(server.PublicNet.IPv4.IP, port)
+	m.Add("__address__", addr)
+
+	m.Add("__meta_hetzner_role", "hcloud")
+	m.Add("__meta_hetzner_server_id", fmt.Sprintf("%d", server.ID))
+	m.Add("__meta_hetzner_server_name", server.Name)
+	m.Add("__meta_hetzner_datacenter", server.Datacenter.Name)
+	m.Add("__meta_hetzner_public_ipv4", server.PublicNet.IPv4.IP)
+	if _, n, _ := net.ParseCIDR(server.PublicNet.IPv6.IP); n != nil {
+		m.Add("__meta_hetzner_public_ipv6_network", n.String())
+	}
+	m.Add("__meta_hetzner_server_status", server.Status)
+
+	m.Add("__meta_hetzner_hcloud_datacenter_location", server.Datacenter.Location.Name)
+	m.Add("__meta_hetzner_hcloud_datacenter_location_network_zone", server.Datacenter.Location.NetworkZone)
+	m.Add("__meta_hetzner_hcloud_server_type", server.ServerType.Name)
+	m.Add("__meta_hetzner_hcloud_cpu_cores", fmt.Sprintf("%d", server.ServerType.Cores))
+	m.Add("__meta_hetzner_hcloud_cpu_type", server.ServerType.CPUType)
+	m.Add("__meta_hetzner_hcloud_memory_size_gb", fmt.Sprintf("%d", int(server.ServerType.Memory)))
+	m.Add("__meta_hetzner_hcloud_disk_size_gb", fmt.Sprintf("%d", server.ServerType.Disk))
+
+	if server.Image != nil {
+		m.Add("__meta_hetzner_hcloud_image_name", server.Image.Name)
+		m.Add("__meta_hetzner_hcloud_image_description", server.Image.Description)
+		m.Add("__meta_hetzner_hcloud_image_os_version", server.Image.OsVersion)
+		m.Add("__meta_hetzner_hcloud_image_os_flavor", server.Image.OsFlavor)
+	}
+
+	for _, privateNet := range server.PrivateNet {
+		networkID := privateNet.ID
+		for _, network := range networks {
+			if networkID == network.ID {
+				labelName := discoveryutils.SanitizeLabelName("__meta_hetzner_hcloud_private_ipv4_" + network.Name)
+				m.Add(labelName, privateNet.IP)
+			}
+		}
+	}
+
+	for labelKey, labelValue := range server.Labels {
+		labelName := discoveryutils.SanitizeLabelName("__meta_hetzner_hcloud_labelpresent_" + labelKey)
+		m.Add(labelName, "true")
+
+		labelName = discoveryutils.SanitizeLabelName("__meta_hetzner_hcloud_label_" + labelKey)
+		m.Add(labelName, labelValue)
+	}
+
+	ms = append(ms, m)
+	return ms
 }
 
-// Datacenter represents the Hetzner datacenter.
-type Datacenter struct {
-	Name     string             `json:"name"`
-	Location DatacenterLocation `json:"location"`
+// getHCloudNetworks returns hcloud networks obtained from the given cfg
+func getHCloudNetworks(cfg *apiConfig) ([]HCloudNetwork, error) {
+	// See https://docs.hetzner.cloud/#networks-get-all-networks
+	var networks []HCloudNetwork
+	page := 1
+	for {
+		path := fmt.Sprintf("/v1/networks?page=%d", page)
+		data, err := cfg.client.GetAPIResponse(path)
+		if err != nil {
+			return nil, fmt.Errorf("cannot query hcloud api for networks: %w", err)
+		}
+		networksPage, nextPage, err := parseHCloudNetworksList(data)
+		if err != nil {
+			return nil, err
+		}
+		networks = append(networks, networksPage...)
+		if nextPage <= page {
+			break
+		}
+		page = nextPage
+	}
+	return networks, nil
 }
 
-// DatacenterLocation represents the datacenter information.
-type DatacenterLocation struct {
-	Name        string `json:"name"`
-	NetworkZone string `json:"network_zone"`
+func parseHCloudNetworksList(data []byte) ([]HCloudNetwork, int, error) {
+	var resp HCloudNetworksList
+	if err := json.Unmarshal(data, &resp); err != nil {
+		return nil, 0, fmt.Errorf("cannot unmarshal HCloudNetworksList from %q: %w", data, err)
+	}
+	return resp.Networks, resp.Meta.Pagination.NextPage, nil
 }
 
-// Image represents the image information.
-type Image struct {
-	Name        string `json:"name"`
-	Description string `json:"description"`
-	OsFlavor    string `json:"os_flavor"`
-	OsVersion   string `json:"os_version"`
+// HCloudNetworksList represents the hetzner cloud networks list.
+//
+// See https://docs.hetzner.cloud/#networks-get-all-networks
+type HCloudNetworksList struct {
+	Meta     HCloudMeta      `json:"meta"`
+	Networks []HCloudNetwork `json:"networks"`
 }
 
-// PublicNet represents the public network information.
-type PublicNet struct {
-	IPv4 IPv4 `json:"ipv4"`
-	IPv6 IPv6 `json:"ipv6"`
+// HCloudNetwork represents the hetzner cloud network information.
+//
+// See https://docs.hetzner.cloud/#networks-get-all-networks
+type HCloudNetwork struct {
+	Name string `json:"name"`
+	ID   int    `json:"id"`
 }
 
-// PrivateNet represents the private network information.
-type PrivateNet struct {
-	ID int    `json:"network"`
-	IP string `json:"ip"`
+// getHCloudServers returns hcloud servers obtained from the given cfg
+func getHCloudServers(cfg *apiConfig) ([]HCloudServer, error) {
+	// See https://docs.hetzner.cloud/#servers-get-all-servers
+	var servers []HCloudServer
+	page := 1
+	for {
+		path := fmt.Sprintf("/v1/servers?page=%d", page)
+		data, err := cfg.client.GetAPIResponse(path)
+		if err != nil {
+			return nil, fmt.Errorf("cannot query hcloud api for servers: %w", err)
+		}
+		serversPage, nextPage, err := parseHCloudServerList(data)
+		if err != nil {
+			return nil, err
+		}
+		servers = append(servers, serversPage...)
+		if nextPage <= page {
+			break
+		}
+		page = nextPage
+	}
+	return servers, nil
 }
 
-// IPv4 represents the IPv4 information.
-type IPv4 struct {
-	IP string `json:"ip"`
+func parseHCloudServerList(data []byte) ([]HCloudServer, int, error) {
+	var resp HCloudServerList
+	if err := json.Unmarshal(data, &resp); err != nil {
+		return nil, 0, fmt.Errorf("cannot unmarshal HCloudServerList from %q: %w", data, err)
+	}
+	return resp.Servers, resp.Meta.Pagination.NextPage, nil
 }
 
-// IPv6 represents the IPv6 information.
-type IPv6 struct {
-	IP string `json:"ip"`
+// HCloudServerList represents a list of servers from Hetzner Cloud API.
+//
+// See https://docs.hetzner.cloud/#servers-get-all-servers
+type HCloudServerList struct {
+	Meta    HCloudMeta     `json:"meta"`
+	Servers []HCloudServer `json:"servers"`
+}
+
+// HCloudServer represents the structure of server data.
+//
+// See https://docs.hetzner.cloud/#servers-get-all-servers
+type HCloudServer struct {
+	ID         int                `json:"id"`
+	Name       string             `json:"name"`
+	Status     string             `json:"status"`
+	PublicNet  HCloudPublicNet    `json:"public_net"`
+	PrivateNet []HCloudPrivateNet `json:"private_net"`
+	ServerType HCloudServerType   `json:"server_type"`
+	Datacenter HCloudDatacenter   `json:"datacenter"`
+	Image      *HCloudImage       `json:"image"`
+	Labels     map[string]string  `json:"labels"`
 }
 
-// ServerType represents the server type information.
-type ServerType struct {
+// HCloudServerType represents the server type information.
+//
+// See https://docs.hetzner.cloud/#servers-get-all-servers
+type HCloudServerType struct {
 	Name    string  `json:"name"`
 	Cores   int     `json:"cores"`
 	CPUType string  `json:"cpu_type"`
@@ -77,114 +193,72 @@ type ServerType struct {
 	Disk    int     `json:"disk"`
 }
 
-// HcloudNetwork represents the hetzner cloud network information.
-type HcloudNetwork struct {
-	Name string `json:"name"`
-	ID   int    `json:"id"`
+// HCloudDatacenter represents the Hetzner datacenter.
+//
+// See https://docs.hetzner.cloud/#servers-get-all-servers
+type HCloudDatacenter struct {
+	Name     string                   `json:"name"`
+	Location HCloudDatacenterLocation `json:"location"`
 }
 
-// HcloudNetworksList represents the hetzner cloud networks list.
-type HcloudNetworksList struct {
-	Networks []HcloudNetwork `json:"networks"`
+// HCloudDatacenterLocation represents the datacenter information.
+//
+// See https://docs.hetzner.cloud/#servers-get-all-servers
+type HCloudDatacenterLocation struct {
+	Name        string `json:"name"`
+	NetworkZone string `json:"network_zone"`
 }
 
-// getHcloudServerLabels returns labels for hcloud servers obtained from the given cfg
-func getHcloudServerLabels(cfg *apiConfig) ([]*promutils.Labels, error) {
-	networks, err := getHcloudNetworks(cfg)
-	if err != nil {
-		return nil, err
-	}
-	servers, err := getServers(cfg)
-	if err != nil {
-		return nil, err
-	}
-	var ms []*promutils.Labels
-	for _, server := range servers.Servers {
-		ms = server.appendTargetLabels(ms, cfg.port, networks)
-	}
-	return ms, nil
+// HCloudPublicNet represents the public network information.
+//
+// See https://docs.hetzner.cloud/#servers-get-all-servers
+type HCloudPublicNet struct {
+	IPv4 HCloudIPv4 `json:"ipv4"`
+	IPv6 HCloudIPv6 `json:"ipv6"`
 }
 
-// getHcloudNetworks returns hcloud networks obtained from the given cfg
-func getHcloudNetworks(cfg *apiConfig) (*HcloudNetworksList, error) {
-	n, err := cfg.client.GetAPIResponse("/networks")
-	if err != nil {
-		return nil, fmt.Errorf("cannot query hcloud api for networks: %w", err)
-	}
-	networks, err := parseHcloudNetworksList(n)
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal HcloudServerList from %q: %w", n, err)
-	}
-	return networks, nil
+// HCloudIPv4 represents the IPv4 information.
+//
+// See https://docs.hetzner.cloud/#servers-get-all-servers
+type HCloudIPv4 struct {
+	IP string `json:"ip"`
 }
 
-// getServers returns hcloud servers obtained from the given cfg
-func getServers(cfg *apiConfig) (*HcloudServerList, error) {
-	s, err := cfg.client.GetAPIResponse("/servers")
-	if err != nil {
-		return nil, fmt.Errorf("cannot query hcloud api for servers: %w", err)
-	}
-	servers, err := parseHcloudServerList(s)
-	if err != nil {
-		return nil, err
-	}
-	return servers, nil
+// HCloudIPv6 represents the IPv6 information.
+//
+// See https://docs.hetzner.cloud/#servers-get-all-servers
+type HCloudIPv6 struct {
+	IP string `json:"ip"`
 }
 
-// parseHcloudNetworks parses HcloudNetworksList from data.
-func parseHcloudNetworksList(data []byte) (*HcloudNetworksList, error) {
-	var networks HcloudNetworksList
-	err := json.Unmarshal(data, &networks)
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal HcloudNetworksList from %q: %w", data, err)
-	}
-	return &networks, nil
+// HCloudPrivateNet represents the private network information.
+//
+// See https://docs.hetzner.cloud/#servers-get-all-servers
+type HCloudPrivateNet struct {
+	ID int    `json:"network"`
+	IP string `json:"ip"`
 }
 
-// parseHcloudServerList parses HcloudServerList from data.
-func parseHcloudServerList(data []byte) (*HcloudServerList, error) {
-	var servers HcloudServerList
-	err := json.Unmarshal(data, &servers)
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal HcloudServerList from %q: %w", data, err)
-	}
-	return &servers, nil
+// HCloudImage represents the image information.
+//
+// See https://docs.hetzner.cloud/#servers-get-all-servers
+type HCloudImage struct {
+	Name        string `json:"name"`
+	Description string `json:"description"`
+	OsFlavor    string `json:"os_flavor"`
+	OsVersion   string `json:"os_version"`
 }
 
-func (server *HcloudServer) appendTargetLabels(ms []*promutils.Labels, port int, networks *HcloudNetworksList) []*promutils.Labels {
-	addr := discoveryutils.JoinHostPort(server.PublicNet.IPv4.IP, port)
-	m := promutils.NewLabels(24)
-	m.Add("__address__", addr)
-	m.Add("__meta_hetzner_server_id", fmt.Sprintf("%d", server.ID))
-	m.Add("__meta_hetzner_server_name", server.Name)
-	m.Add("__meta_hetzner_server_status", server.Status)
-	m.Add("__meta_hetzner_public_ipv4", server.PublicNet.IPv4.IP)
-	m.Add("__meta_hetzner_public_ipv6_network", server.PublicNet.IPv6.IP)
-	m.Add("__meta_hetzner_datacenter", server.Datacenter.Name)
-	m.Add("__meta_hetzner_hcloud_image_name", server.Image.Name)
-	m.Add("__meta_hetzner_hcloud_image_description", server.Image.Description)
-	m.Add("__meta_hetzner_hcloud_image_os_flavor", server.Image.OsFlavor)
-	m.Add("__meta_hetzner_hcloud_image_os_version", server.Image.OsVersion)
-	m.Add("__meta_hetzner_hcloud_datacenter_location", server.Datacenter.Location.Name)
-	m.Add("__meta_hetzner_hcloud_datacenter_location_network_zone", server.Datacenter.Location.NetworkZone)
-	m.Add("__meta_hetzner_hcloud_server_type", server.ServerType.Name)
-	m.Add("__meta_hetzner_hcloud_cpu_cores", fmt.Sprintf("%d", server.ServerType.Cores))
-	m.Add("__meta_hetzner_hcloud_cpu_type", server.ServerType.CPUType)
-	m.Add("__meta_hetzner_hcloud_memory_size_gb", fmt.Sprintf("%d", int(server.ServerType.Memory)))
-	m.Add("__meta_hetzner_hcloud_disk_size_gb", fmt.Sprintf("%d", server.ServerType.Disk))
-
-	for _, privateNet := range server.PrivateNet {
-		for _, network := range networks.Networks {
-			if privateNet.ID == network.ID {
-				m.Add(discoveryutils.SanitizeLabelName("__meta_hetzner_hcloud_private_ipv4_"+network.Name), privateNet.IP)
-			}
-		}
-	}
-	for labelKey, labelValue := range server.Labels {
-		m.Add(discoveryutils.SanitizeLabelName("__meta_hetzner_hcloud_label_"+labelKey), labelValue)
-		m.Add(discoveryutils.SanitizeLabelName("__meta_hetzner_hcloud_labelpresent_"+labelKey), fmt.Sprintf("%t", true))
+// HCloudMeta represents hetzner cloud meta-information.
+//
+// See https://docs.hetzner.cloud/#pagination
+type HCloudMeta struct {
+	Pagination HCloudPagination `json:"pagination"`
+}
 
-	}
-	ms = append(ms, m)
-	return ms
+// HCloudPagination represents hetzner cloud pagination information.
+//
+// See https://docs.hetzner.cloud/#pagination
+type HCloudPagination struct {
+	NextPage int `json:"next_page"`
 }
diff --git a/lib/promscrape/discovery/hetzner/hetzner.go b/lib/promscrape/discovery/hetzner/hetzner.go
index 9662f3887..cc980175c 100644
--- a/lib/promscrape/discovery/hetzner/hetzner.go
+++ b/lib/promscrape/discovery/hetzner/hetzner.go
@@ -12,7 +12,7 @@ import (
 )
 
 // SDCheckInterval defines interval for targets refresh.
-var SDCheckInterval = flag.Duration("promscrape.hetznerSDCheckInterval", time.Minute, "Interval for checking for changes in hetzner. "+
+var SDCheckInterval = flag.Duration("promscrape.hetznerSDCheckInterval", time.Minute, "Interval for checking for changes in Hetzner API. "+
 	"This works only if hetzner_sd_configs is configured in '-promscrape.config' file. "+
 	"See https://docs.victoriametrics.com/sd_configs.html#hetzner_sd_configs for details")
 
@@ -20,15 +20,14 @@ var SDCheckInterval = flag.Duration("promscrape.hetznerSDCheckInterval", time.Mi
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#hetzner_sd_config
 type SDConfig struct {
-	Role              string                     `yaml:"role,omitempty"`
+	Role              string                     `yaml:"role"`
 	Port              *int                       `yaml:"port,omitempty"`
-	Token             *promauth.Secret           `yaml:"token"`
 	HTTPClientConfig  promauth.HTTPClientConfig  `yaml:",inline"`
 	ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"`
 	ProxyURL          *proxy.URL                 `yaml:"proxy_url,omitempty"`
 }
 
-// GetLabels returns hcloud or hetzner robot labels according to sdc.
+// GetLabels returns Hetzner target labels according to sdc.
 func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
 	cfg, err := getAPIConfig(sdc, baseDir)
 	if err != nil {
@@ -38,9 +37,10 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
 	case "robot":
 		return getRobotServerLabels(cfg)
 	case "hcloud":
-		return getHcloudServerLabels(cfg)
+		return getHCloudServerLabels(cfg)
 	default:
-		return nil, fmt.Errorf("skipping unexpected role=%q; must be one of `robot` or `hcloud`", sdc.Role)
+		// The sdc.Role must be already verified by getAPIConfig().
+		panic(fmt.Errorf("BUG: unexpected role=%q; must be one of `robot` or `hcloud`", sdc.Role))
 	}
 }
 
diff --git a/lib/promscrape/discovery/hetzner/robot.go b/lib/promscrape/discovery/hetzner/robot.go
index 3cbb74075..8ab921fa2 100644
--- a/lib/promscrape/discovery/hetzner/robot.go
+++ b/lib/promscrape/discovery/hetzner/robot.go
@@ -10,88 +10,97 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
-type robotServersList struct {
-	Servers []RobotServerResponse
-}
-
-// RobotServerResponse represents hetzner robot server response.
-type RobotServerResponse struct {
-	Server RobotServer `json:"server"`
-}
-
-// RobotServer represents the structure of hetzner robot server data.
-type RobotServer struct {
-	ServerIP     string        `json:"server_ip"`
-	ServerIPV6   string        `json:"server_ipv6_net"`
-	ServerNumber int           `json:"server_number"`
-	ServerName   string        `json:"server_name"`
-	DC           string        `json:"dc"`
-	Status       string        `json:"status"`
-	Product      string        `json:"product"`
-	Canceled     bool          `json:"cancelled"`
-	Subnet       []RobotSubnet `json:"subnet"`
-}
-
-// RobotSubnet represents the structure of hetzner robot subnet data.
-type RobotSubnet struct {
-	IP   string `json:"ip"`
-	Mask string `json:"mask"`
-}
-
 func getRobotServerLabels(cfg *apiConfig) ([]*promutils.Labels, error) {
 	servers, err := getRobotServers(cfg)
 	if err != nil {
 		return nil, err
 	}
 	var ms []*promutils.Labels
-	for _, server := range servers.Servers {
-		ms = server.appendTargetLabels(ms, cfg.port)
+	for i := range servers {
+		ms = appendRobotTargetLabels(ms, &servers[i], cfg.port)
 	}
 	return ms, nil
 }
 
-// parseRobotServersList parses robotServersList from data.
-func parseRobotServersList(data []byte) (*robotServersList, error) {
-	var servers robotServersList
-	err := json.Unmarshal(data, &servers.Servers)
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal robotServersList from %q: %w", data, err)
+func appendRobotTargetLabels(ms []*promutils.Labels, server *RobotServer, port int) []*promutils.Labels {
+	m := promutils.NewLabels(16)
+
+	addr := discoveryutils.JoinHostPort(server.ServerIP, port)
+	m.Add("__address__", addr)
+
+	m.Add("__meta_hetzner_role", "robot")
+	m.Add("__meta_hetzner_server_id", fmt.Sprintf("%d", server.ServerNumber))
+	m.Add("__meta_hetzner_server_name", server.ServerName)
+	m.Add("__meta_hetzner_datacenter", strings.ToLower(server.DC))
+	m.Add("__meta_hetzner_public_ipv4", server.ServerIP)
+	for _, subnet := range server.Subnet {
+		ip := net.ParseIP(subnet.IP)
+		if ip.To4() == nil {
+			m.Add("__meta_hetzner_public_ipv6_network", fmt.Sprintf("%s/%s", subnet.IP, subnet.Mask))
+			break
+		}
 	}
-	return &servers, nil
+	m.Add("__meta_hetzner_server_status", server.Status)
+
+	m.Add("__meta_hetzner_robot_product", server.Product)
+	m.Add("__meta_hetzner_robot_cancelled", fmt.Sprintf("%t", server.Canceled))
+
+	ms = append(ms, m)
+	return ms
 }
 
-func getRobotServers(cfg *apiConfig) (*robotServersList, error) {
-	s, err := cfg.client.GetAPIResponse("/server")
+func getRobotServers(cfg *apiConfig) ([]RobotServer, error) {
+	// See https://robot.hetzner.com/doc/webservice/en.html#server
+	data, err := cfg.client.GetAPIResponse("/server")
 	if err != nil {
 		return nil, fmt.Errorf("cannot query hetzner robot api for servers: %w", err)
 	}
-	servers, err := parseRobotServersList(s)
+	servers, err := parseRobotServers(data)
 	if err != nil {
 		return nil, err
 	}
 	return servers, nil
 }
 
-func (server *RobotServerResponse) appendTargetLabels(ms []*promutils.Labels, port int) []*promutils.Labels {
-	addr := discoveryutils.JoinHostPort(server.Server.ServerIP, port)
-	m := promutils.NewLabels(16)
-	m.Add("__address__", addr)
-	m.Add("__meta_hetzner_server_id", fmt.Sprintf("%d", server.Server.ServerNumber))
-	m.Add("__meta_hetzner_server_name", server.Server.ServerName)
-	m.Add("__meta_hetzner_server_status", server.Server.Status)
-	m.Add("__meta_hetzner_public_ipv4", server.Server.ServerIP)
-	m.Add("__meta_hetzner_datacenter", strings.ToLower(server.Server.DC))
-	m.Add("__meta_hetzner_robot_product", server.Server.Product)
-	m.Add("__meta_hetzner_robot_cancelled", fmt.Sprintf("%t", server.Server.Canceled))
-
-	for _, subnet := range server.Server.Subnet {
-		ip := net.ParseIP(subnet.IP)
-		if ip.To4() == nil {
-			m.Add("__meta_hetzner_public_ipv6_network", fmt.Sprintf("%s/%s", subnet.IP, subnet.Mask))
-			break
-		}
+func parseRobotServers(data []byte) ([]RobotServer, error) {
+	var serverEntries []RobotServerEntry
+	err := json.Unmarshal(data, &serverEntries)
+	if err != nil {
+		return nil, fmt.Errorf("cannot unmarshal RobotServer list from %q: %w", data, err)
+	}
+	servers := make([]RobotServer, len(serverEntries))
+	for i := range serverEntries {
+		servers[i] = serverEntries[i].Server
 	}
+	return servers, nil
+}
 
-	ms = append(ms, m)
-	return ms
+// RobotServerEntry represents a single server entry in hetzner robot server response.
+//
+// See https://robot.hetzner.com/doc/webservice/en.html#server
+type RobotServerEntry struct {
+	Server RobotServer `json:"server"`
+}
+
+// RobotServer represents the structure of hetzner robot server data.
+//
+// See https://robot.hetzner.com/doc/webservice/en.html#server
+type RobotServer struct {
+	ServerIP     string        `json:"server_ip"`
+	ServerIPV6   string        `json:"server_ipv6_net"`
+	ServerNumber int           `json:"server_number"`
+	ServerName   string        `json:"server_name"`
+	DC           string        `json:"dc"`
+	Status       string        `json:"status"`
+	Product      string        `json:"product"`
+	Canceled     bool          `json:"cancelled"`
+	Subnet       []RobotSubnet `json:"subnet"`
+}
+
+// RobotSubnet represents the structure of hetzner robot subnet data.
+//
+// See https://robot.hetzner.com/doc/webservice/en.html#server
+type RobotSubnet struct {
+	IP   string `json:"ip"`
+	Mask string `json:"mask"`
 }
diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go
index fd72e9bc7..648051b21 100644
--- a/lib/promscrape/scraper.go
+++ b/lib/promscrape/scraper.go
@@ -134,13 +134,13 @@ func runScraper(configFile string, pushData func(at *auth.Token, wr *prompbmarsh
 	scs.add("eureka_sd_configs", *eureka.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getEurekaSDScrapeWork(swsPrev) })
 	scs.add("file_sd_configs", *fileSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getFileSDScrapeWork(swsPrev) })
 	scs.add("gce_sd_configs", *gce.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getGCESDScrapeWork(swsPrev) })
+	scs.add("hetzner_sd_configs", *hetzner.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getHetznerSDScrapeWork(swsPrev) })
 	scs.add("http_sd_configs", *http.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getHTTPDScrapeWork(swsPrev) })
 	scs.add("kubernetes_sd_configs", *kubernetes.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getKubernetesSDScrapeWork(swsPrev) })
 	scs.add("kuma_sd_configs", *kuma.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getKumaSDScrapeWork(swsPrev) })
 	scs.add("nomad_sd_configs", *nomad.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getNomadSDScrapeWork(swsPrev) })
 	scs.add("openstack_sd_configs", *openstack.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getOpenStackSDScrapeWork(swsPrev) })
 	scs.add("yandexcloud_sd_configs", *yandexcloud.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getYandexCloudSDScrapeWork(swsPrev) })
-	scs.add("hetzner_sd_configs", *hetzner.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getHetznerSDScrapeWork(swsPrev) })
 	scs.add("static_configs", 0, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getStaticScrapeWork() })
 
 	var tickerCh <-chan time.Time
diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go
index 5858ae880..a5fa45aa5 100644
--- a/lib/promscrape/scrapework.go
+++ b/lib/promscrape/scrapework.go
@@ -337,11 +337,13 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}, globalStopCh <-chan struct{})
 				// Do not send staleness markers on graceful shutdown as Prometheus does.
 				// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2013#issuecomment-1006994079
 			default:
-				// Send staleness markers to all the metrics scraped last time from the target
-				// when the given target disappears as Prometheus does.
-				// Use the current real timestamp for staleness markers, so queries
-				// stop returning data just after the time the target disappears.
-				sw.sendStaleSeries(lastScrape, "", t, true)
+				if !sw.Config.NoStaleMarkers {
+					// Send staleness markers to all the metrics scraped last time from the target
+					// when the given target disappears as Prometheus does.
+					// Use the current real timestamp for staleness markers, so queries
+					// stop returning data just after the time the target disappears.
+					sw.sendStaleSeries(lastScrape, "", t, true)
+				}
 			}
 			if sw.seriesLimiter != nil {
 				sw.seriesLimiter.MustStop()
@@ -487,7 +489,8 @@ func (sw *scrapeWork) processScrapedData(scrapeTimestamp, realTimestamp int64, b
 		bodyString = ""
 	}
 	seriesAdded := 0
-	if !areIdenticalSeries {
+	// cannot track the number of new time series when stale markers are disabled.
+	if !sw.Config.NoStaleMarkers && !areIdenticalSeries {
 		// The returned value for seriesAdded may be bigger than the real number of added series
 		// if some series were removed during relabeling.
 		// This is a trade-off between performance and accuracy.
@@ -517,7 +520,7 @@ func (sw *scrapeWork) processScrapedData(scrapeTimestamp, realTimestamp int64, b
 		writeRequestCtxPool.Put(wc)
 	}
 	// body must be released only after wc is released, since wc refers to body.
-	if !areIdenticalSeries {
+	if !sw.Config.NoStaleMarkers && !areIdenticalSeries {
 		// Send stale markers for disappeared metrics with the real scrape timestamp
 		// in order to guarantee that query doesn't return data after this time for the disappeared metrics.
 		sw.sendStaleSeries(lastScrape, bodyString, realTimestamp, false)
@@ -627,7 +630,8 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
 		scrapesFailed.Inc()
 	}
 	seriesAdded := 0
-	if !areIdenticalSeries {
+	// cannot track the number of new time series when stale markers are disabled.
+	if !sw.Config.NoStaleMarkers && !areIdenticalSeries {
 		// The returned value for seriesAdded may be bigger than the real number of added series
 		// if some series were removed during relabeling.
 		// This is a trade-off between performance and accuracy.
@@ -647,7 +651,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
 	sw.prevBodyLen = sbr.bodyLen
 	wc.reset()
 	writeRequestCtxPool.Put(wc)
-	if !areIdenticalSeries {
+	if !sw.Config.NoStaleMarkers && !areIdenticalSeries {
 		// Send stale markers for disappeared metrics with the real scrape timestamp
 		// in order to guarantee that query doesn't return data after this time for the disappeared metrics.
 		sw.sendStaleSeries(lastScrape, bodyString, realTimestamp, false)
@@ -788,9 +792,6 @@ func (sw *scrapeWork) sendStaleSeries(lastScrape, currScrape string, timestamp i
 	defer func() {
 		<-sendStaleSeriesConcurrencyLimitCh
 	}()
-	if sw.Config.NoStaleMarkers {
-		return
-	}
 	bodyString := lastScrape
 	if currScrape != "" {
 		bodyString = parser.GetRowsDiff(lastScrape, currScrape)


Test patch:
diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go
index 915e7a661..c19cd0c51 100644
--- a/app/vmselect/promql/exec_test.go
+++ b/app/vmselect/promql/exec_test.go
@@ -8050,10 +8050,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_rate()`, func(t *testing.T) {
 		t.Parallel()
-		q := `rollup_rate((2000-time())[600s])`
+		q := `rollup_rate((2200-time())[600s])`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{5, 4, 3, 2, 1, 0},
+			Values:     []float64{6, 5, 4, 3, 2, 1},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -8062,7 +8062,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{6, 5, 4, 3, 2, 1},
+			Values:     []float64{7, 6, 5, 4, 3, 2},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{{
@@ -8071,7 +8071,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r3 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{4, 3, 2, 1, 0, -1},
+			Values:     []float64{5, 4, 3, 2, 1, 0},
 			Timestamps: timestampsExpected,
 		}
 		r3.MetricName.Tags = []storage.Tag{{
@@ -8083,10 +8083,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_rate(q, "max")`, func(t *testing.T) {
 		t.Parallel()
-		q := `rollup_rate((2000-time())[600s], "max")`
+		q := `rollup_rate((2200-time())[600s], "max")`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{6, 5, 4, 3, 2, 1},
+			Values:     []float64{7, 6, 5, 4, 3, 2},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -8094,10 +8094,10 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_rate(q, "avg")`, func(t *testing.T) {
 		t.Parallel()
-		q := `rollup_rate((2000-time())[600s], "avg")`
+		q := `rollup_rate((2200-time())[600s], "avg")`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{5, 4, 3, 2, 1, 0},
+			Values:     []float64{6, 5, 4, 3, 2, 1},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go
index 766a33eae..e0ba2b8f0 100644
--- a/app/vmselect/promql/rollup_test.go
+++ b/app/vmselect/promql/rollup_test.go
@@ -125,7 +125,7 @@ func TestRemoveCounterResets(t *testing.T) {
 	// removeCounterResets doesn't expect negative values, so it doesn't work properly with them.
 	values = []float64{-100, -200, -300, -400}
 	removeCounterResets(values)
-	valuesExpected = []float64{-100, -300, -600, -1000}
+	valuesExpected = []float64{-100, -100, -100, -100}
 	timestampsExpected := []int64{0, 1, 2, 3}
 	testRowsEqual(t, values, timestampsExpected, valuesExpected, timestampsExpected)
 
@@ -136,6 +136,17 @@ func TestRemoveCounterResets(t *testing.T) {
 	valuesExpected = []float64{100, 100, 125, 125, 145, 195}
 	timestampsExpected = []int64{0, 1, 2, 3, 4, 5}
 	testRowsEqual(t, values, timestampsExpected, valuesExpected, timestampsExpected)
+
+	// verify results always increase monotonically with possible float operations precision error
+	values = []float64{34.094223, 2.7518, 2.140669, 0.044878, 1.887095, 2.546569, 2.490149, 0.045, 0.035684, 0.062454, 0.058296}
+	removeCounterResets(values)
+	var prev float64
+	for i, v := range values {
+		if v < prev {
+			t.Fatalf("error: unexpected value keep getting bigger %d; cur %v; pre %v\n", i, v, prev)
+		}
+		prev = v
+	}
 }
 
 func TestDeltaValues(t *testing.T) {
diff --git a/lib/promscrape/discovery/hetzner/hcloud_test.go b/lib/promscrape/discovery/hetzner/hcloud_test.go
index d4903e2af..b4ae981b6 100644
--- a/lib/promscrape/discovery/hetzner/hcloud_test.go
+++ b/lib/promscrape/discovery/hetzner/hcloud_test.go
@@ -8,7 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
-func TestParseHcloudNetworksList(t *testing.T) {
+func TestParseHCloudNetworksList(t *testing.T) {
 	data := `{
 		"meta": {
 		  "pagination": {
@@ -57,21 +57,25 @@ func TestParseHcloudNetworksList(t *testing.T) {
 	  }
 `
 
-	net, err := parseHcloudNetworksList([]byte(data))
+	nets, nextPage, err := parseHCloudNetworksList([]byte(data))
 	if err != nil {
 		t.Fatalf("unexpected error when parsing data: %s", err)
 	}
-	netExpected := &HcloudNetworksList{
-		Networks: []HcloudNetwork{
-			{Name: "mynet", ID: 4711},
+	netsExpected := []HCloudNetwork{
+		{
+			Name: "mynet",
+			ID:   4711,
 		},
 	}
-	if !reflect.DeepEqual(net, netExpected) {
-		t.Fatalf("unexpected parseHcloudNetworksList parsed;\ngot\n%+v\nwant\n%+v", net, netExpected)
+	if !reflect.DeepEqual(nets, netsExpected) {
+		t.Fatalf("unexpected parseHCloudNetworksList parsed;\ngot\n%+v\nwant\n%+v", nets, netsExpected)
+	}
+	if nextPage != 4 {
+		t.Fatalf("unexpected nextPage; got %d; want 4", nextPage)
 	}
 }
 
-func TestParseHcloudServerListResponse(t *testing.T) {
+func TestParseHCloudServerListResponse(t *testing.T) {
 	data := `{
 		"meta": {
 		  "pagination": {
@@ -246,71 +250,72 @@ func TestParseHcloudServerListResponse(t *testing.T) {
 		]
 	  }
 `
-	sl, err := parseHcloudServerList([]byte(data))
+	sl, nextPage, err := parseHCloudServerList([]byte(data))
 	if err != nil {
-		t.Fatalf("unexpected error parseHcloudServerList when parsing data: %s", err)
+		t.Fatalf("unexpected error parseHCloudServerList when parsing data: %s", err)
 	}
-	slExpected := &HcloudServerList{
-		Servers: []HcloudServer{
-			{
-				ID:     42,
-				Name:   "my-resource",
-				Status: "running",
-				PublicNet: PublicNet{
-					IPv4: IPv4{
-						IP: "1.2.3.4",
-					},
-					IPv6: IPv6{
-						IP: "2001:db8::/64",
-					},
-				},
-				PrivateNet: []PrivateNet{
-					{
-						ID: 4711,
-						IP: "10.0.0.2",
-					},
+	slExpected := []HCloudServer{
+		{
+			ID:     42,
+			Name:   "my-resource",
+			Status: "running",
+			PublicNet: HCloudPublicNet{
+				IPv4: HCloudIPv4{
+					IP: "1.2.3.4",
 				},
-				ServerType: ServerType{
-					Name:    "cx11",
-					Cores:   1,
-					CPUType: "shared",
-					Memory:  1.0,
-					Disk:    25,
+				IPv6: HCloudIPv6{
+					IP: "2001:db8::/64",
 				},
-				Datacenter: Datacenter{
-					Name: "fsn1-dc8",
-					Location: DatacenterLocation{
-						Name:        "fsn1",
-						NetworkZone: "eu-central",
-					},
+			},
+			PrivateNet: []HCloudPrivateNet{
+				{
+					ID: 4711,
+					IP: "10.0.0.2",
 				},
-				Image: Image{
-					Name:        "ubuntu-20.04",
-					Description: "Ubuntu 20.04 Standard 64 bit",
-					OsFlavor:    "ubuntu",
-					OsVersion:   "20.04",
+			},
+			ServerType: HCloudServerType{
+				Name:    "cx11",
+				Cores:   1,
+				CPUType: "shared",
+				Memory:  1.0,
+				Disk:    25,
+			},
+			Datacenter: HCloudDatacenter{
+				Name: "fsn1-dc8",
+				Location: HCloudDatacenterLocation{
+					Name:        "fsn1",
+					NetworkZone: "eu-central",
 				},
-				Labels: map[string]string{},
 			},
+			Image: &HCloudImage{
+				Name:        "ubuntu-20.04",
+				Description: "Ubuntu 20.04 Standard 64 bit",
+				OsFlavor:    "ubuntu",
+				OsVersion:   "20.04",
+			},
+			Labels: map[string]string{},
 		},
 	}
 	if !reflect.DeepEqual(sl, slExpected) {
-		t.Fatalf("unexpected parseHcloudServerList parsed;\ngot\n%+v\nwant\n%+v", sl, slExpected)
+		t.Fatalf("unexpected parseHCloudServerList parsed;\ngot\n%+v\nwant\n%+v", sl, slExpected)
+	}
+	if nextPage != 4 {
+		t.Fatalf("unexpected nextPage; got %d; want 4", nextPage)
 	}
 
-	server := sl.Servers[0]
-	var ms []*promutils.Labels
 	port := 123
-	networks := &HcloudNetworksList{
-		Networks: []HcloudNetwork{
-			{Name: "mynet", ID: 4711},
+	networks := []HCloudNetwork{
+		{
+			Name: "mynet",
+			ID:   4711,
 		},
 	}
-	labelss := server.appendTargetLabels(ms, port, networks)
+	labelss := appendHCloudTargetLabels(nil, &sl[0], networks, port)
 
 	expectedLabels := []*promutils.Labels{
 		promutils.NewLabelsFromMap(map[string]string{
 			"__address__":                                            "1.2.3.4:123",
+			"__meta_hetzner_role":                                    "hcloud",
 			"__meta_hetzner_server_id":                               "42",
 			"__meta_hetzner_server_name":                             "my-resource",
 			"__meta_hetzner_server_status":                           "running",
diff --git a/lib/promscrape/discovery/hetzner/robot_test.go b/lib/promscrape/discovery/hetzner/robot_test.go
index 6d06b790c..b7dc8a5c7 100644
--- a/lib/promscrape/discovery/hetzner/robot_test.go
+++ b/lib/promscrape/discovery/hetzner/robot_test.go
@@ -53,58 +53,50 @@ func TestParseRobotServerListResponse(t *testing.T) {
 		}
 	  ]
 `
-	rsl, err := parseRobotServersList([]byte(data))
+	rsl, err := parseRobotServers([]byte(data))
 	if err != nil {
 		t.Fatalf("unexpected error parseRobotServersList when parsing data: %s", err)
 	}
-	rslExpected := &robotServersList{
-		Servers: []RobotServerResponse{
-			{
-				Server: RobotServer{
-					ServerIP:     "123.123.123.123",
-					ServerIPV6:   "2a01:f48:111:4221::",
-					ServerNumber: 321,
-					ServerName:   "server1",
-					Product:      "DS 3000",
-					DC:           "NBG1-DC1",
-					Status:       "ready",
-					Canceled:     false,
-					Subnet: []RobotSubnet{
-						{
-							IP:   "2a01:4f8:111:4221::",
-							Mask: "64",
-						},
-					},
-				},
-			},
-			{
-				Server: RobotServer{
-					ServerIP:     "123.123.123.124",
-					ServerIPV6:   "2a01:f48:111:4221::",
-					ServerNumber: 421,
-					ServerName:   "server2",
-					Product:      "X5",
-					DC:           "FSN1-DC10",
-					Status:       "ready",
-					Canceled:     false,
-					Subnet:       nil,
+	rslExpected := []RobotServer{
+		{
+			ServerIP:     "123.123.123.123",
+			ServerIPV6:   "2a01:f48:111:4221::",
+			ServerNumber: 321,
+			ServerName:   "server1",
+			Product:      "DS 3000",
+			DC:           "NBG1-DC1",
+			Status:       "ready",
+			Canceled:     false,
+			Subnet: []RobotSubnet{
+				{
+					IP:   "2a01:4f8:111:4221::",
+					Mask: "64",
 				},
 			},
 		},
+		{
+			ServerIP:     "123.123.123.124",
+			ServerIPV6:   "2a01:f48:111:4221::",
+			ServerNumber: 421,
+			ServerName:   "server2",
+			Product:      "X5",
+			DC:           "FSN1-DC10",
+			Status:       "ready",
+			Canceled:     false,
+			Subnet:       nil,
+		},
 	}
 	if !reflect.DeepEqual(rsl, rslExpected) {
 		t.Fatalf("unexpected parseRobotServersList parsed;\ngot\n%+v\nwant\n%+v", rsl, rslExpected)
 	}
 
-	server := rsl.Servers[0]
-	var ms []*promutils.Labels
 	port := 123
-
-	labelss := server.appendTargetLabels(ms, port)
+	labelss := appendRobotTargetLabels(nil, &rsl[0], port)
 
 	expectedLabels := []*promutils.Labels{
 		promutils.NewLabelsFromMap(map[string]string{
 			"__address__":                        "123.123.123.123:123",
+			"__meta_hetzner_role":                "robot",
 			"__meta_hetzner_server_id":           "321",
 			"__meta_hetzner_server_name":         "server1",
 			"__meta_hetzner_server_status":       "ready",