Repository: VictoriaMetrics/VictoriaMetrics
Pull request #5585 (2024-01-09 08:14:51)

app/vmauth: adds metric_labels and backend_errors counter
it must improve observability for user requests with new metric - per user backend errors counter. it's needed to calculate requests fail rate to the configured backends. metric_labels configuration allows to perform additional aggregations on top of multiple users from configuration section. It could be multiple clients or clients with separate read/write tokens https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5565

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5585?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) Report
Attention: `12 lines` in your changes are missing coverage. Please review.
> Comparison is base [(`105c6b2`)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/105c6b2eb7ca10ee750e633247aca549db735104?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) 60.16% compared to head [(`e82d709`)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5585?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) 60.15%.
> Report is 3 commits behind head on master.

| [Files](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5585?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) | Patch % | Lines |
|---|---|---|
| [app/vmauth/auth\_config.go](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5585?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtYXV0aC9hdXRoX2NvbmZpZy5nbw==) | 82.05% | [2 Missing and 5 partials :warning: ](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5585?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) |
| [app/vmauth/main.go](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5585?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtYXV0aC9tYWluLmdv) | 0.00% | [5 Missing :warning: ](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5585?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #5585      +/-   ##
==========================================
- Coverage   60.16%   60.15%   -0.01%     
==========================================
  Files         401      401              
  Lines       75494    75522      +28     
==========================================
+ Hits        45422    45433      +11     
- Misses      27541    27552      +11     
- Partials     2531     2537       +6     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5585?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).


Patch:
diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go
index 59e004997..413e97a7b 100644
--- a/app/vmauth/auth_config.go
+++ b/app/vmauth/auth_config.go
@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"os"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -16,6 +17,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/cespare/xxhash/v2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
@@ -60,12 +62,15 @@ type UserInfo struct {
 	TLSInsecureSkipVerify  *bool       `yaml:"tls_insecure_skip_verify,omitempty"`
 	TLSCAFile              string      `yaml:"tls_ca_file,omitempty"`
 
+	MetricLabels map[string]string `yaml:"metric_labels,omitempty"`
+
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
 
 	httpTransport *http.Transport
 
 	requests         *metrics.Counter
+	backendErrors    *metrics.Counter
 	requestsDuration *metrics.Summary
 }
 
@@ -462,10 +467,12 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 // authConfigData needs to be updated each time authConfig is updated.
 var authConfigData atomic.Pointer[[]byte]
 
-var authConfig atomic.Pointer[AuthConfig]
-var authUsers atomic.Pointer[map[string]*UserInfo]
-var authConfigWG sync.WaitGroup
-var stopCh chan struct{}
+var (
+	authConfig   atomic.Pointer[AuthConfig]
+	authUsers    atomic.Pointer[map[string]*UserInfo]
+	authConfigWG sync.WaitGroup
+	stopCh       chan struct{}
+)
 
 // loadAuthConfig loads and applies the config from *authConfigPath.
 // It returns bool value to identify if new config was applied.
@@ -527,16 +534,23 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
-		ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total`)
-		ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds`)
+
+		metricLabels, err := ui.getMetricLabels()
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse metric_labels for unauthorized_user: %w", err)
+		}
+		ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total` + metricLabels)
+		ui.backendErrors = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_request_backend_errors_total` + metricLabels)
+		ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds` + metricLabels)
 		ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
-		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total`)
-		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`, func() float64 {
+		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total` + metricLabels)
+		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`+metricLabels, func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_current`, func() float64 {
+		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
+
 		tr, err := getTransport(ui.TLSInsecureSkipVerify, ui.TLSCAFile)
 		if err != nil {
 			return nil, fmt.Errorf("cannot initialize HTTP transport: %w", err)
@@ -572,25 +586,24 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 			return nil, err
 		}
 
-		name := ui.name()
-		if ui.BearerToken != "" {
-			if ui.Password != "" {
-				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
-			}
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
-			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
+		if ui.BearerToken != "" && ui.Password != "" {
+			return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
 		}
-		if ui.Username != "" {
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
-			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
+
+		metricLabels, err := ui.getMetricLabels()
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
 		}
+		ui.requests = metrics.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
+		ui.backendErrors = metrics.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
+		ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
 		mcr := ui.getMaxConcurrentRequests()
 		ui.concurrencyLimitCh = make(chan struct{}, mcr)
-		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_concurrent_requests_limit_reached_total{username=%q}`, name))
-		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_capacity{username=%q}`, name), func() float64 {
+		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_user_concurrent_requests_limit_reached_total` + metricLabels)
+		_ = metrics.GetOrCreateGauge(`vmauth_user_concurrent_requests_capacity`+metricLabels, func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 {
+		_ = metrics.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
 
@@ -606,6 +619,29 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 	return byAuthToken, nil
 }
 
+var labelNameRegexp = regexp.MustCompile("^[a-zA-Z_:.][a-zA-Z0-9_:.]*$")
+
+func (ui *UserInfo) getMetricLabels() (string, error) {
+	name := ui.name()
+	if len(name) == 0 && len(ui.MetricLabels) == 0 {
+		// fast path
+		return "", nil
+	}
+	labels := make([]string, 0, len(ui.MetricLabels)+1)
+	if len(name) > 0 {
+		labels = append(labels, fmt.Sprintf(`username=%q`, name))
+	}
+	for k, v := range ui.MetricLabels {
+		if !labelNameRegexp.MatchString(k) {
+			return "", fmt.Errorf("incorrect label name=%q, it must match regex=%q for user=%q", k, labelNameRegexp, name)
+		}
+		labels = append(labels, fmt.Sprintf(`%s=%q`, k, v))
+	}
+	sort.Strings(labels)
+	labelsStr := "{" + strings.Join(labels, ",") + "}"
+	return labelsStr, nil
+}
+
 func (ui *UserInfo) initURLs() error {
 	retryStatusCodes := defaultRetryStatusCodes.Values()
 	loadBalancingPolicy := *defaultLoadBalancingPolicy
@@ -676,7 +712,8 @@ func (ui *UserInfo) name() string {
 		return ui.Username
 	}
 	if ui.BearerToken != "" {
-		return "bearer_token"
+		h := xxhash.Sum64([]byte(ui.BearerToken))
+		return fmt.Sprintf("bearer_token:hash:%016X", h)
 	}
 	return ""
 }
diff --git a/app/vmauth/example_config.yml b/app/vmauth/example_config.yml
index e1c814e12..4bdbd372f 100644
--- a/app/vmauth/example_config.yml
+++ b/app/vmauth/example_config.yml
@@ -10,6 +10,11 @@ users:
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 
+  # Adds labels to the exported metrics for given user section
+  # label name must be prometheus compatible and match regex: `^[a-zA-Z_:.][a-zA-Z0-9_:.]*$`
+  metric_labels:
+    backend_dc: eu
+    access_team: dev
   # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
   # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
   # The `X-Server-Hostname:` http header is removed from the proxied response.
diff --git a/app/vmauth/main.go b/app/vmauth/main.go
index 5fa311c6b..2c56ea9c7 100644
--- a/app/vmauth/main.go
+++ b/app/vmauth/main.go
@@ -150,12 +150,20 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		if err := ui.beginConcurrencyLimit(); err != nil {
 			handleConcurrencyLimitError(w, r, err)
 			<-concurrencyLimitCh
+
+			// Requests failed because of concurrency limit must be counted as errors,
+			// since this usually means the backend cannot keep up with the current load.
+			ui.backendErrors.Inc()
 			return
 		}
 	default:
 		concurrentRequestsLimitReached.Inc()
 		err := fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh))
 		handleConcurrencyLimitError(w, r, err)
+
+		// Requests failed because of concurrency limit must be counted as errors,
+		// since this usually means the backend cannot keep up with the current load.
+		ui.backendErrors.Inc()
 		return
 	}
 	processRequest(w, r, ui)
@@ -201,7 +209,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		} else { // Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
 		}
-		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui.httpTransport)
+		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
 		bu.put()
 		if ok {
 			return
@@ -213,15 +221,16 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		StatusCode: http.StatusServiceUnavailable,
 	}
 	httpserver.Errorf(w, r, "%s", err)
+	ui.backendErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, transport *http.Transport) bool {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := sanitizeRequestHeaders(r)
 	req.URL = targetURL
 	req.Host = targetURL.Host
 	updateHeadersByConfig(req.Header, hc.RequestHeaders)
-	res, err := transport.RoundTrip(req)
+	res, err := ui.httpTransport.RoundTrip(req)
 	rtb, rtbOK := req.Body.(*readTrackingBody)
 	if err != nil {
 		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
@@ -229,6 +238,10 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 			requestURI := httpserver.GetRequestURI(r)
 			logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
+			if errors.Is(err, context.DeadlineExceeded) {
+				// Timed out request must be counted as errors, since this usually means that the backend is slow.
+				ui.backendErrors.Inc()
+			}
 			return true
 		}
 		if !rtbOK || !rtb.canRetry() {
@@ -238,6 +251,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
+			ui.backendErrors.Inc()
 			return true
 		}
 		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
@@ -393,8 +407,10 @@ func getTransport(insecureSkipVerifyP *bool, caFile string) (*http.Transport, er
 	return tr, nil
 }
 
-var transportMap = make(map[string]*http.Transport)
-var transportMapLock sync.Mutex
+var (
+	transportMap     = make(map[string]*http.Transport)
+	transportMapLock sync.Mutex
+)
 
 func appendTransportKey(dst []byte, insecureSkipVerify bool, caFile string) []byte {
 	dst = encoding.MarshalBool(dst, insecureSkipVerify)
diff --git a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx
index 51b10a4fc..87fdd6234 100644
--- a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx
+++ b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx
@@ -8,6 +8,7 @@ import { QueryStats } from "../../../api/types";
 import { partialWarning, seriesFetchedWarning } from "./warningText";
 import { AutocompleteOptions } from "../../Main/Autocomplete/Autocomplete";
 import { useQueryDispatch } from "../../../state/query/QueryStateContext";
+import useDeviceDetect from "../../../hooks/useDeviceDetect";
 
 export interface QueryEditorProps {
   onChange: (query: string) => void;
@@ -35,6 +36,7 @@ const QueryEditor: FC<QueryEditorProps> = ({
   label,
   disabled = false
 }) => {
+  const { isMobile } = useDeviceDetect();
 
   const [openAutocomplete, setOpenAutocomplete] = useState(false);
   const [caretPosition, setCaretPosition] = useState([0, 0]);
@@ -115,7 +117,7 @@ const QueryEditor: FC<QueryEditorProps> = ({
         value={value}
         label={label}
         type={"textarea"}
-        autofocus={!!value}
+        autofocus={!isMobile}
         error={error}
         warning={warning}
         onKeyDown={handleKeyDown}
diff --git a/app/vmui/packages/vmui/src/components/ExploreMetrics/ExploreMetricsHeader/ExploreMetricsHeader.tsx b/app/vmui/packages/vmui/src/components/ExploreMetrics/ExploreMetricsHeader/ExploreMetricsHeader.tsx
index 0d4151cb1..ae4d6818b 100644
--- a/app/vmui/packages/vmui/src/components/ExploreMetrics/ExploreMetricsHeader/ExploreMetricsHeader.tsx
+++ b/app/vmui/packages/vmui/src/components/ExploreMetrics/ExploreMetricsHeader/ExploreMetricsHeader.tsx
@@ -71,7 +71,7 @@ const ExploreMetricsHeader: FC<ExploreMetricsHeaderProps> = ({
             label="Job"
             placeholder="Please select job"
             onChange={onChangeJob}
-            autofocus={!job}
+            autofocus={!job && !!jobs.length && !isMobile}
             searchable
           />
         </div>
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3a814b64e..2c3b8be5e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -41,6 +41,8 @@ The sandbox cluster installation is running under the constant load generated by
   - `-remoteRead.oauth2.endpointParams` for `-remoteRead.url`
   - `-remoteWrite.oauth2.endpointParams` for `-remoteWrite.url`
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy incoming requests to different backends based on the requested host via `src_hosts` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth.html#generic-http-proxy-for-different-backends).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): expose `vmauth_user_request_backend_errors_total` and `vmauth_unauthorized_user_request_backend_errors_total` [metrics](https://docs.victoriametrics.com/vmauth.html#monitoring), which track the number of failed requests because of backend errors. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5565).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add an ability to specify additional labels for [per-user metrics](https://docs.victoriametrics.com/vmauth.html#monitoring) via `metric_labels` section. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5565).
 * FEATURE: all VictoriaMetrics components: break HTTP client connection if an error occurs after the server at `-httpListenAddr` already sent response status code. Previously such an error couldn't be detected at client side. Now the client will get an error about invalid chunked response. The error message is simultaneously written to the server log and in the last line of the response. This should help detecting errors when migrating data between VictoriaMetrics instances by [vmctl](https://docs.victoriametrics.com/vmctl.html). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5645).
 * FEATURE: all VictoriaMetrics components: add ability to specify arbitrary HTTP headers to send with every request to `-pushmetrics.url`. See [`push metrics` docs](https://docs.victoriametrics.com/#push-metrics).
 * FEATURE: all VictoriaMetrics components: add `-metrics.exposeMetadata` command-line flag, which allows displaying `TYPE` and `HELP` metadata at `/metrics` page exposed at `-httpListenAddr`. This may be needed when the `/metrics` page is scraped by collector, which requires the `TYPE` and `HELP` metadata such as [Google Cloud Managed Prometheus](https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type).
diff --git a/docs/vmauth.md b/docs/vmauth.md
index c8400c7b7..11bc45b5d 100644
--- a/docs/vmauth.md
+++ b/docs/vmauth.md
@@ -455,7 +455,7 @@ The following [metrics](#monitoring) related to concurrency limits are exposed b
   because of the global concurrency limit has been reached.
 - `vmauth_user_concurrent_requests_capacity{username="..."}` - the limit on the number of concurrent requests for the given `username`.
 - `vmauth_user_concurrent_requests_current{username="..."}` - the current number of concurrent requests for the given `username`.
-- `vmauth_user_concurrent_requests_limit_reached_total{username="foo"}` - the number of requests rejected with `429 Too Many Requests` error
+- `vmauth_user_concurrent_requests_limit_reached_total{username="..."}` - the number of requests rejected with `429 Too Many Requests` error
   because of the concurrency limit has been reached for the given `username`.
 - `vmauth_unauthorized_user_concurrent_requests_capacity` - the limit on the number of concurrent requests for unauthorized users (if `unauthorized_user` section is used).
 - `vmauth_unauthorized_user_concurrent_requests_current` - the current number of concurrent requests for unauthorized users (if `unauthorized_user` section is used).
@@ -465,10 +465,10 @@ The following [metrics](#monitoring) related to concurrency limits are exposed b
 ## Backend TLS setup
 
 By default `vmauth` uses system settings when performing requests to HTTPS backends specified via `url_prefix` option
-in the [`-auth.config`](https://docs.victoriametrics.com/vmauth.html#auth-config). These settings can be overridden with the following command-line flags:
+in the [`-auth.config`](#auth-config). These settings can be overridden with the following command-line flags:
 
 - `-backend.tlsInsecureSkipVerify` allows skipping TLS verification when connecting to HTTPS backends.
-  This global setting can be overridden at per-user level inside [`-auth.config`](https://docs.victoriametrics.com/vmauth.html#auth-config)
+  This global setting can be overridden at per-user level inside [`-auth.config`](#auth-config)
   via `tls_insecure_skip_verify` option. For example:
 
   ```yml
@@ -479,7 +479,7 @@ in the [`-auth.config`](https://docs.victoriametrics.com/vmauth.html#auth-config
 
 - `-backend.tlsCAFile` allows specifying the path to TLS Root CA, which will be used for TLS verification when connecting to HTTPS backends.
   The `-backend.tlsCAFile` may point either to local file or to `http` / `https` url.
-  This global setting can be overridden at per-user level inside [`-auth.config`](https://docs.victoriametrics.com/vmauth.html#auth-config)
+  This global setting can be overridden at per-user level inside [`-auth.config`](#auth-config)
   via `tls_ca_file` option. For example:
 
   ```yml
@@ -695,12 +695,21 @@ It is recommended protecting the following endpoints with authKeys:
 `vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page
 either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later.
 
-`vmauth` exports `vmauth_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric 
-and `vmauth_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) metric 
-with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file.
-It is possible to override or hide the value in the label by specifying `name` field. 
-For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` 
-instead of `vmauth_user_requests_total{username="secret_user"}`:
+`vmauth` exports the following metrics per each defined user in [`-auth.config`](#auth-config):
+
+* `vmauth_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of requests served for the given `username`
+* `vmauth_user_request_backend_errors_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of request errors for the given `username`
+* `vmauth_user_request_duration_seconds` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) - the duration of requests for the given `username`
+* `vmauth_user_concurrent_requests_limit_reached_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of failed requests
+  for the given `username` because of exceeded [concurrency limits](#concurrency-limiting)
+* `vmauth_user_concurrent_requests_capacity` [gauge](https://docs.victoriametrics.com/keyConcepts.html#gauge) - the maximum number of [concurrent requests](#concurrency-limiting)
+  for the given `username`
+* `vmauth_user_concurrent_requests_current` [gauge](https://docs.victoriametrics.com/keyConcepts.html#gauge) - the current number of [concurrent requests](#concurrency-limiting)
+  for the given `username`
+
+By default, per-user metrics contain only `username` label. This label is set to `username` field value at the corresponding user section in the [`-auth.config`](#auth-config) file.
+It is possible to override the `username` label value by specifying `name` field additionally to `username` field.
+For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` instead of `vmauth_user_requests_total{username="secret_user"}`:
 
 ```yml
 users:
@@ -709,10 +718,28 @@ users:
   # other config options here
 ```
 
-For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total` 
-[counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric and 
-`vmauth_unauthorized_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary)
-metric without label (if `unauthorized_user` section of config is used).
+Additional labels for per-user metrics can be specified via `metric_labels` section. For example, the following config
+defines `{dc="eu",team="dev"}` labels additionally to `username="foobar"` label:
+
+```yml
+users:
+- username: "foobar"
+  metric_labels:
+   dc: eu
+   team: dev
+  # other config options here
+```
+
+`vmauth` exports the following metrics if `unauthorized_user` section is defined in [`-auth.config`](#auth-config):
+
+* `vmauth_unauthorized_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of unauthorized requests served
+* `vmauth_unauthorized_user_request_backend_errors_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of unauthorized request errors
+* `vmauth_unauthorized_user_request_duration_seconds` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) - the duration of unauthorized requests
+* `vmauth_unauthorized_user_concurrent_requests_limit_reached_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of failed unauthorized requests
+  because of exceeded [concurrency limits](#concurrency-limiting)
+* `vmauth_unauthorized_user_concurrent_requests_capacity` [gauge](https://docs.victoriametrics.com/keyConcepts.html#gauge) - the maximum number
+  of [concurrent unauthorized requests](#concurrency-limiting)
+* `vmauth_user_concurrent_requests_current` [gauge](https://docs.victoriametrics.com/keyConcepts.html#gauge) - the current number of [concurrent unauthorized requests](#concurrency-limiting)
 
 ## How to build from sources
 


Test patch:
diff --git a/app/vmauth/auth_config_test.go b/app/vmauth/auth_config_test.go
index df96d878d..6c9cb1226 100644
--- a/app/vmauth/auth_config_test.go
+++ b/app/vmauth/auth_config_test.go
@@ -229,6 +229,14 @@ users:
     url_prefix: http://foobar
     headers:
       aaa: bbb
+`)
+	// Invalid metric label name
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+  metric_labels:
+    not-prometheus-compatible: value
 `)
 }
 
@@ -489,7 +497,41 @@ users:
 			}),
 		},
 	})
-
+	// With metric_labels
+	f(`
+users:
+- username: foo-same
+  password: baz
+  url_prefix: http://foo
+  metric_labels:
+    dc: eu
+    team: dev
+- username: foo-same
+  password: bar
+  url_prefix: https://bar/x///
+  metric_labels:
+    backend_env: test
+    team: accounting
+`, map[string]*UserInfo{
+		getAuthToken("", "foo-same", "baz"): {
+			Username:  "foo-same",
+			Password:  "baz",
+			URLPrefix: mustParseURL("http://foo"),
+			MetricLabels: map[string]string{
+				"dc":   "eu",
+				"team": "dev",
+			},
+		},
+		getAuthToken("", "foo-same", "bar"): {
+			Username:  "foo-same",
+			Password:  "bar",
+			URLPrefix: mustParseURL("https://bar/x"),
+			MetricLabels: map[string]string{
+				"backend_env": "test",
+				"team":        "accounting",
+			},
+		},
+	})
 }
 
 func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {
@@ -526,6 +568,86 @@ unauthorized_user:
 	}
 }
 
+func TestUserInfoGetMetricLabels(t *testing.T) {
+	t.Run("empty-labels", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "user1",
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-username", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "user1",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-name", func(t *testing.T) {
+		ui := &UserInfo{
+			Name:        "user1",
+			BearerToken: "abc",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-bearer-token", func(t *testing.T) {
+		ui := &UserInfo{
+			BearerToken: "abc",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="bearer_token:hash:44BC2CF5AD770999"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("invalid-label", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "foo",
+			MetricLabels: map[string]string{
+				",{": "aaaa",
+			},
+		}
+		_, err := ui.getMetricLabels()
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	})
+}
+
 func isSetBool(boolP *bool, expectedValue bool) bool {
 	if boolP == nil {
 		return false