Repository: VictoriaMetrics/VictoriaMetrics
Pull request #6401 (2024-06-04 11:23:43)

fix discovering backend IPs when `url_prefix` contains hostname with `srv+` prefix
**log**
```log
2024-06-04T11:16:51.445Z        info    app/vmauth/main.go:80   started vmauth in 0.001 seconds
2024-06-04T11:16:51.445Z        info    lib/httpserver/httpserver.go:119        starting server at http://127.0.0.1:8427/
2024-06-04T11:16:51.445Z        info    lib/httpserver/httpserver.go:120        pprof handlers are exposed at http://127.0.0.1:8427/debug/pprof/
2024-06-04T11:16:52.899Z        warn    app/vmauth/auth_config.go:353   cannot discover backend SRV records for http://srv+localhost:8080: lookup localhost on 10.100.10.4:53: server misbehaving; use it literally
panic: runtime error: integer divide by zero

goroutine 9 [running]:
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver.handlerWrapper.func1()
        /Users/lhhdz/wd/projects/go/VictoriaMetrics/lib/httpserver/httpserver.go:291 +0x58
panic({0x103115100?, 0x10338d700?})
        /Users/lhhdz/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.3.darwin-arm64/src/runtime/panic.go:770 +0x124
main.getLeastLoadedBackendURL({0x0?, 0x22?, 0x1400014757b?}, 0x1400013c120?)
        /Users/lhhdz/wd/projects/go/VictoriaMetrics/app/vmauth/auth_config.go:473 +0x210
main.(*URLPrefix).getBackendURL(0x140000aa080)
        /Users/lhhdz/wd/projects/go/VictoriaMetrics/app/vmauth/auth_config.go:312 +0xb8
main.processRequest({0x10317f6e0, 0x1400000e048}, 0x14000000360, 0x140000a8000)
        /Users/lhhdz/wd/projects/go/VictoriaMetrics/app/vmauth/main.go:208 +0x380
main.processUserRequest({0x10317f6e0, 0x1400000e048}, 0x14000000360, 0x140000a8000)
        /Users/lhhdz/wd/projects/go/VictoriaMetrics/app/vmauth/main.go:175 +0x20c
main.requestHandler({0x10317f6e0, 0x1400000e048}, 0x14000000360)
        /Users/lhhdz/wd/projects/go/VictoriaMetrics/app/vmauth/main.go:114 +0x22c
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver.handlerWrapper(0x14000034400, {0x10317f860, 0x1400013e0a0}, 0x14000000360, 0x103179bf8)
        /Users/lhhdz/wd/projects/go/VictoriaMetrics/lib/httpserver/httpserver.go:416 +0xf44
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver.gzipHandler.func1({0x10317f860?, 0x1400013e0a0?}, 0x14000147968?)
        /Users/lhhdz/wd/projects/go/VictoriaMetrics/lib/httpserver/httpserver.go:250 +0x3c
net/http.HandlerFunc.ServeHTTP(0x1033910d0?, {0x10317f860?, 0x1400013e0a0?}, 0x4?)
        /Users/lhhdz/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.3.darwin-arm64/src/net/http/server.go:2166 +0x38
github.com/klauspost/compress/gzhttp.NewWrapper.func1.1({0x10317f4d0, 0x1400012e1c0}, 0x14000000360)
        /Users/lhhdz/wd/projects/go/VictoriaMetrics/vendor/github.com/klauspost/compress/gzhttp/compress.go:495 +0x474
net/http.HandlerFunc.ServeHTTP(0x0?, {0x10317f4d0?, 0x1400012e1c0?}, 0x14000147b50?)
        /Users/lhhdz/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.3.darwin-arm64/src/net/http/server.go:2166 +0x38
net/http.serverHandler.ServeHTTP({0x1400022d590?}, {0x10317f4d0?, 0x1400012e1c0?}, 0x6?)
        /Users/lhhdz/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.3.darwin-arm64/src/net/http/server.go:3137 +0xbc
net/http.(*conn).serve(0x140002127e0, {0x10317fcf8, 0x1400022d500})
        /Users/lhhdz/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.3.darwin-arm64/src/net/http/server.go:2039 +0x508
created by net/http.(*Server).Serve in goroutine 20
        /Users/lhhdz/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.3.darwin-arm64/src/net/http/server.go:3285 +0x3f0

Process finished with the exit code 1
```


codecov[bot]: ## [Codecov](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/6401?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) Report
Attention: Patch coverage is `0%` with `2 lines` in your changes missing coverage. Please review.
> Project coverage is 55.43%. Comparing base [(`faeabfc`)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/faeabfc730d6ab6970d10d5800daf600eaba38c4?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) to head [(`4eee9be`)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/4eee9be233f317b0409fc2c18df7f95631076f39?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).
> Report is 768 commits behind head on cluster.

| [Files](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/6401?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) | Patch % | Lines |
|---|---|---|
| [app/vmauth/auth\_config.go](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/6401?src=pr&el=tree&filepath=app%2Fvmauth%2Fauth_config.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtYXV0aC9hdXRoX2NvbmZpZy5nbw==) | 0.00% | [2 Missing :warning: ](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/6401?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) |

<details><summary>Additional details and impacted files</summary>


```diff
@@             Coverage Diff             @@
##           cluster    #6401      +/-   ##
===========================================
- Coverage    59.08%   55.43%   -3.65%     
===========================================
  Files          416      619     +203     
  Lines        78511    85894    +7383     
===========================================
+ Hits         46385    47613    +1228     
- Misses       29567    35227    +5660     
- Partials      2559     3054     +495     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/6401?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).

Haleygo: @shichanglin5 Thanks for the pull request, please check [one more fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6401/commits/290e93d9edaeee42133eeca15689d9144fbe3db1).
shichanglin5: @Haleygo 更新了
Haleygo: @hagen1778 please check https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6401/commits/8aa9db9a58479a81005679fc8cd4bfd4ece6ec45

Patch:
diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go
index b4afeee31..c299a6f66 100644
--- a/app/vmauth/auth_config.go
+++ b/app/vmauth/auth_config.go
@@ -353,7 +353,7 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 				logger.Warnf("cannot discover backend SRV records for %s: %s; use it literally", bu, err)
 				resolvedAddrs = []string{host}
 			} else {
-				resolvedAddrs := make([]string, len(addrs))
+				resolvedAddrs = make([]string, len(addrs))
 				for i, addr := range addrs {
 					resolvedAddrs[i] = fmt.Sprintf("%s:%d", addr.Target, addr.Port)
 				}
@@ -380,6 +380,7 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 	var busNew []*backendURL
 	for _, bu := range up.busOriginal {
 		host := bu.Hostname()
+		host = strings.TrimPrefix(host, "srv+")
 		port := bu.Port()
 		for _, addr := range hostToAddrs[host] {
 			buCopy := *bu
@@ -1011,6 +1012,8 @@ func (up *URLPrefix) sanitizeAndInitialize() error {
 		}
 	}
 	up.bus.Store(&bus)
+	up.nextDiscoveryDeadline.Store(0)
+	up.n.Store(0)
 
 	return nil
 }
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 28d4f2095..bb8bdf0e2 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -90,6 +90,7 @@ Released at 2024-06-07
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): correctly apply `-inmemoryDataFlushInterval` when it's set to minimum supported value 1s.
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): reduce the default value for `-maxLabelValueLen` command-line flag from `16KiB` to `1KiB`. This should prevent from issues like [this one](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6176) when time series with too long labels are ingested into VictoriaMetrics.
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly release memory used for metrics during config reload. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6247).
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): fix discovering backend IPs when `url_prefix` contains hostname with srv+ prefix. Thanks to @shichanglin5 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6401). 
 * BUGFIX: [dashboards](https://grafana.com/orgs/victoriametrics): fix `AnnotationQueryRunner` error in Grafana when executing annotations query against Prometheus backend. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6309) for details.
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): filter deleted label names and values from [`/api/v1/labels`](https://docs.victoriametrics.com/url-examples/#apiv1labels) and [`/api/v1/label/.../values`](https://docs.victoriametrics.com/url-examples/#apiv1labelvalues) responses when `match[]` filter matches small number of time series. The issue was introduced [v1.81.0](https://docs.victoriametrics.com/changelog_2022/#v1810).
 * BUGFIX: [vmalert-tool](https://docs.victoriametrics.com/vmalert-tool/): fix float values template in `input_series`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6391).
diff --git a/lib/netutil/netutil.go b/lib/netutil/netutil.go
index 39ffc609c..150b27723 100644
--- a/lib/netutil/netutil.go
+++ b/lib/netutil/netutil.go
@@ -9,6 +9,22 @@ import (
 	"time"
 )
 
+type resolver interface {
+	LookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*net.SRV, err error)
+	LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error)
+	LookupMX(ctx context.Context, name string) ([]*net.MX, error)
+}
+
+// Resolver is default DNS resolver.
+var Resolver resolver
+
+func init() {
+	Resolver = &net.Resolver{
+		PreferGo:     true,
+		StrictErrors: true,
+	}
+}
+
 // IsTrivialNetworkError returns true if the err can be ignored during logging.
 func IsTrivialNetworkError(err error) bool {
 	// Suppress trivial network errors, which could occur at remote side.
@@ -43,12 +59,6 @@ func DialMaybeSRV(ctx context.Context, network, addr string) (net.Conn, error) {
 	return Dialer.DialContext(ctx, network, addr)
 }
 
-// Resolver is default DNS resolver.
-var Resolver = &net.Resolver{
-	PreferGo:     true,
-	StrictErrors: true,
-}
-
 // Dialer is default network dialer.
 var Dialer = &net.Dialer{
 	Timeout:   30 * time.Second,


Test patch:
diff --git a/app/vmauth/target_url_test.go b/app/vmauth/target_url_test.go
index 56871428d..89fb52a7e 100644
--- a/app/vmauth/target_url_test.go
+++ b/app/vmauth/target_url_test.go
@@ -1,11 +1,15 @@
 package main
 
 import (
+	"context"
 	"fmt"
+	"net"
 	"net/url"
 	"reflect"
 	"strings"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 func TestDropPrefixParts(t *testing.T) {
@@ -92,7 +96,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		u = normalizeURL(u)
 		up, hc := ui.getURLPrefixAndHeaders(u, nil)
 		if up == nil {
-			t.Fatalf("cannot determie backend: %s", err)
+			t.Fatalf("cannot match available backend: %s", err)
 		}
 		bu := up.getBackendURL()
 		target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts)
@@ -256,6 +260,53 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 	f(ui, `/api/v1/query?query=up{env="prod"}`, `http://vmselect/1/prometheus/api/v1/query?query=up%7Benv%3D%22prod%22%7D`, "", "", nil, "least_loaded", 0)
 	f(ui, `/api/v1/query?query=up{foo="bar",env="dev",pod!=""}`, `http://vmselect/0/prometheus/api/v1/query?query=up%7Bfoo%3D%22bar%22%2Cenv%3D%22dev%22%2Cpod%21%3D%22%22%7D`, "", "", nil, "least_loaded", 0)
 	f(ui, `/api/v1/query?query=up{foo="bar"}`, `http://default-server/api/v1/query?query=up%7Bfoo%3D%22bar%22%7D`, "", "", nil, "least_loaded", 0)
+
+	customResolver := &fakeResolver{
+		Resolver: &net.Resolver{},
+		lookupSRVResults: map[string][]*net.SRV{
+			"vmselect": {
+				{
+					Target: "10.6.142.50",
+					Port:   8481,
+				},
+				{
+					Target: "10.6.142.51",
+					Port:   8481,
+				},
+			},
+		},
+		lookupIPAddrResults: map[string][]net.IPAddr{
+			"vminsert": {
+				{
+					IP: net.ParseIP("10.6.142.52"),
+				},
+			},
+		},
+	}
+	netutil.Resolver = customResolver
+
+	// Discover backendURL
+	allowed := true
+	ui = &UserInfo{
+		URLMaps: []URLMap{
+			{
+				SrcPaths:  getRegexs([]string{"/select/.+"}),
+				URLPrefix: mustParseURL("http://srv+vmselect"),
+			},
+			{
+				SrcPaths:  getRegexs([]string{"/insert/.+"}),
+				URLPrefix: mustParseURL("http://vminsert:8480"),
+			},
+		},
+		DiscoverBackendIPs: &allowed,
+		URLPrefix:          mustParseURL("http://non-exist-dns-addr"),
+	}
+	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up", "", "", nil, "least_loaded", 0)
+	// url_prefix counter will be reset, still go to 10.6.142.51
+	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up", "", "", nil, "least_loaded", 0)
+	f(ui, `/insert/0/prometheus/api/v1/write`, "http://10.6.142.52:8480/insert/0/prometheus/api/v1/write", "", "", nil, "least_loaded", 0)
+	// unsuccessful dns resolve
+	f(ui, `/test`, "http://non-exist-dns-addr/test", "", "", nil, "least_loaded", 0)
 }
 
 func TestCreateTargetURLFailure(t *testing.T) {
@@ -295,3 +346,29 @@ func headersToString(hs []*Header) string {
 	}
 	return strings.Join(a, "\n")
 }
+
+type fakeResolver struct {
+	Resolver            *net.Resolver
+	lookupSRVResults    map[string][]*net.SRV
+	lookupIPAddrResults map[string][]net.IPAddr
+}
+
+func (r *fakeResolver) LookupSRV(_ context.Context, _, _, name string) (string, []*net.SRV, error) {
+	if results, ok := r.lookupSRVResults[name]; ok {
+		return name, results, nil
+	}
+
+	return name, nil, fmt.Errorf("no srv results found for host: %s", name)
+}
+
+func (r *fakeResolver) LookupIPAddr(_ context.Context, host string) ([]net.IPAddr, error) {
+	if results, ok := r.lookupIPAddrResults[host]; ok {
+		return results, nil
+	}
+
+	return nil, fmt.Errorf("no results found for host: %s", host)
+}
+
+func (r *fakeResolver) LookupMX(_ context.Context, _ string) ([]*net.MX, error) {
+	return nil, nil
+}