Repository: redis/go-redis
Pull request #3334: Ensure context isn't exhausted via concurrent query as opposed to sentinel query
Author: login: gnpaone
Created: 2025-04-05T13:56:19Z | Merged: 2025-04-16T19:32:40Z

Switch from sequential sentinel query to concurrent query to avoid context exhaustion and connect to new master redis after redis failover


Diff:
diff --git a/sentinel.go b/sentinel.go
index a132af2fe5..0638663536 100644
--- a/sentinel.go
+++ b/sentinel.go
@@ -566,29 +566,60 @@ func (c *sentinelFailover) MasterAddr(ctx context.Context) (string, error) {
 		}
 	}
 
-	for i, sentinelAddr := range c.sentinelAddrs {
-		sentinel := NewSentinelClient(c.opt.sentinelOptions(sentinelAddr))
+	var (
+		masterAddr string
+		wg         sync.WaitGroup
+		once       sync.Once
+		errCh      = make(chan error, len(c.sentinelAddrs))
+	)
 
-		masterAddr, err := sentinel.GetMasterAddrByName(ctx, c.opt.MasterName).Result()
-		if err != nil {
-			_ = sentinel.Close()
-			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-				return "", err
-			}
-			internal.Logger.Printf(ctx, "sentinel: GetMasterAddrByName master=%q failed: %s",
-				c.opt.MasterName, err)
-			continue
-		}
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 
-		// Push working sentinel to the top.
-		c.sentinelAddrs[0], c.sentinelAddrs[i] = c.sentinelAddrs[i], c.sentinelAddrs[0]
-		c.setSentinel(ctx, sentinel)
+	for i, sentinelAddr := range c.sentinelAddrs {
+		wg.Add(1)
+		go func(i int, addr string) {
+			defer wg.Done()
+			sentinelCli := NewSentinelClient(c.opt.sentinelOptions(addr))
+			addrVal, err := sentinelCli.GetMasterAddrByName(ctx, c.opt.MasterName).Result()
+			if err != nil {
+				if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+					// Report immediately and return
+					errCh <- err
+					return
+				}
+				internal.Logger.Printf(ctx, "sentinel: GetMasterAddrByName addr=%s, master=%q failed: %s",
+					addr, c.opt.MasterName, err)
+				_ = sentinelCli.Close()
+				return
+			}
 
-		addr := net.JoinHostPort(masterAddr[0], masterAddr[1])
-		return addr, nil
+			once.Do(func() {
+				masterAddr = net.JoinHostPort(addrVal[0], addrVal[1])
+				// Push working sentinel to the top
+				c.sentinelAddrs[0], c.sentinelAddrs[i] = c.sentinelAddrs[i], c.sentinelAddrs[0]
+				c.setSentinel(ctx, sentinelCli)
+				internal.Logger.Printf(ctx, "sentinel: selected addr=%s masterAddr=%s", addr, masterAddr)
+				cancel()
+			})
+		}(i, sentinelAddr)
 	}
 
-	return "", errors.New("redis: all sentinels specified in configuration are unreachable")
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		if masterAddr != "" {
+			return masterAddr, nil
+		}
+		return "", errors.New("redis: all sentinels specified in configuration are unreachable")
+	case err := <-errCh:
+		return "", err
+	}
 }
 
 func (c *sentinelFailover) replicaAddrs(ctx context.Context, useDisconnected bool) ([]string, error) {
diff --git a/sentinel_test.go b/sentinel_test.go
index 07c7628a06..cde7f956d1 100644
--- a/sentinel_test.go
+++ b/sentinel_test.go
@@ -3,6 +3,7 @@ package redis_test
 import (
 	"context"
 	"net"
+	"time"
 
 	. "github.com/bsm/ginkgo/v2"
 	. "github.com/bsm/gomega"
@@ -32,6 +33,24 @@ var _ = Describe("Sentinel PROTO 2", func() {
 	})
 })
 
+var _ = Describe("Sentinel resolution", func() {
+	It("should resolve master without context exhaustion", func() {
+		shortCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
+		defer cancel()
+
+		client := redis.NewFailoverClient(&redis.FailoverOptions{
+			MasterName:    sentinelName,
+			SentinelAddrs: sentinelAddrs,
+			MaxRetries: -1,
+		})
+
+		err := client.Ping(shortCtx).Err()
+		Expect(err).NotTo(HaveOccurred(), "expected master to resolve without context exhaustion")
+
+		_ = client.Close()
+	})
+})
+
 var _ = Describe("Sentinel", func() {
 	var client *redis.Client
 	var master *redis.Client


Reviews:
user: login: ndyakov
body: Please add a test , it may be a setup of three sentinels, two of which are unreachable. Define the context in such a way, to get `context.DeadlineExceeded` error on `GetMasterAddrByName`. Such setup should fail prior to your PR and succeeded with the introduced change, correct? 
state: CHANGES_REQUESTED
author_association: MEMBER
submitted_at: 2025-04-07T10:46:00Z
user: login: kwenzh
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-10T12:07:55Z
user: login: gnpaone
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T12:14:07Z
user: login: gnpaone
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T04:11:45Z
user: login: ndyakov
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-16T15:38:22Z

Review comments:
diff_hunk: @@ -566,28 +566,52 @@ func (c *sentinelFailover) MasterAddr(ctx context.Context) (string, error) {
 		}
 	}
 
+	var (
+		masterAddr string
+		wg         sync.WaitGroup
+		once       sync.Once
+		done       = make(chan struct{})
+	)
+
 	for i, sentinelAddr := range c.sentinelAddrs {
-		sentinel := NewSentinelClient(c.opt.sentinelOptions(sentinelAddr))
+		wg.Add(1)
+		go func(i int, addr string) {
+			defer wg.Done()
+			select {
+			case <-done:
+				return
+			default:
+				sentinelCli := NewSentinelClient(c.opt.sentinelOptions(addr))
+				addrVal, err := sentinelCli.GetMasterAddrByName(ctx, c.opt.MasterName).Result()
+				if err != nil {
+					internal.Logger.Printf(ctx, "sentinel: GetMasterAddrByName addr=%s, master=%q failed: %s",
+						addr, c.opt.MasterName, err)
+					_ = sentinelCli.Close()
+					return
+				}
 
-		masterAddr, err := sentinel.GetMasterAddrByName(ctx, c.opt.MasterName).Result()
-		if err != nil {
-			_ = sentinel.Close()
-			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-				return "", err
+				once.Do(func() {
+					masterAddr = net.JoinHostPort(addrVal[0], addrVal[1])
+					// Push working sentinel to the top
+					c.sentinelAddrs[0], c.sentinelAddrs[i] = c.sentinelAddrs[i], c.sentinelAddrs[0]
+					c.setSentinel(ctx, sentinelCli)
+					internal.Logger.Printf(ctx, "sentinel: selected addr=%s masterAddr=%s", addr, masterAddr)
+					close(done)
+				})
 			}
-			internal.Logger.Printf(ctx, "sentinel: GetMasterAddrByName master=%q failed: %s",
-				c.opt.MasterName, err)
-			continue
-		}
+		}(i, sentinelAddr)
+	}
 
-		// Push working sentinel to the top.
-		c.sentinelAddrs[0], c.sentinelAddrs[i] = c.sentinelAddrs[i], c.sentinelAddrs[0]
-		c.setSentinel(ctx, sentinel)
+	go func() {
+		wg.Wait()
+		once.Do(func() { close(done) })
+	}()
 
-		addr := net.JoinHostPort(masterAddr[0], masterAddr[1])
-		return addr, nil
-	}
+	<-done
path: sentinel.go
user: login: ndyakov
body: I don't see a reason to have yet another goroutine here just to make sure the channel is closed. It should be perfectly fine to wait on the `wg` in the main thread and forget about the `channel`:
- As far as I know the GC will sweep it even if it was closed here, since it will be unreachable
- I don't image the performance benefit will be significant (e.g. the difference between waiting for all sentinel goroutines to return and call `wg.Done` in their defer statement).
- if found, the `masterAddr` should be set either way. 

created_at: 2025-04-07T10:41:17Z
author_association: MEMBER
diff_hunk: @@ -566,28 +566,52 @@ func (c *sentinelFailover) MasterAddr(ctx context.Context) (string, error) {
 		}
 	}
 
+	var (
+		masterAddr string
+		wg         sync.WaitGroup
+		once       sync.Once
+		done       = make(chan struct{})
+	)
+
 	for i, sentinelAddr := range c.sentinelAddrs {
-		sentinel := NewSentinelClient(c.opt.sentinelOptions(sentinelAddr))
+		wg.Add(1)
+		go func(i int, addr string) {
+			defer wg.Done()
+			select {
+			case <-done:
+				return
+			default:
+				sentinelCli := NewSentinelClient(c.opt.sentinelOptions(addr))
+				addrVal, err := sentinelCli.GetMasterAddrByName(ctx, c.opt.MasterName).Result()
+				if err != nil {
+					internal.Logger.Printf(ctx, "sentinel: GetMasterAddrByName addr=%s, master=%q failed: %s",
+						addr, c.opt.MasterName, err)
+					_ = sentinelCli.Close()
+					return
+				}
 
-		masterAddr, err := sentinel.GetMasterAddrByName(ctx, c.opt.MasterName).Result()
-		if err != nil {
-			_ = sentinel.Close()
-			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-				return "", err
+				once.Do(func() {
+					masterAddr = net.JoinHostPort(addrVal[0], addrVal[1])
+					// Push working sentinel to the top
+					c.sentinelAddrs[0], c.sentinelAddrs[i] = c.sentinelAddrs[i], c.sentinelAddrs[0]
+					c.setSentinel(ctx, sentinelCli)
+					internal.Logger.Printf(ctx, "sentinel: selected addr=%s masterAddr=%s", addr, masterAddr)
+					close(done)
+				})
 			}
-			internal.Logger.Printf(ctx, "sentinel: GetMasterAddrByName master=%q failed: %s",
-				c.opt.MasterName, err)
-			continue
-		}
+		}(i, sentinelAddr)
+	}
 
-		// Push working sentinel to the top.
-		c.sentinelAddrs[0], c.sentinelAddrs[i] = c.sentinelAddrs[i], c.sentinelAddrs[0]
-		c.setSentinel(ctx, sentinel)
+	go func() {
+		wg.Wait()
+		once.Do(func() { close(done) })
+	}()
 
-		addr := net.JoinHostPort(masterAddr[0], masterAddr[1])
-		return addr, nil
-	}
+	<-done
path: sentinel.go
user: login: kwenzh
body: > I don't see a reason to have yet another goroutine here just to make sure the channel is closed. It should be perfectly fine to wait on the `wg` in the main thread and forget about the `channel`:
> 
> * As far as I know the GC will sweep it even if it was closed here, since it will be unreachable
> * I don't image the performance benefit will be significant (e.g. the difference between waiting for all sentinel goroutines to return and call `wg.Done` in their defer statement).
> * if found, the `masterAddr` should be set either way.

look like is not necessary to add a goruntine to do wg.Wait()
created_at: 2025-04-10T12:07:55Z
author_association: NONE
diff_hunk: @@ -566,28 +566,52 @@ func (c *sentinelFailover) MasterAddr(ctx context.Context) (string, error) {
 		}
 	}
 
+	var (
+		masterAddr string
+		wg         sync.WaitGroup
+		once       sync.Once
+		done       = make(chan struct{})
+	)
+
 	for i, sentinelAddr := range c.sentinelAddrs {
-		sentinel := NewSentinelClient(c.opt.sentinelOptions(sentinelAddr))
+		wg.Add(1)
+		go func(i int, addr string) {
+			defer wg.Done()
+			select {
+			case <-done:
+				return
+			default:
+				sentinelCli := NewSentinelClient(c.opt.sentinelOptions(addr))
+				addrVal, err := sentinelCli.GetMasterAddrByName(ctx, c.opt.MasterName).Result()
+				if err != nil {
+					internal.Logger.Printf(ctx, "sentinel: GetMasterAddrByName addr=%s, master=%q failed: %s",
+						addr, c.opt.MasterName, err)
+					_ = sentinelCli.Close()
+					return
+				}
 
-		masterAddr, err := sentinel.GetMasterAddrByName(ctx, c.opt.MasterName).Result()
-		if err != nil {
-			_ = sentinel.Close()
-			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-				return "", err
+				once.Do(func() {
+					masterAddr = net.JoinHostPort(addrVal[0], addrVal[1])
+					// Push working sentinel to the top
+					c.sentinelAddrs[0], c.sentinelAddrs[i] = c.sentinelAddrs[i], c.sentinelAddrs[0]
+					c.setSentinel(ctx, sentinelCli)
+					internal.Logger.Printf(ctx, "sentinel: selected addr=%s masterAddr=%s", addr, masterAddr)
+					close(done)
+				})
 			}
-			internal.Logger.Printf(ctx, "sentinel: GetMasterAddrByName master=%q failed: %s",
-				c.opt.MasterName, err)
-			continue
-		}
+		}(i, sentinelAddr)
+	}
 
-		// Push working sentinel to the top.
-		c.sentinelAddrs[0], c.sentinelAddrs[i] = c.sentinelAddrs[i], c.sentinelAddrs[0]
-		c.setSentinel(ctx, sentinel)
+	go func() {
+		wg.Wait()
+		once.Do(func() { close(done) })
+	}()
 
-		addr := net.JoinHostPort(masterAddr[0], masterAddr[1])
-		return addr, nil
-	}
+	<-done
path: sentinel.go
user: login: gnpaone
body: Yeah, I removed it. Working on the test
created_at: 2025-04-10T12:14:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -566,28 +566,52 @@ func (c *sentinelFailover) MasterAddr(ctx context.Context) (string, error) {
 		}
 	}
 
+	var (
+		masterAddr string
+		wg         sync.WaitGroup
+		once       sync.Once
+		done       = make(chan struct{})
+	)
+
 	for i, sentinelAddr := range c.sentinelAddrs {
-		sentinel := NewSentinelClient(c.opt.sentinelOptions(sentinelAddr))
+		wg.Add(1)
+		go func(i int, addr string) {
+			defer wg.Done()
+			select {
+			case <-done:
+				return
+			default:
+				sentinelCli := NewSentinelClient(c.opt.sentinelOptions(addr))
+				addrVal, err := sentinelCli.GetMasterAddrByName(ctx, c.opt.MasterName).Result()
+				if err != nil {
+					internal.Logger.Printf(ctx, "sentinel: GetMasterAddrByName addr=%s, master=%q failed: %s",
+						addr, c.opt.MasterName, err)
+					_ = sentinelCli.Close()
+					return
+				}
 
-		masterAddr, err := sentinel.GetMasterAddrByName(ctx, c.opt.MasterName).Result()
-		if err != nil {
-			_ = sentinel.Close()
-			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-				return "", err
+				once.Do(func() {
+					masterAddr = net.JoinHostPort(addrVal[0], addrVal[1])
+					// Push working sentinel to the top
+					c.sentinelAddrs[0], c.sentinelAddrs[i] = c.sentinelAddrs[i], c.sentinelAddrs[0]
+					c.setSentinel(ctx, sentinelCli)
+					internal.Logger.Printf(ctx, "sentinel: selected addr=%s masterAddr=%s", addr, masterAddr)
+					close(done)
+				})
 			}
-			internal.Logger.Printf(ctx, "sentinel: GetMasterAddrByName master=%q failed: %s",
-				c.opt.MasterName, err)
-			continue
-		}
+		}(i, sentinelAddr)
+	}
 
-		// Push working sentinel to the top.
-		c.sentinelAddrs[0], c.sentinelAddrs[i] = c.sentinelAddrs[i], c.sentinelAddrs[0]
-		c.setSentinel(ctx, sentinel)
+	go func() {
+		wg.Wait()
+		once.Do(func() { close(done) })
+	}()
 
-		addr := net.JoinHostPort(masterAddr[0], masterAddr[1])
-		return addr, nil
-	}
+	<-done
path: sentinel.go
user: login: gnpaone
body: I've added a test, please check if that's sufficient?
created_at: 2025-04-11T04:11:45Z
author_association: CONTRIBUTOR

PR comments:
user: login: gnpaone
created_at: 2025-04-05T13:57:09Z
body: Hey @ndyakov tried to fix your review points in https://github.com/redis/go-redis/pull/3174
author_association: CONTRIBUTOR
user: login: kwenzh
created_at: 2025-04-10T12:06:31Z
body: > Please add a test , it may be a setup of three sentinels, two of which are unreachable. Define the context in such a way, to get `context.DeadlineExceeded` error on `GetMasterAddrByName`. Such setup should fail prior to your PR and succeeded with the introduced change, correct?

The current problem is that the faulty sentinel node is in the front, which will cause context exhaustion. Even if the last sentinel can correctly provide the master address, don't we consider this situation?
author_association: NONE
user: login: kwenzh
created_at: 2025-04-11T03:01:40Z
body: I think that once any sentinel returns the master address correctly, it should be considered finished and other coroutines should be notified of the end. Even if the context timeout occurs afterwards, we should return the master address that has been queried, right?
author_association: NONE
user: login: kwenzh
created_at: 2025-04-11T03:26:50Z
body: > I think that once any sentinel returns the master address correctly, it should be considered finished and other coroutines should be notified of the end. Even if the context timeout occurs afterwards, we should return the master address that has been queried, right?

with issue： https://github.com/redis/go-redis/issues/3172
author_association: NONE
user: login: gnpaone
created_at: 2025-04-11T04:10:35Z
body: > > I think that once any sentinel returns the master address correctly, it should be considered finished and other coroutines should be notified of the end. Even if the context timeout occurs afterwards, we should return the master address that has been queried, right?
> 
> with issue： #3172

Maybe should I use something like

```go
ctx, cancel := context.WithCancel(ctx)
defer cancel()

...

cancel()
```

to stop goroutines early, once success is achieved ie sentinel returns the master address correctly?
author_association: CONTRIBUTOR
user: login: kwenzh
created_at: 2025-04-11T08:06:09Z
body: > once success is achieved ie sentinel returns the master address correctly?

yeah，once success returns the master address correctly，That's exactly what I meant！
author_association: NONE
user: login: gnpaone
created_at: 2025-04-13T12:03:35Z
body: Done
author_association: CONTRIBUTOR
user: login: ndyakov
created_at: 2025-04-15T13:18:32Z
body: @gnpaone can you please add the issue in the description of this PR. I will review the latest changes later today and get back to you if there are any comments. 
author_association: MEMBER
user: login: gnpaone
created_at: 2025-04-16T11:25:52Z
body: Updated the description @ndyakov 
author_association: CONTRIBUTOR