Repository: twmb/franz-go
Pull request #1026: Bugs
Author: login: twmb
Created: 2025-05-16T20:18:32Z | Merged: 2025-05-20T14:27:01Z



Diff:
diff --git a/pkg/kgo/broker.go b/pkg/kgo/broker.go
index 163638087..65eb342a0 100644
--- a/pkg/kgo/broker.go
+++ b/pkg/kgo/broker.go
@@ -550,7 +550,10 @@ func (b *broker) loadConnection(ctx context.Context, req kmsg.Request) (*brokerC
 		return *pcxn, nil
 	}
 
+	var tries int
 	start := time.Now()
+doConnect:
+	tries++
 	conn, err := b.connect(ctx)
 	defer func() {
 		since := time.Since(start)
@@ -572,7 +575,17 @@ func (b *broker) loadConnection(ctx context.Context, req kmsg.Request) (*brokerC
 		conn:   conn,
 		deadCh: make(chan struct{}),
 	}
-	if err = cxn.init(isProduceCxn); err != nil {
+	if err = cxn.init(isProduceCxn, tries); err != nil {
+		// EventHubs does not handle v4 and resets the connection. We
+		// retry twice. On the first and second attempt, we try our max
+		// version possible (as should be allowed). On the third try,
+		// we downgrade to v0.
+		if er := (*errApiVersionsReset)(nil); errors.As(err, &er) {
+			if tries < 3 {
+				tries++
+				goto doConnect
+			}
+		}
 		b.cl.cfg.logger.Log(LogLevelDebug, "connection initialization failed", "addr", b.addr, "broker", logID(b.meta.NodeID), "err", err)
 		cxn.closeConn()
 		return nil, err
@@ -719,11 +732,11 @@ type brokerCxn struct {
 	deadCh chan struct{}
 }
 
-func (cxn *brokerCxn) init(isProduceCxn bool) error {
+func (cxn *brokerCxn) init(isProduceCxn bool, tries int) error {
 	hasVersions := cxn.b.loadVersions() != nil
 	if !hasVersions {
 		if cxn.b.cl.cfg.maxVersions == nil || cxn.b.cl.cfg.maxVersions.HasKey(18) {
-			if err := cxn.requestAPIVersions(); err != nil {
+			if err := cxn.requestAPIVersions(tries); err != nil {
 				if !errors.Is(err, ErrClientClosed) && !isRetryableBrokerErr(err) {
 					cxn.cl.cfg.logger.Log(LogLevelError, "unable to request api versions", "broker", logID(cxn.b.meta.NodeID), "err", err)
 				}
@@ -749,14 +762,15 @@ func (cxn *brokerCxn) init(isProduceCxn bool) error {
 	return nil
 }
 
-func (cxn *brokerCxn) requestAPIVersions() error {
-	maxVersion := int16(3)
-
-	// If the user configured a max versions, we check that the key exists
-	// before entering this function. Thus, we expect exists to be true,
-	// but we still doubly check it for sanity (as well as userMax, which
-	// can only be non-negative based off of LookupMaxKeyVersion's API).
-	if cxn.cl.cfg.maxVersions != nil {
+func (cxn *brokerCxn) requestAPIVersions(tries int) error {
+	maxVersion := int16(4)
+	if tries >= 3 { // on the third try, we pin to v0; see above in cxn initialization
+		maxVersion = 0
+	} else if cxn.cl.cfg.maxVersions != nil {
+		// If the user configured a max versions, we check that the key exists
+		// before entering this function. Thus, we expect exists to be true,
+		// but we still doubly check it for sanity (as well as userMax, which
+		// can only be non-negative based off of LookupMaxKeyVersion's API).
 		userMax, exists := cxn.cl.cfg.maxVersions.LookupMaxKeyVersion(18) // 18 == api versions
 		if exists && userMax >= 0 {
 			maxVersion = userMax
@@ -779,6 +793,9 @@ start:
 	// api versions does *not* use flexible response headers; see comment in promisedResp
 	rawResp, err := cxn.readResponse(nil, req.Key(), req.GetVersion(), corrID, false, rt, bytesWritten, writeWait, timeToWrite, readEnqueue)
 	if err != nil {
+		if strings.HasSuffix(err.Error(), "connection reset by peer") {
+			return &errApiVersionsReset{err}
+		}
 		return err
 	}
 	if len(rawResp) < 2 {
diff --git a/pkg/kgo/errors.go b/pkg/kgo/errors.go
index 8c3b93d32..ef88ab3a6 100644
--- a/pkg/kgo/errors.go
+++ b/pkg/kgo/errors.go
@@ -369,3 +369,10 @@ func errCodeMessage(code int16, errMessage *string) error {
 	}
 	return nil
 }
+
+type errApiVersionsReset struct {
+	err error
+}
+
+func (e *errApiVersionsReset) Error() string { return e.err.Error() }
+func (e *errApiVersionsReset) Unwrap() error { return e.err }
diff --git a/pkg/kgo/sink.go b/pkg/kgo/sink.go
index 33562f8c4..14fdf08de 100644
--- a/pkg/kgo/sink.go
+++ b/pkg/kgo/sink.go
@@ -835,7 +835,12 @@ func (s *sink) handleReqRespBatch(
 	// Since we have received a response and we are the first batch, we can
 	// at this point re-enable failing from load errors.
 	//
-	// We do not need a lock since the owner is locked.
+	// We do not need the mutex lock on the batch. We already have the
+	// recBuf mu (guarding most concurrency). The only place batch fields
+	// are accessed & modified without the recBuf mu is when writing a
+	// batch, and we only ever use a batch in inflight request at a time
+	// (regardless of the partition  being canceled or moving to a
+	// different sink).
 	batch.canFailFromLoadErrs = true
 
 	// By default, we assume we errored. Non-error updates this back
@@ -1475,12 +1480,12 @@ func (recBuf *recBuf) bumpRepeatedLoadErr(err error) {
 		return
 	}
 	batch0 := recBuf.batches[0]
-	batch0.tries++
 
 	// We need to lock the batch as well because there could be a buffered
 	// request about to be written. Writing requests only grabs the batch
 	// mu, not the recBuf mu.
 	batch0.mu.Lock()
+	batch0.tries++
 	var (
 		canFail        = !recBuf.cl.idempotent() || batch0.canFailFromLoadErrs // we can only fail if we are not idempotent or if we have no outstanding requests
 		batch0Fail     = batch0.maybeFailErr(&recBuf.cl.cfg) != nil            // timeout, retries, or aborting
diff --git a/pkg/kgo/txn.go b/pkg/kgo/txn.go
index 9543cb0c9..531a34d86 100644
--- a/pkg/kgo/txn.go
+++ b/pkg/kgo/txn.go
@@ -720,8 +720,9 @@ func (cl *Client) EndTransaction(ctx context.Context, commit TransactionEndTry)
 		req.ProducerID = id
 		req.ProducerEpoch = epoch
 		req.Commit = bool(commit)
-		if !cl.producer.tx890p2 { // v5 is only supported with KIP-890 part 2
-			ctx = context.WithValue(ctx, ctxPinReq, &pinReq{pinMax: true, max: 4})
+		ctx := ctx // capture a local ctx variable in case we introduce concurrency above later
+		if !cl.producer.tx890p2 {
+			ctx = context.WithValue(ctx, ctxPinReq, &pinReq{pinMax: true, max: 4}) // v5 is only supported with KIP-890 part 2
 		}
 		resp, err := req.RequestWith(ctx, cl)
 		if err != nil {
@@ -1064,6 +1065,7 @@ func (g *groupConsumer) commitTxn(ctx context.Context, tx890p2 bool, req *kmsg.T
 		g.cl.cfg.logger.Log(LogLevelDebug, "issuing txn offset commit", "uncommitted", req)
 
 		start := time.Now()
+		ctx := ctx // capture a local ctx variable; do not use the shared one that is concurrently read above
 		if !tx890p2 {
 			ctx = context.WithValue(ctx, ctxPinReq, &pinReq{pinMax: true, max: 4}) // v5 is only supported with KIP-890 part 2
 		}


Reviews:
user: login: stampy88
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-17T14:18:01Z
user: login: stampy88
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-17T14:18:11Z
user: login: stampy88
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-17T14:19:32Z
user: login: twmb
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-19T21:38:23Z

Review comments:
diff_hunk: @@ -749,8 +762,11 @@ func (cxn *brokerCxn) init(isProduceCxn bool) error {
 	return nil
 }
 
-func (cxn *brokerCxn) requestAPIVersions() error {
-	maxVersion := int16(3)
+func (cxn *brokerCxn) requestAPIVersions(tries int) error {
+	maxVersion := int16(4)
+	if tries >= 3 { // on the third try, we pin to v0; see above in cxn initialization
+		maxVersion = 0
+	}
path: pkg/kgo/broker.go
user: login: stampy88
body: the `if cxn.cl.cfg.maxVersions != nil` needs to be an `else if` to the `if tries >= 3 `, otherwise it is always getting the  `cxn.cl.cfg.maxVersions` since that is defaulted to `kversion.Stable()`
created_at: 2025-05-17T14:18:01Z
author_association: NONE
diff_hunk: @@ -749,8 +762,11 @@ func (cxn *brokerCxn) init(isProduceCxn bool) error {
 	return nil
 }
 
-func (cxn *brokerCxn) requestAPIVersions() error {
-	maxVersion := int16(3)
+func (cxn *brokerCxn) requestAPIVersions(tries int) error {
+	maxVersion := int16(4)
+	if tries >= 3 { // on the third try, we pin to v0; see above in cxn initialization
+		maxVersion = 0
+	}
path: pkg/kgo/broker.go
user: login: stampy88
body: doing that allowed me to connect
created_at: 2025-05-17T14:18:11Z
author_association: NONE
diff_hunk: @@ -749,8 +762,11 @@ func (cxn *brokerCxn) init(isProduceCxn bool) error {
 	return nil
 }
 
-func (cxn *brokerCxn) requestAPIVersions() error {
-	maxVersion := int16(3)
+func (cxn *brokerCxn) requestAPIVersions(tries int) error {
+	maxVersion := int16(4)
+	if tries >= 3 { // on the third try, we pin to v0; see above in cxn initialization
+		maxVersion = 0
+	}
path: pkg/kgo/broker.go
user: login: stampy88
body: here's the success output:
```
[DEBUG] opening connection to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[INFO] immediate metadata update triggered; why: querying metadata for consumer initialization
[DEBUG] connection opened to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] issuing api versions request; broker: seed_0, version: 4
[DEBUG] wrote ApiVersions v4; broker: seed_0, bytes_written: 39, write_wait: 17.532µs, time_to_write: 13.649µs, err: <nil>
[DEBUG] read ApiVersions v4; broker: seed_0, bytes_read: 0, read_wait: 50.809µs, time_to_read: 64.013612ms, err: read tcp 192.168.0.36:49024->172.202.80.16:9093: read: connection reset by peer
[DEBUG] opening connection to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] connection opened to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] issuing api versions request; broker: seed_0, version: 0
[DEBUG] wrote ApiVersions v0; broker: seed_0, bytes_written: 25, write_wait: 29.789µs, time_to_write: 46.496µs, err: <nil>
[DEBUG] read ApiVersions v0; broker: seed_0, bytes_read: 164, read_wait: 58.92µs, time_to_read: 56.555529ms, err: <nil>
[DEBUG] issuing SASLHandshakeRequest; broker: seed_0
[DEBUG] wrote SASLHandshake v1; broker: seed_0, bytes_written: 32, write_wait: 51.832µs, time_to_write: 87.786µs, err: <nil>
[DEBUG] read SASLHandshake v1; broker: seed_0, bytes_read: 34, read_wait: 32.447µs, time_to_read: 56.214164ms, err: <nil>
[DEBUG] beginning sasl authentication; broker: seed_0, addr: XXXX.servicebus.windows.net:9093, mechanism: PLAIN, authenticate: true
[DEBUG] issuing SASLAuthenticate; broker: seed_0, version: 2, step: 0
[DEBUG] wrote SASLAuthenticate v2; broker: seed_0, bytes_written: 205, write_wait: 64.825µs, time_to_write: 129.626µs, err: <nil>
[DEBUG] read SASLAuthenticate v2; broker: seed_0, bytes_read: 22, read_wait: 41.829µs, time_to_read: 56.39973ms, err: <nil>
[DEBUG] connection initialized successfully; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] wrote Metadata v5; broker: seed_0, bytes_written: 30, write_wait: 740.239902ms, time_to_write: 128.326µs, err: <nil>
[DEBUG] wrote Metadata v5; broker: seed_0, bytes_written: 42, write_wait: 739.975064ms, time_to_write: 75.084µs, err: <nil>
[DEBUG] read Metadata v5; broker: seed_0, bytes_read: 501, read_wait: 211.798µs, time_to_read: 426.95047ms, err: <nil>
[DEBUG] read Metadata v5; broker: seed_0, bytes_read: 303, read_wait: 427.185489ms, time_to_read: 25.579µs, err: <nil>
created_at: 2025-05-17T14:19:32Z
author_association: NONE
diff_hunk: @@ -749,8 +762,11 @@ func (cxn *brokerCxn) init(isProduceCxn bool) error {
 	return nil
 }
 
-func (cxn *brokerCxn) requestAPIVersions() error {
-	maxVersion := int16(3)
+func (cxn *brokerCxn) requestAPIVersions(tries int) error {
+	maxVersion := int16(4)
+	if tries >= 3 { // on the third try, we pin to v0; see above in cxn initialization
+		maxVersion = 0
+	}
path: pkg/kgo/broker.go
user: login: twmb
body: Good find, pushed. PTAL and if everything is good, I'll finally release this (hopefully last) bugfix release.
created_at: 2025-05-19T21:38:22Z
author_association: OWNER

PR comments:
user: login: stampy88
created_at: 2025-05-17T00:07:28Z
body: @twmb get a broken pipe on the second attempt on the `cxn.writeRequest(nil, time.Now(), req)` that returns you from the `requestAPIVersions` method. 

```
[DEBUG] opening connection to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[INFO] immediate metadata update triggered; why: querying metadata for consumer initialization
[DEBUG] connection opened to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] issuing api versions request; broker: seed_0, version: 4
[DEBUG] wrote ApiVersions v4; broker: seed_0, bytes_written: 61, write_wait: 66.334µs, time_to_write: 64.303µs, err: <nil>
[DEBUG] read ApiVersions v4; broker: seed_0, bytes_read: 0, read_wait: 49.028µs, time_to_read: 55.838997ms, err: read tcp 192.168.0.36:35724->172.202.80.16:9093: read: connection reset by peer
[DEBUG] issuing api versions request; broker: seed_0, version: 4
[DEBUG] wrote ApiVersions v4; broker: seed_0, bytes_written: 0, write_wait: 28.262µs, time_to_write: 118.755µs, err: write tcp 192.168.0.36:35724->172.202.80.16:9093: write: broken pipe
[DEBUG] connection initialization failed; addr: XXXX.servicebus.windows.net:9093, broker: seed_0, err: write tcp 192.168.0.36:35724->172.202.80.16:9093: write: broken pipe
[DEBUG] opening connection to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] connection opened to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] issuing api versions request; broker: seed_0, version: 4
[DEBUG] wrote ApiVersions v4; broker: seed_0, bytes_written: 61, write_wait: 22.226µs, time_to_write: 52.14µs, err: <nil>
[DEBUG] read ApiVersions v4; broker: seed_0, bytes_read: 0, read_wait: 48.51µs, time_to_read: 61.016297ms, err: read tcp 192.168.0.36:35732->172.202.80.16:9093: read: connection reset by peer
[DEBUG] issuing api versions request; broker: seed_0, version: 4
[DEBUG] wrote ApiVersions v4; broker: seed_0, bytes_written: 0, write_wait: 36.951µs, time_to_write: 82.008µs, err: write tcp 192.168.0.36:35732->172.202.80.16:9093: write: broken pipe
[DEBUG] connection initialization failed; addr: XXXX.servicebus.windows.net:9093, broker: seed_0, err: write tcp 192.168.0.36:35732->172.202.80.16:9093: write: broken pipe
[DEBUG] opening connection to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
panic: problem connecting to broker: write tcp 192.168.0.36:35732->172.202.80.16:9093: write: broken pipe
author_association: NONE
user: login: twmb
created_at: 2025-05-17T00:10:38Z
body: Ah duh I need to re-dial.
Re-dialing is a more invasive change.
author_association: OWNER
user: login: twmb
created_at: 2025-05-17T00:22:46Z
body: Ok, pushed an update that retries from the point of actually opening a connection. @stampy88 no promises on this one but if you're able to test, lmk.
author_association: OWNER
user: login: stampy88
created_at: 2025-05-17T14:09:07Z
body: @twmb no dice. I am debugging it now
```
[DEBUG] opening connection to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[INFO] immediate metadata update triggered; why: querying metadata for consumer initialization
[DEBUG] connection opened to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] issuing api versions request; broker: seed_0, version: 4
[DEBUG] wrote ApiVersions v4; broker: seed_0, bytes_written: 61, write_wait: 34.66µs, time_to_write: 54.997µs, err: <nil>
[DEBUG] read ApiVersions v4; broker: seed_0, bytes_read: 0, read_wait: 39.376µs, time_to_read: 57.790489ms, err: read tcp 192.168.0.36:38996->172.202.80.16:9093: read: connection reset by peer
[DEBUG] opening connection to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] connection opened to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] issuing api versions request; broker: seed_0, version: 4
[DEBUG] wrote ApiVersions v4; broker: seed_0, bytes_written: 61, write_wait: 26.701µs, time_to_write: 84.776µs, err: <nil>
[DEBUG] read ApiVersions v4; broker: seed_0, bytes_read: 0, read_wait: 69.701µs, time_to_read: 66.237244ms, err: read tcp 192.168.0.36:39002->172.202.80.16:9093: read: connection reset by peer
[DEBUG] connection initialization failed; addr: XXXX.servicebus.windows.net:9093, broker: seed_0, err: read tcp 192.168.0.36:39002->172.202.80.16:9093: read: connection reset by peer
[DEBUG] opening connection to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] connection opened to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] issuing api versions request; broker: seed_0, version: 4
[DEBUG] wrote ApiVersions v4; broker: seed_0, bytes_written: 61, write_wait: 23.956µs, time_to_write: 56.119µs, err: <nil>
[DEBUG] read ApiVersions v4; broker: seed_0, bytes_read: 0, read_wait: 95.266µs, time_to_read: 58.980771ms, err: read tcp 192.168.0.36:39018->172.202.80.16:9093: read: connection reset by peer
[DEBUG] opening connection to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] connection opened to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
[DEBUG] issuing api versions request; broker: seed_0, version: 4
[DEBUG] wrote ApiVersions v4; broker: seed_0, bytes_written: 61, write_wait: 11.096µs, time_to_write: 23.759µs, err: <nil>
[DEBUG] read ApiVersions v4; broker: seed_0, bytes_read: 0, read_wait: 17.329µs, time_to_read: 56.786844ms, err: read tcp 192.168.0.36:39028->172.202.80.16:9093: read: connection reset by peer
[DEBUG] connection initialization failed; addr: XXXX.servicebus.windows.net:9093, broker: seed_0, err: read tcp 192.168.0.36:39028->172.202.80.16:9093: read: connection reset by peer
[DEBUG] opening connection to broker; addr: XXXX.servicebus.windows.net:9093, broker: seed_0
panic: problem connecting to broker: read tcp 192.168.0.36:39028->172.202.80.16:9093: read: connection reset by peer
author_association: NONE
user: login: stampy88
created_at: 2025-05-20T14:40:51Z
body: @twmb tested it out and it looks good!
author_association: NONE