Repository: vitessio/vitess (Go)

Issue:
Bug Report: VReplication Does Not Surface COM_BINLOG_DUMP_GTID Errors
### Overview of the Issue

VReplication ends up making `COM_BINLOG_DUMP_GTID` mysql protocol requests to a vttablet's mysqld in order to stream binary log events from a source. Currently if it gets an error it logs messages about that connection between the tablet's binlog component and the mysqld being lost/closed and returns a generic EOF error to the caller/client, for example:
```
vttablet: rpc error: code = Unknown desc = stream (at source tablet) error @ c7594820-936c-11ed-b025-12af3911a716:1-71: unexpected server EOF
```
In most cases the issue *is an ephemeral error* and reconnecting resolves it. In other cases, the problem persists and all the client sees is that no progress is being made and things seem stuck with what appears to be connection related issues. 

The most common error that gets us to this stuck state is this (from the mysqld error log):
```
[Warning] [MY-011809] [Server] Cannot replicate to server with server_uuid='...' because the present server has purged required binary logs. The connecting server needs to replicate the missing transactions from elsewhere, or be replaced by a new server created from a more recent backup. To prevent this error in the future, consider increasing the binary log expiration period on the present server. The missing transactions are '...'.
```

Currently the only way to detect the situation is by looking at the relevant mysqld's error log and seeing the above message repeatedly there.

We should try and surface this error/failure in VReplication if at all possible so that it's clear that there's an unrecoverable error that the user needs to address.

### How To Repeat
```
git checkout main && make build && pushd examples/local

./101_initial_cluster.sh; ./201_customer_tablets.sh; ./202_move_tables.sh

vtctlclient Workflow -- customer.commerce2customer stop

mysql < ../common/insert_commerce_data.sql

for id in 100 101 102; do
  mysql -u root -h localhost --socket=${VTDATAROOT}/vt_0000000${id}/mysql.sock --binary-as-hex=false -e "flush binary logs; select sleep(10); purge binary logs before now()"
done

vtctlclient Workflow -- customer.commerce2customer start
sleep 10
vtctlclient Workflow -- customer.commerce2customer show

./401_teardown.sh
popd
```

The final relevant output from the `show` is:
```
"State": "Error",
"Message": "vttablet: rpc error: code = Unknown desc = stream (at source tablet) error @ c7594820-936c-11ed-b025-12af3911a716:1-71: unexpected server EOF",
```



Fix (patch):
diff --git a/go/vt/binlog/binlog_connection.go b/go/vt/binlog/binlog_connection.go
index 49c25e23d58..1cdb2d6cacc 100644
--- a/go/vt/binlog/binlog_connection.go
+++ b/go/vt/binlog/binlog_connection.go
@@ -99,16 +99,16 @@ func connectForReplication(cp dbconfigs.Connector) (*mysql.Conn, error) {
 
 // StartBinlogDumpFromCurrent requests a replication binlog dump from
 // the current position.
-func (bc *BinlogConnection) StartBinlogDumpFromCurrent(ctx context.Context) (mysql.Position, <-chan mysql.BinlogEvent, error) {
+func (bc *BinlogConnection) StartBinlogDumpFromCurrent(ctx context.Context) (mysql.Position, <-chan mysql.BinlogEvent, <-chan error, error) {
 	ctx, bc.cancel = context.WithCancel(ctx)
 
 	position, err := bc.Conn.PrimaryPosition()
 	if err != nil {
-		return mysql.Position{}, nil, fmt.Errorf("failed to get primary position: %v", err)
+		return mysql.Position{}, nil, nil, fmt.Errorf("failed to get primary position: %v", err)
 	}
 
-	c, err := bc.StartBinlogDumpFromPosition(ctx, "", position)
-	return position, c, err
+	c, e, err := bc.StartBinlogDumpFromPosition(ctx, "", position)
+	return position, c, e, err
 }
 
 // StartBinlogDumpFromPosition requests a replication binlog dump from
@@ -120,33 +120,42 @@ func (bc *BinlogConnection) StartBinlogDumpFromCurrent(ctx context.Context) (mys
 // by canceling the context.
 //
 // Note the context is valid and used until eventChan is closed.
-func (bc *BinlogConnection) StartBinlogDumpFromPosition(ctx context.Context, binlogFilename string, startPos mysql.Position) (<-chan mysql.BinlogEvent, error) {
+func (bc *BinlogConnection) StartBinlogDumpFromPosition(ctx context.Context, binlogFilename string, startPos mysql.Position) (<-chan mysql.BinlogEvent, <-chan error, error) {
 	ctx, bc.cancel = context.WithCancel(ctx)
 
 	log.Infof("sending binlog dump command: startPos=%v, serverID=%v", startPos, bc.serverID)
 	if err := bc.SendBinlogDumpCommand(bc.serverID, binlogFilename, startPos); err != nil {
 		log.Errorf("couldn't send binlog dump command: %v", err)
-		return nil, err
+		return nil, nil, err
 	}
 
-	return bc.streamEvents(ctx), nil
+	c, e := bc.streamEvents(ctx)
+
+	return c, e, nil
 }
 
-// streamEvents returns a channel on which events are streamed.
-func (bc *BinlogConnection) streamEvents(ctx context.Context) chan mysql.BinlogEvent {
+// streamEvents returns a channel on which events are streamed and a channel on
+// which errors are propagated.
+func (bc *BinlogConnection) streamEvents(ctx context.Context) (chan mysql.BinlogEvent, chan error) {
 	// FIXME(alainjobart) I think we can use a buffered channel for better performance.
 	eventChan := make(chan mysql.BinlogEvent)
+	errChan := make(chan error)
 
 	// Start reading events.
 	bc.wg.Add(1)
 	go func() {
 		defer func() {
 			close(eventChan)
+			close(errChan)
 			bc.wg.Done()
 		}()
 		for {
 			event, err := bc.Conn.ReadBinlogEvent()
 			if err != nil {
+				select {
+				case errChan <- err:
+				case <-ctx.Done():
+				}
 				if sqlErr, ok := err.(*mysql.SQLError); ok && sqlErr.Number() == mysql.CRServerLost {
 					// CRServerLost = Lost connection to MySQL server during query
 					// This is not necessarily an error. It could just be that we closed
@@ -165,7 +174,7 @@ func (bc *BinlogConnection) streamEvents(ctx context.Context) chan mysql.BinlogE
 			}
 		}
 	}()
-	return eventChan
+	return eventChan, errChan
 }
 
 // StartBinlogDumpFromBinlogBeforeTimestamp requests a replication
@@ -196,21 +205,22 @@ func (bc *BinlogConnection) streamEvents(ctx context.Context) chan mysql.BinlogE
 // by canceling the context.
 //
 // Note the context is valid and used until eventChan is closed.
-func (bc *BinlogConnection) StartBinlogDumpFromBinlogBeforeTimestamp(ctx context.Context, timestamp int64) (<-chan mysql.BinlogEvent, error) {
+func (bc *BinlogConnection) StartBinlogDumpFromBinlogBeforeTimestamp(ctx context.Context, timestamp int64) (<-chan mysql.BinlogEvent, <-chan error, error) {
 	ctx, bc.cancel = context.WithCancel(ctx)
 
 	filename, err := bc.findFileBeforeTimestamp(ctx, timestamp)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// Start dumping the logs. The position is '4' to skip the
 	// Binlog File Header. See this page for more info:
 	// https://dev.mysql.com/doc/internals/en/binlog-file.html
 	if err := bc.Conn.WriteComBinlogDump(bc.serverID, filename, 4, 0); err != nil {
-		return nil, fmt.Errorf("failed to send the ComBinlogDump command: %v", err)
+		return nil, nil, fmt.Errorf("failed to send the ComBinlogDump command: %v", err)
 	}
-	return bc.streamEvents(ctx), nil
+	e, c := bc.streamEvents(ctx)
+	return e, c, nil
 }
 
 func (bc *BinlogConnection) findFileBeforeTimestamp(ctx context.Context, timestamp int64) (filename string, err error) {
diff --git a/go/vt/binlog/binlog_streamer.go b/go/vt/binlog/binlog_streamer.go
index 1101c214eb6..6bf2c26bf9c 100644
--- a/go/vt/binlog/binlog_streamer.go
+++ b/go/vt/binlog/binlog_streamer.go
@@ -207,6 +207,7 @@ func (bls *Streamer) Stream(ctx context.Context) (err error) {
 	}
 
 	var events <-chan mysql.BinlogEvent
+	var errs <-chan error
 	if bls.timestamp != 0 {
 		// MySQL 5.6 only: We are going to start reading the
 		// logs from the beginning of a binlog file. That is
@@ -214,7 +215,7 @@ func (bls *Streamer) Stream(ctx context.Context) (err error) {
 		// contains the starting GTIDSet, and we will save
 		// that as the current position.
 		bls.usePreviousGTIDs = true
-		events, err = bls.conn.StartBinlogDumpFromBinlogBeforeTimestamp(ctx, bls.timestamp)
+		events, errs, err = bls.conn.StartBinlogDumpFromBinlogBeforeTimestamp(ctx, bls.timestamp)
 	} else if !bls.startPos.IsZero() {
 		// MySQL 5.6 only: we are starting from a random
 		// binlog position. It turns out we will receive a
@@ -223,16 +224,17 @@ func (bls *Streamer) Stream(ctx context.Context) (err error) {
 		// the starting position we pass in, it seems it is
 		// just the PREVIOUS_GTIDS_EVENT from the file we're reading.
 		// So we have to skip it.
-		events, err = bls.conn.StartBinlogDumpFromPosition(ctx, "", bls.startPos)
+		events, errs, err = bls.conn.StartBinlogDumpFromPosition(ctx, "", bls.startPos)
 	} else {
-		bls.startPos, events, err = bls.conn.StartBinlogDumpFromCurrent(ctx)
+		bls.startPos, events, errs, err = bls.conn.StartBinlogDumpFromCurrent(ctx)
 	}
 	if err != nil {
 		return err
 	}
+
 	// parseEvents will loop until the events channel is closed, the
 	// service enters the SHUTTING_DOWN state, or an error occurs.
-	stopPos, err = bls.parseEvents(ctx, events)
+	stopPos, err = bls.parseEvents(ctx, events, errs)
 	return err
 }
 
@@ -243,7 +245,7 @@ func (bls *Streamer) Stream(ctx context.Context) (err error) {
 // If the sendTransaction func returns io.EOF, parseEvents returns ErrClientEOF.
 // If the events channel is closed, parseEvents returns ErrServerEOF.
 // If the context is done, returns ctx.Err().
-func (bls *Streamer) parseEvents(ctx context.Context, events <-chan mysql.BinlogEvent) (mysql.Position, error) {
+func (bls *Streamer) parseEvents(ctx context.Context, events <-chan mysql.BinlogEvent, errs <-chan error) (mysql.Position, error) {
 	var statements []FullBinlogStatement
 	var format mysql.BinlogFormat
 	var gtid mysql.GTID
@@ -297,6 +299,8 @@ func (bls *Streamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog
 				log.Infof("reached end of binlog event stream")
 				return pos, ErrServerEOF
 			}
+		case err = <-errs:
+			return pos, err
 		case <-ctx.Done():
 			log.Infof("stopping early due to binlog Streamer service shutdown or client disconnect")
 			return pos, ctx.Err()
diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go
index d34c933d5b2..7cc98794312 100644
--- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go
+++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go
@@ -194,16 +194,16 @@ func (vs *vstreamer) replicate(ctx context.Context) error {
 	}
 	defer conn.Close()
 
-	events, err := conn.StartBinlogDumpFromPosition(vs.ctx, "", vs.pos)
+	events, errs, err := conn.StartBinlogDumpFromPosition(vs.ctx, "", vs.pos)
 	if err != nil {
 		return wrapError(err, vs.pos, vs.vse)
 	}
-	err = vs.parseEvents(vs.ctx, events)
+	err = vs.parseEvents(vs.ctx, events, errs)
 	return wrapError(err, vs.pos, vs.vse)
 }
 
 // parseEvents parses and sends events.
-func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.BinlogEvent) error {
+func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.BinlogEvent, errs <-chan error) error {
 	// bufferAndTransmit uses bufferedEvents and curSize to buffer events.
 	var (
 		bufferedEvents []*binlogdatapb.VEvent
@@ -388,6 +388,8 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog
 				// Increment this counter for testing.
 				vschemaUpdateCount.Add(1)
 			}
+		case err := <-errs:
+			return err
 		case <-ctx.Done():
 			return nil
 		case <-hbTimer.C: