Repository: libp2p/go-libp2p
Pull request #2573 (2023-09-15 07:54:16)

swarm: use happy eyeballs ranking for TCP dials
The required change is much simpler than my previous tries. Closing other open PRs in favour of this. 

sukunrt: @marten-seemann Addressed review comments. I'll take webtransport up in a later PR. 

Patch:
diff --git a/core/transport/transport.go b/core/transport/transport.go
index 89a9608d..d56a3cff 100644
--- a/core/transport/transport.go
+++ b/core/transport/transport.go
@@ -5,6 +5,7 @@ package transport
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net"
 
 	"github.com/libp2p/go-libp2p/core/network"
@@ -124,3 +125,47 @@ type Upgrader interface {
 	// Upgrade upgrades the multiaddr/net connection into a full libp2p-transport connection.
 	Upgrade(ctx context.Context, t Transport, maconn manet.Conn, dir network.Direction, p peer.ID, scope network.ConnManagementScope) (CapableConn, error)
 }
+
+// DialUpdater provides updates on in progress dials.
+type DialUpdater interface {
+	// DialWithUpdates dials a remote peer and provides updates on the passed channel.
+	DialWithUpdates(context.Context, ma.Multiaddr, peer.ID, chan<- DialUpdate) (CapableConn, error)
+}
+
+// DialUpdateKind indicates the type of DialUpdate event.
+type DialUpdateKind int
+
+const (
+	// UpdateKindDialFailed indicates dial failed.
+	UpdateKindDialFailed DialUpdateKind = iota
+	// UpdateKindDialSuccessful indicates dial succeeded.
+	UpdateKindDialSuccessful
+	// UpdateKindHandshakeProgressed indicates successful completion of the TCP 3-way
+	// handshake
+	UpdateKindHandshakeProgressed
+)
+
+func (k DialUpdateKind) String() string {
+	switch k {
+	case UpdateKindDialFailed:
+		return "DialFailed"
+	case UpdateKindDialSuccessful:
+		return "DialSuccessful"
+	case UpdateKindHandshakeProgressed:
+		return "UpdateKindHandshakeProgressed"
+	default:
+		return fmt.Sprintf("DialUpdateKind<Unknown-%d>", k)
+	}
+}
+
+// DialUpdate is used by DialUpdater to provide dial updates.
+type DialUpdate struct {
+	// Kind is the kind of update event.
+	Kind DialUpdateKind
+	// Addr is the peer's address.
+	Addr ma.Multiaddr
+	// Conn is the resulting connection on success.
+	Conn CapableConn
+	// Err is the reason for dial failure.
+	Err error
+}
diff --git a/p2p/net/swarm/dial_ranker.go b/p2p/net/swarm/dial_ranker.go
index 3725884e..7e58876b 100644
--- a/p2p/net/swarm/dial_ranker.go
+++ b/p2p/net/swarm/dial_ranker.go
@@ -58,8 +58,19 @@ func NoDelayDialRanker(addrs []ma.Multiaddr) []network.AddrDelay {
 //  3. If a QUIC or WebTransport address is present, TCP addresses dials are delayed relative to the last QUIC dial:
 //     We prefer to end up with a QUIC connection. For public addresses, the delay introduced is 250ms (PublicTCPDelay),
 //     and for private addresses 30ms (PrivateTCPDelay).
+//  4. For the TCP addresses we follow a strategy similar to QUIC with an optimisation for handling the long TCP
+//     handshake time described in 6. If both IPv6 TCP and IPv4 TCP addresses are present, we do a Happy Eyeballs
+//     style ranking. First dial the IPv6 TCP address with the lowest port. After this, dial the IPv4 TCP address
+//     with the lowest port delayed by 250ms (PublicTCPDelay) for public addresses, and 30ms (PrivateTCPDelay)
+//     for local addresses. After this we dial all the rest of the addresses delayed by 250ms (PublicTCPDelay) for
+//     public addresses, and 30ms (PrivateTCPDelay) for local addresses.
+//  5. If only one of TCP IPv6 or TCP IPv4 addresses are present, dial the TCP address with the lowest port
+//     first. After this we dial the rest of the TCP addresses delayed by 250ms (PublicTCPDelay) for public
+//     addresses, and 30ms (PrivateTCPDelay) for local addresses.
+//  6. When a TCP socket is connected and awaiting security and muxer upgrade, we stop new dials for 2*PrivateTCPDelay
+//     to allow for the upgrade to complete.
 //
-// We dial lowest ports first for QUIC addresses as they are more likely to be the listen port.
+// We dial lowest ports first as they are more likely to be the listen port.
 func DefaultDialRanker(addrs []ma.Multiaddr) []network.AddrDelay {
 	relay, addrs := filterAddrs(addrs, isRelayAddr)
 	pvt, addrs := filterAddrs(addrs, manet.IsPrivateAddr)
@@ -88,22 +99,57 @@ func DefaultDialRanker(addrs []ma.Multiaddr) []network.AddrDelay {
 // addresses relative to direct addresses.
 func getAddrDelay(addrs []ma.Multiaddr, tcpDelay time.Duration, quicDelay time.Duration,
 	offset time.Duration) []network.AddrDelay {
+	if len(addrs) == 0 {
+		return nil
+	}
 
 	sort.Slice(addrs, func(i, j int) bool { return score(addrs[i]) < score(addrs[j]) })
 
-	// If the first address is (QUIC, IPv6), make the second address (QUIC, IPv4).
-	happyEyeballs := false
-	if len(addrs) > 0 {
+	// addrs is now sorted by (Transport, IPVersion). Reorder addrs for happy eyeballs dialing.
+	// For QUIC and TCP, if we have both IPv6 and IPv4 addresses, move the
+	// highest priority IPv4 address to the second position.
+	happyEyeballsQUIC := false
+	happyEyeballsTCP := false
+	// tcpStartIdx is the index of the first TCP Address
+	var tcpStartIdx int
+	{
+		i := 0
+		// If the first QUIC address is IPv6 move the first QUIC IPv4 address to second position
 		if isQUICAddr(addrs[0]) && isProtocolAddr(addrs[0], ma.P_IP6) {
-			for i := 1; i < len(addrs); i++ {
-				if isQUICAddr(addrs[i]) && isProtocolAddr(addrs[i], ma.P_IP4) {
-					// make IPv4 address the second element
-					if i > 1 {
-						a := addrs[i]
-						copy(addrs[2:], addrs[1:i])
+			for j := 1; j < len(addrs); j++ {
+				if isQUICAddr(addrs[j]) && isProtocolAddr(addrs[j], ma.P_IP4) {
+					// The first IPv4 address is at position j
+					// Move the jth element at position 1 shifting the affected elements
+					if j > 1 {
+						a := addrs[j]
+						copy(addrs[2:], addrs[1:j])
 						addrs[1] = a
 					}
-					happyEyeballs = true
+					happyEyeballsQUIC = true
+					i = j + 1
+					break
+				}
+			}
+		}
+
+		for tcpStartIdx = i; tcpStartIdx < len(addrs); tcpStartIdx++ {
+			if isProtocolAddr(addrs[tcpStartIdx], ma.P_TCP) {
+				break
+			}
+		}
+
+		// If the first TCP address is IPv6 move the first TCP IPv4 address to second position
+		if tcpStartIdx < len(addrs) && isProtocolAddr(addrs[tcpStartIdx], ma.P_IP6) {
+			for j := tcpStartIdx + 1; j < len(addrs); j++ {
+				if isProtocolAddr(addrs[j], ma.P_TCP) && isProtocolAddr(addrs[j], ma.P_IP4) {
+					// First TCP IPv4 address is at position j, move it to position tcpStartIdx+1
+					// which is the second priority TCP address
+					if j > tcpStartIdx+1 {
+						a := addrs[j]
+						copy(addrs[tcpStartIdx+2:], addrs[tcpStartIdx+1:j])
+						addrs[tcpStartIdx+1] = a
+					}
+					happyEyeballsTCP = true
 					break
 				}
 			}
@@ -111,25 +157,42 @@ func getAddrDelay(addrs []ma.Multiaddr, tcpDelay time.Duration, quicDelay time.D
 	}
 
 	res := make([]network.AddrDelay, 0, len(addrs))
-
-	var totalTCPDelay time.Duration
+	var tcpFirstDialDelay time.Duration
 	for i, addr := range addrs {
 		var delay time.Duration
 		switch {
 		case isQUICAddr(addr):
-			// For QUIC addresses we dial an IPv6 address, then after quicDelay an IPv4
-			// address, then after quicDelay we dial rest of the addresses.
+			// We dial an IPv6 address, then after quicDelay an IPv4
+			// address, then after a further quicDelay we dial the rest of the addresses.
 			if i == 1 {
 				delay = quicDelay
 			}
-			if i > 1 && happyEyeballs {
-				delay = 2 * quicDelay
-			} else if i > 1 {
-				delay = quicDelay
+			if i > 1 {
+				// If we have happy eyeballs for QUIC, dials after the second position
+				// will be delayed by 2*quicDelay
+				if happyEyeballsQUIC {
+					delay = 2 * quicDelay
+				} else {
+					delay = quicDelay
+				}
 			}
-			totalTCPDelay = delay + tcpDelay
+			tcpFirstDialDelay = delay + tcpDelay
 		case isProtocolAddr(addr, ma.P_TCP):
-			delay = totalTCPDelay
+			// We dial an IPv6 address, then after tcpDelay an IPv4
+			// address, then after a further tcpDelay we dial the rest of the addresses.
+			if i == tcpStartIdx+1 {
+				delay = tcpDelay
+			}
+			if i > tcpStartIdx+1 {
+				// If we have happy eyeballs for TCP, dials after the second position
+				// will be delayed by 2*tcpDelay
+				if happyEyeballsTCP {
+					delay = 2 * tcpDelay
+				} else {
+					delay = tcpDelay
+				}
+			}
+			delay += tcpFirstDialDelay
 		}
 		res = append(res, network.AddrDelay{Addr: addr, Delay: offset + delay})
 	}
diff --git a/p2p/net/swarm/dial_worker.go b/p2p/net/swarm/dial_worker.go
index f51733b1..6d6dcf65 100644
--- a/p2p/net/swarm/dial_worker.go
+++ b/p2p/net/swarm/dial_worker.go
@@ -8,8 +8,10 @@ import (
 
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
+	tpt "github.com/libp2p/go-libp2p/core/transport"
 
 	ma "github.com/multiformats/go-multiaddr"
+	manet "github.com/multiformats/go-multiaddr/net"
 )
 
 // dialRequest is structure used to request dials to the peer associated with a
@@ -62,6 +64,8 @@ type addrDial struct {
 	createdAt time.Time
 	// dialRankingDelay is the delay in dialing this address introduced by the ranking logic
 	dialRankingDelay time.Duration
+	// expectedTCPUpgradeTime is the expected time by which security upgrade will complete
+	expectedTCPUpgradeTime time.Time
 }
 
 // dialWorker synchronises concurrent dials to a peer. It ensures that we make at most one dial to a
@@ -77,7 +81,7 @@ type dialWorker struct {
 	// we dial an address at most once
 	trackedDials map[string]*addrDial
 	// resch is used to receive response for dials to the peers addresses.
-	resch chan dialResult
+	resch chan tpt.DialUpdate
 
 	connected bool // true when a connection has been successfully established
 
@@ -96,7 +100,7 @@ func newDialWorker(s *Swarm, p peer.ID, reqch <-chan dialRequest, cl Clock) *dia
 		reqch:           reqch,
 		pendingRequests: make(map[*pendRequest]struct{}),
 		trackedDials:    make(map[string]*addrDial),
-		resch:           make(chan dialResult),
+		resch:           make(chan tpt.DialUpdate),
 		cl:              cl,
 	}
 }
@@ -123,12 +127,18 @@ func (w *dialWorker) loop() {
 			<-dialTimer.Ch()
 		}
 		timerRunning = false
-		if dq.len() > 0 {
+		if dq.Len() > 0 {
 			if dialsInFlight == 0 && !w.connected {
 				// if there are no dials in flight, trigger the next dials immediately
 				dialTimer.Reset(startTime)
 			} else {
-				dialTimer.Reset(startTime.Add(dq.top().Delay))
+				resetTime := startTime.Add(dq.top().Delay)
+				for _, ad := range w.trackedDials {
+					if !ad.expectedTCPUpgradeTime.IsZero() && ad.expectedTCPUpgradeTime.After(resetTime) {
+						resetTime = ad.expectedTCPUpgradeTime
+					}
+				}
+				dialTimer.Reset(resetTime)
 			}
 			timerRunning = true
 		}
@@ -303,16 +313,29 @@ loop:
 			// Update all requests waiting on this address. On success, complete the request.
 			// On error, record the error
 
-			dialsInFlight--
 			ad, ok := w.trackedDials[string(res.Addr.Bytes())]
 			if !ok {
 				log.Errorf("SWARM BUG: no entry for address %s in trackedDials", res.Addr)
 				if res.Conn != nil {
 					res.Conn.Close()
 				}
+				dialsInFlight--
 				continue
 			}
 
+			// TCP Connection has been established. Wait for connection upgrade on this address
+			// before making new dials.
+			if res.Kind == tpt.UpdateKindHandshakeProgressed {
+				// Only wait for public addresses to complete dialing since private dials
+				// are quick any way
+				if manet.IsPublicAddr(res.Addr) {
+					ad.expectedTCPUpgradeTime = w.cl.Now().Add(PublicTCPDelay)
+				}
+				scheduleNextDial()
+				continue
+			}
+			dialsInFlight--
+			ad.expectedTCPUpgradeTime = time.Time{}
 			if res.Conn != nil {
 				// we got a connection, add it to the swarm
 				conn, err := w.s.addConn(res.Conn, network.DirOutbound)
@@ -417,7 +440,7 @@ func newDialQueue() *dialQueue {
 // Add adds adelay to the queue. If another element exists in the queue with
 // the same address, it replaces that element.
 func (dq *dialQueue) Add(adelay network.AddrDelay) {
-	for i := 0; i < dq.len(); i++ {
+	for i := 0; i < dq.Len(); i++ {
 		if dq.q[i].Addr.Equal(adelay.Addr) {
 			if dq.q[i].Delay == adelay.Delay {
 				// existing element is the same. nothing to do
@@ -430,7 +453,7 @@ func (dq *dialQueue) Add(adelay network.AddrDelay) {
 		}
 	}
 
-	for i := 0; i < dq.len(); i++ {
+	for i := 0; i < dq.Len(); i++ {
 		if dq.q[i].Delay > adelay.Delay {
 			dq.q = append(dq.q, network.AddrDelay{}) // extend the slice
 			copy(dq.q[i+1:], dq.q[i:])
@@ -443,13 +466,13 @@ func (dq *dialQueue) Add(adelay network.AddrDelay) {
 
 // NextBatch returns all the elements in the queue with the highest priority
 func (dq *dialQueue) NextBatch() []network.AddrDelay {
-	if dq.len() == 0 {
+	if dq.Len() == 0 {
 		return nil
 	}
 
 	// i is the index of the second highest priority element
 	var i int
-	for i = 0; i < dq.len(); i++ {
+	for i = 0; i < dq.Len(); i++ {
 		if dq.q[i].Delay != dq.q[0].Delay {
 			break
 		}
@@ -464,7 +487,7 @@ func (dq *dialQueue) top() network.AddrDelay {
 	return dq.q[0]
 }
 
-// len returns the number of elements in the queue
-func (dq *dialQueue) len() int {
+// Len returns the number of elements in the queue
+func (dq *dialQueue) Len() int {
 	return len(dq.q)
 }
diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go
index ccfe7d23..f5638bbe 100644
--- a/p2p/net/swarm/limiter.go
+++ b/p2p/net/swarm/limiter.go
@@ -13,17 +13,11 @@ import (
 	ma "github.com/multiformats/go-multiaddr"
 )
 
-type dialResult struct {
-	Conn transport.CapableConn
-	Addr ma.Multiaddr
-	Err  error
-}
-
 type dialJob struct {
 	addr    ma.Multiaddr
 	peer    peer.ID
 	ctx     context.Context
-	resp    chan dialResult
+	resp    chan transport.DialUpdate
 	timeout time.Duration
 }
 
@@ -45,7 +39,7 @@ type dialLimiter struct {
 	waitingOnPeerLimit map[peer.ID][]*dialJob
 }
 
-type dialfunc func(context.Context, peer.ID, ma.Multiaddr) (transport.CapableConn, error)
+type dialfunc func(context.Context, peer.ID, ma.Multiaddr, chan<- transport.DialUpdate) (transport.CapableConn, error)
 
 func newDialLimiter(df dialfunc) *dialLimiter {
 	fd := ConcurrentFdDials
@@ -216,9 +210,13 @@ func (dl *dialLimiter) executeDial(j *dialJob) {
 	dctx, cancel := context.WithTimeout(j.ctx, j.timeout)
 	defer cancel()
 
-	con, err := dl.dialFunc(dctx, j.peer, j.addr)
+	con, err := dl.dialFunc(dctx, j.peer, j.addr, j.resp)
+	kind := transport.UpdateKindDialSuccessful
+	if err != nil {
+		kind = transport.UpdateKindDialFailed
+	}
 	select {
-	case j.resp <- dialResult{Conn: con, Addr: j.addr, Err: err}:
+	case j.resp <- transport.DialUpdate{Kind: kind, Conn: con, Addr: j.addr, Err: err}:
 	case <-j.ctx.Done():
 		if con != nil {
 			con.Close()
diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go
index 3bf80091..288ad9cc 100644
--- a/p2p/net/swarm/swarm_dial.go
+++ b/p2p/net/swarm/swarm_dial.go
@@ -403,7 +403,7 @@ func (s *Swarm) resolveAddrs(ctx context.Context, pi peer.AddrInfo) ([]ma.Multia
 	return resolved, nil
 }
 
-func (s *Swarm) dialNextAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr, resch chan dialResult) error {
+func (s *Swarm) dialNextAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr, resch chan transport.DialUpdate) error {
 	// check the dial backoff
 	if forceDirect, _ := network.GetForceDirectDial(ctx); !forceDirect {
 		if s.backf.Backoff(p, addr) {
@@ -505,7 +505,7 @@ func (s *Swarm) filterKnownUndialables(p peer.ID, addrs []ma.Multiaddr) (goodAdd
 // limitedDial will start a dial to the given peer when
 // it is able, respecting the various different types of rate
 // limiting that occur without using extra goroutines per addr
-func (s *Swarm) limitedDial(ctx context.Context, p peer.ID, a ma.Multiaddr, resp chan dialResult) {
+func (s *Swarm) limitedDial(ctx context.Context, p peer.ID, a ma.Multiaddr, resp chan transport.DialUpdate) {
 	timeout := s.dialTimeout
 	if manet.IsPrivateAddr(a) && s.dialTimeoutLocal < s.dialTimeout {
 		timeout = s.dialTimeoutLocal
@@ -520,7 +520,7 @@ func (s *Swarm) limitedDial(ctx context.Context, p peer.ID, a ma.Multiaddr, resp
 }
 
 // dialAddr is the actual dial for an addr, indirectly invoked through the limiter
-func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (transport.CapableConn, error) {
+func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr, updCh chan<- transport.DialUpdate) (transport.CapableConn, error) {
 	// Just to double check. Costs nothing.
 	if s.local == p {
 		return nil, ErrDialToSelf
@@ -538,7 +538,13 @@ func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (tra
 	}
 
 	start := time.Now()
-	connC, err := tpt.Dial(ctx, addr, p)
+	var connC transport.CapableConn
+	var err error
+	if du, ok := tpt.(transport.DialUpdater); ok {
+		connC, err = du.DialWithUpdates(ctx, addr, p, updCh)
+	} else {
+		connC, err = tpt.Dial(ctx, addr, p)
+	}
 
 	// We're recording any error as a failure here.
 	// Notably, this also applies to cancelations (i.e. if another dial attempt was faster).
diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go
index f277b3f8..d52bb960 100644
--- a/p2p/transport/tcp/tcp.go
+++ b/p2p/transport/tcp/tcp.go
@@ -131,6 +131,7 @@ type TcpTransport struct {
 }
 
 var _ transport.Transport = &TcpTransport{}
+var _ transport.DialUpdater = &TcpTransport{}
 
 // NewTCPTransport creates a tcp transport object that tracks dialers and listeners
 // created. It represents an entire TCP stack (though it might not necessarily be).
@@ -176,13 +177,17 @@ func (t *TcpTransport) maDial(ctx context.Context, raddr ma.Multiaddr) (manet.Co
 
 // Dial dials the peer at the remote address.
 func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (transport.CapableConn, error) {
+	return t.DialWithUpdates(ctx, raddr, p, nil)
+}
+
+func (t *TcpTransport) DialWithUpdates(ctx context.Context, raddr ma.Multiaddr, p peer.ID, updateChan chan<- transport.DialUpdate) (transport.CapableConn, error) {
 	connScope, err := t.rcmgr.OpenConnection(network.DirOutbound, true, raddr)
 	if err != nil {
 		log.Debugw("resource manager blocked outgoing connection", "peer", p, "addr", raddr, "error", err)
 		return nil, err
 	}
 
-	c, err := t.dialWithScope(ctx, raddr, p, connScope)
+	c, err := t.dialWithScope(ctx, raddr, p, connScope, updateChan)
 	if err != nil {
 		connScope.Done()
 		return nil, err
@@ -190,7 +195,7 @@ func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID)
 	return c, nil
 }
 
-func (t *TcpTransport) dialWithScope(ctx context.Context, raddr ma.Multiaddr, p peer.ID, connScope network.ConnManagementScope) (transport.CapableConn, error) {
+func (t *TcpTransport) dialWithScope(ctx context.Context, raddr ma.Multiaddr, p peer.ID, connScope network.ConnManagementScope, updateChan chan<- transport.DialUpdate) (transport.CapableConn, error) {
 	if err := connScope.SetPeer(p); err != nil {
 		log.Debugw("resource manager blocked outgoing connection for peer", "peer", p, "addr", raddr, "error", err)
 		return nil, err
@@ -212,6 +217,13 @@ func (t *TcpTransport) dialWithScope(ctx context.Context, raddr ma.Multiaddr, p
 			return nil, err
 		}
 	}
+	if updateChan != nil {
+		select {
+		case updateChan <- transport.DialUpdate{Kind: transport.UpdateKindHandshakeProgressed, Addr: raddr}:
+		default:
+			// It is better to skip the update than to delay upgrading the connection
+		}
+	}
 	direction := network.DirOutbound
 	if ok, isClient, _ := network.GetSimultaneousConnect(ctx); ok && !isClient {
 		direction = network.DirInbound


Test patch:
diff --git a/p2p/net/swarm/dial_ranker_test.go b/p2p/net/swarm/dial_ranker_test.go
index ec3d10a0..5ef3cc27 100644
--- a/p2p/net/swarm/dial_ranker_test.go
+++ b/p2p/net/swarm/dial_ranker_test.go
@@ -161,6 +161,7 @@ func TestDelayRankerTCPDelay(t *testing.T) {
 	t1 := ma.StringCast("/ip4/1.2.3.5/tcp/1/")
 	t1v6 := ma.StringCast("/ip6/1::2/tcp/1")
 	t2 := ma.StringCast("/ip4/1.2.3.4/tcp/2")
+	t3 := ma.StringCast("/ip4/1.2.3.4/tcp/3")
 
 	testCase := []struct {
 		name   string
@@ -169,36 +170,63 @@ func TestDelayRankerTCPDelay(t *testing.T) {
 	}{
 		{
 			name:  "quic-with-tcp-ip6-ip4",
-			addrs: []ma.Multiaddr{q1v1, q1v16, q2v16, q3v16, q2v1, t1, t2},
+			addrs: []ma.Multiaddr{q1v1, q1v16, q2v16, q3v16, q2v1, t1, t1v6, t2, t3},
 			output: []network.AddrDelay{
 				{Addr: q1v16, Delay: 0},
 				{Addr: q1v1, Delay: PublicQUICDelay},
 				{Addr: q2v16, Delay: 2 * PublicQUICDelay},
 				{Addr: q3v16, Delay: 2 * PublicQUICDelay},
 				{Addr: q2v1, Delay: 2 * PublicQUICDelay},
-				{Addr: t1, Delay: 3 * PublicQUICDelay},
-				{Addr: t2, Delay: 3 * PublicQUICDelay},
+				{Addr: t1v6, Delay: 3 * PublicQUICDelay},
+				{Addr: t1, Delay: 4 * PublicQUICDelay},
+				{Addr: t2, Delay: 5 * PublicQUICDelay},
+				{Addr: t3, Delay: 5 * PublicQUICDelay},
 			},
 		},
 		{
 			name:  "quic-ip4-with-tcp",
-			addrs: []ma.Multiaddr{q1v1, t1, t2, t1v6},
+			addrs: []ma.Multiaddr{q1v1, t2, t1v6, t1},
+			output: []network.AddrDelay{
+				{Addr: q1v1, Delay: 0},
+				{Addr: t1v6, Delay: PublicQUICDelay},
+				{Addr: t1, Delay: 2 * PublicQUICDelay},
+				{Addr: t2, Delay: 3 * PublicQUICDelay},
+			},
+		},
+		{
+			name:  "quic-ip4-with-tcp-ipv4",
+			addrs: []ma.Multiaddr{q1v1, t2, t3, t1},
 			output: []network.AddrDelay{
 				{Addr: q1v1, Delay: 0},
 				{Addr: t1, Delay: PublicTCPDelay},
-				{Addr: t2, Delay: PublicTCPDelay},
+				{Addr: t2, Delay: 2 * PublicQUICDelay},
+				{Addr: t3, Delay: 2 * PublicTCPDelay},
+			},
+		},
+		{
+			name:  "quic-ip4-with-two-tcp",
+			addrs: []ma.Multiaddr{q1v1, t1v6, t2},
+			output: []network.AddrDelay{
+				{Addr: q1v1, Delay: 0},
 				{Addr: t1v6, Delay: PublicTCPDelay},
+				{Addr: t2, Delay: 2 * PublicTCPDelay},
 			},
 		},
 		{
 			name:  "tcp-ip4-ip6",
-			addrs: []ma.Multiaddr{t1, t2, t1v6},
+			addrs: []ma.Multiaddr{t1, t2, t1v6, t3},
 			output: []network.AddrDelay{
 				{Addr: t1v6, Delay: 0},
-				{Addr: t1, Delay: 0},
-				{Addr: t2, Delay: 0},
+				{Addr: t1, Delay: PublicTCPDelay},
+				{Addr: t2, Delay: 2 * PublicTCPDelay},
+				{Addr: t3, Delay: 2 * PublicTCPDelay},
 			},
 		},
+		{
+			name:   "empty",
+			addrs:  []ma.Multiaddr{},
+			output: []network.AddrDelay{},
+		},
 	}
 	for _, tc := range testCase {
 		t.Run(tc.name, func(t *testing.T) {
diff --git a/p2p/net/swarm/dial_worker_test.go b/p2p/net/swarm/dial_worker_test.go
index 6679bd95..e70bc3ce 100644
--- a/p2p/net/swarm/dial_worker_test.go
+++ b/p2p/net/swarm/dial_worker_test.go
@@ -484,8 +484,8 @@ func TestDialQueueNextBatch(t *testing.T) {
 					}
 				}
 			}
-			if q.len() != 0 {
-				t.Errorf("expected queue to be empty at end. got: %d", q.len())
+			if q.Len() != 0 {
+				t.Errorf("expected queue to be empty at end. got: %d", q.Len())
 			}
 		})
 	}
@@ -1081,3 +1081,61 @@ func TestDialWorkerLoopAddrDedup(t *testing.T) {
 		t.Errorf("expected a fail response")
 	}
 }
+
+func TestDialWorkerLoopTCPConnUpgradeWait(t *testing.T) {
+	s1 := makeSwarmWithNoListenAddrs(t, WithDialTimeout(10*time.Second))
+	s2 := makeSwarmWithNoListenAddrs(t, WithDialTimeout(10*time.Second))
+	defer s1.Close()
+	defer s2.Close()
+	// Connection to a1 will fail but a1 is a public address so we can test waiting for tcp
+	// connection established dial update. ipv4only.arpa reserved address.
+	a1 := ma.StringCast(fmt.Sprintf("/ip4/192.0.0.170/tcp/%d", 10001))
+	// Connection to a2 will succeed.
+	a2 := ma.StringCast(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", 10002))
+	s2.Listen(a2)
+
+	s1.Peerstore().AddAddrs(s2.LocalPeer(), []ma.Multiaddr{a1, a2}, peerstore.PermanentAddrTTL)
+
+	rankerCalled := make(chan struct{})
+	s1.dialRanker = func(addrs []ma.Multiaddr) []network.AddrDelay {
+		defer close(rankerCalled)
+		return []network.AddrDelay{{Addr: a1, Delay: 0}, {Addr: a2, Delay: 100 * time.Millisecond}}
+	}
+
+	reqch := make(chan dialRequest)
+	resch := make(chan dialResponse, 2)
+	cl := newMockClock()
+	worker := newDialWorker(s1, s2.LocalPeer(), reqch, cl)
+	go worker.loop()
+	defer worker.wg.Wait()
+	defer close(reqch)
+
+	reqch <- dialRequest{ctx: context.Background(), resch: resch}
+
+	<-rankerCalled
+	// Wait a bit to let the loop make the dial attempt to a1
+	time.Sleep(1 * time.Second)
+	// Send conn established for a1
+	worker.resch <- transport.DialUpdate{Kind: transport.UpdateKindHandshakeProgressed, Addr: a1}
+	// Dial to a2 shouldn't happen even if a2 is scheduled to dial by now
+	cl.AdvanceBy(200 * time.Millisecond)
+	select {
+	case r := <-resch:
+		t.Fatalf("didn't expect any event on resch %s %s", r.err, r.conn)
+	case <-time.After(500 * time.Millisecond):
+	}
+
+	// Dial to a2 should happen now
+	// This number is high because there's a race between this goroutine advancing the clock
+	// and the worker loop goroutine processing the TCPConnectionEstablished event.
+	// In case it processes the event after the previous clock advancement we need to wait
+	// 2 * PublicTCPDelay.
+	cl.AdvanceBy(2 * PublicTCPDelay)
+	select {
+	case r := <-resch:
+		require.NoError(t, r.err)
+		require.NotNil(t, r.conn)
+	case <-time.After(3 * time.Second):
+		t.Errorf("expected a fail response")
+	}
+}
diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go
index 046b4c75..8e61958c 100644
--- a/p2p/net/swarm/limiter_test.go
+++ b/p2p/net/swarm/limiter_test.go
@@ -39,7 +39,7 @@ func tcpPortOver(a ma.Multiaddr, n int) bool {
 	return pnum > n
 }
 
-func tryDialAddrs(ctx context.Context, l *dialLimiter, p peer.ID, addrs []ma.Multiaddr, res chan dialResult) {
+func tryDialAddrs(ctx context.Context, l *dialLimiter, p peer.ID, addrs []ma.Multiaddr, res chan transport.DialUpdate) {
 	for _, a := range addrs {
 		l.AddDialJob(&dialJob{
 			ctx:  ctx,
@@ -51,7 +51,7 @@ func tryDialAddrs(ctx context.Context, l *dialLimiter, p peer.ID, addrs []ma.Mul
 }
 
 func hangDialFunc(hang chan struct{}) dialfunc {
-	return func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.CapableConn, error) {
+	return func(ctx context.Context, p peer.ID, a ma.Multiaddr, _ chan<- transport.DialUpdate) (transport.CapableConn, error) {
 		if mafmt.UTP.Matches(a) {
 			return transport.CapableConn(nil), nil
 		}
@@ -79,7 +79,7 @@ func TestLimiterBasicDials(t *testing.T) {
 	bads := []ma.Multiaddr{addrWithPort(1), addrWithPort(2), addrWithPort(3), addrWithPort(4)}
 	good := addrWithPort(20)
 
-	resch := make(chan dialResult)
+	resch := make(chan transport.DialUpdate)
 	pid := peer.ID("testpeer")
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -130,7 +130,7 @@ func TestFDLimiting(t *testing.T) {
 	goodTCP := addrWithPort(20)
 
 	ctx := context.Background()
-	resch := make(chan dialResult)
+	resch := make(chan transport.DialUpdate)
 
 	// take all fd limit tokens with hang dials
 	for _, pid := range pids {
@@ -188,7 +188,7 @@ func TestFDLimiting(t *testing.T) {
 func TestTokenRedistribution(t *testing.T) {
 	var lk sync.Mutex
 	hangchs := make(map[peer.ID]chan struct{})
-	df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.CapableConn, error) {
+	df := func(ctx context.Context, p peer.ID, a ma.Multiaddr, _ chan<- transport.DialUpdate) (transport.CapableConn, error) {
 		if tcpPortOver(a, 10) {
 			return (transport.CapableConn)(nil), nil
 		}
@@ -205,7 +205,7 @@ func TestTokenRedistribution(t *testing.T) {
 	pids := []peer.ID{"testpeer1", "testpeer2"}
 
 	ctx := context.Background()
-	resch := make(chan dialResult)
+	resch := make(chan transport.DialUpdate)
 
 	// take all fd limit tokens with hang dials
 	for _, pid := range pids {
@@ -281,7 +281,7 @@ func TestTokenRedistribution(t *testing.T) {
 }
 
 func TestStressLimiter(t *testing.T) {
-	df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.CapableConn, error) {
+	df := func(ctx context.Context, p peer.ID, a ma.Multiaddr, _ chan<- transport.DialUpdate) (transport.CapableConn, error) {
 		if tcpPortOver(a, 1000) {
 			return transport.CapableConn(nil), nil
 		}
@@ -305,7 +305,7 @@ func TestStressLimiter(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 
-			resp := make(chan dialResult)
+			resp := make(chan transport.DialUpdate)
 			time.Sleep(time.Duration(rand.Intn(10)) * time.Millisecond)
 			for _, i := range rand.Perm(len(addresses)) {
 				l.AddDialJob(&dialJob{
@@ -335,7 +335,7 @@ func TestStressLimiter(t *testing.T) {
 }
 
 func TestFDLimitUnderflow(t *testing.T) {
-	df := func(ctx context.Context, p peer.ID, addr ma.Multiaddr) (transport.CapableConn, error) {
+	df := func(ctx context.Context, p peer.ID, addr ma.Multiaddr, _ chan<- transport.DialUpdate) (transport.CapableConn, error) {
 		select {
 		case <-ctx.Done():
 		case <-time.After(5 * time.Second):
@@ -361,7 +361,7 @@ func TestFDLimitUnderflow(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 
-			resp := make(chan dialResult)
+			resp := make(chan transport.DialUpdate)
 			l.AddDialJob(&dialJob{
 				addr: addrs[i],
 				ctx:  ctx,
diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go
index 26385faa..d96c3431 100644
--- a/p2p/transport/tcp/tcp_test.go
+++ b/p2p/transport/tcp/tcp_test.go
@@ -17,6 +17,7 @@ import (
 	ttransport "github.com/libp2p/go-libp2p/p2p/transport/testsuite"
 
 	ma "github.com/multiformats/go-multiaddr"
+	manet "github.com/multiformats/go-multiaddr/net"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
 )
@@ -147,6 +148,54 @@ func TestTcpTransportCantListenUtp(t *testing.T) {
 	envReuseportVal = true
 }
 
+func TestDialWithUpdates(t *testing.T) {
+	peerA, ia := makeInsecureMuxer(t)
+	_, ib := makeInsecureMuxer(t)
+
+	ua, err := tptu.New(ia, muxers, nil, nil, nil)
+	require.NoError(t, err)
+	ta, err := NewTCPTransport(ua, nil)
+	require.NoError(t, err)
+	ln, err := ta.Listen(ma.StringCast("/ip4/127.0.0.1/tcp/0"))
+	require.NoError(t, err)
+	defer ln.Close()
+
+	ub, err := tptu.New(ib, muxers, nil, nil, nil)
+	require.NoError(t, err)
+	tb, err := NewTCPTransport(ub, nil)
+	require.NoError(t, err)
+
+	updCh := make(chan transport.DialUpdate, 1)
+	conn, err := tb.DialWithUpdates(context.Background(), ln.Multiaddr(), peerA, updCh)
+	upd := <-updCh
+	require.Equal(t, upd.Kind, transport.UpdateKindHandshakeProgressed)
+	require.NotNil(t, conn)
+	require.NoError(t, err)
+
+	acceptAndClose := func() manet.Listener {
+		li, err := manet.Listen(ma.StringCast("/ip4/127.0.0.1/tcp/0"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		go func() {
+			conn, err := li.Accept()
+			if err != nil {
+				return
+			}
+			conn.Close()
+		}()
+		return li
+	}
+	li := acceptAndClose()
+	defer li.Close()
+	// This dial will fail as acceptAndClose will not upgrade the connection
+	conn, err = tb.DialWithUpdates(context.Background(), li.Multiaddr(), peerA, updCh)
+	upd = <-updCh
+	require.Equal(t, upd.Kind, transport.UpdateKindHandshakeProgressed)
+	require.Nil(t, conn)
+	require.Error(t, err)
+}
+
 func makeInsecureMuxer(t *testing.T) (peer.ID, []sec.SecureTransport) {
 	t.Helper()
 	priv, _, err := crypto.GenerateKeyPair(crypto.Ed25519, 256)