Repository: tailscale/tailscale (Go)

Issue:
Hairpinning detection fails about 50% of the time
Version: 1.7.356

I have previously reported a bug that incorrectly claims my Apple Airport Extreme router doesn't support NAT hairpinning, even though I've almost completely sure I have seen it doing that correctly in the past.

Anyway, I just noticed today that my node showed up as 'HairPinning: true' in the admin panel. I re-ran `tailscale netcheck` and it said false the first time, and true the second time. Indeed, the results are very flakey:

```
$ for a in $(seq 100); do tailscale netcheck 2>/dev/null | grep HairPinning; done | sort | uniq -c
     57 	* HairPinning: false
     43 	* HairPinning: true
```

(Ratios are similar I sleep between netcheck calls)

Although it's possible that my router really does fail to do hairpinning ~57% of the time, I think it's more likely that our detection is wrong.


Discussion:
I noticed this as well. If I run netcheck with --verbose, I can see that there's _always_ a received STUN packet, and yet sometimes it's recognized:

```
# tailscale netcheck --verbose |& grep -Ev ':3478|-'
2023/03/19 14:07:47 netcheck: sent haircheck to $my_public_ipv4:56029
2023/03/19 14:07:47 netcheck: received STUN packet from $hairpin_ipv4:57757
2023/03/19 14:07:48 netcheck: hairCheck done  
2023/03/19 14:07:48 netcheck: portMap done
2023/03/19 14:07:48 netcheck: [v2] checkCaptivePortal url="http://derp15d.tailscale.com/generate_204" status_code=204 va
lid_response=true
2023/03/19 14:07:48 netcheck: [v1] report: udp=true v6=true mapvarydest=false hair=true portmap= v4a=$my_public_ipv4:56029 ...
```

and sometimes it is not:

```
# tailscale netcheck --verbose |& grep -Ev ':3478|-'
2023/03/19 14:07:53 netcheck: sent haircheck to $my_public_ipv4:39023
2023/03/19 14:07:53 netcheck: received STUN packet from $hairpin_ipv4:50753
2023/03/19 14:07:53 netcheck: hairCheck timeout
2023/03/19 14:07:53 netcheck: hairCheck done
2023/03/19 14:07:53 netcheck: portMap done
2023/03/19 14:07:53 netcheck: [v2] checkCaptivePortal url="http://derp1h.tailscale.com/generate_204" status_code=204 valid_response=true
2023/03/19 14:07:53 netcheck: [v1] report: udp=true v6=true mapvarydest=false hair=false portmap= v4a=$my_public_ipv4:39023 ...
```

ahh, this is because `hairpinCheckTimeout` triggers before `waitHairCheck` even starts so both `rs.gotHairSTUN` and `rs.hairTimeout` have events, and it's 50/50 which event is handled first.

eg: 
```
15:26:25.912333 netcheck: sent haircheck to $my_public_ipv4:34681
15:26:25.913305 netcheck: received STUN packet from $hairpin_ipv4:36505
15:26:25.913354 netcheck: rs.gotHairSTUN <- src
15:26:26.013346 netcheck: hairpinCheckTimeout triggered: close(rs.hairTimeout)
15:26:26.352838 netcheck: waitHairCheck
15:26:26.352872 netcheck: <-rs.gotHairSTUN
15:26:26.352897 netcheck: hairCheck done
```

vs

```
15:26:23.338085 netcheck: sent haircheck to $my_public_ipv4:40544
15:26:23.339257 netcheck: received STUN packet from $hairpin_ipv4:59298
15:26:23.339583 netcheck: rs.gotHairSTUN <- src
15:26:23.438379 netcheck: hairpinCheckTimeout triggered: close(rs.hairTimeout)
15:26:23.653563 netcheck: waitHairCheck
15:26:23.653594 netcheck: <-rs.hairTimeout
15:26:23.653620 netcheck: hairCheck done
```

maybe the same channel could be used for both events?  That way they would be queued in the correct order.



Patch:
diff --git a/net/netcheck/netcheck.go b/net/netcheck/netcheck.go
index 1897fa372d856..d53eb7ee28000 100644
--- a/net/netcheck/netcheck.go
+++ b/net/netcheck/netcheck.go
@@ -638,20 +638,23 @@ func (rs *reportState) waitHairCheck(ctx context.Context) {
 		return
 	}
 
+	// First, check whether we have a value before we check for timeouts.
+	select {
+	case <-rs.gotHairSTUN:
+		ret.HairPinning.Set(true)
+		return
+	default:
+	}
+
+	// Now, wait for a response or a timeout.
 	select {
 	case <-rs.gotHairSTUN:
 		ret.HairPinning.Set(true)
 	case <-rs.hairTimeout:
 		rs.c.vlogf("hairCheck timeout")
 		ret.HairPinning.Set(false)
-	default:
-		select {
-		case <-rs.gotHairSTUN:
-			ret.HairPinning.Set(true)
-		case <-rs.hairTimeout:
-			ret.HairPinning.Set(false)
-		case <-ctx.Done():
-		}
+	case <-ctx.Done():
+		rs.c.vlogf("hairCheck context timeout")
 	}
 }
 


Test patch:
diff --git a/net/netcheck/netcheck_test.go b/net/netcheck/netcheck_test.go
index 797889926ec39..4d4bc4a2f10b1 100644
--- a/net/netcheck/netcheck_test.go
+++ b/net/netcheck/netcheck_test.go
@@ -47,6 +47,113 @@ func TestHairpinSTUN(t *testing.T) {
 	}
 }
 
+func TestHairpinWait(t *testing.T) {
+	makeClient := func(t *testing.T) (*Client, *reportState) {
+		tx := stun.NewTxID()
+		c := &Client{}
+		req := stun.Request(tx)
+		if !stun.Is(req) {
+			t.Fatal("expected STUN message")
+		}
+
+		var err error
+		rs := &reportState{
+			c:           c,
+			hairTX:      tx,
+			gotHairSTUN: make(chan netip.AddrPort, 1),
+			hairTimeout: make(chan struct{}),
+			report:      newReport(),
+		}
+		rs.pc4Hair, err = net.ListenUDP("udp4", &net.UDPAddr{
+			IP:   net.ParseIP("127.0.0.1"),
+			Port: 0,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		c.curState = rs
+		return c, rs
+	}
+
+	ll, err := net.ListenPacket("udp", "localhost:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ll.Close()
+	dstAddr := netip.MustParseAddrPort(ll.LocalAddr().String())
+
+	t.Run("Success", func(t *testing.T) {
+		c, rs := makeClient(t)
+		req := stun.Request(rs.hairTX)
+
+		// Start a hairpin check to ourselves.
+		rs.startHairCheckLocked(dstAddr)
+
+		// Fake receiving the stun check from ourselves after some period of time.
+		src := netip.MustParseAddrPort(rs.pc4Hair.LocalAddr().String())
+		c.handleHairSTUNLocked(req, src)
+
+		rs.waitHairCheck(context.Background())
+
+		// Verify that we set HairPinning
+		if got := rs.report.HairPinning; !got.EqualBool(true) {
+			t.Errorf("wanted HairPinning=true, got %v", got)
+		}
+	})
+
+	t.Run("LateReply", func(t *testing.T) {
+		c, rs := makeClient(t)
+		req := stun.Request(rs.hairTX)
+
+		// Start a hairpin check to ourselves.
+		rs.startHairCheckLocked(dstAddr)
+
+		// Wait until we've timed out, to mimic the race in #1795.
+		<-rs.hairTimeout
+
+		// Fake receiving the stun check from ourselves after some period of time.
+		src := netip.MustParseAddrPort(rs.pc4Hair.LocalAddr().String())
+		c.handleHairSTUNLocked(req, src)
+
+		// Wait for a hairpin response
+		rs.waitHairCheck(context.Background())
+
+		// Verify that we set HairPinning
+		if got := rs.report.HairPinning; !got.EqualBool(true) {
+			t.Errorf("wanted HairPinning=true, got %v", got)
+		}
+	})
+
+	t.Run("Timeout", func(t *testing.T) {
+		_, rs := makeClient(t)
+
+		// Start a hairpin check to ourselves.
+		rs.startHairCheckLocked(dstAddr)
+
+		ctx, cancel := context.WithTimeout(context.Background(), hairpinCheckTimeout*50)
+		defer cancel()
+
+		// Wait in the background
+		waitDone := make(chan struct{})
+		go func() {
+			rs.waitHairCheck(ctx)
+			close(waitDone)
+		}()
+
+		// If we do nothing, then we time out; confirm that we set
+		// HairPinning to false in this case.
+		select {
+		case <-waitDone:
+			if got := rs.report.HairPinning; !got.EqualBool(false) {
+				t.Errorf("wanted HairPinning=false, got %v", got)
+			}
+		case <-ctx.Done():
+			t.Fatalf("timed out waiting for hairpin channel")
+		}
+	})
+}
+
 func TestBasic(t *testing.T) {
 	stunAddr, cleanup := stuntest.Serve(t)
 	defer cleanup()