Repository: etcd-io/etcd
File: tests/integration/v3_lease_test.go (Go)

Code before:
}

func setupLeaseForwardingCluster(t *testing.T) (pb.LeaseClient, *integration.Member) {
	t.Helper()
	cluster := integration.NewCluster(t, &integration.ClusterConfig{Size: 3})
	t.Cleanup(func() { cluster.Terminate(t) })

	leaderIdx := cluster.WaitLeader(t)
	followerIdx := (leaderIdx + 1) % 3
	return integration.ToGRPC(cluster.Client(leaderIdx)).Lease, cluster.Members[followerIdx]
}

func getLeaseKeepAliveMetric(t *testing.T, member *integration.Member, grpcCode string) int64 {
	t.Helper()
	metricVal, err := member.Metric(
		"grpc_server_handled_total",
		`grpc_method="LeaseKeepAlive"`,
		fmt.Sprintf(`grpc_code="%v"`, grpcCode),
	)
	require.NoError(t, err)
	count, err := strconv.ParseInt(metricVal, 10, 32)
	require.NoError(t, err)
	return count
}

func enableLeaseRenewFailpoint(t *testing.T) {
	t.Helper()
	failpointName := "beforeServeHTTPLeaseRenew"
	require.NoError(t, gofail.Enable(failpointName, `sleep("8s")`))
	t.Cleanup(func() {
		terr := gofail.Disable(failpointName)
		if terr != nil && !errors.Is(terr, gofail.ErrDisabled) {
			t.Fatalf("Failed to disable failpoint %v, got error: %v", failpointName, terr)
		}
	})
}

// TestV3LeaseCheckpoint ensures a lease checkpoint results in a remaining TTL being persisted
// across leader elections.
func TestV3LeaseCheckpoint(t *testing.T) {
	tcs := []struct {
		name                  string
		checkpointingEnabled  bool
		ttl                   time.Duration
		checkpointingInterval time.Duration
		leaderChanges         int
		clusterSize           int
		expectTTLIsGT         time.Duration
		expectTTLIsLT         time.Duration
	}{
		{

Review comment (suggestion):
```suggestion
func sleepBeforeServingLeaseRenew(t *testing.T, duration time.Duration) {
```

Pass the duration so at the test level it's clear for how long server is stuck.

Code after:

		grantResp, err := leaderClient.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30})
		require.NoError(t, err)
		leaseID := grantResp.ID

		keepAliveClient, err := integration.ToGRPC(follower.Client).Lease.LeaseKeepAlive(t.Context())
		require.NoError(t, err)
		defer keepAliveClient.CloseSend()

		require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID}))
		_, err = keepAliveClient.Recv()
		require.NoError(t, err)

		sleepBeforeServingLeaseRenew(t, sleepDuration)

		prevUnavailableCount := getLeaseKeepAliveMetric(t, follower, "Unavailable")
		require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID}))

		_, err = keepAliveClient.Recv()
		require.Equal(t, rpctypes.ErrGRPCTimeout, err)

		require.Eventually(t, func() bool {
			return getLeaseKeepAliveMetric(t, follower, "Unavailable") == prevUnavailableCount+1
		}, 3*time.Second, 100*time.Millisecond)
	})
}

func setupLeaseForwardingCluster(t *testing.T) (pb.LeaseClient, *integration.Member) {
	t.Helper()
	cluster := integration.NewCluster(t, &integration.ClusterConfig{Size: 3})
	t.Cleanup(func() { cluster.Terminate(t) })

	leaderIdx := cluster.WaitLeader(t)
	followerIdx := (leaderIdx + 1) % 3
	return integration.ToGRPC(cluster.Client(leaderIdx)).Lease, cluster.Members[followerIdx]
}

func getLeaseKeepAliveMetric(t *testing.T, member *integration.Member, grpcCode string) int64 {
	t.Helper()
	metricVal, err := member.Metric(
		"grpc_server_handled_total",
		`grpc_method="LeaseKeepAlive"`,
		fmt.Sprintf(`grpc_code="%v"`, grpcCode),
	)
	require.NoError(t, err)
	count, err := strconv.ParseInt(metricVal, 10, 32)
	require.NoError(t, err)
	return count
}

func sleepBeforeServingLeaseRenew(t *testing.T, duration time.Duration) {