Repository: etcd-io/etcd
File: tests/integration/v3_lease_test.go (Go)

Code before:
		t.Skip("please run 'make gofail-enable' before running the test")
	}
	integration.BeforeTest(t)

	// Shows current behavior: client cancel during forwarding incorrectly returns Unavailable.
	t.Run("client cancels while forwarding", func(t *testing.T) {
		leaderClient, follower := setupLeaseForwardingCluster(t)

		grantResp, err := leaderClient.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30})
		require.NoError(t, err)
		leaseID := grantResp.ID

		ctx, cancel := context.WithCancel(t.Context())
		keepAliveClient, err := integration.ToGRPC(follower.Client).Lease.LeaseKeepAlive(ctx)
		require.NoError(t, err)
		defer keepAliveClient.CloseSend()

		require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID}))
		_, err = keepAliveClient.Recv()
		require.NoError(t, err)

		enableLeaseRenewFailpoint(t)

		// Use server metrics to verify behavior since client.Recv() always returns Canceled
		// after cancel() regardless of the actual server response.
		prevCanceledCount := getLeaseKeepAliveMetric(t, follower, "Canceled")
		prevUnavailableCount := getLeaseKeepAliveMetric(t, follower, "Unavailable")

		require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID}))
		time.Sleep(50 * time.Millisecond)
		cancel()

		require.Eventually(t, func() bool {
			return getLeaseKeepAliveMetric(t, follower, "Unavailable") == prevUnavailableCount+1
		}, 3*time.Second, 100*time.Millisecond)
		require.Equal(t, prevCanceledCount, getLeaseKeepAliveMetric(t, follower, "Canceled"))
	})

	t.Run("forwarding times out", func(t *testing.T) {
		leaderClient, follower := setupLeaseForwardingCluster(t)

		grantResp, err := leaderClient.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30})
		require.NoError(t, err)
		leaseID := grantResp.ID

		keepAliveClient, err := integration.ToGRPC(follower.Client).Lease.LeaseKeepAlive(t.Context())
		require.NoError(t, err)
		defer keepAliveClient.CloseSend()

		require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID}))
		_, err = keepAliveClient.Recv()

Review comment (bug):
For contrasting I recommend adding a third scenario that describes a situation without failpoint and shows what is the baseline behavior you expect. So you cover both cases A implies B and validate that ~A doesn't imply B. 

Code after:
		t.Skip("please run 'make gofail-enable' before running the test")
	}
	integration.BeforeTest(t)
	// Longer than leaseHandler.ServeHTTP()'s default timeout duration
	sleepDuration := 8 * time.Second

	t.Run("forwarding succeeds", func(t *testing.T) {
		leaderClient, follower := setupLeaseForwardingCluster(t)

		grantResp, err := leaderClient.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30})
		require.NoError(t, err)
		leaseID := grantResp.ID

		keepAliveClient, err := integration.ToGRPC(follower.Client).Lease.LeaseKeepAlive(t.Context())
		require.NoError(t, err)
		defer keepAliveClient.CloseSend()

		require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID}))
		resp, err := keepAliveClient.Recv()
		require.NoError(t, err)
		require.Equal(t, leaseID, resp.ID)
		require.Positive(t, resp.TTL)
	})

	// Shows current behavior: client cancel during forwarding incorrectly returns Unavailable.
	t.Run("client cancels while forwarding", func(t *testing.T) {
		leaderClient, follower := setupLeaseForwardingCluster(t)

		grantResp, err := leaderClient.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30})
		require.NoError(t, err)
		leaseID := grantResp.ID

		ctx, cancel := context.WithCancel(t.Context())
		keepAliveClient, err := integration.ToGRPC(follower.Client).Lease.LeaseKeepAlive(ctx)
		require.NoError(t, err)
		defer keepAliveClient.CloseSend()

		require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID}))
		_, err = keepAliveClient.Recv()
		require.NoError(t, err)

		sleepBeforeServingLeaseRenew(t, sleepDuration)

		// Use server metrics to verify behavior since client.Recv() always returns Canceled
		// after cancel() regardless of the actual server response.
		prevCanceledCount := getLeaseKeepAliveMetric(t, follower, "Canceled")
		prevUnavailableCount := getLeaseKeepAliveMetric(t, follower, "Unavailable")

		require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID}))
		time.Sleep(50 * time.Millisecond)
		cancel()