Repository: grpc/grpc-go
File: xds/internal/xdsclient/metrics_test.go (Go)

Code before:
		WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
		MetricsRecorder:    tmr,
	})
	if err != nil {
		t.Fatalf("Failed to create an xDS client: %v", err)
	}
	defer close()

	const listenerResourceName = "test-listener-resource"

	// Watch for the listener on the above management server.
	xdsresource.WatchListener(client, listenerResourceName, noopListenerWatcher{})
	// Verify that an ADS stream is opened and an LDS request with the above
	// resource name is sent.
	select {
	case <-streamOpened:
	case <-ctx.Done():
		t.Fatal("Timeout when waiting for ADS stream to open")
	}

	// Close the listener and ensure that the ADS stream breaks. This should
	// cause a server failure count to emit eventually.
	lis.Stop()
	select {
	case <-ctx.Done():
		t.Fatal("Timeout when waiting for ADS stream to close")
	default:
	}
	// Restart to prevent the attempt to create a new ADS stream after back off.
	lis.Restart()

	mdWant := stats.MetricsData{
		Handle:    xdsClientServerFailureMetric.Descriptor(),
		IntIncr:   1,
		LabelKeys: []string{"grpc.target", "grpc.xds.server"},
		LabelVals: []string{"Test/ServerFailureMetrics_BeforeResponseRecv", mgmtServer.Address},
	}
	if err := tmr.WaitForInt64Count(ctx, mdWant); err != nil {
		t.Fatal(err.Error())
	}
}

// TestServerFailureMetrics_AfterResponseRecv configures an xDS client, and a
// management server to send a valid LDS updates, and verifies that the
// server failure metric is not emitted. It then closes the management server
// listener to close the ADS stream and verifies that the server failure metric
// is still not emitted because the the ADS stream was closed after having
// received a response on the stream.
func (s) TestServerFailureMetrics_AfterResponseRecv(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()

Review comment (suggestion):
This code is still racy because the backoff could have completed before the listener is restarted. There is an option that can be used to set a really long backoff duration instead.
https://github.com/grpc/grpc-go/blob/5edab9e55414068e74320716117a2659c5d2174e/xds/internal/xdsclient/pool.go#L60-L63

Code after:
		WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
		MetricsRecorder:    tmr,
	})
	if err != nil {
		t.Fatalf("Failed to create an xDS client: %v", err)
	}
	defer close()

	const listenerResourceName = "test-listener-resource"

	// Watch for the listener on the above management server.
	xdsresource.WatchListener(client, listenerResourceName, noopListenerWatcher{})
	// Verify that an ADS stream is opened and an LDS request with the above
	// resource name is sent.
	select {
	case <-streamOpened:
	case <-ctx.Done():
		t.Fatal("Timeout when waiting for ADS stream to open")
	}

	// Close the listener and ensure that the ADS stream breaks. This should
	// cause a server failure count to emit eventually.
	lis.Stop()

	// Restart to prevent the attempt to create a new ADS stream after back off.
	lis.Restart()

	mdWant := stats.MetricsData{
		Handle:    xdsClientServerFailureMetric.Descriptor(),
		IntIncr:   1,
		LabelKeys: []string{"grpc.target", "grpc.xds.server"},
		LabelVals: []string{"Test/ServerFailureMetrics_BeforeResponseRecv", mgmtServer.Address},
	}
	if err := tmr.WaitForInt64Count(ctx, mdWant); err != nil {
		t.Fatal(err.Error())
	}
}

// TestServerFailureMetrics_AfterResponseRecv configures an xDS client, and a
// management server to send a valid LDS updates, and verifies that the
// server failure metric is not emitted. It then closes the management server
// listener to close the ADS stream and verifies that the server failure metric
// is still not emitted because the the ADS stream was closed after having
// received a response on the stream.
func (s) TestServerFailureMetrics_AfterResponseRecv(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()

	tmr := stats.NewTestMetricsRecorder()
	l, err := testutils.LocalTCPListener()
	if err != nil {