Repository: grpc/grpc-go
File: xds/internal/xdsclient/tests/fallback_test.go (Go)

Code before:
	defer cc.Close()
	cc.Connect()
	client := testgrpc.NewTestServiceClient(cc)

	// Secondary2 fallback, RPCs reach backend3.
	secondary2Lis.Restart()
	if err := waitForRPCsToReachBackend(ctx, client, backend3.Address); err != nil {
		t.Fatal(err)
	}

	// Secondary1 becomes available, RPCs go to backend2.
	secondary1Lis.Restart()
	secondary1ManagementServer.Update(ctx, e2e.UpdateOptions{
		NodeID:    nodeID,
		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, "route-s1")},
		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig("route-s1", serviceName, "cluster-s1")},
		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster("cluster-s1", "endpoints-s1", e2e.SecurityLevelNone)},
		Endpoints: []*v3endpointpb.ClusterLoadAssignment{
			e2e.DefaultEndpoint("endpoints-s1", "localhost", []uint32{testutils.ParsePort(t, backend2.Address)}),
		},
	})
	// Wait for switch from secondary2 to secondary1.
	if conn, err := secondary2WrappedLis.NewConnCh.Receive(ctx); err == nil {
		cw := conn.(*testutils.ConnWrapper)
		if _, err := cw.CloseCh.Receive(ctx); err != nil {
			t.Fatalf("Failed to get connection to secondary2: %v", err)
		}
	}
	if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
		t.Fatal(err)
	}

	// Primary becomes available, RPCs go to backend1.
	primaryLis.Restart()
	primaryManagementServer.Update(ctx, e2e.DefaultClientResources(e2e.ResourceParams{
		DialTarget: serviceName,
		NodeID:     nodeID,
		Host:       "localhost",
		Port:       testutils.ParsePort(t, backend1.Address),
		SecLevel:   e2e.SecurityLevelNone,
	}))
	if conn, err := secondary1WrappedLis.NewConnCh.Receive(ctx); err == nil {
		cw := conn.(*testutils.ConnWrapper)
		cw.CloseCh.Receive(ctx)
	}
	if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
		t.Fatal(err)
	}
}


Review comment (bug):
The error message is wrong. Here we are checking if connection to secondary2 is closed or not because we not have secondary1 running. Change the error message.

Code after:
	client := testgrpc.NewTestServiceClient(cc)

	// Verify that connection attempts were made to primaryWrappedLis and
	// secondaryWrappedLis, before using tertiaryWrappedLis to make
	// successful RPCs to backend3.
	if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
		t.Fatalf("Timeout when waiting for connection to primary: %v", err)
	}

	// Stop primary, client should connect to secondary.
	primaryLis.Stop()
	if _, err := secondaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
		t.Fatalf("Timeout when waiting for connection to secondary after primary stopped: %v", err)
	}

	// Stop secondary, client should connect to tertiary.
	secondaryLis.Stop()
	tertiaryConn, err := tertiaryWrappedLis.NewConnCh.Receive(ctx)
	if err != nil {
		t.Fatalf("Timeout when waiting for connection to tertiary after secondary stopped: %v", err)
	}

	// Tertiary has all resources, RPCs should succeed to backend3.
	if err := waitForRPCsToReachBackend(ctx, client, backend3.Address); err != nil {
		t.Fatal(err)
	}

	// Secondary1 becomes available, RPCs go to backend2.
	secondaryLis.Restart()
	secondaryManagementServer.Update(ctx, e2e.UpdateOptions{
		NodeID:    nodeID,
		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, "route-s1")},
		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig("route-s1", serviceName, "cluster-s1")},
		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster("cluster-s1", "endpoints-s1", e2e.SecurityLevelNone)},
		Endpoints: []*v3endpointpb.ClusterLoadAssignment{
			e2e.DefaultEndpoint("endpoints-s1", "localhost", []uint32{testutils.ParsePort(t, backend2.Address)}),
		},
	})

	secondaryConn, err := secondaryWrappedLis.NewConnCh.Receive(ctx)
	if err != nil {
		t.Fatalf("Timeout when waiting for new connection to secondary: %v", err)
	}
	if _, err := tertiaryConn.(*testutils.ConnWrapper).CloseCh.Receive(ctx); err != nil {
		t.Fatalf("Timeout when waiting for connection to the tertiary to be closed after promotion to secondary: %v", err)
	}
	if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
		t.Fatal(err)
	}

	// Primary becomes available, RPCs go to backend1.