Repository: etcd-io/etcd
File: tests/integration/clientv3/lease/lease_test.go (Go)

Code before:
	}

	var prevUnavailableCount, prevCanceledCount int64
	if !integration.ThroughProxy {
		prevUnavailableCount = getLeaseKeepAliveMetric(t, clus.Members[0], "Unavailable")
		prevCanceledCount = getLeaseKeepAliveMetric(t, clus.Members[0], "Canceled")
	}
	clus.Members[1].Stop(t)
	// kaReqLeader may issue multiple requests while waiting for the first
	// response from proxy server; drain any stray keepalive responses
	time.Sleep(100 * time.Millisecond)
	for {
		<-kaReqLeader
		if len(kaReqLeader) == 0 {
			break
		}
	}

	select {
	case resp, ok := <-kaReqLeader:
		require.Falsef(t, ok, "expected closed require leader, got response %+v", resp)
	case <-time.After(5 * time.Second):
		t.Fatal("keepalive with require leader took too long to close")
	}

	if !integration.ThroughProxy {
		require.Eventuallyf(t, func() bool {
			return getLeaseKeepAliveMetric(t, clus.Members[0], "Unavailable") > prevUnavailableCount
		}, 3*time.Second, 100*time.Millisecond,
			"expected Unavailable metric to increase after leader loss, prev count: %d", prevUnavailableCount)
		currentCanceledCount := getLeaseKeepAliveMetric(t, clus.Members[0], "Canceled")
		require.Equalf(t, prevCanceledCount, currentCanceledCount,
			"Canceled metric should not change, expected %d, got %d", prevCanceledCount, currentCanceledCount)
	}

	select {
	case _, ok := <-kaWait:
		require.Truef(t, ok, "got closed channel with no require leader, expected non-closed")
	case <-time.After(10 * time.Millisecond):
		// wait some to detect any closes happening soon after kaReqLeader closing
	}
}

func getLeaseKeepAliveMetric(t *testing.T, member *integration.Member, grpcCode string) int64 {
	t.Helper()
	metricVal, err := member.Metric(
		"grpc_server_handled_total",
		`grpc_method="LeaseKeepAlive"`,
		fmt.Sprintf(`grpc_code="%v"`, grpcCode),
	)
	require.NoError(t, err)

Review comment (suggestion):
Don't think we had any cases of integration tests collecting metrics to test things in etcd. 

grpc proxy doesn't and will never proxy server metrics, so there is no reason to fetch metrics from proxy.

Could we just have Metric function go directly the member instead of proxy? If someone wants proxy metrics they should implement separate ProxyMetrics function.

Code after:
	}

	prevUnavailableCount := getLeaseKeepAliveMetric(t, clus.Members[0], "Unavailable")
	prevCanceledCount := getLeaseKeepAliveMetric(t, clus.Members[0], "Canceled")

	clus.Members[1].Stop(t)
	// kaReqLeader may issue multiple requests while waiting for the first
	// response from proxy server; drain any stray keepalive responses
	time.Sleep(100 * time.Millisecond)
	for {
		<-kaReqLeader
		if len(kaReqLeader) == 0 {
			break
		}
	}

	select {
	case resp, ok := <-kaReqLeader:
		require.Falsef(t, ok, "expected closed require leader, got response %+v", resp)
	case <-time.After(5 * time.Second):
		t.Fatal("keepalive with require leader took too long to close")
	}

	require.Eventuallyf(t, func() bool {
		return getLeaseKeepAliveMetric(t, clus.Members[0], "Unavailable") > prevUnavailableCount
	}, 3*time.Second, 100*time.Millisecond,
		"expected Unavailable metric to increase after leader loss, prev count: %d", prevUnavailableCount)
	// Ensure the error is Unavailable, not Canceled
	currentCanceledCount := getLeaseKeepAliveMetric(t, clus.Members[0], "Canceled")
	require.Equalf(t, prevCanceledCount, currentCanceledCount,
		"Canceled metric should not change, expected %d, got %d", prevCanceledCount, currentCanceledCount)

	select {
	case _, ok := <-kaWait:
		require.Truef(t, ok, "got closed channel with no require leader, expected non-closed")
	case <-time.After(10 * time.Millisecond):
		// wait some to detect any closes happening soon after kaReqLeader closing
	}
}

func getLeaseKeepAliveMetric(t *testing.T, member *integration.Member, grpcCode string) int64 {
	t.Helper()
	metricVal, err := member.Metric(
		"grpc_server_handled_total",
		`grpc_method="LeaseKeepAlive"`,
		fmt.Sprintf(`grpc_code="%v"`, grpcCode),
	)
	require.NoError(t, err)
	count, err := strconv.ParseInt(metricVal, 10, 64)
	require.NoError(t, err)
	return count