Repository: etcd-io/etcd
File: tests/e2e/reproduce_19406_test.go (Go)

Code before:
	// Compaction runs in batches. During each batch, it acquires a lock, releases it at the end,
	// and then waits for a compactionSleepInterval before starting the next batch. This pause
	// allows PUT requests to be processed.
	// Therefore, the total compaction sleep interval larger or equal to
	// (compaction iteration number - 1) * compactionSleepInterval
	httpEndpoint := clus.EndpointsHTTP()[0]
	totalKeys := produceKeyNum + 1
	pauseDuration, totalDuration, err := GetEtcdCompactionMetrics(t, httpEndpoint)
	require.NoError(t, err)
	actualSleepInterval := time.Duration(totalDuration-pauseDuration) * time.Millisecond
	expectSleepInterval := compactionSleepInterval * time.Duration(totalKeys)
	t.Logf("db_compaction_pause_duration: %.2f db_compaction_total_duration: %.2f, totalKeys: %d",
		pauseDuration, totalDuration, totalKeys)
	require.GreaterOrEqualf(t, actualSleepInterval, expectSleepInterval,
		"expect total compact sleep interval larger than (%v) but got (%v)",
		expectSleepInterval, actualSleepInterval)
}

func GetEtcdCompactionMetrics(t *testing.T, httpEndpoint string) (pauseDuration, totalDuration float64, err error) {
	// make an http request to fetch all Prometheus metrics
	url := httpEndpoint + "/metrics"
	resp, err := http.Get(url)
	require.NoErrorf(t, err, "failed to open url %s", url)
	b, err := io.ReadAll(resp.Body)
	resp.Body.Close()
	require.NoErrorf(t, err, "failed to read %s", url)

	for _, l := range strings.Split(string(b), "\n") {
		if strings.Contains(l, `etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_sum`) {
			parts := strings.Fields(l)
			if len(parts) == 2 {
				pauseDuration, err = strconv.ParseFloat(parts[1], 64)
				if err != nil {
					return 0, 0, fmt.Errorf("invalid pause sum value: %w", err)
				}
			}
		}
		if strings.Contains(l, `etcd_debugging_mvcc_db_compaction_total_duration_milliseconds_sum`) {
			parts := strings.Fields(l)
			if len(parts) == 2 {
				totalDuration, err = strconv.ParseFloat(parts[1], 64)
				if err != nil {
					return 0, 0, fmt.Errorf("invalid pause sum value: %w", err)
				}
			}
		}
	}
	return pauseDuration, totalDuration, nil
}


Review comment (suggestion):
Please consider to move `getMetrics` to e2e/framework, and reuse it.

https://github.com/etcd-io/etcd/blob/ddeaba726c2ba98e083bc382b1fc87a9e5e8669d/tests/e2e/metrics_test.go#L344-L359

Code after:
	// Therefore, the total compaction sleep interval larger or equal to
	// (compaction iteration number - 1) * compactionSleepInterval
	httpEndpoint := clus.EndpointsHTTP()[0]
	totalKeys := produceKeyNum + 1
	pauseDuration, totalDuration := getEtcdCompactionMetrics(t, httpEndpoint)
	require.NoError(t, err)
	actualSleepInterval := time.Duration(totalDuration-pauseDuration) * time.Millisecond
	expectSleepInterval := compactionSleepInterval * time.Duration(totalKeys)
	t.Logf("db_compaction_pause_duration: %.2f db_compaction_total_duration: %.2f, totalKeys: %d",
		pauseDuration, totalDuration, totalKeys)
	require.GreaterOrEqualf(t, actualSleepInterval, expectSleepInterval,
		"expect total compact sleep interval larger than (%v) but got (%v)",
		expectSleepInterval, actualSleepInterval)
}

func getEtcdCompactionMetrics(t *testing.T, httpEndpoint string) (pauseDuration, totalDuration float64) {
	metricsURL, err := url.JoinPath(httpEndpoint, "metrics")
	require.NoError(t, err)

	// Fetch metrics from the endpoint
	metricFamilies, err := e2e.GetMetrics(metricsURL)
	require.NoError(t, err)

	// Extract sum from histogram metric
	getHistogramSum := func(name string) float64 {
		mf, ok := metricFamilies[name]
		require.Truef(t, ok, "metric %q not found", name)
		require.NotEmptyf(t, mf.Metric, "metric %q has no data", name)

		hist := mf.Metric[0].GetHistogram()
		require.NotEmptyf(t, hist, "metric %q is not a histogram", name)

		return hist.GetSampleSum()
	}

	pauseDuration = getHistogramSum("etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds")
	totalDuration = getHistogramSum("etcd_debugging_mvcc_db_compaction_total_duration_milliseconds")

	return pauseDuration, totalDuration
}