Repository: prometheus/prometheus
File: notifier/manager_test.go (Go)

Code before:
		})
	}

	checkNoErr := func() {
		t.Helper()
		select {
		case err := <-errc:
			require.NoError(t, err)
		default:
		}
	}

	// start send loops
	for _, ams := range h.alertmanagers {
		ams.startSendLoops(ams.ams)
	}
	defer func() {
		for _, ams := range h.alertmanagers {
			ams.cleanSendLoops(ams.ams...)
		}
	}()

	h.Send(alerts...)
	require.Eventually(t, func() bool {
		return prom_testutil.ToFloat64(h.metrics.sent.WithLabelValues(server1.URL)) == DefaultMaxBatchSize
	}, time.Second*10, time.Millisecond*10)
	checkNoErr()

	// the only am in set 1 is down
	status1.Store(int32(http.StatusNotFound))
	h.Send(alerts...)
	// wait for all send loops to process before changing any status
	require.Eventually(t, func() bool {
		return prom_testutil.ToFloat64(h.metrics.errors.WithLabelValues(server1.URL)) == DefaultMaxBatchSize &&
			prom_testutil.ToFloat64(h.metrics.sent.WithLabelValues(server2.URL)) == DefaultMaxBatchSize*2 &&
			prom_testutil.ToFloat64(h.metrics.sent.WithLabelValues(server3.URL)) == DefaultMaxBatchSize*2
	}, time.Second*10, time.Millisecond*10)
	checkNoErr()

	// fix the am
	status1.Store(int32(http.StatusOK))

	// only one of the ams in set 2 is down
	status2.Store(int32(http.StatusInternalServerError))
	h.Send(alerts...)
	// wait for all send loops to either send or fail with errors depending on their status
	require.Eventually(t, func() bool {
		return prom_testutil.ToFloat64(h.metrics.errors.WithLabelValues(server2.URL)) == DefaultMaxBatchSize &&
			prom_testutil.ToFloat64(h.metrics.sent.WithLabelValues(server1.URL)) == DefaultMaxBatchSize*2 &&
			prom_testutil.ToFloat64(h.metrics.sent.WithLabelValues(server3.URL)) == DefaultMaxBatchSize*3
	}, time.Second*10, time.Millisecond*10)

Review comment (bug):
a timeout bigger that time.Second*2 is still needed? (asking in case those are leftovers from previous debuging)

Code after:
		ams.startSendLoops(ams.ams)
	}
	defer func() {
		for _, ams := range h.alertmanagers {
			ams.cleanSendLoops(ams.ams...)
		}
	}()

	h.Send(alerts...)
	require.Eventually(t, func() bool {
		return prom_testutil.ToFloat64(h.metrics.sent.WithLabelValues(server1.URL)) == DefaultMaxBatchSize
	}, time.Second*2, time.Millisecond*10)
	checkNoErr()

	// The only am in set 1 is down.
	status1.Store(int32(http.StatusNotFound))
	h.Send(alerts...)
	// Wait for all send loops to process before changing any status.
	require.Eventually(t, func() bool {
		return prom_testutil.ToFloat64(h.metrics.errors.WithLabelValues(server1.URL)) == DefaultMaxBatchSize &&
			prom_testutil.ToFloat64(h.metrics.sent.WithLabelValues(server2.URL)) == DefaultMaxBatchSize*2 &&
			prom_testutil.ToFloat64(h.metrics.sent.WithLabelValues(server3.URL)) == DefaultMaxBatchSize*2
	}, time.Second*2, time.Millisecond*10)
	checkNoErr()

	// Fix the am.
	status1.Store(int32(http.StatusOK))

	// Only one of the ams in set 2 is down.
	status2.Store(int32(http.StatusInternalServerError))
	h.Send(alerts...)
	// Wait for all send loops to either send or fail with errors depending on their status.
	require.Eventually(t, func() bool {
		return prom_testutil.ToFloat64(h.metrics.errors.WithLabelValues(server2.URL)) == DefaultMaxBatchSize &&
			prom_testutil.ToFloat64(h.metrics.sent.WithLabelValues(server1.URL)) == DefaultMaxBatchSize*2 &&
			prom_testutil.ToFloat64(h.metrics.sent.WithLabelValues(server3.URL)) == DefaultMaxBatchSize*3
	}, time.Second*2, time.Millisecond*10)
	checkNoErr()

	// Both ams in set 2 are down.
	status3.Store(int32(http.StatusInternalServerError))
	h.Send(alerts...)
	require.Eventually(t, func() bool {
		return prom_testutil.ToFloat64(h.metrics.errors.WithLabelValues(server2.URL)) == DefaultMaxBatchSize*2 &&
			prom_testutil.ToFloat64(h.metrics.errors.WithLabelValues(server3.URL)) == DefaultMaxBatchSize
	}, time.Second*2, time.Millisecond*10)
	checkNoErr()
}

func TestHandlerSendAllRemapPerAm(t *testing.T) {
	var (