Repository: moira-alert/moira
Pull request #1149: feat: db methods for delivery check and new metric for sender
Author: login: AleksandrMatsko
Created: 2025-02-28T10:40:02Z | Merged: 2025-03-25T07:52:45Z

# PR Summary

- Add new database interface which should be used in senders, that support delivery checks
- Add metrics to indicate that delivery checks has stopped



Diff:
diff --git a/database/redis/delivery.go b/database/redis/delivery.go
new file mode 100644
index 000000000..662e84a0f
--- /dev/null
+++ b/database/redis/delivery.go
@@ -0,0 +1,61 @@
+package redis
+
+import (
+	"errors"
+
+	"github.com/go-redis/redis/v8"
+	"github.com/moira-alert/moira/database"
+)
+
+// AddDeliveryChecksData adds given data to sorted by timestamp set relative to contact type.
+func (connector *DbConnector) AddDeliveryChecksData(contactType string, timestamp int64, data string) error {
+	client := connector.Client()
+	ctx := connector.Context()
+
+	return client.ZAdd(
+		ctx,
+		deliveryCheckKeyWithContactType(contactType),
+		&redis.Z{
+			Score:  float64(timestamp),
+			Member: data,
+		}).Err()
+}
+
+// GetDeliveryChecksData reads data from for given tim range relative to contact type.
+func (connector *DbConnector) GetDeliveryChecksData(contactType string, from string, to string) ([]string, error) {
+	client := connector.Client()
+	ctx := connector.Context()
+
+	res, err := client.ZRangeByScore(
+		ctx,
+		deliveryCheckKeyWithContactType(contactType),
+		&redis.ZRangeBy{
+			Min:    from,
+			Max:    to,
+			Offset: 0,
+			Count:  -1,
+		}).Result()
+	if err != nil {
+		if errors.Is(err, redis.Nil) {
+			return nil, database.ErrNil
+		}
+
+		return nil, err
+	}
+
+	return res, nil
+}
+
+// RemoveDeliveryChecksData removes data from for given time range relative to contact type.
+func (connector *DbConnector) RemoveDeliveryChecksData(contactType string, from string, to string) (int64, error) {
+	client := connector.Client()
+	ctx := connector.Context()
+
+	return client.ZRemRangeByScore(ctx, deliveryCheckKeyWithContactType(contactType), from, to).Result()
+}
+
+const deliveryCheckKey = "moira-delivery-check"
+
+func deliveryCheckKeyWithContactType(contactType string) string {
+	return deliveryCheckKey + ":" + contactType
+}
diff --git a/database/redis/delivery_test.go b/database/redis/delivery_test.go
new file mode 100644
index 000000000..6190ed9dd
--- /dev/null
+++ b/database/redis/delivery_test.go
@@ -0,0 +1,70 @@
+package redis
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+
+	logging "github.com/moira-alert/moira/logging/zerolog_adapter"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDeliveryChecksDataManipulation(t *testing.T) {
+	logger, _ := logging.GetLogger("dataBase")
+	dataBase := NewTestDatabase(logger)
+	dataBase.Flush()
+	defer dataBase.Flush()
+
+	const testContactType = "test_contact_type"
+	var testTimestamp int64 = 12345678
+	storedData := make([]string, 0)
+
+	Convey("Test delivery checks manipulation", t, func() {
+		Convey("store delivery checks data", func() {
+			for i := range 6 {
+				data := fmt.Sprintf("data_%v", i)
+				storedData = append(storedData, data)
+
+				err := dataBase.AddDeliveryChecksData(testContactType, testTimestamp+int64(i), data)
+				So(err, ShouldBeNil)
+			}
+		})
+
+		Convey("get delivery checks", func() {
+			Convey("all checks", func() {
+				gotRes, err := dataBase.GetDeliveryChecksData(testContactType, "-inf", "+inf")
+				So(err, ShouldBeNil)
+				So(gotRes, ShouldResemble, storedData)
+			})
+
+			Convey("with fixed from", func() {
+				gotRes, err := dataBase.GetDeliveryChecksData(testContactType, strconv.FormatInt(testTimestamp+2, 10), "+inf")
+				So(err, ShouldBeNil)
+				So(gotRes, ShouldResemble, storedData[2:])
+			})
+
+			Convey("with fixed to", func() {
+				gotRes, err := dataBase.GetDeliveryChecksData(testContactType, "-inf", strconv.FormatInt(testTimestamp+4, 10))
+				So(err, ShouldBeNil)
+				So(gotRes, ShouldResemble, storedData[:5])
+			})
+
+			Convey("with fixed from and to", func() {
+				gotRes, err := dataBase.GetDeliveryChecksData(testContactType, strconv.FormatInt(testTimestamp+2, 10), strconv.FormatInt(testTimestamp+4, 10))
+				So(err, ShouldBeNil)
+				So(gotRes, ShouldResemble, storedData[2:5])
+			})
+		})
+
+		Convey("remove delivery checks", func() {
+			removedCount, err := dataBase.RemoveDeliveryChecksData(testContactType, "-inf", strconv.FormatInt(testTimestamp+3, 10))
+			So(err, ShouldBeNil)
+			So(removedCount, ShouldEqual, 4)
+
+			gotRes, err := dataBase.GetDeliveryChecksData(testContactType, strconv.FormatInt(testTimestamp+2, 10), strconv.FormatInt(testTimestamp+6, 10))
+			So(err, ShouldBeNil)
+			So(gotRes, ShouldResemble, storedData[4:])
+		})
+	})
+}
diff --git a/generate_mocks.sh b/generate_mocks.sh
index 75e9eca37..51610b083 100755
--- a/generate_mocks.sh
+++ b/generate_mocks.sh
@@ -9,6 +9,7 @@ go install go.uber.org/mock/mockgen@v0.4.0
 mockgen -destination=mock/moira-alert/locks.go -package=mock_moira_alert github.com/moira-alert/moira Lock
 mockgen -destination=mock/moira-alert/mutex.go -package=mock_moira_alert github.com/moira-alert/moira Mutex
 mockgen -destination=mock/moira-alert/database.go -package=mock_moira_alert github.com/moira-alert/moira Database
+mockgen -destination=mock/moira-alert/delivery_checker_database.go -package=mock_moira_alert github.com/moira-alert/moira DeliveryCheckerDatabase
 mockgen -destination=mock/moira-alert/image_store.go -package=mock_moira_alert github.com/moira-alert/moira ImageStore
 mockgen -destination=mock/moira-alert/logger.go -package=mock_moira_alert github.com/moira-alert/moira Logger
 mockgen -destination=mock/moira-alert/event_builder.go -package=mock_moira_alert github.com/moira-alert/moira/logging EventBuilder
diff --git a/interfaces.go b/interfaces.go
index 06797369c..3caec6ead 100644
--- a/interfaces.go
+++ b/interfaces.go
@@ -159,6 +159,19 @@ type Database interface {
 	CleanUpAbandonedRetentions() error
 	RemoveMetricsByPrefix(pattern string) error
 	RemoveAllMetrics() error
+
+	// Delivery checks
+	DeliveryCheckerDatabase
+}
+
+// DeliveryCheckerDatabase is used by senders that can track if the notification was delivered.
+type DeliveryCheckerDatabase interface {
+	// AddDeliveryChecksData must be used to store data need for performing delivery checks.
+	AddDeliveryChecksData(contactType string, timestamp int64, data string) error
+	// GetDeliveryChecksData must be used to get data need for performing delivery checks.
+	GetDeliveryChecksData(contactType string, from string, to string) ([]string, error)
+	// RemoveDeliveryChecksData must remove already used data for performing delivery checks.
+	RemoveDeliveryChecksData(contactType string, from string, to string) (int64, error)
 }
 
 // Lock implements lock abstraction.
diff --git a/metrics/notifier.go b/metrics/notifier.go
index 7cdbe43ce..e7cdfa6ee 100644
--- a/metrics/notifier.go
+++ b/metrics/notifier.go
@@ -6,41 +6,43 @@ import (
 
 // NotifierMetrics is a collection of metrics used in notifier.
 type NotifierMetrics struct {
-	SubsMalformed                       Meter
-	EventsReceived                      Meter
-	EventsMalformed                     Meter
-	EventsProcessingFailed              Meter
-	EventsByState                       MetersCollection
-	SendingFailed                       Meter
-	ContactsSendingNotificationsOK      MetersCollection
-	ContactsSendingNotificationsFailed  MetersCollection
-	ContactsDroppedNotifications        MetersCollection
-	ContactsDeliveryNotificationsOK     MetersCollection
-	ContactsDeliveryNotificationsFailed MetersCollection
-	PlotsBuildDurationMs                Histogram
-	PlotsEvaluateTriggerDurationMs      Histogram
-	fetchNotificationsDurationMs        Histogram
-	notifierIsAlive                     Meter
+	SubsMalformed                              Meter
+	EventsReceived                             Meter
+	EventsMalformed                            Meter
+	EventsProcessingFailed                     Meter
+	EventsByState                              MetersCollection
+	SendingFailed                              Meter
+	ContactsSendingNotificationsOK             MetersCollection
+	ContactsSendingNotificationsFailed         MetersCollection
+	ContactsDroppedNotifications               MetersCollection
+	ContactsDeliveryNotificationsOK            MetersCollection
+	ContactsDeliveryNotificationsFailed        MetersCollection
+	ContactsDeliveryNotificationsChecksStopped MetersCollection
+	PlotsBuildDurationMs                       Histogram
+	PlotsEvaluateTriggerDurationMs             Histogram
+	fetchNotificationsDurationMs               Histogram
+	notifierIsAlive                            Meter
 }
 
 // ConfigureNotifierMetrics is notifier metrics configurator.
 func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics {
 	return &NotifierMetrics{
-		SubsMalformed:                       registry.NewMeter("subs", "malformed"),
-		EventsReceived:                      registry.NewMeter("events", "received"),
-		EventsMalformed:                     registry.NewMeter("events", "malformed"),
-		EventsProcessingFailed:              registry.NewMeter("events", "failed"),
-		EventsByState:                       NewMetersCollection(registry),
-		SendingFailed:                       registry.NewMeter("sending", "failed"),
-		ContactsSendingNotificationsOK:      NewMetersCollection(registry),
-		ContactsSendingNotificationsFailed:  NewMetersCollection(registry),
-		ContactsDroppedNotifications:        NewMetersCollection(registry),
-		ContactsDeliveryNotificationsOK:     NewMetersCollection(registry),
-		ContactsDeliveryNotificationsFailed: NewMetersCollection(registry),
-		PlotsBuildDurationMs:                registry.NewHistogram("plots", "build", "duration", "ms"),
-		PlotsEvaluateTriggerDurationMs:      registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"),
-		fetchNotificationsDurationMs:        registry.NewHistogram("fetch", "notifications", "duration", "ms"),
-		notifierIsAlive:                     registry.NewMeter("", "alive"),
+		SubsMalformed:                              registry.NewMeter("subs", "malformed"),
+		EventsReceived:                             registry.NewMeter("events", "received"),
+		EventsMalformed:                            registry.NewMeter("events", "malformed"),
+		EventsProcessingFailed:                     registry.NewMeter("events", "failed"),
+		EventsByState:                              NewMetersCollection(registry),
+		SendingFailed:                              registry.NewMeter("sending", "failed"),
+		ContactsSendingNotificationsOK:             NewMetersCollection(registry),
+		ContactsSendingNotificationsFailed:         NewMetersCollection(registry),
+		ContactsDroppedNotifications:               NewMetersCollection(registry),
+		ContactsDeliveryNotificationsOK:            NewMetersCollection(registry),
+		ContactsDeliveryNotificationsFailed:        NewMetersCollection(registry),
+		ContactsDeliveryNotificationsChecksStopped: NewMetersCollection(registry),
+		PlotsBuildDurationMs:                       registry.NewHistogram("plots", "build", "duration", "ms"),
+		PlotsEvaluateTriggerDurationMs:             registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"),
+		fetchNotificationsDurationMs:               registry.NewHistogram("fetch", "notifications", "duration", "ms"),
+		notifierIsAlive:                            registry.NewMeter("", "alive"),
 	}
 }
 
diff --git a/metrics/sender.go b/metrics/sender.go
index e1414c207..e10b530a2 100644
--- a/metrics/sender.go
+++ b/metrics/sender.go
@@ -2,8 +2,9 @@ package metrics
 
 // SenderMetrics should be used for sender which can understand if the notification was delivered or not.
 type SenderMetrics struct {
-	ContactDeliveryNotificationOK     Meter
-	ContactDeliveryNotificationFailed Meter
+	ContactDeliveryNotificationOK           Meter
+	ContactDeliveryNotificationFailed       Meter
+	ContactDeliveryNotificationCheckStopped Meter
 }
 
 // ConfigureSenderMetrics configures SenderMetrics using NotifierMetrics with given graphiteIdent for senderContactType.
@@ -13,5 +14,7 @@ func ConfigureSenderMetrics(notifierMetrics *NotifierMetrics, graphiteIdent stri
 			RegisterMeter(senderContactType, graphiteIdent, "delivery_ok"),
 		ContactDeliveryNotificationFailed: notifierMetrics.ContactsDeliveryNotificationsFailed.
 			RegisterMeter(senderContactType, graphiteIdent, "delivery_failed"),
+		ContactDeliveryNotificationCheckStopped: notifierMetrics.ContactsDeliveryNotificationsChecksStopped.
+			RegisterMeter(senderContactType, graphiteIdent, "delivery_checks_stopped"),
 	}
 }
diff --git a/mock/moira-alert/database.go b/mock/moira-alert/database.go
index 4bce2497f..26d6e4e70 100644
--- a/mock/moira-alert/database.go
+++ b/mock/moira-alert/database.go
@@ -55,6 +55,20 @@ func (mr *MockDatabaseMockRecorder) AcquireTriggerCheckLock(arg0, arg1 any) *gom
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireTriggerCheckLock", reflect.TypeOf((*MockDatabase)(nil).AcquireTriggerCheckLock), arg0, arg1)
 }
 
+// AddDeliveryChecksData mocks base method.
+func (m *MockDatabase) AddDeliveryChecksData(arg0 string, arg1 int64, arg2 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "AddDeliveryChecksData", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// AddDeliveryChecksData indicates an expected call of AddDeliveryChecksData.
+func (mr *MockDatabaseMockRecorder) AddDeliveryChecksData(arg0, arg1, arg2 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddDeliveryChecksData", reflect.TypeOf((*MockDatabase)(nil).AddDeliveryChecksData), arg0, arg1, arg2)
+}
+
 // AddNotification mocks base method.
 func (m *MockDatabase) AddNotification(arg0 *moira.ScheduledNotification) error {
 	m.ctrl.T.Helper()
@@ -446,6 +460,21 @@ func (mr *MockDatabaseMockRecorder) GetContacts(arg0 any) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetContacts", reflect.TypeOf((*MockDatabase)(nil).GetContacts), arg0)
 }
 
+// GetDeliveryChecksData mocks base method.
+func (m *MockDatabase) GetDeliveryChecksData(arg0, arg1, arg2 string) ([]string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDeliveryChecksData", arg0, arg1, arg2)
+	ret0, _ := ret[0].([]string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetDeliveryChecksData indicates an expected call of GetDeliveryChecksData.
+func (mr *MockDatabaseMockRecorder) GetDeliveryChecksData(arg0, arg1, arg2 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeliveryChecksData", reflect.TypeOf((*MockDatabase)(nil).GetDeliveryChecksData), arg0, arg1, arg2)
+}
+
 // GetMetricRetention mocks base method.
 func (m *MockDatabase) GetMetricRetention(arg0 string) (int64, error) {
 	m.ctrl.T.Helper()
@@ -1207,6 +1236,21 @@ func (mr *MockDatabaseMockRecorder) RemoveContact(arg0 any) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveContact", reflect.TypeOf((*MockDatabase)(nil).RemoveContact), arg0)
 }
 
+// RemoveDeliveryChecksData mocks base method.
+func (m *MockDatabase) RemoveDeliveryChecksData(arg0, arg1, arg2 string) (int64, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RemoveDeliveryChecksData", arg0, arg1, arg2)
+	ret0, _ := ret[0].(int64)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// RemoveDeliveryChecksData indicates an expected call of RemoveDeliveryChecksData.
+func (mr *MockDatabaseMockRecorder) RemoveDeliveryChecksData(arg0, arg1, arg2 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveDeliveryChecksData", reflect.TypeOf((*MockDatabase)(nil).RemoveDeliveryChecksData), arg0, arg1, arg2)
+}
+
 // RemoveMetricRetention mocks base method.
 func (m *MockDatabase) RemoveMetricRetention(arg0 string) error {
 	m.ctrl.T.Helper()
diff --git a/mock/moira-alert/delivery_checker_database.go b/mock/moira-alert/delivery_checker_database.go
new file mode 100644
index 000000000..1c67a7161
--- /dev/null
+++ b/mock/moira-alert/delivery_checker_database.go
@@ -0,0 +1,83 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/moira-alert/moira (interfaces: DeliveryCheckerDatabase)
+//
+// Generated by this command:
+//
+//	mockgen -destination=mock/moira-alert/delivery_checker_database.go -package=mock_moira_alert github.com/moira-alert/moira DeliveryCheckerDatabase
+//
+
+// Package mock_moira_alert is a generated GoMock package.
+package mock_moira_alert
+
+import (
+	reflect "reflect"
+
+	gomock "go.uber.org/mock/gomock"
+)
+
+// MockDeliveryCheckerDatabase is a mock of DeliveryCheckerDatabase interface.
+type MockDeliveryCheckerDatabase struct {
+	ctrl     *gomock.Controller
+	recorder *MockDeliveryCheckerDatabaseMockRecorder
+}
+
+// MockDeliveryCheckerDatabaseMockRecorder is the mock recorder for MockDeliveryCheckerDatabase.
+type MockDeliveryCheckerDatabaseMockRecorder struct {
+	mock *MockDeliveryCheckerDatabase
+}
+
+// NewMockDeliveryCheckerDatabase creates a new mock instance.
+func NewMockDeliveryCheckerDatabase(ctrl *gomock.Controller) *MockDeliveryCheckerDatabase {
+	mock := &MockDeliveryCheckerDatabase{ctrl: ctrl}
+	mock.recorder = &MockDeliveryCheckerDatabaseMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockDeliveryCheckerDatabase) EXPECT() *MockDeliveryCheckerDatabaseMockRecorder {
+	return m.recorder
+}
+
+// AddDeliveryChecksData mocks base method.
+func (m *MockDeliveryCheckerDatabase) AddDeliveryChecksData(arg0 string, arg1 int64, arg2 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "AddDeliveryChecksData", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// AddDeliveryChecksData indicates an expected call of AddDeliveryChecksData.
+func (mr *MockDeliveryCheckerDatabaseMockRecorder) AddDeliveryChecksData(arg0, arg1, arg2 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddDeliveryChecksData", reflect.TypeOf((*MockDeliveryCheckerDatabase)(nil).AddDeliveryChecksData), arg0, arg1, arg2)
+}
+
+// GetDeliveryChecksData mocks base method.
+func (m *MockDeliveryCheckerDatabase) GetDeliveryChecksData(arg0, arg1, arg2 string) ([]string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetDeliveryChecksData", arg0, arg1, arg2)
+	ret0, _ := ret[0].([]string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetDeliveryChecksData indicates an expected call of GetDeliveryChecksData.
+func (mr *MockDeliveryCheckerDatabaseMockRecorder) GetDeliveryChecksData(arg0, arg1, arg2 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeliveryChecksData", reflect.TypeOf((*MockDeliveryCheckerDatabase)(nil).GetDeliveryChecksData), arg0, arg1, arg2)
+}
+
+// RemoveDeliveryChecksData mocks base method.
+func (m *MockDeliveryCheckerDatabase) RemoveDeliveryChecksData(arg0, arg1, arg2 string) (int64, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RemoveDeliveryChecksData", arg0, arg1, arg2)
+	ret0, _ := ret[0].(int64)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// RemoveDeliveryChecksData indicates an expected call of RemoveDeliveryChecksData.
+func (mr *MockDeliveryCheckerDatabaseMockRecorder) RemoveDeliveryChecksData(arg0, arg1, arg2 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveDeliveryChecksData", reflect.TypeOf((*MockDeliveryCheckerDatabase)(nil).RemoveDeliveryChecksData), arg0, arg1, arg2)
+}
diff --git a/notifier/registrator.go b/notifier/registrator.go
index 104635dbc..91f4dbea9 100644
--- a/notifier/registrator.go
+++ b/notifier/registrator.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/moira-alert/moira"
 	"github.com/moira-alert/moira/metrics"
@@ -48,12 +49,32 @@ var (
 	ErrMissingContactType = errors.New("failed to retrieve sender contact type from sender settings")
 )
 
+const (
+	deliveryCheckWorkerLockPrefix = "moira-delivery-check-lock:"
+	deliveryCheckLockTTL          = 30 * time.Second
+)
+
+func workerDeliveryCheckLockKey(contactType string) string {
+	return deliveryCheckWorkerLockPrefix + contactType
+}
+
 // RegisterSenders watch on senders config and register all configured senders.
 func (notifier *StandardNotifier) RegisterSenders(connector moira.Database) error { //nolint
 	var err error
 	for _, senderSettings := range notifier.config.Senders {
 		senderSettings["front_uri"] = notifier.config.FrontURL
-		switch senderSettings["sender_type"] {
+
+		senderType, ok := senderSettings["sender_type"].(string)
+		if !ok {
+			return ErrMissingSenderType
+		}
+
+		senderContactType, ok := senderSettings["contact_type"].(string)
+		if !ok {
+			return ErrMissingContactType
+		}
+
+		switch senderType {
 		case mailSender:
 			err = notifier.RegisterSender(senderSettings, &mail.Sender{})
 		case pushoverSender:
@@ -73,6 +94,9 @@ func (notifier *StandardNotifier) RegisterSenders(connector moira.Database) erro
 		case twilioSmsSender, twilioVoiceSender:
 			err = notifier.RegisterSender(senderSettings, &twilio.Sender{})
 		case webhookSender:
+			workerLock := connector.NewLock(workerDeliveryCheckLockKey(senderContactType), deliveryCheckLockTTL)
+			_ = workerLock
+
 			err = notifier.RegisterSender(senderSettings, &webhook.Sender{})
 		case opsgenieSender:
 			err = notifier.RegisterSender(senderSettings, &opsgenie.Sender{ImageStores: notifier.imageStores})
@@ -85,7 +109,7 @@ func (notifier *StandardNotifier) RegisterSenders(connector moira.Database) erro
 		// case "phone":
 		// 	err = notifier.RegisterSender(senderSettings, &kontur.SmsSender{})
 		default:
-			return fmt.Errorf("unknown sender type [%s]", senderSettings["sender_type"])
+			return fmt.Errorf("unknown sender type [%s]", senderType)
 		}
 
 		if err != nil {
diff --git a/notifier/registrator_test.go b/notifier/registrator_test.go
index ce020bdc2..04d78ab70 100644
--- a/notifier/registrator_test.go
+++ b/notifier/registrator_test.go
@@ -64,14 +64,14 @@ func TestRegisterSender(t *testing.T) {
 		})
 
 		Convey("Successfully register sender with damaged enable_metrics", func() {
-			sendersOkMetrics := mock_metrics.NewMockMetersCollection(mockCtrl)
-			sendersFailedMetrics := mock_metrics.NewMockMetersCollection(mockCtrl)
-			sendersDroppedNotifications := mock_metrics.NewMockMetersCollection(mockCtrl)
+			contactsSendingNotificationsOKMetrics := mock_metrics.NewMockMetersCollection(mockCtrl)
+			contactsSendingNotificationsFailedMetrics := mock_metrics.NewMockMetersCollection(mockCtrl)
+			contactsDroppedNotifications := mock_metrics.NewMockMetersCollection(mockCtrl)
 
 			notifierMetrics := &metrics.NotifierMetrics{
-				ContactsSendingNotificationsOK:     sendersOkMetrics,
-				ContactsSendingNotificationsFailed: sendersFailedMetrics,
-				ContactsDroppedNotifications:       sendersDroppedNotifications,
+				ContactsSendingNotificationsOK:     contactsSendingNotificationsOKMetrics,
+				ContactsSendingNotificationsFailed: contactsSendingNotificationsFailedMetrics,
+				ContactsDroppedNotifications:       contactsDroppedNotifications,
 			}
 			standardNotifier.metrics = notifierMetrics
 
@@ -82,9 +82,9 @@ func TestRegisterSender(t *testing.T) {
 				"enable_metrics": "abracdabra",
 			}
 
-			sendersOkMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_ok").Times(1)
-			sendersFailedMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_failed").Times(1)
-			sendersDroppedNotifications.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "notifications_dropped").Times(1)
+			contactsSendingNotificationsOKMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_ok").Times(1)
+			contactsSendingNotificationsFailedMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_failed").Times(1)
+			contactsDroppedNotifications.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "notifications_dropped").Times(1)
 			sender.EXPECT().Init(senderSettings, standardNotifier.logger, standardNotifier.config.Location, standardNotifier.config.DateTimeFormat)
 
 			err := standardNotifier.RegisterSender(senderSettings, sender)
@@ -92,18 +92,20 @@ func TestRegisterSender(t *testing.T) {
 		})
 
 		Convey("Register sender with additional metrics", func() {
-			sendersOkMetrics := mock_metrics.NewMockMetersCollection(mockCtrl)
-			sendersFailedMetrics := mock_metrics.NewMockMetersCollection(mockCtrl)
-			sendersDroppedNotifications := mock_metrics.NewMockMetersCollection(mockCtrl)
-			sendersDeliveryOK := mock_metrics.NewMockMetersCollection(mockCtrl)
-			sendersDeliveryFailed := mock_metrics.NewMockMetersCollection(mockCtrl)
+			contactsSendingNotificationsOKMetrics := mock_metrics.NewMockMetersCollection(mockCtrl)
+			contactsSendingNotificationsFailedMetrics := mock_metrics.NewMockMetersCollection(mockCtrl)
+			contactsDroppedNotifications := mock_metrics.NewMockMetersCollection(mockCtrl)
+			contactsDeliveryNotificationsOK := mock_metrics.NewMockMetersCollection(mockCtrl)
+			contactsDeliveryNotificationsFailed := mock_metrics.NewMockMetersCollection(mockCtrl)
+			contactsDeliveryNotificationsChecksStopped := mock_metrics.NewMockMetersCollection(mockCtrl)
 
 			notifierMetrics := &metrics.NotifierMetrics{
-				ContactsSendingNotificationsOK:      sendersOkMetrics,
-				ContactsSendingNotificationsFailed:  sendersFailedMetrics,
-				ContactsDroppedNotifications:        sendersDroppedNotifications,
-				ContactsDeliveryNotificationsOK:     sendersDeliveryOK,
-				ContactsDeliveryNotificationsFailed: sendersDeliveryFailed,
+				ContactsSendingNotificationsOK:             contactsSendingNotificationsOKMetrics,
+				ContactsSendingNotificationsFailed:         contactsSendingNotificationsFailedMetrics,
+				ContactsDroppedNotifications:               contactsDroppedNotifications,
+				ContactsDeliveryNotificationsOK:            contactsDeliveryNotificationsOK,
+				ContactsDeliveryNotificationsFailed:        contactsDeliveryNotificationsFailed,
+				ContactsDeliveryNotificationsChecksStopped: contactsDeliveryNotificationsChecksStopped,
 			}
 			standardNotifier.metrics = notifierMetrics
 
@@ -114,11 +116,12 @@ func TestRegisterSender(t *testing.T) {
 				"enable_metrics": true,
 			}
 
-			sendersOkMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_ok").Times(1)
-			sendersFailedMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_failed").Times(1)
-			sendersDroppedNotifications.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "notifications_dropped").Times(1)
-			sendersDeliveryOK.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "delivery_ok").Times(1)
-			sendersDeliveryFailed.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "delivery_failed").Times(1)
+			contactsSendingNotificationsOKMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_ok").Times(1)
+			contactsSendingNotificationsFailedMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_failed").Times(1)
+			contactsDroppedNotifications.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "notifications_dropped").Times(1)
+			contactsDeliveryNotificationsOK.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "delivery_ok").Times(1)
+			contactsDeliveryNotificationsFailed.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "delivery_failed").Times(1)
+			contactsDeliveryNotificationsChecksStopped.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "delivery_checks_stopped").Times(1)
 			sender.EXPECT().Init(senderSettings, standardNotifier.logger, standardNotifier.config.Location, standardNotifier.config.DateTimeFormat)
 
 			err := standardNotifier.RegisterSender(senderSettings, sender)
@@ -135,7 +138,8 @@ func TestRegisterSendersWithUnknownType(t *testing.T) {
 		DateTimeFormat:   dateTimeFormat,
 		Senders: []map[string]interface{}{
 			{
-				"sender_type": "some_type",
+				"sender_type":  "some_type",
+				"contact_type": "some_contact_type",
 			},
 		},
 	}


Reviews:
user: login: AleksandrMatsko
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-06T13:22:37Z
user: login: kissken
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T07:25:57Z
user: login: kissken
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T07:33:58Z
user: login: AleksandrMatsko
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T07:48:31Z
user: login: AleksandrMatsko
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T07:55:18Z
user: login: kissken
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T08:01:42Z
user: login: AleksandrMatsko
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T08:41:28Z
user: login: mchrome
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T12:02:06Z
user: login: mchrome
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T12:02:27Z
user: login: Tetrergeru
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-10T14:25:24Z
user: login: Tetrergeru
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-10T14:25:39Z
user: login: AleksandrMatsko
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-14T04:47:13Z
user: login: kissken
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T08:27:13Z
user: login: AleksandrMatsko
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T08:34:22Z
user: login: kissken
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T10:58:01Z
user: login: Tetrergeru
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T10:58:08Z
user: login: kissken
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T10:58:25Z
user: login: Tetrergeru
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-18T10:59:54Z
user: login: kissken
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T11:00:30Z
user: login: AleksandrMatsko
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T11:07:44Z
user: login: AleksandrMatsko
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T11:09:23Z
user: login: kissken
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T11:13:33Z
user: login: kissken
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-18T11:13:48Z
user: login: AleksandrMatsko
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T11:16:40Z

Review comments:
diff_hunk: @@ -514,7 +514,7 @@ func (*Trigger) Render(http.ResponseWriter, *http.Request) error {
 
 // PopulatedDescription returns new trigger description after template populating.
 func (trigger *Trigger) PopulatedDescription(events moira.NotificationEvents) (*string, error) {
-	var emptyString = ""
+	emptyString := ""
path: api/dto/triggers.go
user: login: AleksandrMatsko
body: Это не я, это `gofumpt`
created_at: 2025-03-06T13:22:37Z
author_association: MEMBER
diff_hunk: @@ -159,6 +159,21 @@ type Database interface {
 	CleanUpAbandonedRetentions() error
 	RemoveMetricsByPrefix(pattern string) error
 	RemoveAllMetrics() error
+
+	// Delivery checks
+	DeliveryCheckerDatabase
+}
+
+// DeliveryCheckerDatabase is used by senders that can track if the notification was delivered.
+type DeliveryCheckerDatabase interface {
+	// AddDeliveryChecksData must be used to store data need for performing delivery checks.
+	AddDeliveryChecksData(contactType string, timestamp int64, data string) error
+	// GetDeliveryChecksData must be used to get data need for performing delivery checks.
+	GetDeliveryChecksData(contactType string, from string, to string) ([]string, error)
+	// RemoveDeliveryChecksData must remove already used data for performing delivery checks.
+	RemoveDeliveryChecksData(contactType string, from string, to string) (int64, error)
+	// NewLock creates Lock
+	NewLock(name string, ttl time.Duration) Lock
path: interfaces.go
user: login: kissken
body: ой, я пока не поняла, но отмечу, что глаз зацепился, что мы блокировки куда-то наверх потащили...
created_at: 2025-03-07T07:25:57Z
author_association: MEMBER
diff_hunk: @@ -31,6 +31,7 @@ type Sender struct {
 	client   *http.Client
 	log      moira.Logger
 	metrics  *metrics.SenderMetrics
+	Database moira.DeliveryCheckerDatabase
path: senders/webhook/webhook.go
user: login: kissken
body: мне кажется, должен быть отдельный компонент, который работает с базенкой
created_at: 2025-03-07T07:33:58Z
author_association: MEMBER
diff_hunk: @@ -31,6 +31,7 @@ type Sender struct {
 	client   *http.Client
 	log      moira.Logger
 	metrics  *metrics.SenderMetrics
+	Database moira.DeliveryCheckerDatabase
path: senders/webhook/webhook.go
user: login: AleksandrMatsko
body: Я не до конца понял, можешь раскрыть идею?
Что-то, вроде, прослойки между логикой проверки и самой базой?
created_at: 2025-03-07T07:48:31Z
author_association: MEMBER
diff_hunk: @@ -159,6 +159,21 @@ type Database interface {
 	CleanUpAbandonedRetentions() error
 	RemoveMetricsByPrefix(pattern string) error
 	RemoveAllMetrics() error
+
+	// Delivery checks
+	DeliveryCheckerDatabase
+}
+
+// DeliveryCheckerDatabase is used by senders that can track if the notification was delivered.
+type DeliveryCheckerDatabase interface {
+	// AddDeliveryChecksData must be used to store data need for performing delivery checks.
+	AddDeliveryChecksData(contactType string, timestamp int64, data string) error
+	// GetDeliveryChecksData must be used to get data need for performing delivery checks.
+	GetDeliveryChecksData(contactType string, from string, to string) ([]string, error)
+	// RemoveDeliveryChecksData must remove already used data for performing delivery checks.
+	RemoveDeliveryChecksData(contactType string, from string, to string) (int64, error)
+	// NewLock creates Lock
+	NewLock(name string, ttl time.Duration) Lock
path: interfaces.go
user: login: AleksandrMatsko
body: Понимаю, но я затащил их сюда, т.к. хотел использовать логику worker. Более элегантного решения не придумал(
created_at: 2025-03-07T07:55:18Z
author_association: MEMBER
diff_hunk: @@ -31,6 +31,7 @@ type Sender struct {
 	client   *http.Client
 	log      moira.Logger
 	metrics  *metrics.SenderMetrics
+	Database moira.DeliveryCheckerDatabase
path: senders/webhook/webhook.go
user: login: kissken
body: сейчас, я так поняла, ты будешь подключать базу и реализовывать логику проверялки живости в каждом сендере, но точно ли это так должно быть? 
created_at: 2025-03-07T08:01:42Z
author_association: MEMBER
diff_hunk: @@ -31,6 +31,7 @@ type Sender struct {
 	client   *http.Client
 	log      moira.Logger
 	metrics  *metrics.SenderMetrics
+	Database moira.DeliveryCheckerDatabase
path: senders/webhook/webhook.go
user: login: AleksandrMatsko
body: я понял, подумаю
created_at: 2025-03-07T08:41:28Z
author_association: MEMBER
diff_hunk: @@ -159,6 +159,21 @@ type Database interface {
 	CleanUpAbandonedRetentions() error
 	RemoveMetricsByPrefix(pattern string) error
 	RemoveAllMetrics() error
+
+	// Delivery checks
+	DeliveryCheckerDatabase
+}
+
+// DeliveryCheckerDatabase is used by senders that can track if the notification was delivered.
+type DeliveryCheckerDatabase interface {
+	// AddDeliveryChecksData must be used to store data need for performing delivery checks.
+	AddDeliveryChecksData(contactType string, timestamp int64, data string) error
+	// GetDeliveryChecksData must be used to get data need for performing delivery checks.
+	GetDeliveryChecksData(contactType string, from string, to string) ([]string, error)
+	// RemoveDeliveryChecksData must remove already used data for performing delivery checks.
+	RemoveDeliveryChecksData(contactType string, from string, to string) (int64, error)
+	// NewLock creates Lock
+	NewLock(name string, ttl time.Duration) Lock
path: interfaces.go
user: login: mchrome
body: Добавь в описание зачем NewLock тут нужен, если оставишь так
created_at: 2025-03-10T12:02:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -6,41 +6,43 @@ import (
 
 // NotifierMetrics is a collection of metrics used in notifier.
 type NotifierMetrics struct {
-	SubsMalformed                       Meter
-	EventsReceived                      Meter
-	EventsMalformed                     Meter
-	EventsProcessingFailed              Meter
-	EventsByState                       MetersCollection
-	SendingFailed                       Meter
-	ContactsSendingNotificationsOK      MetersCollection
-	ContactsSendingNotificationsFailed  MetersCollection
-	ContactsDroppedNotifications        MetersCollection
-	ContactsDeliveryNotificationsOK     MetersCollection
-	ContactsDeliveryNotificationsFailed MetersCollection
-	PlotsBuildDurationMs                Histogram
-	PlotsEvaluateTriggerDurationMs      Histogram
-	fetchNotificationsDurationMs        Histogram
-	notifierIsAlive                     Meter
+	SubsMalformed                              Meter
+	EventsReceived                             Meter
+	EventsMalformed                            Meter
+	EventsProcessingFailed                     Meter
+	EventsByState                              MetersCollection
+	SendingFailed                              Meter
+	ContactsSendingNotificationsOK             MetersCollection
path: metrics/notifier.go
user: login: Tetrergeru
body: Метрики внезапно оказались в ПР с базой :D
created_at: 2025-03-10T14:25:23Z
author_association: MEMBER
diff_hunk: @@ -6,41 +6,43 @@ import (
 
 // NotifierMetrics is a collection of metrics used in notifier.
 type NotifierMetrics struct {
-	SubsMalformed                       Meter
-	EventsReceived                      Meter
-	EventsMalformed                     Meter
-	EventsProcessingFailed              Meter
-	EventsByState                       MetersCollection
-	SendingFailed                       Meter
-	ContactsSendingNotificationsOK      MetersCollection
-	ContactsSendingNotificationsFailed  MetersCollection
-	ContactsDroppedNotifications        MetersCollection
-	ContactsDeliveryNotificationsOK     MetersCollection
-	ContactsDeliveryNotificationsFailed MetersCollection
-	PlotsBuildDurationMs                Histogram
-	PlotsEvaluateTriggerDurationMs      Histogram
-	fetchNotificationsDurationMs        Histogram
-	notifierIsAlive                     Meter
+	SubsMalformed                              Meter
+	EventsReceived                             Meter
+	EventsMalformed                            Meter
+	EventsProcessingFailed                     Meter
+	EventsByState                              MetersCollection
+	SendingFailed                              Meter
+	ContactsSendingNotificationsOK             MetersCollection
path: metrics/notifier.go
user: login: Tetrergeru
body: Это не проблема, просто немного неочевидно
created_at: 2025-03-10T14:25:38Z
author_association: MEMBER
diff_hunk: @@ -159,6 +159,21 @@ type Database interface {
 	CleanUpAbandonedRetentions() error
 	RemoveMetricsByPrefix(pattern string) error
 	RemoveAllMetrics() error
+
+	// Delivery checks
+	DeliveryCheckerDatabase
+}
+
+// DeliveryCheckerDatabase is used by senders that can track if the notification was delivered.
+type DeliveryCheckerDatabase interface {
+	// AddDeliveryChecksData must be used to store data need for performing delivery checks.
+	AddDeliveryChecksData(contactType string, timestamp int64, data string) error
+	// GetDeliveryChecksData must be used to get data need for performing delivery checks.
+	GetDeliveryChecksData(contactType string, from string, to string) ([]string, error)
+	// RemoveDeliveryChecksData must remove already used data for performing delivery checks.
+	RemoveDeliveryChecksData(contactType string, from string, to string) (int64, error)
+	// NewLock creates Lock
+	NewLock(name string, ttl time.Duration) Lock
path: interfaces.go
user: login: AleksandrMatsko
body: Избавился от лока
created_at: 2025-03-14T04:47:13Z
author_association: MEMBER
diff_hunk: @@ -48,12 +49,32 @@ var (
 	ErrMissingContactType = errors.New("failed to retrieve sender contact type from sender settings")
 )
 
+const (
+	deliveryCheckWorkerLockPrefix = "moira-delivery-check-lock:"
+	deliveryCheckLockTTL          = 30 * time.Second
path: notifier/registrator.go
user: login: kissken
body: что-то захардкоженное :(
created_at: 2025-03-18T08:27:13Z
author_association: MEMBER
diff_hunk: @@ -48,12 +49,32 @@ var (
 	ErrMissingContactType = errors.New("failed to retrieve sender contact type from sender settings")
 )
 
+const (
+	deliveryCheckWorkerLockPrefix = "moira-delivery-check-lock:"
+	deliveryCheckLockTTL          = 30 * time.Second
path: notifier/registrator.go
user: login: AleksandrMatsko
body: оно везде захардкоженное(
created_at: 2025-03-18T08:34:22Z
author_association: MEMBER
diff_hunk: @@ -48,12 +49,32 @@ var (
 	ErrMissingContactType = errors.New("failed to retrieve sender contact type from sender settings")
 )
 
+const (
+	deliveryCheckWorkerLockPrefix = "moira-delivery-check-lock:"
+	deliveryCheckLockTTL          = 30 * time.Second
path: notifier/registrator.go
user: login: kissken
body: хнычу 
created_at: 2025-03-18T10:58:01Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,61 @@
+package redis
+
+import (
+	"errors"
+
+	"github.com/go-redis/redis/v8"
+	"github.com/moira-alert/moira/database"
+)
+
+// AddDeliveryChecksData adds given data to sorted by timestamp set relative to contact type.
+func (connector *DbConnector) AddDeliveryChecksData(contactType string, timestamp int64, data string) error {
+	client := connector.Client()
+	ctx := connector.Context()
+
+	return client.ZAdd(
+		ctx,
+		deliveryCheckKeyWithContactType(contactType),
+		&redis.Z{
+			Score:  float64(timestamp),
+			Member: data,
+		}).Err()
+}
+
+// GetDeliveryChecksData reads data from for given tim range relative to contact type.
+func (connector *DbConnector) GetDeliveryChecksData(contactType string, from string, to string) ([]string, error) {
path: database/redis/delivery.go
user: login: Tetrergeru
body: А этот `[]string` — это массив json-ов, получается? У нас нет возможность тут их распарсить или они для каждого сендера будут какие-то свои уникальные?
created_at: 2025-03-18T10:58:08Z
author_association: MEMBER
diff_hunk: @@ -73,6 +94,9 @@ func (notifier *StandardNotifier) RegisterSenders(connector moira.Database) erro
 		case twilioSmsSender, twilioVoiceSender:
 			err = notifier.RegisterSender(senderSettings, &twilio.Sender{})
 		case webhookSender:
+			workerLock := connector.NewLock(workerDeliveryCheckLockKey(senderContactType), deliveryCheckLockTTL)
+			_ = workerLock
path: notifier/registrator.go
user: login: kissken
body: я не оч поняла, почему так
created_at: 2025-03-18T10:58:24Z
author_association: MEMBER
diff_hunk: @@ -73,6 +94,9 @@ func (notifier *StandardNotifier) RegisterSenders(connector moira.Database) erro
 		case twilioSmsSender, twilioVoiceSender:
 			err = notifier.RegisterSender(senderSettings, &twilio.Sender{})
 		case webhookSender:
+			workerLock := connector.NewLock(workerDeliveryCheckLockKey(senderContactType), deliveryCheckLockTTL)
+			_ = workerLock
path: notifier/registrator.go
user: login: kissken
body: да, я не очень поняла, логику :(
created_at: 2025-03-18T11:00:30Z
author_association: MEMBER
diff_hunk: @@ -73,6 +94,9 @@ func (notifier *StandardNotifier) RegisterSenders(connector moira.Database) erro
 		case twilioSmsSender, twilioVoiceSender:
 			err = notifier.RegisterSender(senderSettings, &twilio.Sender{})
 		case webhookSender:
+			workerLock := connector.NewLock(workerDeliveryCheckLockKey(senderContactType), deliveryCheckLockTTL)
+			_ = workerLock
path: notifier/registrator.go
user: login: AleksandrMatsko
body: Я здесь специально заинитил Lock, чтобы проверить что для него всё нужное есть. В следующих PR-ах я его заиспользовал.
created_at: 2025-03-18T11:07:44Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,61 @@
+package redis
+
+import (
+	"errors"
+
+	"github.com/go-redis/redis/v8"
+	"github.com/moira-alert/moira/database"
+)
+
+// AddDeliveryChecksData adds given data to sorted by timestamp set relative to contact type.
+func (connector *DbConnector) AddDeliveryChecksData(contactType string, timestamp int64, data string) error {
+	client := connector.Client()
+	ctx := connector.Context()
+
+	return client.ZAdd(
+		ctx,
+		deliveryCheckKeyWithContactType(contactType),
+		&redis.Z{
+			Score:  float64(timestamp),
+			Member: data,
+		}).Err()
+}
+
+// GetDeliveryChecksData reads data from for given tim range relative to contact type.
+func (connector *DbConnector) GetDeliveryChecksData(contactType string, from string, to string) ([]string, error) {
path: database/redis/delivery.go
user: login: AleksandrMatsko
body: > А этот []string — это массив json-ов, получается? У нас нет возможность тут их распарсить или они для каждого сендера будут какие-то свои уникальные?

Предполагал, что будут специфичные для каждого сендера -> поэтому сделал именно массив стрингов
created_at: 2025-03-18T11:09:23Z
author_association: MEMBER
diff_hunk: @@ -73,6 +94,9 @@ func (notifier *StandardNotifier) RegisterSenders(connector moira.Database) erro
 		case twilioSmsSender, twilioVoiceSender:
 			err = notifier.RegisterSender(senderSettings, &twilio.Sender{})
 		case webhookSender:
+			workerLock := connector.NewLock(workerDeliveryCheckLockKey(senderContactType), deliveryCheckLockTTL)
+			_ = workerLock
path: notifier/registrator.go
user: login: kissken
body: ага, вижу, но там такая же подвешенная штука 
created_at: 2025-03-18T11:13:33Z
author_association: MEMBER
diff_hunk: @@ -73,6 +94,9 @@ func (notifier *StandardNotifier) RegisterSenders(connector moira.Database) erro
 		case twilioSmsSender, twilioVoiceSender:
 			err = notifier.RegisterSender(senderSettings, &twilio.Sender{})
 		case webhookSender:
+			workerLock := connector.NewLock(workerDeliveryCheckLockKey(senderContactType), deliveryCheckLockTTL)
+			_ = workerLock
path: notifier/registrator.go
user: login: AleksandrMatsko
body: которая потом файнали заюзается в третьем PR)
created_at: 2025-03-18T11:16:40Z
author_association: MEMBER

PR comments:
user: login: github-actions[bot]
created_at: 2025-03-25T07:54:33Z
body: <!-- add-pr-comment:add-pr-comment -->

Build and push Docker images with tag: 2025-03-25.bc8dbb5
author_association: NONE