Repository: VictoriaMetrics/operator

Issue:
Support incident.io receiver in VMAlertmanagerConfig
Alertmanager just added a new incident.io receiver in their v0.29.0 version.

https://github.com/prometheus/alertmanager/releases/tag/v0.29.0
> [FEATURE] Add incident.io notifier. https://github.com/prometheus/alertmanager/pull/4372

It'd be great to have it supported in `VMAlertmanagerConfig` as we are using the VM operator to manage all our alert deployments and IIO for paging.

It should be implemented in the same way other receivers and keep tag format compatible with alertmanager.

(Similar to https://github.com/VictoriaMetrics/operator/issues/1293)

Discussion:
@swythan: Wow, that was fast! Thanks.@AndrewChubatiuk: incident.io support was added in operator [v0.66.0 ](https://github.com/VictoriaMetrics/operator/releases/tag/v0.66.0) and released in helm chart [0.57.0](https://github.com/VictoriaMetrics/helm-charts/releases/tag/victoria-metrics-operator-0.57.0)

Solution patch:
diff --git a/api/operator/v1beta1/vmalertmanagerconfig_types.go b/api/operator/v1beta1/vmalertmanagerconfig_types.go
index a529be09..1add3bf8 100644
--- a/api/operator/v1beta1/vmalertmanagerconfig_types.go
+++ b/api/operator/v1beta1/vmalertmanagerconfig_types.go
@@ -356,6 +356,8 @@ type Receiver struct {
 	// +optional
 	JiraConfigs []JiraConfig `json:"jira_configs,omitempty" yaml:"jira_configs,omitempty"`
 	// +optional
+	IncidentIOConfigs []IncidentIOConfig `json:"incidentio_configs,omitempty" yaml:"incidentio_configs,omitempty"`
+	// +optional
 	RocketchatConfigs []RocketchatConfig `json:"rocketchat_configs,omitempty" yaml:"rocketchat_configs,omitempty"`
 	// +optional
 	MSTeamsV2Configs []MSTeamsV2Config `json:"msteamsv2_configs,omitempty" yaml:"msteamsv2_configs,omitempty"`
@@ -1038,6 +1040,32 @@ type JiraConfig struct {
 	HTTPConfig *HTTPConfig `json:"http_config,omitempty" yaml:"http_config,omitempty"`
 }
 
+// IncidentIOConfig configures notifications via incident.io.
+// https://prometheus.io/docs/alerting/latest/configuration/#incidentio_config
+// available from v0.66.0 operator version
+// and v0.29.0 alertmanager version
+type IncidentIOConfig struct {
+	// SendResolved controls notify about resolved alerts.
+	// +optional
+	SendResolved *bool `json:"send_resolved,omitempty" yaml:"send_resolved,omitempty"`
+	// The URL to send the incident.io alert. This would typically be provided by the
+	// incident.io team when setting up an alert source.
+	URL string `json:"url,omitempty" yaml:"url,omitempty"`
+	// AlertSourceToken is used to authenticate with incident.io
+	// +optional
+	AlertSourceToken *corev1.SecretKeySelector `yaml:"alert_source_token,omitempty" json:"alert_source_token,omitempty"`
+	// MaxAlerts defines maximum number of alerts to be sent per incident.io message.
+	// +optional
+	MaxAlerts int `json:"max_alerts,omitempty" yaml:"max_alerts,omitempty"`
+	// Timeout is the maximum time allowed to invoke incident.io
+	// +optional
+	Timeout string `json:"timeout,omitempty" yaml:"timeout,omitempty"`
+	// +optional
+	// +kubebuilder:validation:Schemaless
+	// +kubebuilder:pruning:PreserveUnknownFields
+	HTTPConfig *HTTPConfig `json:"http_config,omitempty" yaml:"http_config,omitempty"`
+}
+
 // RocketchatConfig configures notifications via Rocketchat.
 // https://prometheus.io/docs/alerting/latest/configuration/#rocketchat_config
 // available from v0.55.0 operator version
@@ -1826,10 +1854,18 @@ func validateReceiver(recv Receiver) error {
 		}
 	}
 
+	for idx, cfg := range recv.IncidentIOConfigs {
+		if len(cfg.URL) != 0 {
+			if _, err := url.Parse(cfg.URL); err != nil {
+				return fmt.Errorf("at idx=%d for incidentio_configs incorrect url=%q: %w", idx, cfg.URL, err)
+			}
+		}
+	}
+
 	for idx, cfg := range recv.RocketchatConfigs {
 		if cfg.APIURL != nil && *cfg.APIURL != "" {
 			if _, err := url.Parse(*cfg.APIURL); err != nil {
-				return fmt.Errorf("at idx=%d for jira_configs incorrect url=%q: %w", idx, *cfg.APIURL, err)
+				return fmt.Errorf("at idx=%d for rocketchat_configs incorrect url=%q: %w", idx, *cfg.APIURL, err)
 			}
 		}
 	}
diff --git a/api/operator/v1beta1/zz_generated.deepcopy.go b/api/operator/v1beta1/zz_generated.deepcopy.go
index ca75a293..1e0dd838 100644
--- a/api/operator/v1beta1/zz_generated.deepcopy.go
+++ b/api/operator/v1beta1/zz_generated.deepcopy.go
@@ -1566,6 +1566,36 @@ func (in *ImageConfig) DeepCopy() *ImageConfig {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *IncidentIOConfig) DeepCopyInto(out *IncidentIOConfig) {
+	*out = *in
+	if in.SendResolved != nil {
+		in, out := &in.SendResolved, &out.SendResolved
+		*out = new(bool)
+		**out = **in
+	}
+	if in.AlertSourceToken != nil {
+		in, out := &in.AlertSourceToken, &out.AlertSourceToken
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.HTTPConfig != nil {
+		in, out := &in.HTTPConfig, &out.HTTPConfig
+		*out = new(HTTPConfig)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IncidentIOConfig.
+func (in *IncidentIOConfig) DeepCopy() *IncidentIOConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(IncidentIOConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *InhibitRule) DeepCopyInto(out *InhibitRule) {
 	*out = *in
@@ -2398,6 +2428,13 @@ func (in *Receiver) DeepCopyInto(out *Receiver) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.IncidentIOConfigs != nil {
+		in, out := &in.IncidentIOConfigs, &out.IncidentIOConfigs
+		*out = make([]IncidentIOConfig, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.RocketchatConfigs != nil {
 		in, out := &in.RocketchatConfigs, &out.RocketchatConfigs
 		*out = make([]RocketchatConfig, len(*in))
diff --git a/config/crd/overlay/crd.yaml b/config/crd/overlay/crd.yaml
index 64779564..37d3dae3 100644
--- a/config/crd/overlay/crd.yaml
+++ b/config/crd/overlay/crd.yaml
@@ -12859,6 +12859,60 @@ spec:
                             type: string
                         type: object
                       type: array
+                    incidentio_configs:
+                      items:
+                        description: |-
+                          IncidentIOConfig configures notifications via incident.io.
+                          https://prometheus.io/docs/alerting/latest/configuration/#incidentio_config
+                          available from v0.66.0 operator version
+                          and v0.29.0 alertmanager version
+                        properties:
+                          alert_source_token:
+                            description: AlertSourceToken is used to authenticate
+                              with incident.io
+                            properties:
+                              key:
+                                description: The key of the secret to select from.  Must
+                                  be a valid secret key.
+                                type: string
+                              name:
+                                default: ""
+                                description: |-
+                                  Name of the referent.
+                                  This field is effectively required, but due to backwards compatibility is
+                                  allowed to be empty. Instances of this type with an empty value here are
+                                  almost certainly wrong.
+                                  More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                type: string
+                              optional:
+                                description: Specify whether the Secret or its key
+                                  must be defined
+                                type: boolean
+                            required:
+                            - key
+                            type: object
+                            x-kubernetes-map-type: atomic
+                          http_config:
+                            x-kubernetes-preserve-unknown-fields: true
+                          max_alerts:
+                            description: MaxAlerts defines maximum number of alerts
+                              to be sent per incident.io message.
+                            type: integer
+                          send_resolved:
+                            description: SendResolved controls notify about resolved
+                              alerts.
+                            type: boolean
+                          timeout:
+                            description: Timeout is the maximum time allowed to invoke
+                              incident.io
+                            type: string
+                          url:
+                            description: |-
+                              The URL to send the incident.io alert. This would typically be provided by the
+                              incident.io team when setting up an alert source.
+                            type: string
+                        type: object
+                      type: array
                     jira_configs:
                       items:
                         description: |-
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3b45779a..d1cbaa73 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -18,6 +18,7 @@ aliases:
 * FEATURE: [vmoperator](https://docs.victoriametrics.com/operator/): add `VM_ENABLETCP6` variable that runs all operator CRs in IPv6 mode. See [#1581](https://github.com/VictoriaMetrics/operator/issues/1581).
 * FEATURE: [config-reloader](https://github.com/VictoriaMetrics/operator/blob/master/cmd/config-reloader/README.md): set default config reloader image version equal to current operator version. See [#2562](https://github.com/VictoriaMetrics/helm-charts/pull/2562).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): do not set `promscrape.cluster.membersCount` and `promscrape.cluster.memberNum` flags in ingestOnly mode. See [#1594](https://github.com/VictoriaMetrics/operator/issues/1594).
+* FEATURE: [vmalertmanagerconfig](https://docs.victoriametrics.com/operator/resources/vmalertmanagerconfig/): support incident.io receiver. See [#1637](https://github.com/VictoriaMetrics/operator/issues/1637).
 
 * BUGFIX: [vmalertmanager](https://docs.victoriametrics.com/operator/resources/vmalertmanager/): check `mute_time_intervals` in subroutes: See [#1618](https://github.com/VictoriaMetrics/operator/issues/1618).
 
diff --git a/docs/api.md b/docs/api.md
index 861800c1..7ae28f2d 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -1961,7 +1961,7 @@ Appears in: [VMAlertDatasourceSpec](#vmalertdatasourcespec), [VMAlertNotifierSpe
 HTTPConfig defines a client HTTP configuration for VMAlertmanagerConfig objects
 See https://prometheus.io/docs/alerting/latest/configuration/#http_config
 
-Appears in: [DiscordConfig](#discordconfig), [JiraConfig](#jiraconfig), [MSTeamsConfig](#msteamsconfig), [MSTeamsV2Config](#msteamsv2config), [OpsGenieConfig](#opsgenieconfig), [PagerDutyConfig](#pagerdutyconfig), [PushoverConfig](#pushoverconfig), [RocketchatConfig](#rocketchatconfig), [SlackConfig](#slackconfig), [SnsConfig](#snsconfig), [TelegramConfig](#telegramconfig), [VictorOpsConfig](#victoropsconfig), [WeChatConfig](#wechatconfig), [WebexConfig](#webexconfig), [WebhookConfig](#webhookconfig)
+Appears in: [DiscordConfig](#discordconfig), [IncidentIOConfig](#incidentioconfig), [JiraConfig](#jiraconfig), [MSTeamsConfig](#msteamsconfig), [MSTeamsV2Config](#msteamsv2config), [OpsGenieConfig](#opsgenieconfig), [PagerDutyConfig](#pagerdutyconfig), [PushoverConfig](#pushoverconfig), [RocketchatConfig](#rocketchatconfig), [SlackConfig](#slackconfig), [SnsConfig](#snsconfig), [TelegramConfig](#telegramconfig), [VictorOpsConfig](#victoropsconfig), [WeChatConfig](#wechatconfig), [WebexConfig](#webexconfig), [WebhookConfig](#webhookconfig)
 
 | Field | Description |
 | --- | --- |
@@ -2025,6 +2025,27 @@ Appears in: [PagerDutyConfig](#pagerdutyconfig)
 | source<a href="#imageconfig-source" id="imageconfig-source">#</a><br/>_string_ | _(Required)_<br/> |
 
 
+#### IncidentIOConfig
+
+
+
+IncidentIOConfig configures notifications via incident.io.
+https://prometheus.io/docs/alerting/latest/configuration/#incidentio_config
+available from v0.66.0 operator version
+and v0.29.0 alertmanager version
+
+Appears in: [Receiver](#receiver)
+
+| Field | Description |
+| --- | --- |
+| alert_source_token<a href="#incidentioconfig-alert_source_token" id="incidentioconfig-alert_source_token">#</a><br/>_[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#secretkeyselector-v1-core)_ | _(Optional)_<br/>AlertSourceToken is used to authenticate with incident.io |
+| http_config<a href="#incidentioconfig-http_config" id="incidentioconfig-http_config">#</a><br/>_[HTTPConfig](#httpconfig)_ | _(Optional)_<br/> |
+| max_alerts<a href="#incidentioconfig-max_alerts" id="incidentioconfig-max_alerts">#</a><br/>_integer_ | _(Optional)_<br/>MaxAlerts defines maximum number of alerts to be sent per incident.io message. |
+| send_resolved<a href="#incidentioconfig-send_resolved" id="incidentioconfig-send_resolved">#</a><br/>_boolean_ | _(Optional)_<br/>SendResolved controls notify about resolved alerts. |
+| timeout<a href="#incidentioconfig-timeout" id="incidentioconfig-timeout">#</a><br/>_string_ | _(Optional)_<br/>Timeout is the maximum time allowed to invoke incident.io |
+| url<a href="#incidentioconfig-url" id="incidentioconfig-url">#</a><br/>_string_ | _(Required)_<br/>The URL to send the incident.io alert. This would typically be provided by the<br />incident.io team when setting up an alert source. |
+
+
 #### InhibitRule
 
 
@@ -2480,6 +2501,7 @@ Appears in: [VMAlertmanagerConfigSpec](#vmalertmanagerconfigspec)
 | --- | --- |
 | discord_configs<a href="#receiver-discord_configs" id="receiver-discord_configs">#</a><br/>_[DiscordConfig](#discordconfig) array_ | _(Optional)_<br/> |
 | email_configs<a href="#receiver-email_configs" id="receiver-email_configs">#</a><br/>_[EmailConfig](#emailconfig) array_ | _(Optional)_<br/>EmailConfigs defines email notification configurations. |
+| incidentio_configs<a href="#receiver-incidentio_configs" id="receiver-incidentio_configs">#</a><br/>_[IncidentIOConfig](#incidentioconfig) array_ | _(Optional)_<br/> |
 | jira_configs<a href="#receiver-jira_configs" id="receiver-jira_configs">#</a><br/>_[JiraConfig](#jiraconfig) array_ | _(Optional)_<br/> |
 | msteams_configs<a href="#receiver-msteams_configs" id="receiver-msteams_configs">#</a><br/>_[MSTeamsConfig](#msteamsconfig) array_ | _(Optional)_<br/> |
 | msteamsv2_configs<a href="#receiver-msteamsv2_configs" id="receiver-msteamsv2_configs">#</a><br/>_[MSTeamsV2Config](#msteamsv2config) array_ | _(Optional)_<br/> |
diff --git a/docs/env.md b/docs/env.md
index 1c5744f4..fd0f0c09 100644
--- a/docs/env.md
+++ b/docs/env.md
@@ -114,7 +114,7 @@
 | VM_VMALERTMANAGER_CONFIGRELOADERCPU: `10m` <a href="#variables-vm-vmalertmanager-configreloadercpu" id="variables-vm-vmalertmanager-configreloadercpu">#</a><br>Deprecated: use VM_CONFIG_RELOADER_REQUEST_CPU instead |
 | VM_VMALERTMANAGER_CONFIGRELOADERMEMORY: `25Mi` <a href="#variables-vm-vmalertmanager-configreloadermemory" id="variables-vm-vmalertmanager-configreloadermemory">#</a><br>Deprecated: use VM_CONFIG_RELOADER_REQUEST_MEMORY instead |
 | VM_VMALERTMANAGER_ALERTMANAGERDEFAULTBASEIMAGE: `prom/alertmanager` <a href="#variables-vm-vmalertmanager-alertmanagerdefaultbaseimage" id="variables-vm-vmalertmanager-alertmanagerdefaultbaseimage">#</a> |
-| VM_VMALERTMANAGER_ALERTMANAGERVERSION: `v0.28.1` <a href="#variables-vm-vmalertmanager-alertmanagerversion" id="variables-vm-vmalertmanager-alertmanagerversion">#</a> |
+| VM_VMALERTMANAGER_ALERTMANAGERVERSION: `v0.29.0` <a href="#variables-vm-vmalertmanager-alertmanagerversion" id="variables-vm-vmalertmanager-alertmanagerversion">#</a> |
 | VM_VMALERTMANAGER_USEDEFAULTRESOURCES: `true` <a href="#variables-vm-vmalertmanager-usedefaultresources" id="variables-vm-vmalertmanager-usedefaultresources">#</a> |
 | VM_VMALERTMANAGER_RESOURCE_LIMIT_MEM: `256Mi` <a href="#variables-vm-vmalertmanager-resource-limit-mem" id="variables-vm-vmalertmanager-resource-limit-mem">#</a> |
 | VM_VMALERTMANAGER_RESOURCE_LIMIT_CPU: `100m` <a href="#variables-vm-vmalertmanager-resource-limit-cpu" id="variables-vm-vmalertmanager-resource-limit-cpu">#</a> |
diff --git a/internal/config/config.go b/internal/config/config.go
index 79455657..987955fa 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -364,7 +364,7 @@ type BaseOperatorConf struct {
 		// Deprecated: use VM_CONFIG_RELOADER_REQUEST_MEMORY instead
 		ConfigReloaderMemory         string `default:"25Mi" env:"CONFIGRELOADERMEMORY"`
 		AlertmanagerDefaultBaseImage string `default:"prom/alertmanager" env:"ALERTMANAGERDEFAULTBASEIMAGE"`
-		AlertManagerVersion          string `default:"v0.28.1" env:"ALERTMANAGERVERSION"`
+		AlertManagerVersion          string `default:"v0.29.0" env:"ALERTMANAGERVERSION"`
 		UseDefaultResources          bool   `default:"true" env:"USEDEFAULTRESOURCES"`
 		Resource                     struct {
 			Limit struct {
diff --git a/internal/controller/operator/factory/vmalertmanager/config.go b/internal/controller/operator/factory/vmalertmanager/config.go
index ef52b1a9..a2d661bd 100644
--- a/internal/controller/operator/factory/vmalertmanager/config.go
+++ b/internal/controller/operator/factory/vmalertmanager/config.go
@@ -513,6 +513,13 @@ func (cb *configBuilder) buildCfg(receiver vmv1beta1.Receiver) error {
 	}
 	cb.finalizeSection("rocketchat_configs")
 
+	for _, c := range receiver.IncidentIOConfigs {
+		if err := cb.buildIncidentIO(c); err != nil {
+			return err
+		}
+	}
+	cb.finalizeSection("incidentio_configs")
+
 	for _, rcCfg := range receiver.MSTeamsV2Configs {
 		if err := cb.buildTeamsV2(rcCfg); err != nil {
 			return err
@@ -761,6 +768,43 @@ func (cb *configBuilder) buildJira(jira vmv1beta1.JiraConfig) error {
 	return nil
 }
 
+func (cb *configBuilder) buildIncidentIO(rc vmv1beta1.IncidentIOConfig) error {
+	var temp yaml.MapSlice
+	if rc.HTTPConfig != nil {
+		c, err := cb.buildHTTPConfig(rc.HTTPConfig)
+		if err != nil {
+			return err
+		}
+		temp = append(temp, yaml.MapItem{Key: "http_config", Value: c})
+	}
+	toYaml := func(key string, src string) {
+		if len(src) > 0 {
+			temp = append(temp, yaml.MapItem{Key: key, Value: src})
+		}
+	}
+	if rc.AlertSourceToken != nil {
+		secret, err := cb.cache.LoadKeyFromSecret(cb.namespace, rc.AlertSourceToken)
+		if err != nil {
+			return err
+		}
+		toYaml("alert_source_token", secret)
+	}
+	if rc.SendResolved != nil {
+		temp = append(temp, yaml.MapItem{Key: "send_resolved", Value: *rc.SendResolved})
+	}
+	if len(rc.URL) > 0 {
+		toYaml("url", rc.URL)
+	}
+	if len(rc.Timeout) > 0 {
+		toYaml("timeout", rc.Timeout)
+	}
+	if rc.MaxAlerts > 0 {
+		temp = append(temp, yaml.MapItem{Key: "max_alerts", Value: rc.MaxAlerts})
+	}
+	cb.currentYaml = append(cb.currentYaml, temp)
+	return nil
+}
+
 func (cb *configBuilder) buildRocketchat(rc vmv1beta1.RocketchatConfig) error {
 	if rc.TokenID == nil {
 		if cb.globalConfig.Global.RocketchatTokenID == "" && len(cb.globalConfig.Global.RocketchatTokenIDFile) == 0 {



Test patch:
diff --git a/internal/controller/operator/factory/vmalertmanager/config_test.go b/internal/controller/operator/factory/vmalertmanager/config_test.go
index 375c59a6..1eb1a927 100644
--- a/internal/controller/operator/factory/vmalertmanager/config_test.go
+++ b/internal/controller/operator/factory/vmalertmanager/config_test.go
@@ -1232,6 +1232,66 @@ receivers:
       url: https://example.com/action
       msg: some message
 templates: []
+`,
+		},
+		{
+			name: "incidentio",
+			args: args{
+				amcfgs: []*vmv1beta1.VMAlertmanagerConfig{
+					{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "incidentio-dev",
+							Namespace: "default",
+						},
+						Spec: vmv1beta1.VMAlertmanagerConfigSpec{
+							Route: &vmv1beta1.Route{
+								Receiver: "incidentio",
+							},
+							Receivers: []vmv1beta1.Receiver{
+								{
+									Name: "incidentio",
+									IncidentIOConfigs: []vmv1beta1.IncidentIOConfig{
+										{
+											URL: "http://example.com/",
+											AlertSourceToken: &corev1.SecretKeySelector{
+												Key: "TOKEN",
+												LocalObjectReference: corev1.LocalObjectReference{
+													Name: "incidentio-access",
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			predefinedObjects: []runtime.Object{
+				&corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "incidentio-access",
+						Namespace: "default",
+					},
+					Data: map[string][]byte{
+						"TOKEN": []byte(`token value`),
+					},
+				},
+			},
+			want: `route:
+  receiver: blackhole
+  routes:
+  - matchers:
+    - namespace = "default"
+    receiver: default-incidentio-dev-incidentio
+    continue: true
+receivers:
+- name: blackhole
+- name: default-incidentio-dev-incidentio
+  incidentio_configs:
+  - alert_source_token: token value
+    url: http://example.com/
+templates: []
 `,
 		},
 		{