Repository: openshift/configuration-anomaly-detection
Pull request #424: OSD-29424 - Update PD incident title after investigation
Author: login: tnierman
Labels: name: approved
description: Indicates a PR has been approved by an approver from all required OWNERS files.
name: lgtm
description: Indicates that a PR is ready to be merged.
Created: 2025-04-11T21:11:07Z | Merged: 2025-04-12T13:38:44Z

https://issues.redhat.com/browse/OSD-29424

---

These changes update the PagerDuty incident title after an investigation completes to more visibly indicate when CAD has run. The goal is to reduce the potential for duplicated work as a result of on-call not seeing the investigation notes CAD attaches to the incident.

The title is only updated once: subsequent CAD runs will not re-attach the `[CAD Investigated]` prefix to the incident, in order to keep titles useful across multiple investigation attempts


Diff:
diff --git a/cadctl/cmd/investigate/investigate.go b/cadctl/cmd/investigate/investigate.go
index 06840b2a..3bfc2d3c 100644
--- a/cadctl/cmd/investigate/investigate.go
+++ b/cadctl/cmd/investigate/investigate.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1"
 	investigations "github.com/openshift/configuration-anomaly-detection/pkg/investigations"
@@ -47,6 +48,8 @@ var (
 	payloadPath  = "./payload.json"
 )
 
+const pagerdutyTitlePrefix = "[CAD Investigated]"
+
 func init() {
 	InvestigateCmd.Flags().StringVarP(&payloadPath, "payload-path", "p", payloadPath, "the path to the payload")
 	InvestigateCmd.Flags().StringVarP(&logging.LogLevelString, "log-level", "l", "", "the log level [debug,info,warn,error,fatal], default = info")
@@ -151,8 +154,11 @@ func run(cmd *cobra.Command, _ []string) error {
 	logging.Infof("Starting investigation for %s", alertInvestigation.Name())
 	result, err := alertInvestigation.Run(investigationResources)
 	updateMetrics(alertInvestigation.Name(), &result)
+	if err != nil {
+		return err
+	}
 
-	return err
+	return updateIncidentTitle(pdClient)
 }
 
 func handleCADFailure(err error, resources *investigation.Resources, pdClient *pagerduty.SdkClient) {
@@ -231,3 +237,16 @@ func updateMetrics(investigationName string, result *investigation.Investigation
 		metrics.Inc(metrics.LimitedSupportSet, append([]string{investigationName}, result.LimitedSupportSet.Labels...)...)
 	}
 }
+
+func updateIncidentTitle(pdClient *pagerduty.SdkClient) error {
+	currentTitle := pdClient.GetTitle()
+	if strings.Contains(currentTitle, pagerdutyTitlePrefix) {
+		return nil
+	}
+	newTitle := fmt.Sprintf("[CAD Investigated] %s", currentTitle)
+	err := pdClient.UpdateIncidentTitle(newTitle)
+	if err != nil {
+		return fmt.Errorf("failed to update PagerDuty incident title: %w", err)
+	}
+	return nil
+}
diff --git a/pkg/pagerduty/mock/pagerdutymock.go b/pkg/pagerduty/mock/pagerdutymock.go
index fb80ee5b..5295ac5b 100644
--- a/pkg/pagerduty/mock/pagerdutymock.go
+++ b/pkg/pagerduty/mock/pagerdutymock.go
@@ -122,3 +122,17 @@ func (mr *MockClientMockRecorder) SilenceIncidentWithNote(notes any) *gomock.Cal
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SilenceIncidentWithNote", reflect.TypeOf((*MockClient)(nil).SilenceIncidentWithNote), notes)
 }
+
+// UpdateIncidentTitle mocks base method.
+func (m *MockClient) UpdateIncidentTitle(title string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateIncidentTitle", title)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateIncidentTitle indicates an expected call of UpdateIncidentTitle.
+func (mr *MockClientMockRecorder) UpdateIncidentTitle(title any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateIncidentTitle", reflect.TypeOf((*MockClient)(nil).UpdateIncidentTitle), title)
+}
diff --git a/pkg/pagerduty/pagerduty.go b/pkg/pagerduty/pagerduty.go
index 7ecdc60c..f799bb35 100644
--- a/pkg/pagerduty/pagerduty.go
+++ b/pkg/pagerduty/pagerduty.go
@@ -51,6 +51,7 @@ type Client interface {
 	GetServiceID() string
 	EscalateIncidentWithNote(notes string) error
 	EscalateIncident() error
+	UpdateIncidentTitle(title string) error
 }
 
 // SdkClient will hold all the required fields for any SdkClient Operation
@@ -228,22 +229,22 @@ func (c *SdkClient) SetIncidentData(incidentData *IncidentData) {
 	c.incidentData = incidentData
 }
 
-// GetServiceID returns the event type of the webhook
+// GetServiceID returns the ID of the pagerduty service containing the client's incident
 func (c *SdkClient) GetServiceID() string {
 	return c.incidentData.ServiceID
 }
 
-// GetServiceName returns the event type of the webhook
+// GetServiceName returns the pagerduty service containing the client's incident
 func (c *SdkClient) GetServiceName() string {
 	return c.incidentData.ServiceSummary
 }
 
-// GetTitle returns the event type of the webhook
+// GetTitle returns the title of the client's incident
 func (c *SdkClient) GetTitle() string {
 	return c.incidentData.IncidentTitle
 }
 
-// GetIncidentID returns the event type of the webhook
+// GetIncidentID returns the ID of the client's incident
 func (c *SdkClient) GetIncidentID() string {
 	return c.incidentData.IncidentID
 }
@@ -541,3 +542,18 @@ func (c *SdkClient) EscalateIncident() error {
 	}
 	return nil
 }
+
+func (c *SdkClient) UpdateIncidentTitle(title string) error {
+	o := []sdk.ManageIncidentsOptions{
+		{
+			ID:    c.GetIncidentID(),
+			Title: title,
+		},
+	}
+
+	err := c.updateIncident(o)
+	if err != nil {
+		return fmt.Errorf("failed to update incident: %w", err)
+	}
+	return nil
+}


Reviews:
user: login: joshbranham
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T21:17:30Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T22:09:13Z
user: login: typeid
body: /lgtm
/approve
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-12T13:34:43Z

Review comments:
diff_hunk: @@ -231,3 +237,16 @@ func updateMetrics(investigationName string, result *investigation.Investigation
 		metrics.Inc(metrics.LimitedSupportSet, append([]string{investigationName}, result.LimitedSupportSet.Labels...)...)
 	}
 }
+
+func updateIncidentTitle(pdClient *pagerduty.SdkClient) error {
+	currentTitle := pdClient.GetTitle()
+	if strings.Contains(currentTitle, pagerdutyTitlePrefix) {
+		return nil
+	}
+	newTitle := fmt.Sprintf("[CAD Investigated] %s", pdClient.GetTitle())
path: cadctl/cmd/investigate/investigate.go
user: login: joshbranham
body: Let's just use the existing `currentTitle` value here to save an API call
```suggestion
	newTitle := fmt.Sprintf("[CAD Investigated] %s", currentTitle)
```
created_at: 2025-04-11T21:17:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -231,3 +237,16 @@ func updateMetrics(investigationName string, result *investigation.Investigation
 		metrics.Inc(metrics.LimitedSupportSet, append([]string{investigationName}, result.LimitedSupportSet.Labels...)...)
 	}
 }
+
+func updateIncidentTitle(pdClient *pagerduty.SdkClient) error {
+	currentTitle := pdClient.GetTitle()
+	if strings.Contains(currentTitle, pagerdutyTitlePrefix) {
+		return nil
+	}
+	newTitle := fmt.Sprintf("[CAD Investigated] %s", pdClient.GetTitle())
path: cadctl/cmd/investigate/investigate.go
user: login: tnierman
body: Whoops, good catch
created_at: 2025-04-11T22:09:13Z
author_association: MEMBER

PR comments:
user: login: openshift-ci[bot]
created_at: 2025-04-11T22:29:17Z
body: @tnierman: all tests **passed!**


[Full PR test history](https://prow.ci.openshift.org/pr-history?org=openshift&repo=configuration-anomaly-detection&pr=424). [Your PR dashboard](https://prow.ci.openshift.org/pr?query=is:pr+state:open+author:tnierman).

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository. I understand the commands that are listed [here](https://go.k8s.io/bot-commands).
</details>
<!-- test report -->
author_association: CONTRIBUTOR
user: login: codecov-commenter
created_at: 2025-04-11T22:29:34Z
body: ## [Codecov](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift) Report
Attention: Patch coverage is `0%` with `29 lines` in your changes missing coverage. Please review.
> Project coverage is 31.95%. Comparing base [(`179db6a`)](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/commit/179db6ae2797352e6485ce75e9e3c0f256075418?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift) to head [(`bfdb6d9`)](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/commit/bfdb6d9c77a5fc3fa0e4f32266c6ff211d1806a8?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift).

| [Files with missing lines](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift) | Patch % | Lines |
|---|---|---|
| [cadctl/cmd/investigate/investigate.go](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?src=pr&el=tree&filepath=cadctl%2Fcmd%2Finvestigate%2Finvestigate.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift#diff-Y2FkY3RsL2NtZC9pbnZlc3RpZ2F0ZS9pbnZlc3RpZ2F0ZS5nbw==) | 0.00% | [12 Missing :warning: ](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift) |
| [pkg/pagerduty/pagerduty.go](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?src=pr&el=tree&filepath=pkg%2Fpagerduty%2Fpagerduty.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift#diff-cGtnL3BhZ2VyZHV0eS9wYWdlcmR1dHkuZ28=) | 0.00% | [9 Missing :warning: ](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift) |
| [pkg/pagerduty/mock/pagerdutymock.go](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?src=pr&el=tree&filepath=pkg%2Fpagerduty%2Fmock%2Fpagerdutymock.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift#diff-cGtnL3BhZ2VyZHV0eS9tb2NrL3BhZ2VyZHV0eW1vY2suZ28=) | 0.00% | [8 Missing :warning: ](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift) |

<details><summary>Additional details and impacted files</summary>


[![Impacted file tree graph](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424/graphs/tree.svg?width=650&height=150&src=pr&token=6LF1WKP1IZ&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift)](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift)

```diff
@@            Coverage Diff             @@
##             main     #424      +/-   ##
==========================================
- Coverage   32.42%   31.95%   -0.47%     
==========================================
  Files          26       26              
  Lines        1937     1965      +28     
==========================================
  Hits          628      628              
- Misses       1261     1289      +28     
  Partials       48       48              
```

| [Files with missing lines](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift) | Coverage Δ | |
|---|---|---|
| [pkg/pagerduty/mock/pagerdutymock.go](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?src=pr&el=tree&filepath=pkg%2Fpagerduty%2Fmock%2Fpagerdutymock.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift#diff-cGtnL3BhZ2VyZHV0eS9tb2NrL3BhZ2VyZHV0eW1vY2suZ28=) | `35.48% <0.00%> (-5.26%)` | :arrow_down: |
| [pkg/pagerduty/pagerduty.go](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?src=pr&el=tree&filepath=pkg%2Fpagerduty%2Fpagerduty.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift#diff-cGtnL3BhZ2VyZHV0eS9wYWdlcmR1dHkuZ28=) | `62.29% <0.00%> (-2.39%)` | :arrow_down: |
| [cadctl/cmd/investigate/investigate.go](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/424?src=pr&el=tree&filepath=cadctl%2Fcmd%2Finvestigate%2Finvestigate.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift#diff-Y2FkY3RsL2NtZC9pbnZlc3RpZ2F0ZS9pbnZlc3RpZ2F0ZS5nbw==) | `0.00% <0.00%> (ø)` | |

</details>
<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: openshift-ci[bot]
created_at: 2025-04-12T13:35:02Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/openshift/configuration-anomaly-detection/pull/424#" title="Author self-approved">tnierman</a>*, *<a href="https://github.com/openshift/configuration-anomaly-detection/pull/424#pullrequestreview-2762164360" title="Approved">typeid</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=openshift%2Fconfiguration-anomaly-detection).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/openshift/configuration-anomaly-detection/blob/main/OWNERS)~~ [tnierman,typeid]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: CONTRIBUTOR