Repository: openshift/configuration-anomaly-detection
Pull request #404: OSD-18645 -  Initial implementation for CannotRetrieveUpdatesSRE
Author: login: anispate
Labels: name: approved
description: Indicates a PR has been approved by an approver from all required OWNERS files.
name: lgtm
description: Indicates that a PR is ready to be merged.
Created: 2025-04-04T18:42:45Z | Merged: 2025-05-28T15:58:38Z

[OSD-18645](https://issues.redhat.com/browse/OSD-18645) - CAD implementation for CannotRetrieveUpdatesSRE

Sample ticket: https://redhat.pagerduty.com/incidents/Q1S45W54TK1QKU#:~:text=%E2%9A%A0%EF%B8%8F%20ClusterVersion%20error%20detected,primary%20for%20review 

Updated sample ticket: https://redhat.pagerduty.com/incidents/Q2UVRI8YGLPP3G

Updated sample ticket: https://redhat.pagerduty.com/incidents/Q01H4ZQHM90EML


Diff:
diff --git a/pkg/investigations/cannotretrieveupdatessre/README.md b/pkg/investigations/cannotretrieveupdatessre/README.md
new file mode 100644
index 00000000..b47ad493
--- /dev/null
+++ b/pkg/investigations/cannotretrieveupdatessre/README.md
@@ -0,0 +1,13 @@
+# cannotretrieveupdatessre Investigation
+
+Investigates the CannotRetrieveUpdatesSRE alert by running network verifier and updating the notes with investigation details to the PagerDuty alert about the cluster version status.
+
+## Investigation Logic
+
+The `CannotRetrieveUpdatesSRE` investigation is designed to diagnose issues where an OpenShift cluster cannot retrieve updates from its configured channel. It performs two main checks:
+1. **Network Verification**: Uses the `networkverifier` package to ensure the cluster can reach required update endpoints.
+2. **ClusterVersion Check**: Examines the `ClusterVersion` resource for conditions indicating update retrieval failures, such as `VersionNotFound`.
+
+## Testing
+
+Refer to the [testing README](./testing/README.md) for instructions on testing this investigation
diff --git a/pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go b/pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
new file mode 100644
index 00000000..10fa0628
--- /dev/null
+++ b/pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
@@ -0,0 +1,117 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		deferErr := k8scli.Clean()
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s", r.Cluster.ID(), failureReason)
+		case networkverifier.Success:
+			notes.AppendSuccess("Network verifier passed")
+		}
+	}
+
+	// Check ClusterVersion
+	clusterVersion, err := getClusterVersion(k8scli)
+	if err != nil {
+		notes.AppendWarning("Failed to get ClusterVersion: %s", err.Error())
+	} else {
+		notes.AppendSuccess("ClusterVersion found: %s", clusterVersion.Status.Desired.Version)
+
+		failureReason := getUpdateRetrievalFailures(clusterVersion)
+		if failureReason != "" {
+			logging.Warnf("Detected ClusterVersion issue: %s", failureReason)
+			notes.AppendWarning("ClusterVersion related issue detected: %s. Current version %s not found in channel %s",
+				failureReason, clusterVersion.Status.Desired.Version, clusterVersion.Spec.Channel)
+		}
+	}
+	notes.AppendWarning("Alert escalated to on-call primary for review and please check the ClusterVersion.")
+	return result, r.PdClient.EscalateIncidentWithNote(notes.String())
+}
+
+func getClusterVersion(k8scli client.Client) (*configv1.ClusterVersion, error) {
+	clusterVersion := &configv1.ClusterVersion{}
+	err := k8scli.Get(context.TODO(), client.ObjectKey{Name: "version"}, clusterVersion)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get ClusterVersion: %w", err)
+	}
+	return clusterVersion, nil
+}
+
+// getUpdateRetrievalFailures checks for update retrieval failures in the ClusterVersion
+func getUpdateRetrievalFailures(clusterVersion *configv1.ClusterVersion) string {
+	for _, condition := range clusterVersion.Status.Conditions {
+		msg, found := checkCondition(condition)
+		if found {
+			return msg
+		}
+	}
+	return ""
+}
+
+func checkCondition(condition configv1.ClusterOperatorStatusCondition) (string, bool) {
+	if condition.Type != "RetrievedUpdates" {
+		return "", false
+	}
+	if condition.Status == configv1.ConditionFalse {
+		return fmt.Sprintf("(Reason: %s). %s", condition.Reason, condition.Message), true
+	}
+	return "", false
+}
+
+func (i *Investigation) Name() string {
+	return alertname
+}
+
+func (i *Investigation) Description() string {
+	return fmt.Sprintf("Investigates '%s' alerts by running network verifier and checking ClusterVersion", alertname)
+}
+
+func (i *Investigation) ShouldInvestigateAlert(alert string) bool {
+	return strings.Contains(alert, alertname)
+}
+
+func (i *Investigation) IsExperimental() bool {
+	return true
+}
diff --git a/pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre_test.go b/pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre_test.go
new file mode 100644
index 00000000..c2c562fe
--- /dev/null
+++ b/pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre_test.go
@@ -0,0 +1,177 @@
+package cannotretrieveupdatessre
+
+import (
+	"strings"
+	"testing"
+
+	configv1 "github.com/openshift/api/config/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func newFakeClient(objs ...client.Object) (client.Client, error) {
+	s := scheme.Scheme
+	err := configv1.AddToScheme(s)
+	if err != nil {
+		return nil, err
+	}
+
+	client := fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build()
+	return client, nil
+}
+
+func TestGetClusterVersion(t *testing.T) {
+	tests := []struct {
+		name            string
+		clusterVersion  *configv1.ClusterVersion
+		expectedVersion string
+		expectError     bool
+	}{
+		{
+			name: "Valid ClusterVersion",
+			clusterVersion: &configv1.ClusterVersion{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "version",
+				},
+				Spec: configv1.ClusterVersionSpec{
+					Channel:   "stable-4.18",
+					ClusterID: "d1ba89f3-fd3e-48d2-91c6-test",
+				},
+				Status: configv1.ClusterVersionStatus{
+					Desired: configv1.Release{Version: "4.18.10"},
+				},
+			},
+			expectedVersion: "4.18.10",
+			expectError:     false,
+		},
+		{
+			name:            "ClusterVersion Not Found",
+			clusterVersion:  nil,
+			expectedVersion: "",
+			expectError:     true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var k8scli client.Client
+			var err error
+			if tt.clusterVersion != nil {
+				k8scli, err = newFakeClient(tt.clusterVersion)
+			} else {
+				k8scli, err = newFakeClient()
+			}
+			if err != nil {
+				t.Fatalf("failed to create a fake client: %v", err)
+			}
+
+			got, err := getClusterVersion(k8scli)
+
+			if tt.expectError && err == nil {
+				t.Errorf("Expected an error, got none")
+			} else if !tt.expectError && err != nil {
+				t.Errorf("Expected no error, got %v", err)
+			}
+
+			if !tt.expectError {
+				if got.Status.Desired.Version != tt.expectedVersion {
+					t.Errorf("Expected version %q, got %q", tt.expectedVersion, got.Status.Desired.Version)
+				}
+			} else {
+				if got != nil {
+					t.Errorf("Expected nil ClusterVersion error, got %v", got)
+				}
+				if err != nil && !apierrors.IsNotFound(err) && !strings.Contains(err.Error(), "failed to get ClusterVersion") {
+					t.Errorf("Expected error to be related about failed to get the ClusterVersion, got %v", err)
+				}
+			}
+		})
+	}
+}
+
+func TestGetUpdateRetrievalFailures(t *testing.T) {
+	tests := []struct {
+		name           string
+		clusterVersion *configv1.ClusterVersion
+		expectedNote   string
+	}{
+		{
+			name: "RemoteFailed condition",
+			clusterVersion: &configv1.ClusterVersion{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "version",
+				},
+				Spec: configv1.ClusterVersionSpec{
+					Channel: "stable-4.18",
+				},
+				Status: configv1.ClusterVersionStatus{
+					Conditions: []configv1.ClusterOperatorStatusCondition{
+						{
+							Type:    "RetrievedUpdates",
+							Status:  configv1.ConditionFalse,
+							Reason:  "RemoteFailed",
+							Message: "Unable to retrieve available updates",
+						},
+					},
+				},
+			},
+			expectedNote: "(Reason: RemoteFailed). Unable to retrieve available updates",
+		},
+		{
+			name: "VersionNotFound condition",
+			clusterVersion: &configv1.ClusterVersion{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "version",
+				},
+				Spec: configv1.ClusterVersionSpec{
+					Channel: "stable-4.18",
+				},
+				Status: configv1.ClusterVersionStatus{
+					Conditions: []configv1.ClusterOperatorStatusCondition{
+						{
+							Type:    "RetrievedUpdates",
+							Status:  configv1.ConditionFalse,
+							Reason:  "VersionNotFound",
+							Message: "Unable to retrieve available updates",
+						},
+					},
+				},
+			},
+			expectedNote: "(Reason: VersionNotFound). Unable to retrieve available updates",
+		},
+		{
+			name: "Happy path",
+			clusterVersion: &configv1.ClusterVersion{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "version",
+				},
+				Spec: configv1.ClusterVersionSpec{
+					Channel: "stable-4.18",
+				},
+				Status: configv1.ClusterVersionStatus{
+					Conditions: []configv1.ClusterOperatorStatusCondition{
+						{
+							Type:    "RetrievedUpdates",
+							Status:  configv1.ConditionTrue,
+							Reason:  "UpdatesRetrieved",
+							Message: "Available updates retrieved successfully",
+						},
+					},
+				},
+			},
+			expectedNote: "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			reason := getUpdateRetrievalFailures(tt.clusterVersion)
+			if reason != tt.expectedNote {
+				t.Errorf("Expected note %q, got %q", tt.expectedNote, reason)
+			}
+		})
+	}
+}
diff --git a/pkg/investigations/CannotRetrieveUpdatesSRE/metadata.yaml b/pkg/investigations/cannotretrieveupdatessre/metadata.yaml
similarity index 100%
rename from pkg/investigations/CannotRetrieveUpdatesSRE/metadata.yaml
rename to pkg/investigations/cannotretrieveupdatessre/metadata.yaml
diff --git a/pkg/investigations/cannotretrieveupdatessre/testing/README.md b/pkg/investigations/cannotretrieveupdatessre/testing/README.md
new file mode 100644
index 00000000..f12c5950
--- /dev/null
+++ b/pkg/investigations/cannotretrieveupdatessre/testing/README.md
@@ -0,0 +1,17 @@
+# Testing CannotRetrieveUpdatesSRE Investigation
+
+### Update the ClusterVersion Channel
+- Below script helps to set the test channel to check the clusterversion change.
+```sh
+#!/bin/bash
+
+# Use test channel for the ClusterVersion
+oc patch clusterversion version --type merge -p '{"spec":{"channel":"stable-4.18-test"}}' --as backplane-cluster-admin
+sleep 30
+
+# Verify
+oc get clusterversion version -o jsonpath='{.spec.channel}' | grep "stable-4.18-test" || { echo "Failed to set the channel"; exit 1; }
+
+# Optional: Revert back to the original change
+#oc patch clusterversion version --type merge -p '{"spec":{"channel":"stable-4.18"}}' --as backplane-cluster-admin
+```
diff --git a/pkg/investigations/registry.go b/pkg/investigations/registry.go
index d312feb5..7e921cd0 100644
--- a/pkg/investigations/registry.go
+++ b/pkg/investigations/registry.go
@@ -2,6 +2,7 @@ package investigations
 
 import (
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/apierrorbudgetburn"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/cannotretrieveupdatessre"
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/ccam"
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/chgm"
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/clustermonitoringerrorbudgetburn"
@@ -22,6 +23,7 @@ var availableInvestigations = []investigation.Investigation{
 	&insightsoperatordown.Investigation{},
 	&upgradeconfigsyncfailureover4hr.Investigation{},
 	&machinehealthcheckunterminatedshortcircuitsre.Investigation{},
+	&cannotretrieveupdatessre.Investigation{},
 }
 
 // GetInvestigation returns the first Investigation that applies to the given alert title.
diff --git a/test/generate_incident.sh b/test/generate_incident.sh
index b0a13dff..6378d73b 100755
--- a/test/generate_incident.sh
+++ b/test/generate_incident.sh
@@ -10,6 +10,7 @@ declare -A alert_mapping=(
     ["InsightsOperatorDown"]="InsightsOperatorDown"
     ["MachineHealthCheckUnterminatedShortCircuitSRE"]="MachineHealthCheckUnterminatedShortCircuitSRE CRITICAL (1)"
     ["ApiErrorBudgetBurn"]="api-ErrorBudgetBurn k8sgpt test CRITICAL (1)"
+    ["CannotRetrieveUpdatesSRE"]="CannotRetrieveUpdatesSRE"
 )
 
 # Function to print help message


Reviews:
user: login: joshbranham
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T20:17:46Z
user: login: joshbranham
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T20:18:16Z
user: login: joshbranham
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:33:01Z
user: login: anispate
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T16:57:17Z
user: login: joshbranham
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T17:11:02Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-15T17:07:41Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-15T17:12:03Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T15:23:27Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T15:25:12Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T15:26:06Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T15:27:28Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T15:56:50Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T15:59:20Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T21:19:24Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T21:21:35Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T21:22:47Z
user: login: anispate
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T21:24:50Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T21:25:32Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T21:34:15Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T21:37:18Z
user: login: typeid
state: CHANGES_REQUESTED
author_association: MEMBER
submitted_at: 2025-05-13T10:52:52Z
user: login: anispate
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-13T14:53:52Z
user: login: typeid
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-14T10:15:27Z
user: login: clcollins
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-27T17:29:18Z
user: login: clcollins
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-27T19:13:08Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-27T20:18:04Z
user: login: tnierman
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-27T20:20:47Z

Review comments:
diff_hunk: @@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/zsh
path: test/generate_incident.sh
user: login: joshbranham
body: This should be
```suggestion
#!/usr/bin/env bash
```
created_at: 2025-04-09T20:17:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1,6 +1,7 @@
 package investigations
 
 import (
+	CannotRetrieveUpdatesSRE "github.com/openshift/configuration-anomaly-detection/pkg/investigations/CannotRetrieveUpdatesSRE"
path: pkg/investigations/registry.go
user: login: joshbranham
body: Let's keep the existing pattern and not import this with a custom capitalized package name
created_at: 2025-04-09T20:18:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,153 @@
+package CannotRetrieveUpdatesSRE
path: pkg/investigations/CannotRetrieveUpdatesSRE/CannotRetrieveUpdatesSRE.go
user: login: joshbranham
body: This should be a lowercase package name like `cannotretrieveupdatessre`
created_at: 2025-04-09T20:51:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,153 @@
+package CannotRetrieveUpdatesSRE
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"k8s.io/apimachinery/pkg/fields"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct {
+	kclient client.Client
+	notes   *notewriter.NoteWriter
+}
+
+// setup initializes the investigation resources
+func (i *Investigation) setup(r *investigation.Resources) error {
+	logging.Infof("Setting up investigation '%s' for cluster %s with remediation name %s",
+		i.Name(), r.Cluster.ID(), r.Name)
+
+	k, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		logging.Errorf("Failed to initialize Kubernetes client: %v", err)
+		return fmt.Errorf("failed to initialize kubernetes client: %w", err)
+	}
+	i.kclient = k
+	i.notes = notewriter.New(r.Name, logging.RawLogger)
+
+	logging.Infof("Successfully set up Kubernetes client and notewriter for remediation %s", r.Name)
+	return nil
+}
+
+// cleanup handles resource cleanup after investigation
+func (i *Investigation) cleanup(r *investigation.Resources) error {
+	logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+	err := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		logging.Errorf("Failed to cleanup Kubernetes client: %v", err)
+		return fmt.Errorf("failed to cleanup kubernetes client: %w", err)
+	}
+	logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+	return nil
+}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (i *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
path: pkg/investigations/CannotRetrieveUpdatesSRE/CannotRetrieveUpdatesSRE.go
user: login: joshbranham
body: `nit`: If this is a public function and the main entry/usage of this package, lets put it first in this file. It helps those reading the code to read it as if it were be executed without having to scroll past a bunch of private functions first.
created_at: 2025-04-09T20:56:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,153 @@
+package CannotRetrieveUpdatesSRE
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"k8s.io/apimachinery/pkg/fields"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct {
+	kclient client.Client
+	notes   *notewriter.NoteWriter
+}
+
+// setup initializes the investigation resources
+func (i *Investigation) setup(r *investigation.Resources) error {
+	logging.Infof("Setting up investigation '%s' for cluster %s with remediation name %s",
+		i.Name(), r.Cluster.ID(), r.Name)
+
+	k, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		logging.Errorf("Failed to initialize Kubernetes client: %v", err)
+		return fmt.Errorf("failed to initialize kubernetes client: %w", err)
+	}
+	i.kclient = k
+	i.notes = notewriter.New(r.Name, logging.RawLogger)
+
+	logging.Infof("Successfully set up Kubernetes client and notewriter for remediation %s", r.Name)
+	return nil
+}
+
+// cleanup handles resource cleanup after investigation
+func (i *Investigation) cleanup(r *investigation.Resources) error {
+	logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+	err := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		logging.Errorf("Failed to cleanup Kubernetes client: %v", err)
+		return fmt.Errorf("failed to cleanup kubernetes client: %w", err)
+	}
+	logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+	return nil
+}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (i *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+
+	// Setup & teardown
+	err := i.setup(r)
+	if err != nil {
+		return result, fmt.Errorf("failed to setup investigation: %w", err)
+	}
+	defer func(r *investigation.Resources) {
+		if err := i.cleanup(r); err != nil {
+			logging.Errorf("Failed to cleanup investigation: %v", err)
+		}
+	}(r)
+
+	if r.Cluster == nil || r.Cluster.ID() == "" {
+		errMsg := "Invalid cluster configuration: cluster or cluster ID is missing"
+		logging.Errorf(errMsg)
+		i.notes.AppendWarning(errMsg)
+		return result, r.PdClient.EscalateIncidentWithNote(i.notes.String())
+	}
+
+	// Run network verification
+	logging.Infof("Running network verification for cluster %s", r.Cluster.ID())
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		logging.Errorf("Network verifier failed: %v", err)
+		i.notes.AppendWarning("Network verifier encountered an error: %v", err)
+	} else {
+		logging.Infof("Network verification completed with result: %v", verifierResult)
+		switch verifierResult {
+		case networkverifier.Success:
+			i.notes.AppendSuccess("Network verifier passed")
+		case networkverifier.Failure:
+			logging.Infof("Network verifier reported failure: %s", failureReason)
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			i.notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s",
+				r.Cluster.ID(), failureReason)
+		}
path: pkg/investigations/CannotRetrieveUpdatesSRE/CannotRetrieveUpdatesSRE.go
user: login: joshbranham
body: This is a bit tight, I would suggest something like the following to make it a bit easier to read each thing occurring in this `else`
```suggestion
		logging.Infof("Network verification completed with result: %v", verifierResult)

		switch verifierResult {
		case networkverifier.Success:
			i.notes.AppendSuccess("Network verifier passed")
		case networkverifier.Failure:
			logging.Infof("Network verifier reported failure: %s", failureReason)
			
			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
			i.notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s",
				r.Cluster.ID(), failureReason)
		}
```
created_at: 2025-04-09T20:59:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,153 @@
+package CannotRetrieveUpdatesSRE
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"k8s.io/apimachinery/pkg/fields"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct {
+	kclient client.Client
+	notes   *notewriter.NoteWriter
+}
+
+// setup initializes the investigation resources
+func (i *Investigation) setup(r *investigation.Resources) error {
+	logging.Infof("Setting up investigation '%s' for cluster %s with remediation name %s",
+		i.Name(), r.Cluster.ID(), r.Name)
+
+	k, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		logging.Errorf("Failed to initialize Kubernetes client: %v", err)
+		return fmt.Errorf("failed to initialize kubernetes client: %w", err)
+	}
+	i.kclient = k
+	i.notes = notewriter.New(r.Name, logging.RawLogger)
+
+	logging.Infof("Successfully set up Kubernetes client and notewriter for remediation %s", r.Name)
+	return nil
+}
+
+// cleanup handles resource cleanup after investigation
+func (i *Investigation) cleanup(r *investigation.Resources) error {
+	logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+	err := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		logging.Errorf("Failed to cleanup Kubernetes client: %v", err)
+		return fmt.Errorf("failed to cleanup kubernetes client: %w", err)
+	}
+	logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+	return nil
+}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (i *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+
+	// Setup & teardown
+	err := i.setup(r)
+	if err != nil {
+		return result, fmt.Errorf("failed to setup investigation: %w", err)
+	}
+	defer func(r *investigation.Resources) {
+		if err := i.cleanup(r); err != nil {
+			logging.Errorf("Failed to cleanup investigation: %v", err)
+		}
+	}(r)
+
+	if r.Cluster == nil || r.Cluster.ID() == "" {
+		errMsg := "Invalid cluster configuration: cluster or cluster ID is missing"
+		logging.Errorf(errMsg)
+		i.notes.AppendWarning(errMsg)
+		return result, r.PdClient.EscalateIncidentWithNote(i.notes.String())
+	}
+
+	// Run network verification
+	logging.Infof("Running network verification for cluster %s", r.Cluster.ID())
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		logging.Errorf("Network verifier failed: %v", err)
+		i.notes.AppendWarning("Network verifier encountered an error: %v", err)
+	} else {
+		logging.Infof("Network verification completed with result: %v", verifierResult)
+		switch verifierResult {
+		case networkverifier.Success:
+			i.notes.AppendSuccess("Network verifier passed")
+		case networkverifier.Failure:
+			logging.Infof("Network verifier reported failure: %s", failureReason)
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			i.notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s",
+				r.Cluster.ID(), failureReason)
+		}
+	}
+
+	// Check ClusterVersion
+	logging.Infof("Checking ClusterVersion for cluster %s", r.Cluster.ID())
+	cvList := &configv1.ClusterVersionList{}
+	listOptions := &client.ListOptions{FieldSelector: fields.SelectorFromSet(fields.Set{"metadata.name": "version"})}
+	err = i.kclient.List(context.TODO(), cvList, listOptions)
+	switch {
+	case err != nil:
+		logging.Errorf("Failed to list ClusterVersion: %v", err)
+		i.notes.AppendWarning("Failed to list ClusterVersion: %v\nThis may indicate cluster access issues", err)
+	case len(cvList.Items) != 1:
+		logging.Warnf("Found %d ClusterVersions, expected 1", len(cvList.Items))
+		i.notes.AppendWarning("Found %d ClusterVersions, expected 1", len(cvList.Items))
+	default:
+		versionCv := cvList.Items[0]
+		logging.Infof("ClusterVersion found: %s", versionCv.Status.Desired.Version)
+		for _, condition := range versionCv.Status.Conditions {
+			logging.Debugf("Checking ClusterVersion condition: Type=%s, Status=%s, Reason=%s, Message=%s",
+				condition.Type, condition.Status, condition.Reason, condition.Message)
+			if condition.Type == "RetrievedUpdates" &&
+				condition.Status == "False" &&
+				condition.Reason == "VersionNotFound" &&
+				strings.Contains(condition.Message, "Unable to retrieve available updates") {
+				i.notes.AppendWarning("ClusterVersion error detected: %s\nThis indicates the current version %s is not found in the specified channel %s",
+					condition.Message, versionCv.Status.Desired.Version, versionCv.Spec.Channel)
+			}
+		}
+		fmt.Printf("Cluster version: %s\n", versionCv.Status.Desired.Version)
+	}
path: pkg/investigations/CannotRetrieveUpdatesSRE/CannotRetrieveUpdatesSRE.go
user: login: joshbranham
body: This would be cleaner without a switch block. Just perform conditional checks for your each of your failures before doing the `default` behavior
created_at: 2025-04-09T21:00:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,153 @@
+package CannotRetrieveUpdatesSRE
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"k8s.io/apimachinery/pkg/fields"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct {
+	kclient client.Client
+	notes   *notewriter.NoteWriter
+}
+
+// setup initializes the investigation resources
+func (i *Investigation) setup(r *investigation.Resources) error {
+	logging.Infof("Setting up investigation '%s' for cluster %s with remediation name %s",
+		i.Name(), r.Cluster.ID(), r.Name)
+
+	k, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		logging.Errorf("Failed to initialize Kubernetes client: %v", err)
+		return fmt.Errorf("failed to initialize kubernetes client: %w", err)
+	}
+	i.kclient = k
+	i.notes = notewriter.New(r.Name, logging.RawLogger)
+
+	logging.Infof("Successfully set up Kubernetes client and notewriter for remediation %s", r.Name)
+	return nil
+}
+
+// cleanup handles resource cleanup after investigation
+func (i *Investigation) cleanup(r *investigation.Resources) error {
+	logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+	err := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		logging.Errorf("Failed to cleanup Kubernetes client: %v", err)
+		return fmt.Errorf("failed to cleanup kubernetes client: %w", err)
+	}
+	logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+	return nil
+}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (i *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+
+	// Setup & teardown
+	err := i.setup(r)
+	if err != nil {
+		return result, fmt.Errorf("failed to setup investigation: %w", err)
+	}
+	defer func(r *investigation.Resources) {
+		if err := i.cleanup(r); err != nil {
+			logging.Errorf("Failed to cleanup investigation: %v", err)
+		}
+	}(r)
+
+	if r.Cluster == nil || r.Cluster.ID() == "" {
+		errMsg := "Invalid cluster configuration: cluster or cluster ID is missing"
+		logging.Errorf(errMsg)
+		i.notes.AppendWarning(errMsg)
+		return result, r.PdClient.EscalateIncidentWithNote(i.notes.String())
+	}
+
+	// Run network verification
+	logging.Infof("Running network verification for cluster %s", r.Cluster.ID())
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		logging.Errorf("Network verifier failed: %v", err)
+		i.notes.AppendWarning("Network verifier encountered an error: %v", err)
+	} else {
+		logging.Infof("Network verification completed with result: %v", verifierResult)
+		switch verifierResult {
+		case networkverifier.Success:
+			i.notes.AppendSuccess("Network verifier passed")
+		case networkverifier.Failure:
+			logging.Infof("Network verifier reported failure: %s", failureReason)
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			i.notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s",
+				r.Cluster.ID(), failureReason)
+		}
+	}
+
+	// Check ClusterVersion
+	logging.Infof("Checking ClusterVersion for cluster %s", r.Cluster.ID())
+	cvList := &configv1.ClusterVersionList{}
+	listOptions := &client.ListOptions{FieldSelector: fields.SelectorFromSet(fields.Set{"metadata.name": "version"})}
+	err = i.kclient.List(context.TODO(), cvList, listOptions)
+	switch {
+	case err != nil:
+		logging.Errorf("Failed to list ClusterVersion: %v", err)
+		i.notes.AppendWarning("Failed to list ClusterVersion: %v\nThis may indicate cluster access issues", err)
+	case len(cvList.Items) != 1:
+		logging.Warnf("Found %d ClusterVersions, expected 1", len(cvList.Items))
+		i.notes.AppendWarning("Found %d ClusterVersions, expected 1", len(cvList.Items))
+	default:
+		versionCv := cvList.Items[0]
+		logging.Infof("ClusterVersion found: %s", versionCv.Status.Desired.Version)
+		for _, condition := range versionCv.Status.Conditions {
+			logging.Debugf("Checking ClusterVersion condition: Type=%s, Status=%s, Reason=%s, Message=%s",
+				condition.Type, condition.Status, condition.Reason, condition.Message)
+			if condition.Type == "RetrievedUpdates" &&
+				condition.Status == "False" &&
+				condition.Reason == "VersionNotFound" &&
+				strings.Contains(condition.Message, "Unable to retrieve available updates") {
+				i.notes.AppendWarning("ClusterVersion error detected: %s\nThis indicates the current version %s is not found in the specified channel %s",
+					condition.Message, versionCv.Status.Desired.Version, versionCv.Spec.Channel)
+			}
+		}
+		fmt.Printf("Cluster version: %s\n", versionCv.Status.Desired.Version)
+	}
path: pkg/investigations/CannotRetrieveUpdatesSRE/CannotRetrieveUpdatesSRE.go
user: login: anispate
body: I tried to do without a switch block but then one of the test was failing and it recommended me to use the switch case that time. This was the test that failed: https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/openshift_configuration-anomaly-detection/404/pull-ci-openshift-configuration-anomaly-detection-main-lint/1909316667297501184 
created_at: 2025-04-10T16:57:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,153 @@
+package CannotRetrieveUpdatesSRE
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"k8s.io/apimachinery/pkg/fields"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct {
+	kclient client.Client
+	notes   *notewriter.NoteWriter
+}
+
+// setup initializes the investigation resources
+func (i *Investigation) setup(r *investigation.Resources) error {
+	logging.Infof("Setting up investigation '%s' for cluster %s with remediation name %s",
+		i.Name(), r.Cluster.ID(), r.Name)
+
+	k, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		logging.Errorf("Failed to initialize Kubernetes client: %v", err)
+		return fmt.Errorf("failed to initialize kubernetes client: %w", err)
+	}
+	i.kclient = k
+	i.notes = notewriter.New(r.Name, logging.RawLogger)
+
+	logging.Infof("Successfully set up Kubernetes client and notewriter for remediation %s", r.Name)
+	return nil
+}
+
+// cleanup handles resource cleanup after investigation
+func (i *Investigation) cleanup(r *investigation.Resources) error {
+	logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+	err := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		logging.Errorf("Failed to cleanup Kubernetes client: %v", err)
+		return fmt.Errorf("failed to cleanup kubernetes client: %w", err)
+	}
+	logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+	return nil
+}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (i *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+
+	// Setup & teardown
+	err := i.setup(r)
+	if err != nil {
+		return result, fmt.Errorf("failed to setup investigation: %w", err)
+	}
+	defer func(r *investigation.Resources) {
+		if err := i.cleanup(r); err != nil {
+			logging.Errorf("Failed to cleanup investigation: %v", err)
+		}
+	}(r)
+
+	if r.Cluster == nil || r.Cluster.ID() == "" {
+		errMsg := "Invalid cluster configuration: cluster or cluster ID is missing"
+		logging.Errorf(errMsg)
+		i.notes.AppendWarning(errMsg)
+		return result, r.PdClient.EscalateIncidentWithNote(i.notes.String())
+	}
+
+	// Run network verification
+	logging.Infof("Running network verification for cluster %s", r.Cluster.ID())
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		logging.Errorf("Network verifier failed: %v", err)
+		i.notes.AppendWarning("Network verifier encountered an error: %v", err)
+	} else {
+		logging.Infof("Network verification completed with result: %v", verifierResult)
+		switch verifierResult {
+		case networkverifier.Success:
+			i.notes.AppendSuccess("Network verifier passed")
+		case networkverifier.Failure:
+			logging.Infof("Network verifier reported failure: %s", failureReason)
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			i.notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s",
+				r.Cluster.ID(), failureReason)
+		}
+	}
+
+	// Check ClusterVersion
+	logging.Infof("Checking ClusterVersion for cluster %s", r.Cluster.ID())
+	cvList := &configv1.ClusterVersionList{}
+	listOptions := &client.ListOptions{FieldSelector: fields.SelectorFromSet(fields.Set{"metadata.name": "version"})}
+	err = i.kclient.List(context.TODO(), cvList, listOptions)
+	switch {
+	case err != nil:
+		logging.Errorf("Failed to list ClusterVersion: %v", err)
+		i.notes.AppendWarning("Failed to list ClusterVersion: %v\nThis may indicate cluster access issues", err)
+	case len(cvList.Items) != 1:
+		logging.Warnf("Found %d ClusterVersions, expected 1", len(cvList.Items))
+		i.notes.AppendWarning("Found %d ClusterVersions, expected 1", len(cvList.Items))
+	default:
+		versionCv := cvList.Items[0]
+		logging.Infof("ClusterVersion found: %s", versionCv.Status.Desired.Version)
+		for _, condition := range versionCv.Status.Conditions {
+			logging.Debugf("Checking ClusterVersion condition: Type=%s, Status=%s, Reason=%s, Message=%s",
+				condition.Type, condition.Status, condition.Reason, condition.Message)
+			if condition.Type == "RetrievedUpdates" &&
+				condition.Status == "False" &&
+				condition.Reason == "VersionNotFound" &&
+				strings.Contains(condition.Message, "Unable to retrieve available updates") {
+				i.notes.AppendWarning("ClusterVersion error detected: %s\nThis indicates the current version %s is not found in the specified channel %s",
+					condition.Message, versionCv.Status.Desired.Version, versionCv.Spec.Channel)
+			}
+		}
+		fmt.Printf("Cluster version: %s\n", versionCv.Status.Desired.Version)
+	}
path: pkg/investigations/CannotRetrieveUpdatesSRE/CannotRetrieveUpdatesSRE.go
user: login: joshbranham
body: I don't follow, what do you mean "it recommended me to use the switch case that time"?

I think this is pointing to a broader issue that your `Run` here has a lot of business logic in one function, including performing multiple "validations" and needing to never return early.

If you break each "validation" into private functions, and call each in order from `Run` then you can handle the failure modes independent and it might be cleaner. Here is some pseudocode to work from. Let me know what you think and if you have any questions 👍 

```golang
func Run() {
	err := checkClusterVersion()
	if err != nil {
		// log warning
		// write warning note for pagerduty
	}
	
	err = runNetworkVerifier()
	if err != nil {
		// log warning
		// write warning note for pagerduty
	}
	
	// cleanup
}

func checkClusterVersion() error {
	cvList, err := getCvs()
	if err != nil {
		return err
	}
	
	if len(cvList) > 1 {
		return errors.New("More than one cluster version found")
	}
	
	// do the rest of the happy path CV validation

    return nil
}

func runNetworkVerifier() error {
	result, err := runVerifier()
	if err != nil || result.Failed {
		return err
	}

	return nil
}
```


created_at: 2025-04-10T17:11:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,169 @@
+package cannotretrieveupdatesre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"k8s.io/apimachinery/pkg/fields"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "cannotretrieveupdatesre"
+)
+
+type Investigation struct {
+	kclient client.Client
+	notes   *notewriter.NoteWriter
+}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (i *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+
+	// Setup
+	err := i.setup(r)
+	if err != nil {
+		return result, fmt.Errorf("failed to setup investigation: %w", err)
+	}
+
+	defer func(r *investigation.Resources) {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		if cleanupErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName); cleanupErr != nil {
+			logging.Errorf("Failed to cleanup Kubernetes client: %v", cleanupErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}(r)
+
+	if err := i.checkClusterValidity(r); err != nil {
+		logging.Errorf("Cluster validation failed: %v", err)
+		return result, r.PdClient.EscalateIncidentWithNote(i.notes.String())
+	}
+
+	if err := i.runNetworkVerifier(r, &result); err != nil {
+		logging.Errorf("Network verification failed: %v", err)
path: pkg/investigations/cannotretrieveupdatesre/cannotRetrieveUpdateSRE.go
user: login: tnierman
body: We should probably handle this case when the network-verifier fails, right?

ie - add a note to the PD incident, prepare an SL (eventually), return a `result` and escalate to on-call, etc
created_at: 2025-04-15T17:07:41Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,169 @@
+package cannotretrieveupdatesre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"k8s.io/apimachinery/pkg/fields"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "cannotretrieveupdatesre"
+)
+
+type Investigation struct {
+	kclient client.Client
+	notes   *notewriter.NoteWriter
+}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (i *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+
+	// Setup
+	err := i.setup(r)
+	if err != nil {
+		return result, fmt.Errorf("failed to setup investigation: %w", err)
+	}
+
+	defer func(r *investigation.Resources) {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		if cleanupErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName); cleanupErr != nil {
+			logging.Errorf("Failed to cleanup Kubernetes client: %v", cleanupErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}(r)
+
+	if err := i.checkClusterValidity(r); err != nil {
+		logging.Errorf("Cluster validation failed: %v", err)
+		return result, r.PdClient.EscalateIncidentWithNote(i.notes.String())
+	}
+
+	if err := i.runNetworkVerifier(r, &result); err != nil {
+		logging.Errorf("Network verification failed: %v", err)
path: pkg/investigations/cannotretrieveupdatesre/cannotRetrieveUpdateSRE.go
user: login: tnierman
body: On second thought, the network-verifier could fail on unrelated domains, so perhaps it _is_ better to continue the investigation :thinking: 
created_at: 2025-04-15T17:12:03Z
author_association: MEMBER
path: pkg/investigations/cannotretrieveupdatessre/testing/README.md
user: login: tnierman
body: Looks like this is still the boilerplate document from running the `bootstrap-new-investigation` make target.

Can you revise this to include detailed steps for another SRE to test your work? Additional objects, scripts, etc can also be added to this `testing/` directory
created_at: 2025-05-12T15:23:27Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,164 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		}
+	}()
+
+	defer func(r *investigation.Resources) {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		if cleanupErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName); cleanupErr != nil {
+			logging.Errorf("Failed to cleanup Kubernetes client: %v", cleanupErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}(r)
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: tnierman
body: Looks like you're cleaning up twice here
created_at: 2025-05-12T15:25:12Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,164 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		}
+	}()
+
+	defer func(r *investigation.Resources) {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		if cleanupErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName); cleanupErr != nil {
+			logging.Errorf("Failed to cleanup Kubernetes client: %v", cleanupErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}(r)
+
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		logging.Error("Network verifier ran into an error: %s", err.Error())
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: tnierman
body: I don't believe we need to both log and note this - using `notes.AppendWarning` should be sufficient to add this message to the logs as well
created_at: 2025-05-12T15:26:05Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,164 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		}
+	}()
+
+	defer func(r *investigation.Resources) {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		if cleanupErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName); cleanupErr != nil {
+			logging.Errorf("Failed to cleanup Kubernetes client: %v", cleanupErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}(r)
+
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		logging.Error("Network verifier ran into an error: %s", err.Error())
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			// We do not return as we want the alert to be escalated either no matter what.
+			logging.Error("could not add failure reason incident notes")
+		}
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: tnierman
body: Calling `AddNote()` here without returning will mean that ~~the network verifier results~~ the network verifier error response potentially gets added to the incident twice: once here and again below, no?

Would it make more sense to `notes.AppendWarning` in this block, and just add those notes at the end?
created_at: 2025-05-12T15:27:28Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,164 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		}
+	}()
+
+	defer func(r *investigation.Resources) {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		if cleanupErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName); cleanupErr != nil {
+			logging.Errorf("Failed to cleanup Kubernetes client: %v", cleanupErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}(r)
+
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		logging.Error("Network verifier ran into an error: %s", err.Error())
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			// We do not return as we want the alert to be escalated either no matter what.
+			logging.Error("could not add failure reason incident notes")
+		}
+	}
+
+	switch verifierResult {
+	case networkverifier.Failure:
+		logging.Infof("Network verifier reported failure: %s", failureReason)
+		// XXX: metrics.Inc(metrics.ServicelogPrepared, investigationName)
+		result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+		notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s", r.Cluster.ID(), failureReason)
+
+		// In the future, we want to send a service log in this case
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			logging.Error("could not add issues to incident notes")
+		}
+	case networkverifier.Success:
+		notes.AppendSuccess("Network verifier passed")
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			logging.Error("could not add passed message to incident notes")
+		}
+	}
+
+	// Check ClusterVersion
+	clusterVersion, note, err := checkClusterVersion(k8scli, r.Cluster.ID())
+	if err != nil {
+		notes.AppendWarning("Failure checking ClusterVersion: %s", err.Error())
+		notes.AppendWarning("Alert escalated to on-call primary for review.")
+		logging.Infof("Escalating incident with notes for cluster %s", r.Cluster.ID())
+		err = r.PdClient.EscalateIncidentWithNote(notes.String())
+		if err != nil {
+			logging.Errorf("Failed to escalate incident to PagerDuty: %v", err)
+			return result, fmt.Errorf("failed to escalate incident: %w", err)
+		}
+		return result, err
+	}
+	if note != "" {
+		notes.AppendWarning(note)
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			logging.Error("could not add notes to the incident")
+		}
+	}
+	if clusterVersion != "" {
+		notes.AppendSuccess("ClusterVersion found: %s", clusterVersion)
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			logging.Error("could not add passed message to incident notes")
+		}
+	}
+
+	notes.AppendWarning("Alert escalated to on-call primary for review.")
+	logging.Infof("Escalating incident with notes for cluster %s", r.Cluster.ID())
+	err = r.PdClient.EscalateIncidentWithNote(notes.String())
+	if err != nil {
+		logging.Errorf("Failed to escalate incident to PagerDuty: %v", err)
+		return result, fmt.Errorf("failed to escalate incident: %w", err)
+	}
+	logging.Infof("Investigation completed and escalated successfully for cluster %s", r.Cluster.ID())
+
+	return result, nil
+}
+
+// checkClusterVersion retrieves the cluster version
+func checkClusterVersion(k8scli client.Client, clusterID string) (version string, note string, err error) {
+	logging.Infof("Checking ClusterVersion for cluster %s", clusterID)
+	clusterVersion := &configv1.ClusterVersion{}
+	err = k8scli.Get(context.TODO(), client.ObjectKey{Name: "version"}, clusterVersion)
+	if err != nil {
+		return "", "Failed to get ClusterVersion: cluster access issues detected", fmt.Errorf("failed to get ClusterVersion: %w", err)
+	}
+	logging.Infof("ClusterVersion channel: %s", clusterVersion.Spec.Channel)
+	logging.Infof("ClusterVersion found: %s", clusterVersion.Status.Desired.Version)
+	logging.Debugf("ClusterVersion conditions: %+v", clusterVersion.Status.Conditions)
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: tnierman
body: Do we need to log this info? Or is it leftover from debugging? 

We also log the condition below, so it seems rather repetitive 
created_at: 2025-05-12T15:56:49Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,164 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		}
+	}()
+
+	defer func(r *investigation.Resources) {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		if cleanupErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName); cleanupErr != nil {
+			logging.Errorf("Failed to cleanup Kubernetes client: %v", cleanupErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}(r)
+
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		logging.Error("Network verifier ran into an error: %s", err.Error())
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			// We do not return as we want the alert to be escalated either no matter what.
+			logging.Error("could not add failure reason incident notes")
+		}
+	}
+
+	switch verifierResult {
+	case networkverifier.Failure:
+		logging.Infof("Network verifier reported failure: %s", failureReason)
+		// XXX: metrics.Inc(metrics.ServicelogPrepared, investigationName)
+		result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+		notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s", r.Cluster.ID(), failureReason)
+
+		// In the future, we want to send a service log in this case
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			logging.Error("could not add issues to incident notes")
+		}
+	case networkverifier.Success:
+		notes.AppendSuccess("Network verifier passed")
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			logging.Error("could not add passed message to incident notes")
+		}
+	}
+
+	// Check ClusterVersion
+	clusterVersion, note, err := checkClusterVersion(k8scli, r.Cluster.ID())
+	if err != nil {
+		notes.AppendWarning("Failure checking ClusterVersion: %s", err.Error())
+		notes.AppendWarning("Alert escalated to on-call primary for review.")
+		logging.Infof("Escalating incident with notes for cluster %s", r.Cluster.ID())
+		err = r.PdClient.EscalateIncidentWithNote(notes.String())
+		if err != nil {
+			logging.Errorf("Failed to escalate incident to PagerDuty: %v", err)
+			return result, fmt.Errorf("failed to escalate incident: %w", err)
+		}
+		return result, err
+	}
+	if note != "" {
+		notes.AppendWarning(note)
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			logging.Error("could not add notes to the incident")
+		}
+	}
+	if clusterVersion != "" {
+		notes.AppendSuccess("ClusterVersion found: %s", clusterVersion)
+		err = r.PdClient.AddNote(notes.String())
+		if err != nil {
+			logging.Error("could not add passed message to incident notes")
+		}
+	}
+
+	notes.AppendWarning("Alert escalated to on-call primary for review.")
+	logging.Infof("Escalating incident with notes for cluster %s", r.Cluster.ID())
+	err = r.PdClient.EscalateIncidentWithNote(notes.String())
+	if err != nil {
+		logging.Errorf("Failed to escalate incident to PagerDuty: %v", err)
+		return result, fmt.Errorf("failed to escalate incident: %w", err)
+	}
+	logging.Infof("Investigation completed and escalated successfully for cluster %s", r.Cluster.ID())
+
+	return result, nil
+}
+
+// checkClusterVersion retrieves the cluster version
+func checkClusterVersion(k8scli client.Client, clusterID string) (version string, note string, err error) {
+	logging.Infof("Checking ClusterVersion for cluster %s", clusterID)
+	clusterVersion := &configv1.ClusterVersion{}
+	err = k8scli.Get(context.TODO(), client.ObjectKey{Name: "version"}, clusterVersion)
+	if err != nil {
+		return "", "Failed to get ClusterVersion: cluster access issues detected", fmt.Errorf("failed to get ClusterVersion: %w", err)
+	}
+	logging.Infof("ClusterVersion channel: %s", clusterVersion.Spec.Channel)
+	logging.Infof("ClusterVersion found: %s", clusterVersion.Status.Desired.Version)
+	logging.Debugf("ClusterVersion conditions: %+v", clusterVersion.Status.Conditions)
+
+	for _, condition := range clusterVersion.Status.Conditions {
+		logging.Debugf("Checking ClusterVersion condition: Type=%s, Status=%s, Reason=%s, Message=%q",
+			condition.Type, condition.Status, condition.Reason, condition.Message)
+		if condition.Type == "RetrievedUpdates" && condition.Status == "False" {
+			if (condition.Reason == "VersionNotFound" || condition.Reason == "RemoteFailed") &&
+				strings.Contains(strings.TrimSpace(condition.Message), "Unable to retrieve available updates") {
+				logging.Warnf("Detected ClusterVersion error: Reason=%s, Message=%s", condition.Reason, condition.Message)
+				return "", fmt.Sprintf("ClusterVersion error detected: %s. Current version %s not found in channel %s",
+						condition.Message, clusterVersion.Status.Desired.Version, clusterVersion.Spec.Channel),
+					fmt.Errorf("clusterversion validation failed: %s", condition.Reason)
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: tnierman
body: I find it a tad bit confusing that we're returning an error here: it's not the state we want to see the `clusterversion` in, but I'd expect an error return to mean that we _couldn't_ check the `clusterversion`, not that we were able to check the clusterversion but didn't like the state we saw it in
created_at: 2025-05-12T15:59:20Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,5 @@
+# Testing CannotRetrieveUpdatesSRE Investigation
+
+TODO:
+- Add a test script or test objects to this  directory for future maintainers to use
+- Edit this README file and add detailed instructions on how to use the script/objects to recreate the conditions for the investigation. Be sure to include any assumptions or prerequisites about the environment (disable hive syncsetting, etc)
path: pkg/investigations/cannotretrieveupdatessre/testing/README.md
user: login: tnierman
body: This README still contains just the default text. Can we populate it with instructions on how to test this investigation?
created_at: 2025-05-12T21:19:23Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,124 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			logging.Infof("Network verifier reported failure: %s", failureReason)
+			// XXX: metrics.Inc(metrics.ServicelogPrepared, investigationName)
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: tnierman
body: ```suggestion

```
created_at: 2025-05-12T21:21:35Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,124 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			logging.Infof("Network verifier reported failure: %s", failureReason)
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: tnierman
body: ```suggestion

```
We probably can exclude this logging line since we're noting the failure below too?
created_at: 2025-05-12T21:22:46Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,124 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			logging.Infof("Network verifier reported failure: %s", failureReason)
+			// XXX: metrics.Inc(metrics.ServicelogPrepared, investigationName)
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: anispate
body: I was going to remove it but it exists on other cpd.go and insightoperatordown.go file so, i kept it.
created_at: 2025-05-12T21:24:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,124 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			logging.Infof("Network verifier reported failure: %s", failureReason)
+			// XXX: metrics.Inc(metrics.ServicelogPrepared, investigationName)
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s", r.Cluster.ID(), failureReason)
+		case networkverifier.Success:
+			notes.AppendSuccess("Network verifier passed")
+		}
+	}
+
+	// Check ClusterVersion
+	clusterVersion, note, err := checkClusterVersion(k8scli, r.Cluster.ID())
+	if err != nil {
+		notes.AppendWarning("Failure checking ClusterVersion: %s", err.Error())
+	} else {
+		if note != "" {
+			notes.AppendWarning(note)
+		}
+		if clusterVersion != "" {
+			notes.AppendSuccess("ClusterVersion found: %s", clusterVersion)
+		}
+	}
+
+	notes.AppendWarning("Alert escalated to on-call primary for review.")
+	logging.Infof("Escalating incident with notes for cluster %s", r.Cluster.ID())
+	err = r.PdClient.EscalateIncidentWithNote(notes.String())
+	if err != nil {
+		logging.Errorf("Failed to escalate incident to PagerDuty: %v", err)
+		return result, fmt.Errorf("failed to escalate incident: %w", err)
+	}
+	logging.Infof("Investigation completed and escalated successfully for cluster %s", r.Cluster.ID())
+
+	return result, nil
+}
+
+// checkClusterVersion retrieves the cluster version
+func checkClusterVersion(k8scli client.Client, clusterID string) (version string, note string, err error) {
+	logging.Infof("Checking ClusterVersion for cluster %s", clusterID)
+	clusterVersion := &configv1.ClusterVersion{}
+	err = k8scli.Get(context.TODO(), client.ObjectKey{Name: "version"}, clusterVersion)
+	if err != nil {
+		return "", "Failed to get ClusterVersion: cluster access issues detected", fmt.Errorf("failed to get ClusterVersion: %w", err)
+	}
+	logging.Infof("ClusterVersion channel: %s", clusterVersion.Spec.Channel)
+	logging.Infof("ClusterVersion found: %s", clusterVersion.Status.Desired.Version)
+
+	for _, condition := range clusterVersion.Status.Conditions {
+		if condition.Type == "RetrievedUpdates" && condition.Status == "False" {
+			if (condition.Reason == "VersionNotFound" || condition.Reason == "RemoteFailed") &&
+				strings.Contains(strings.TrimSpace(condition.Message), "Unable to retrieve available updates") {
+				logging.Warnf("Detected ClusterVersion issue: Reason=%s, Message=%s", condition.Reason, condition.Message)
+				return "", fmt.Sprintf("ClusterVersion issue detected: %s. Current version %s not found in channel %s",
+						condition.Message, clusterVersion.Status.Desired.Version, clusterVersion.Spec.Channel),
+					fmt.Errorf("clusterversion has undesirable state: %s", condition.Reason)
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: tnierman
body: The way this function is written and called [above](https://github.com/openshift/configuration-anomaly-detection/pull/404/files#diff-761565fefa17061f704f9cb537f6fbf1a357cc959e9fef51467520bb78f8132bR61-R71), returning an error here won't allow the note to be added to the PD incident.

I'd prefer if we didn't return an error at all, honestly, because that should be reserved for cases where we couldn't check the state of the cluster, rather than cases where we did check the state of the cluster, but didn't like what we saw.
created_at: 2025-05-12T21:25:32Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,124 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			logging.Infof("Network verifier reported failure: %s", failureReason)
+			// XXX: metrics.Inc(metrics.ServicelogPrepared, investigationName)
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s", r.Cluster.ID(), failureReason)
+		case networkverifier.Success:
+			notes.AppendSuccess("Network verifier passed")
+		}
+	}
+
+	// Check ClusterVersion
+	clusterVersion, note, err := checkClusterVersion(k8scli, r.Cluster.ID())
+	if err != nil {
+		notes.AppendWarning("Failure checking ClusterVersion: %s", err.Error())
+	} else {
+		if note != "" {
+			notes.AppendWarning(note)
+		}
+		if clusterVersion != "" {
+			notes.AppendSuccess("ClusterVersion found: %s", clusterVersion)
+		}
+	}
+
+	notes.AppendWarning("Alert escalated to on-call primary for review.")
+	logging.Infof("Escalating incident with notes for cluster %s", r.Cluster.ID())
+	err = r.PdClient.EscalateIncidentWithNote(notes.String())
+	if err != nil {
+		logging.Errorf("Failed to escalate incident to PagerDuty: %v", err)
+		return result, fmt.Errorf("failed to escalate incident: %w", err)
+	}
+	logging.Infof("Investigation completed and escalated successfully for cluster %s", r.Cluster.ID())
+
+	return result, nil
+}
+
+// checkClusterVersion retrieves the cluster version
+func checkClusterVersion(k8scli client.Client, clusterID string) (version string, note string, err error) {
+	logging.Infof("Checking ClusterVersion for cluster %s", clusterID)
+	clusterVersion := &configv1.ClusterVersion{}
+	err = k8scli.Get(context.TODO(), client.ObjectKey{Name: "version"}, clusterVersion)
+	if err != nil {
+		return "", "Failed to get ClusterVersion: cluster access issues detected", fmt.Errorf("failed to get ClusterVersion: %w", err)
+	}
+	logging.Infof("ClusterVersion channel: %s", clusterVersion.Spec.Channel)
+	logging.Infof("ClusterVersion found: %s", clusterVersion.Status.Desired.Version)
+
+	for _, condition := range clusterVersion.Status.Conditions {
+		if condition.Type == "RetrievedUpdates" && condition.Status == "False" {
+			if (condition.Reason == "VersionNotFound" || condition.Reason == "RemoteFailed") &&
+				strings.Contains(strings.TrimSpace(condition.Message), "Unable to retrieve available updates") {
+				logging.Warnf("Detected ClusterVersion issue: Reason=%s, Message=%s", condition.Reason, condition.Message)
+				return "", fmt.Sprintf("ClusterVersion issue detected: %s. Current version %s not found in channel %s",
+						condition.Message, clusterVersion.Status.Desired.Version, clusterVersion.Spec.Channel),
+					fmt.Errorf("clusterversion has undesirable state: %s", condition.Reason)
+			}
+		}
+	}
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: tnierman
body: ```suggestion
	for _, condition := range clusterVersion.Status.Conditions {
		if condition.Type == "RetrievedUpdates" {
		    note, err := checkRetrievedUpdatesCondition(condition)
		    return clusterVersion.Status.DesiredVersion, note, err
		}
	}
```
It looks like we're only looking to analyze the state of one condition here, correct? Would it make more sense to break some of this out into it's own function? 

Something like the following may make the nested conditionals more readable:

```
func checkRetrievedUpdatesCondition(condition corev1.StatusCondition) (string, err) {
    if condition.Status == corev1.ConditionFalse {
        return ...
    }
    if (condition.Reason == ...) {
        return ...
    }
}
```

Additionally, we won't waste CPU cycles continuing to loop through conditions we don't need to analyze once we find the one we're after.
created_at: 2025-05-12T21:34:15Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,124 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			logging.Infof("Network verifier reported failure: %s", failureReason)
+			// XXX: metrics.Inc(metrics.ServicelogPrepared, investigationName)
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: tnierman
body: I think we should remove it for now :slightly_smiling_face:  If/when we add metrics to CAD, we'll have to scan the codebase for instances of servicelogs being sent anyway
created_at: 2025-05-12T21:37:18Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,136 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: typeid
body: **issue:** I think this will fail as backplane will look for the `metadata.yaml` in the path `pkg/investigations/CannotRetrieveUpdatesSRE/metadata.yaml`, but the path is lowercase. 
created_at: 2025-05-13T10:42:33Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,136 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s", r.Cluster.ID(), failureReason)
+		case networkverifier.Success:
+			notes.AppendSuccess("Network verifier passed")
+		}
+	}
+
+	// Check ClusterVersion
+	clusterVersion, note, err := checkClusterVersion(k8scli, r.Cluster.ID())
+	if err != nil {
+		notes.AppendWarning("Failure checking ClusterVersion: %s", err.Error())
+		if note != "" {
+			notes.AppendWarning(note)
+		}
+	} else {
+		if note != "" {
+			notes.AppendWarning(note)
+		}
+		if clusterVersion != "" {
+			notes.AppendSuccess("ClusterVersion found: %s", clusterVersion)
+		}
+	}
+
+	notes.AppendWarning("Alert escalated to on-call primary for review.")
+	logging.Infof("Escalating incident with notes for cluster %s", r.Cluster.ID())
+	err = r.PdClient.EscalateIncidentWithNote(notes.String())
+	if err != nil {
+		logging.Errorf("Failed to escalate incident to PagerDuty: %v", err)
+		return result, fmt.Errorf("failed to escalate incident: %w", err)
+	}
+	logging.Infof("Investigation completed and escalated successfully for cluster %s", r.Cluster.ID())
+
+	return result, nil
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: typeid
body: We don't need to add in the PD note that we escalated the alert to SRE, it should be obvious if the SRE has the alert on their board ;) 

There might be a few superfluous logs here as well, the following should suffice: 

```suggestion
	return result, r.PdClient.EscalateIncidentWithNote(notes.String())
```
created_at: 2025-05-13T10:45:55Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,136 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		logging.Infof("Cleaning up investigation resources for cluster %s", r.Cluster.ID())
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, remediationName)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		} else {
+			logging.Infof("Cleanup completed successfully for cluster %s", r.Cluster.ID())
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s", r.Cluster.ID(), failureReason)
+		case networkverifier.Success:
+			notes.AppendSuccess("Network verifier passed")
+		}
+	}
+
+	// Check ClusterVersion
+	clusterVersion, note, err := checkClusterVersion(k8scli, r.Cluster.ID())
+	if err != nil {
+		notes.AppendWarning("Failure checking ClusterVersion: %s", err.Error())
+		if note != "" {
+			notes.AppendWarning(note)
+		}
+	} else {
+		if note != "" {
+			notes.AppendWarning(note)
+		}
+		if clusterVersion != "" {
+			notes.AppendSuccess("ClusterVersion found: %s", clusterVersion)
+		}
+	}
+
+	notes.AppendWarning("Alert escalated to on-call primary for review.")
+	logging.Infof("Escalating incident with notes for cluster %s", r.Cluster.ID())
+	err = r.PdClient.EscalateIncidentWithNote(notes.String())
+	if err != nil {
+		logging.Errorf("Failed to escalate incident to PagerDuty: %v", err)
+		return result, fmt.Errorf("failed to escalate incident: %w", err)
+	}
+	logging.Infof("Investigation completed and escalated successfully for cluster %s", r.Cluster.ID())
+
+	return result, nil
+}
+
+// checkClusterVersion retrieves the cluster version
+func checkClusterVersion(k8scli client.Client, clusterID string) (version string, note string, err error) {
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: typeid
body: This function does more than the name suggests:
- it retrieves the cluster version
- checks if the updates could be retrieved
- pre-formats a note

Could we separate the logic here for the functions to be more clear cut on what they are doing? 

E.g.

`func getClusterVersion()` -> no logs, just gets the cluster version object
`func getUpdateRetrievalFailures(clusterversion)` -> looks for update retrieval failures in the clusterversion 

Logs and note creation should be outside of this logic, ideally in the main investigation function logic. This way, we could even move the above functions in common packages as they are re-usable. 
created_at: 2025-05-13T10:52:27Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,136 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: anispate
body: But, i have updated the path as part of the PR to the lowercase for the metadata.yaml as part of the PR so, would it still cause an issue?
created_at: 2025-05-13T14:53:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,136 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: typeid
body: `remediationName` uses camel case in your PR, correct? And the path is not camel case in this PR, so it's a mismatch as far as I can tell
created_at: 2025-05-14T10:15:27Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,117 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		deferErr := k8scli.Clean()
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s", r.Cluster.ID(), failureReason)
+		case networkverifier.Success:
+			notes.AppendSuccess("Network verifier passed")
+		}
+	}
+
+	// Check ClusterVersion
+	clusterVersion, err := getClusterVersion(k8scli)
+	if err != nil {
+		notes.AppendWarning("Failed to get ClusterVersion: %s", err.Error())
+	} else {
+		notes.AppendSuccess("ClusterVersion found: %s", clusterVersion.Status.Desired.Version)
+
+		failureReason := getUpdateRetrievalFailures(clusterVersion)
+		if failureReason != "" {
+			logging.Warnf("Detected ClusterVersion issue: %s", failureReason)
+			notes.AppendWarning("ClusterVersion related issue detected: %s. Current version %s not found in channel %s",
+				failureReason, clusterVersion.Status.Desired.Version, clusterVersion.Spec.Channel)
+		}
+	}
+	notes.AppendWarning("Alert escalated to on-call primary for review and please check the ClusterVersion.")
+	return result, r.PdClient.EscalateIncidentWithNote(notes.String())
+}
+
+func getClusterVersion(k8scli client.Client) (*configv1.ClusterVersion, error) {
+	clusterVersion := &configv1.ClusterVersion{}
+	err := k8scli.Get(context.TODO(), client.ObjectKey{Name: "version"}, clusterVersion)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get ClusterVersion: %w", err)
+	}
+	return clusterVersion, nil
+}
+
+// getUpdateRetrievalFailures checks for update retrieval failures in the ClusterVersion
+func getUpdateRetrievalFailures(clusterVersion *configv1.ClusterVersion) string {
+	for _, condition := range clusterVersion.Status.Conditions {
+		msg, found := checkCondition(condition)
+		if found {
+			return msg
+		}
+	}
+	return ""
+}
+
+func checkCondition(condition configv1.ClusterOperatorStatusCondition) (string, bool) {
+	if condition.Type != "RetrievedUpdates" {
+		return "", false
+	}
+	if condition.Status == configv1.ConditionFalse {
+		return fmt.Sprintf("(Reason: %s). %s", condition.Reason, condition.Message), true
+	}
+	return "", false
+}
+
+func (i *Investigation) Name() string {
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: clcollins
body: Are these CAD conventions?  Just for my own personal edification.  :)
created_at: 2025-05-27T17:29:18Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,117 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		deferErr := k8scli.Clean()
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s", r.Cluster.ID(), failureReason)
+		case networkverifier.Success:
+			notes.AppendSuccess("Network verifier passed")
+		}
+	}
+
+	// Check ClusterVersion
+	clusterVersion, err := getClusterVersion(k8scli)
+	if err != nil {
+		notes.AppendWarning("Failed to get ClusterVersion: %s", err.Error())
+	} else {
+		notes.AppendSuccess("ClusterVersion found: %s", clusterVersion.Status.Desired.Version)
+
+		failureReason := getUpdateRetrievalFailures(clusterVersion)
+		if failureReason != "" {
+			logging.Warnf("Detected ClusterVersion issue: %s", failureReason)
+			notes.AppendWarning("ClusterVersion related issue detected: %s. Current version %s not found in channel %s",
+				failureReason, clusterVersion.Status.Desired.Version, clusterVersion.Spec.Channel)
+		}
+	}
+	notes.AppendWarning("Alert escalated to on-call primary for review and please check the ClusterVersion.")
+	return result, r.PdClient.EscalateIncidentWithNote(notes.String())
+}
+
+func getClusterVersion(k8scli client.Client) (*configv1.ClusterVersion, error) {
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: clcollins
body: This function looks like it would be useful in other investigations.  Is there a more central package this could be placed into, like `utils` perhaps?
created_at: 2025-05-27T19:05:48Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,156 @@
+package cannotretrieveupdatessre
+
+import (
+	"testing"
+
+	configv1 "github.com/openshift/api/config/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func newFakeClient(objs ...client.Object) (client.Client, error) {
+	s := scheme.Scheme
+	err := configv1.AddToScheme(s)
+	if err != nil {
+		return nil, err
+	}
+
+	client := fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build()
+	return client, nil
+}
+
+func TestGetClusterVersion(t *testing.T) {
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre_test.go
user: login: clcollins
body: I'm open for disagreement, but this seems like a backwards test.  The function you're testing is _super_ simple, and you're mocking the k8sCli client anyway, so the only logic ~not~ from your own code is the error handling, but the only test case is one where you're expecting no error.  

So you're really only testing the fake client itself, which I'm not sure is valuable.
created_at: 2025-05-27T19:10:25Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,156 @@
+package cannotretrieveupdatessre
+
+import (
+	"testing"
+
+	configv1 "github.com/openshift/api/config/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func newFakeClient(objs ...client.Object) (client.Client, error) {
+	s := scheme.Scheme
+	err := configv1.AddToScheme(s)
+	if err != nil {
+		return nil, err
+	}
+
+	client := fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build()
+	return client, nil
+}
+
+func TestGetClusterVersion(t *testing.T) {
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre_test.go
user: login: clcollins
body: The easiest fix would probably be to just add another test returning an error from the fake client, and ensuring it returns the error you're expecting.  It doesn't _hurt_ to have the existing test, I guess.
created_at: 2025-05-27T19:11:43Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,117 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		deferErr := k8scli.Clean()
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s", r.Cluster.ID(), failureReason)
+		case networkverifier.Success:
+			notes.AppendSuccess("Network verifier passed")
+		}
+	}
+
+	// Check ClusterVersion
+	clusterVersion, err := getClusterVersion(k8scli)
+	if err != nil {
+		notes.AppendWarning("Failed to get ClusterVersion: %s", err.Error())
+	} else {
+		notes.AppendSuccess("ClusterVersion found: %s", clusterVersion.Status.Desired.Version)
+
+		failureReason := getUpdateRetrievalFailures(clusterVersion)
+		if failureReason != "" {
+			logging.Warnf("Detected ClusterVersion issue: %s", failureReason)
+			notes.AppendWarning("ClusterVersion related issue detected: %s. Current version %s not found in channel %s",
+				failureReason, clusterVersion.Status.Desired.Version, clusterVersion.Spec.Channel)
+		}
+	}
+	notes.AppendWarning("Alert escalated to on-call primary for review and please check the ClusterVersion.")
+	return result, r.PdClient.EscalateIncidentWithNote(notes.String())
+}
+
+func getClusterVersion(k8scli client.Client) (*configv1.ClusterVersion, error) {
+	clusterVersion := &configv1.ClusterVersion{}
+	err := k8scli.Get(context.TODO(), client.ObjectKey{Name: "version"}, clusterVersion)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get ClusterVersion: %w", err)
+	}
+	return clusterVersion, nil
+}
+
+// getUpdateRetrievalFailures checks for update retrieval failures in the ClusterVersion
+func getUpdateRetrievalFailures(clusterVersion *configv1.ClusterVersion) string {
+	for _, condition := range clusterVersion.Status.Conditions {
+		msg, found := checkCondition(condition)
+		if found {
+			return msg
+		}
+	}
+	return ""
+}
+
+func checkCondition(condition configv1.ClusterOperatorStatusCondition) (string, bool) {
+	if condition.Type != "RetrievedUpdates" {
+		return "", false
+	}
+	if condition.Status == configv1.ConditionFalse {
+		return fmt.Sprintf("(Reason: %s). %s", condition.Reason, condition.Message), true
+	}
+	return "", false
+}
+
+func (i *Investigation) Name() string {
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: tnierman
body: They're required methods for an `Investigation` to satisfy [this interface](https://github.com/openshift/configuration-anomaly-detection/blob/main/pkg/investigations/investigation/investigation.go#L24-L32), yeah
created_at: 2025-05-27T20:18:04Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,117 @@
+package cannotretrieveupdatessre
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	alertname       = "CannotRetrieveUpdatesSRE"
+	remediationName = "CannotRetrieveUpdatesSRE"
+)
+
+type Investigation struct{}
+
+// Run executes the investigation for the CannotRetrieveUpdatesSRE alert
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
+	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, remediationName)
+	if err != nil {
+		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
+	}
+	defer func() {
+		deferErr := k8scli.Clean()
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		}
+	}()
+
+	// Run network verifier
+	verifierResult, failureReason, err := networkverifier.Run(r.Cluster, r.ClusterDeployment, r.AwsClient)
+	if err != nil {
+		notes.AppendWarning("NetworkVerifier failed to run:\n\t %s", err.Error())
+	} else {
+		switch verifierResult {
+		case networkverifier.Failure:
+			result.ServiceLogPrepared = investigation.InvestigationStep{Performed: true, Labels: nil}
+			notes.AppendWarning("NetworkVerifier found unreachable targets. \n \n Verify and send service log if necessary: \n osdctl servicelog post %s -t https://raw.githubusercontent.com/openshift/managed-notifications/master/osd/required_network_egresses_are_blocked.json -p URLS=%s", r.Cluster.ID(), failureReason)
+		case networkverifier.Success:
+			notes.AppendSuccess("Network verifier passed")
+		}
+	}
+
+	// Check ClusterVersion
+	clusterVersion, err := getClusterVersion(k8scli)
+	if err != nil {
+		notes.AppendWarning("Failed to get ClusterVersion: %s", err.Error())
+	} else {
+		notes.AppendSuccess("ClusterVersion found: %s", clusterVersion.Status.Desired.Version)
+
+		failureReason := getUpdateRetrievalFailures(clusterVersion)
+		if failureReason != "" {
+			logging.Warnf("Detected ClusterVersion issue: %s", failureReason)
+			notes.AppendWarning("ClusterVersion related issue detected: %s. Current version %s not found in channel %s",
+				failureReason, clusterVersion.Status.Desired.Version, clusterVersion.Spec.Channel)
+		}
+	}
+	notes.AppendWarning("Alert escalated to on-call primary for review and please check the ClusterVersion.")
+	return result, r.PdClient.EscalateIncidentWithNote(notes.String())
+}
+
+func getClusterVersion(k8scli client.Client) (*configv1.ClusterVersion, error) {
path: pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go
user: login: tnierman
body: +1 - fwiw other CAD maintainers have expressed a desire to subdivide the [`pkg/investigation/utils/` packages](https://github.com/openshift/configuration-anomaly-detection/tree/main/pkg/investigations/utils) into focused units. I think currently, we only have `node` and `machine` utilities packages, so I'd recommend creating a new one (maybe called `version`?)
created_at: 2025-05-27T20:20:47Z
author_association: MEMBER

PR comments:
user: login: codecov-commenter
created_at: 2025-04-07T19:10:45Z
body: ## [Codecov](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/404?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift) Report
:x: Patch coverage is `27.11864%` with `43 lines` in your changes missing coverage. Please review.
:white_check_mark: Project coverage is 31.96%. Comparing base ([`71ed508`](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/commit/71ed50883c3ded79b4f73a7ecda47d69dc62ea74?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift)) to head ([`503c320`](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/commit/503c32031e93f56010595264bacbc7e22fa961b1?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift)).
:warning: Report is 384 commits behind head on main.

| [Files with missing lines](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/404?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift) | Patch % | Lines |
|---|---|---|
| [...nnotretrieveupdatessre/cannotretrieveupdatessre.go](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/404?src=pr&el=tree&filepath=pkg%2Finvestigations%2Fcannotretrieveupdatessre%2Fcannotretrieveupdatessre.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift#diff-cGtnL2ludmVzdGlnYXRpb25zL2Nhbm5vdHJldHJpZXZldXBkYXRlc3NyZS9jYW5ub3RyZXRyaWV2ZXVwZGF0ZXNzcmUuZ28=) | 27.11% | [42 Missing and 1 partial :warning: ](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/404?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift) |

<details><summary>Additional details and impacted files</summary>



[![Impacted file tree graph](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/404/graphs/tree.svg?width=650&height=150&src=pr&token=6LF1WKP1IZ&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift)](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/404?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift)

```diff
@@            Coverage Diff             @@
##             main     #404      +/-   ##
==========================================
- Coverage   32.08%   31.96%   -0.12%     
==========================================
  Files          35       36       +1     
  Lines        2425     2484      +59     
==========================================
+ Hits          778      794      +16     
- Misses       1587     1629      +42     
- Partials       60       61       +1     
```

| [Files with missing lines](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/404?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift) | Coverage Δ | |
|---|---|---|
| [pkg/investigations/registry.go](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/404?src=pr&el=tree&filepath=pkg%2Finvestigations%2Fregistry.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift#diff-cGtnL2ludmVzdGlnYXRpb25zL3JlZ2lzdHJ5Lmdv) | `0.00% <ø> (ø)` | |
| [...nnotretrieveupdatessre/cannotretrieveupdatessre.go](https://app.codecov.io/gh/openshift/configuration-anomaly-detection/pull/404?src=pr&el=tree&filepath=pkg%2Finvestigations%2Fcannotretrieveupdatessre%2Fcannotretrieveupdatessre.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openshift#diff-cGtnL2ludmVzdGlnYXRpb25zL2Nhbm5vdHJldHJpZXZldXBkYXRlc3NyZS9jYW5ub3RyZXRyaWV2ZXVwZGF0ZXNzcmUuZ28=) | `27.11% <27.11%> (ø)` | |
</details>
<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: tnierman
created_at: 2025-05-28T14:57:32Z
body: /lgtm
/approve
author_association: MEMBER
user: login: openshift-ci[bot]
created_at: 2025-05-28T15:00:43Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/openshift/configuration-anomaly-detection/pull/404#" title="Author self-approved">anispate</a>*, *<a href="https://github.com/openshift/configuration-anomaly-detection/pull/404#issuecomment-2916659916" title="Approved">tnierman</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=openshift%2Fconfiguration-anomaly-detection).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/openshift/configuration-anomaly-detection/blob/main/OWNERS)~~ [tnierman]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: CONTRIBUTOR
user: login: openshift-ci[bot]
created_at: 2025-05-28T15:53:20Z
body: @anispate: all tests **passed!**


[Full PR test history](https://prow.ci.openshift.org/pr-history?org=openshift&repo=configuration-anomaly-detection&pr=404). [Your PR dashboard](https://prow.ci.openshift.org/pr?query=is:pr+state:open+author:anispate).

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository. I understand the commands that are listed [here](https://go.k8s.io/bot-commands).
</details>
<!-- test report -->
author_association: CONTRIBUTOR