Repository: numaproj/numaflow

Issue:
MonoVertex and Pipeline Status should indicate "PodsHealthy=false" if container restarted recently
**Describe the bug**
MonoVertex Status isn't indicating that the PodsHealthy Condition is false even though the Pod Container is continually restarting. 

If the container restarted in the last few minutes due to error, that should also warrant that Condition to be false. 

---


<!-- Issue Author: Don't delete this message to encourage other users to support your issue! -->

**Message from the maintainers**:

Impacted by this bug? Give it a 👍. We often sort issues this way to know what to prioritize.

>For quick help and support, join our [slack](https://numaproj.slack.com/join/shared_invite/zt-19svuv47m-YKHhsQ~~KK9mBv1E7pNzfg#/shared-invite/email) channel.


Discussion:

Solution patch:
diff --git a/pkg/reconciler/util.go b/pkg/reconciler/util.go
index 8edad6b1..c0225a60 100644
--- a/pkg/reconciler/util.go
+++ b/pkg/reconciler/util.go
@@ -19,6 +19,7 @@ package reconciler
 import (
 	"fmt"
 	"slices"
+	"time"
 
 	appv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -49,6 +50,7 @@ func CheckPodsStatus(pods *corev1.PodList) (healthy bool, reason string, message
 }
 
 func isPodHealthy(pod *corev1.Pod) (healthy bool, reason string) {
+	var lastRestartTime time.Time
 	for _, c := range pod.Status.ContainerStatuses {
 		if c.State.Waiting != nil && slices.Contains(unhealthyWaitingStatus, c.State.Waiting.Reason) {
 			return false, c.State.Waiting.Reason
@@ -56,6 +58,19 @@ func isPodHealthy(pod *corev1.Pod) (healthy bool, reason string) {
 		if c.State.Terminated != nil && c.State.Terminated.Reason == "Error" {
 			return false, c.State.Terminated.Reason
 		}
+		if x := c.LastTerminationState.Terminated; x != nil && !x.FinishedAt.Time.IsZero() {
+			if lastRestartTime.IsZero() || x.FinishedAt.Time.After(lastRestartTime) {
+				// Only check OOM or exit with Error
+				// TODO: revisit later if needed.
+				if x.ExitCode == 137 || (x.ExitCode == 143 && x.Reason == "Error") {
+					lastRestartTime = x.FinishedAt.Time
+				}
+			}
+		}
+	}
+	// Container restart happened in the last 2 mins
+	if !lastRestartTime.IsZero() && lastRestartTime.Add(2*time.Minute).After(time.Now()) {
+		return false, "RecentRestart"
 	}
 	return true, ""
 }
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 8c3b7ef5..cc37b922 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -824,9 +824,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.2"
+version = "1.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
+checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c"
 dependencies = [
  "jobserver",
  "libc",
@@ -2081,7 +2081,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
 dependencies = [
  "cfg-if",
- "windows-targets 0.52.6",
+ "windows-targets 0.48.5",
 ]
 
 [[package]]
@@ -3177,15 +3177,14 @@ dependencies = [
 
 [[package]]
 name = "ring"
-version = "0.17.8"
+version = "0.17.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
+checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee"
 dependencies = [
  "cc",
  "cfg-if",
  "getrandom",
  "libc",
- "spin",
  "untrusted",
  "windows-sys 0.52.0",
 ]
@@ -3687,12 +3686,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "spin"
-version = "0.9.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
-
 [[package]]
 name = "spki"
 version = "0.7.3"



Test patch:
diff --git a/pkg/reconciler/util_test.go b/pkg/reconciler/util_test.go
index 72a09da1..e23841e1 100644
--- a/pkg/reconciler/util_test.go
+++ b/pkg/reconciler/util_test.go
@@ -18,6 +18,7 @@ package reconciler
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	appv1 "k8s.io/api/apps/v1"
@@ -67,6 +68,53 @@ func TestCheckVertexPodsStatus(t *testing.T) {
 		assert.Equal(t, "NoPodsFound", reason)
 		assert.True(t, done)
 	})
+
+	t.Run("Test Vertex status as true with non-recent restart", func(t *testing.T) {
+		pods := corev1.PodList{Items: []corev1.Pod{
+			{ObjectMeta: metav1.ObjectMeta{Name: "test-pod"}, Status: corev1.PodStatus{
+				ContainerStatuses: []corev1.ContainerStatus{
+					{
+						State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "Running"}},
+						LastTerminationState: corev1.ContainerState{
+							Terminated: &corev1.ContainerStateTerminated{
+								FinishedAt: metav1.Time{
+									Time: time.Now().Add(-3 * time.Minute),
+								},
+							},
+						},
+					},
+				}},
+			}},
+		}
+		done, reason, message := CheckPodsStatus(&pods)
+		assert.Equal(t, "All pods are healthy", message)
+		assert.Equal(t, "Running", reason)
+		assert.True(t, done)
+	})
+
+	t.Run("Test Vertex status as false with recent restart", func(t *testing.T) {
+		pods := corev1.PodList{Items: []corev1.Pod{
+			{ObjectMeta: metav1.ObjectMeta{Name: "test-pod"}, Status: corev1.PodStatus{
+				ContainerStatuses: []corev1.ContainerStatus{
+					{
+						State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "Running"}},
+						LastTerminationState: corev1.ContainerState{
+							Terminated: &corev1.ContainerStateTerminated{
+								FinishedAt: metav1.Time{
+									Time: time.Now().Add(-1 * time.Minute),
+								},
+								ExitCode: 137,
+							},
+						},
+					},
+				}},
+			}},
+		}
+		done, reason, message := CheckPodsStatus(&pods)
+		assert.Equal(t, "Pod test-pod is unhealthy", message)
+		assert.Equal(t, "PodRecentRestart", reason)
+		assert.False(t, done)
+	})
 }
 
 var (