Repository: LangStream/langstream (language: python)

Issue:
Status of deployed app is always ERROR_DEPLOYING
This is due to the pod status check that expects the pod running. In this case, the pod is a job and it's completed


Discussion / hints:
I have a pod that is always reported with this error

```

  write-to-astra-sink-1:
      status:
        status: "ERROR"
        reason: "Exception(s) during workflow execution. Details:\n - com.datastax.oss.sga.deployer.k8s.controllers.agents.AgentController$StsDependantResource\
          \ -> io.fabric8.kubernetes.client.KubernetesClientException: Failure executing:\
          \ PATCH at: https://10.96.0.1:443/apis/apps/v1/namespaces/sga-default/statefulsets/test-write-to-astra-sink-1?fieldManager=agent-controller&force=true.\
          \ Message: Operation cannot be fulfilled on statefulsets.apps \"test-write-to-astra-sink-1\"\
          : the object has been modified; please apply your changes to the latest\
          \ version and try again. Received status: Status(apiVersion=v1, code=409,\
          \ details=StatusDetails(causes=[], group=apps, kind=statefulsets, name=test-write-to-astra-sink-1,\
          \ retryAfterSeconds=null, uid=null, additionalProperties={}), kind=Status,\
          \ message=Operation cannot be fulfilled on statefulsets.apps \"test-write-to-astra-sink-1\"\
          : the object has been modified; please apply your changes to the latest\
          \ version and try again, metadata=ListMeta(_continue=null, remainingItemCount=null,\
          \ resourceVersion=null, selfLink=null, additionalProperties={}), reason=Conflict,\
          \ status=Failure, additionalProperties={}).\n\tat io.fabric8.kubernetes.client.KubernetesClientException.copyAsCause(KubernetesClientException.java:238)\n\
          \tat io.fabric8.kubernetes.client.dsl.internal.OperationSupport.waitForResult(OperationSupport.java:518)\n\
          \tat io.fabric8.kubernetes.client.dsl.internal.OperationSupport.handleResponse(OperationSupport.java:535)\n\
          \tat io.fabric8.kubernetes.client.dsl.internal.OperationSupport.handlePatch(OperationSupport.java:430)\n\
          \tat io.fabric8.kubernetes.client.dsl.internal.OperationSupport.handlePatch(OperationSupport.java:408)\n\
          \tat io.fabric8.kubernetes.client.dsl.internal.BaseOperation.handlePatch(BaseOperation.java:713)\n\
          \tat io.fabric8.kubernetes.client.dsl.internal.HasMetadataOperation.lambda$patch$2(HasMetadataOperation.java:232)\n\
          \tat io.fabric8.kubernetes.client.dsl.internal.HasMetadataOperation.patch(HasMetadataOperation.java:237)\n\
          \tat io.fabric8.kubernetes.client.dsl.internal.HasMetadataOperation.patch(HasMetadataOperation.java:252)\n\
          \tat io.fabric8.kubernetes.client.dsl.internal.BaseOperation.serverSideApply(BaseOperation.java:1132)\n\
          \tat io.fabric8.kubernetes.client.dsl.internal.BaseOperation.serverSideApply(BaseOperation.java:92)\n\
          \tat io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependentResource.update(KubernetesDependentResource.java:145)\n\

```


The pod is running well

On the operator I see these logs:
```
10:11:07 INFO  [io.jav.ope.pro.dep.AbstractDependentResource] (pool-12-thread-2) Updating 'test-write-to-astra-sink-1' StatefulSet for primary ResourceID{name='test-write-to-astra-sink-1', namespace='sga-default'}                  
10:11:08 ERROR [io.jav.ope.pro.eve.ReconciliationDispatcher] (ReconcilerExecutor-agent-controller-44) Error during event processing ExecutionScope{ resource id: ResourceID{name='test-write-to-astra-sink-1', namespace='sga-default'}
, version: 2684} failed.: io.javaoperatorsdk.operator.AggregatedOperatorException: Exception(s) during workflow execution. Details:                                                                                                    
 - com.datastax.oss.sga.deployer.k8s.controllers.agents.AgentController$StsDependantResource -> io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: PATCH at: https://10.96.0.1:443/apis/apps/v1/namespaces/sga-
default/statefulsets/test-write-to-astra-sink-1?fieldManager=agent-controller&force=true. Message: Operation cannot be fulfilled on statefulsets.apps "test-write-to-astra-sink-1": the object has been modified; please apply your cha
nges to the latest version and try again. Received status: Status(apiVersion=v1, code=409, details=StatusDetails(causes=[], group=apps, kind=statefulsets, name=test-write-to-astra-sink-1, retryAfterSeconds=null, uid=null, additiona
lProperties={}), kind=Status, message=Operation cannot be fulfilled on statefulsets.apps "test-write-to-astra-sink-1": the object has been modified; please apply your changes to the latest version and try again, metadata=ListMeta(_
continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Conflict, status=Failure, additionalProperties={}).                                                                      
    at io.fabric8.kubernetes.client.KubernetesClientException.copyAsCause(KubernetesClientException.java:238)                                                                                                                          
    at io.fabric8.kubernetes.client.dsl.internal.OperationSupport.waitForResult(OperationSupport.java:518)                                                                                                                             
    at io.fabric8.kubernetes.client.dsl.internal.OperationSupport.handleResponse(OperationSupport.java:535)                                                                                                                            
    at io.fabric8.kubernetes.client.dsl.internal.OperationSupport.handlePatch(OperationSupport.java:430)                                                                                                                               
    at io.fabric8.kubernetes.client.dsl.internal.OperationSupport.handlePatch(OperationSupport.java:408)                                                                                                                               
    at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.handlePatch(BaseOperation.java:713)                                                                                                                                     
    at io.fabric8.kubernetes.client.dsl.internal.HasMetadataOperation.lambda$patch$2(HasMetadataOperation.java:232)                                                                                                                    
    at io.fabric8.kubernetes.client.dsl.internal.HasMetadataOperation.patch(HasMetadataOperation.java:237)                                                                                                                             
    at io.fabric8.kubernetes.client.dsl.internal.HasMetadataOperation.patch(HasMetadataOperation.java:252)                                                                                                                             
    at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.serverSideApply(BaseOperation.java:1132)                                                                                                                                
    at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.serverSideApply(BaseOperation.java:92)                                                                                                                                  
    at io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependentResource.update(KubernetesDependentResource.java:145)                                                                                            
    at io.javaoperatorsdk.operator.processing.dependent.kubernetes.CRUDKubernetesDependentResource.update(CRUDKubernetesDependentResource.java:16)                                                                                     
    at io.javaoperatorsdk.operator.processing.dependent.AbstractDependentResource.handleUpdate(AbstractDependentResource.java:144)                                                                                                     
    at io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependentResource.handleUpdate(KubernetesDependentResource.java:122)                                                                                      
    at io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependentResource.handleUpdate(KubernetesDependentResource.java:32)                                                                                       
    at io.javaoperatorsdk.operator.processing.dependent.AbstractDependentResource.reconcile(AbstractDependentResource.java:72)                                 
```


and
```
10:11:08 ERROR [io.jav.ope.pro.eve.EventProcessor] (ReconcilerExecutor-agent-controller-44) Exhausted retries for ExecutionScope{ resource id: ResourceID{name='test-write-to-astra-sink-1', namespace='sga-default'}, version: 2684}
```
I caused this situation while the docker image for the "runtime" was not available and the pod was in error "ImageNeverPull"

Fix patch:
diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/agents/AgentResourcesFactory.java b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/agents/AgentResourcesFactory.java
index c47e4be74..e5951c93b 100644
--- a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/agents/AgentResourcesFactory.java
+++ b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/agents/AgentResourcesFactory.java
@@ -673,6 +673,11 @@ private static Map<String, ApplicationStatus.AgentWorkerStatus> getPodStatuses(
                                                 .INITIALIZING();
                                         case ERROR -> ApplicationStatus.AgentWorkerStatus.ERROR(
                                                 podStatus.getUrl(), podStatus.getMessage());
+                                        default -> throw new RuntimeException(
+                                                "Unexpected pod state: "
+                                                        + podStatus.getState()
+                                                        + " "
+                                                        + e.getKey());
                                     };
                             if (agentRunnerSpec != null) {
                                 status =
diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/apps/AppResourcesFactory.java b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/apps/AppResourcesFactory.java
index e61892300..15b6c1d63 100644
--- a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/apps/AppResourcesFactory.java
+++ b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/apps/AppResourcesFactory.java
@@ -269,7 +269,7 @@ private static ApplicationLifecycleStatus getStatusFromJob(
                 KubeUtil.getPodsStatuses(pods).values().iterator().next();
 
         return switch (podStatus.getState()) {
-            case RUNNING, WAITING -> customResource.getStatus().getStatus();
+            case RUNNING, WAITING, COMPLETED -> customResource.getStatus().getStatus();
             case ERROR -> delete
                     ? ApplicationLifecycleStatus.errorDeleting(podStatus.getMessage())
                     : ApplicationLifecycleStatus.errorDeploying(podStatus.getMessage());
diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/KubeUtil.java b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/KubeUtil.java
index d276c6924..8b328b66a 100644
--- a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/KubeUtil.java
+++ b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/main/java/ai/langstream/deployer/k8s/util/KubeUtil.java
@@ -101,10 +101,12 @@ public static boolean isStatefulSetReady(StatefulSet sts) {
     public static class PodStatus {
         public static final PodStatus RUNNING = new PodStatus(State.RUNNING, null, null);
         public static final PodStatus WAITING = new PodStatus(State.WAITING, null, null);
+        public static final PodStatus COMPLETED = new PodStatus(State.COMPLETED, null, null);
 
         public enum State {
             RUNNING,
             WAITING,
+            COMPLETED,
             ERROR
         }
 
@@ -131,13 +133,13 @@ public static Map<String, PodStatus> getPodsStatuses(List<Pod> pods) {
                     pod.getStatus().getInitContainerStatuses();
             if (initContainerStatuses != null && !initContainerStatuses.isEmpty()) {
                 for (ContainerStatus containerStatus : initContainerStatuses) {
-                    status = getStatusFromContainerState(containerStatus.getLastState(), true);
+                    status = getStatusFromContainerState(containerStatus.getLastState());
                     if (status == null) {
-                        status = getStatusFromContainerState(containerStatus.getState(), true);
+                        status = getStatusFromContainerState(containerStatus.getState());
                     }
                 }
             }
-            if (status == null) {
+            if (status == null || status == PodStatus.COMPLETED) {
                 final List<ContainerStatus> containerStatuses =
                         pod.getStatus().getContainerStatuses();
 
@@ -146,9 +148,9 @@ public static Map<String, PodStatus> getPodsStatuses(List<Pod> pods) {
                 } else {
                     // only one container per pod
                     final ContainerStatus containerStatus = containerStatuses.get(0);
-                    status = getStatusFromContainerState(containerStatus.getLastState(), false);
+                    status = getStatusFromContainerState(containerStatus.getLastState());
                     if (status == null) {
-                        status = getStatusFromContainerState(containerStatus.getState(), false);
+                        status = getStatusFromContainerState(containerStatus.getState());
                     }
                     if (status == null) {
                         status = PodStatus.RUNNING;
@@ -173,13 +175,14 @@ public static Map<String, PodStatus> getPodsStatuses(List<Pod> pods) {
         return podStatuses;
     }
 
-    private static PodStatus getStatusFromContainerState(ContainerState state, boolean isInit) {
+    private static PodStatus getStatusFromContainerState(ContainerState state) {
         if (state == null) {
             return null;
         }
         if (state.getTerminated() != null) {
-            if (isInit && state.getTerminated().getReason().equals("Completed")) {
-                return null;
+            log.info("Found terminated container state {}", state.getTerminated());
+            if ("Completed".equals(state.getTerminated().getReason())) {
+                return PodStatus.COMPLETED;
             }
             if (state.getTerminated().getMessage() != null) {
                 return new PodStatus(
diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/controllers/apps/AppController.java b/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/controllers/apps/AppController.java
index 733c86b41..8a87e6c8f 100644
--- a/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/controllers/apps/AppController.java
+++ b/langstream-k8s-deployer/langstream-k8s-deployer-operator/src/main/java/ai/langstream/deployer/k8s/controllers/apps/AppController.java
@@ -116,6 +116,8 @@ private Duration handleJob(ApplicationCustomResource application, boolean delete
             createJob(application, delete);
             if (!delete) {
                 application.getStatus().setStatus(ApplicationLifecycleStatus.DEPLOYING);
+            } else {
+                application.getStatus().setStatus(ApplicationLifecycleStatus.DELETING);
             }
             return DEFAULT_RESCHEDULE_DURATION;
         } else {


Test patch:
diff --git a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/util/KubeUtilTest.java b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/util/KubeUtilTest.java
index 2a0c26d74..4360a9999 100644
--- a/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/util/KubeUtilTest.java
+++ b/langstream-k8s-deployer/langstream-k8s-deployer-core/src/test/java/ai/langstream/deployer/k8s/util/KubeUtilTest.java
@@ -92,4 +92,162 @@ public void testOOMKilled() {
         assertEquals(KubeUtil.PodStatus.State.ERROR, status.getState());
         assertEquals("OOMKilled", status.getMessage());
     }
+
+    @Test
+    public void testDeployerJobCompleted() {
+        String podYaml =
+                """
+                apiVersion: v1
+                kind: Pod
+                metadata:
+                  name: pod1
+                  namespace: langstream-default
+                spec: {}
+                status:
+                  containerStatuses:
+                  - containerID: docker://ab5142a1750327f815212b5af5d49b3c6ffa24d10e76b3eacb4b05d781dfda80
+                    image: ghcr.io/langstream/langstream-runtime:0.0.8
+                    imageID: docker-pullable://ghcr.io/langstream/langstream-runtime@sha256:8e82f1dfb02afe9137adf4a5dff4770705ff5b6607b70206288d164b116ba10d
+                    lastState: {}
+                    name: deployer
+                    ready: false
+                    restartCount: 0
+                    started: false
+                    state:
+                      terminated:
+                        containerID: docker://ab5142a1750327f815212b5af5d49b3c6ffa24d10e76b3eacb4b05d781dfda80
+                        exitCode: 0
+                        finishedAt: "2023-09-01T17:32:05Z"
+                        reason: Completed
+                        startedAt: "2023-09-01T17:32:02Z"
+                  initContainerStatuses:
+                  - containerID: docker://973604d477c3b301a5ac716eb4d41a4294b92560d8b3adb94f47976b28f0f5f0
+                    image: ghcr.io/langstream/langstream-runtime:0.0.8
+                    imageID: docker-pullable://ghcr.io/langstream/langstream-runtime@sha256:8e82f1dfb02afe9137adf4a5dff4770705ff5b6607b70206288d164b116ba10d
+                    lastState: {}
+                    name: deployer-init-config
+                    ready: true
+                    restartCount: 0
+                    state:
+                      terminated:
+                        containerID: docker://973604d477c3b301a5ac716eb4d41a4294b92560d8b3adb94f47976b28f0f5f0
+                        exitCode: 0
+                        finishedAt: "2023-09-01T17:32:01Z"
+                        reason: Completed
+                        startedAt: "2023-09-01T17:32:01Z"
+                  phase: Succeeded
+                  startTime: "2023-09-01T17:29:19Z"
+                """;
+
+        final Pod pod = SerializationUtil.readYaml(podYaml, Pod.class);
+        final KubeUtil.PodStatus status = KubeUtil.getPodsStatuses(List.of(pod)).get("pod1");
+        assertEquals(KubeUtil.PodStatus.State.COMPLETED, status.getState());
+        assertNull(status.getMessage());
+    }
+
+    @Test
+    public void testDeployerJobRunning() {
+        String podYaml =
+                """
+                        apiVersion: v1
+                        kind: Pod
+                        metadata:
+                          name: pod1
+                          namespace: langstream-default
+                        spec: {}
+                        status:
+                          containerStatuses:
+                          - containerID: docker://b6f25d48eae76c2a03c9b5b56e020543eaf444c415124cc07b0952c9ae84bc77
+                            image: ghcr.io/langstream/langstream-runtime:0.0.8
+                            imageID: docker-pullable://ghcr.io/langstream/langstream-runtime@sha256:8e82f1dfb02afe9137adf4a5dff4770705ff5b6607b70206288d164b116ba10d
+                            lastState: {}
+                            name: deployer
+                            ready: true
+                            restartCount: 0
+                            started: true
+                            state:
+                              running:
+                                startedAt: "2023-09-01T21:09:38Z"
+                          hostIP: 192.168.49.2
+                          initContainerStatuses:
+                          - containerID: docker://a36105bd1d2f17ba799b7c24098ccfb6e64f2026b2b11f02b6446bf64b3e94a5
+                            image: ghcr.io/langstream/langstream-runtime:0.0.8
+                            imageID: docker-pullable://ghcr.io/langstream/langstream-runtime@sha256:8e82f1dfb02afe9137adf4a5dff4770705ff5b6607b70206288d164b116ba10d
+                            lastState: {}
+                            name: deployer-init-config
+                            ready: true
+                            restartCount: 0
+                            state:
+                              terminated:
+                                containerID: docker://a36105bd1d2f17ba799b7c24098ccfb6e64f2026b2b11f02b6446bf64b3e94a5
+                                exitCode: 0
+                                finishedAt: "2023-09-01T21:09:37Z"
+                                reason: Completed
+                                startedAt: "2023-09-01T21:09:37Z"
+                          phase: Running
+                          podIP: 10.244.0.55
+                          podIPs:
+                          - ip: 10.244.0.55
+                          qosClass: Burstable
+                          startTime: "2023-09-01T21:09:36Z"
+                        """;
+
+        final Pod pod = SerializationUtil.readYaml(podYaml, Pod.class);
+        final KubeUtil.PodStatus status = KubeUtil.getPodsStatuses(List.of(pod)).get("pod1");
+        assertEquals(KubeUtil.PodStatus.State.RUNNING, status.getState());
+        assertNull(status.getMessage());
+    }
+
+    @Test
+    public void testDeployerJobInitializing() {
+        String podYaml =
+                """
+                        apiVersion: v1
+                        kind: Pod
+                        metadata:
+                          name: pod1
+                          namespace: langstream-default
+                        spec: {}
+                        status:
+                          containerStatuses:
+                          - containerID: docker://b6f25d48eae76c2a03c9b5b56e020543eaf444c415124cc07b0952c9ae84bc77
+                            image: ghcr.io/langstream/langstream-runtime:0.0.8
+                            imageID: docker-pullable://ghcr.io/langstream/langstream-runtime@sha256:8e82f1dfb02afe9137adf4a5dff4770705ff5b6607b70206288d164b116ba10d
+                            lastState: {}
+                            name: deployer
+                            ready: true
+                            restartCount: 0
+                            started: true
+                            state:
+                              running:
+                                startedAt: "2023-09-01T21:09:38Z"
+                          hostIP: 192.168.49.2
+                          initContainerStatuses:
+                          - containerID: docker://a36105bd1d2f17ba799b7c24098ccfb6e64f2026b2b11f02b6446bf64b3e94a5
+                            image: ghcr.io/langstream/langstream-runtime:0.0.8
+                            imageID: docker-pullable://ghcr.io/langstream/langstream-runtime@sha256:8e82f1dfb02afe9137adf4a5dff4770705ff5b6607b70206288d164b116ba10d
+                            lastState: {}
+                            name: deployer-init-config
+                            ready: true
+                            restartCount: 0
+                            state:
+                              terminated:
+                                containerID: docker://a36105bd1d2f17ba799b7c24098ccfb6e64f2026b2b11f02b6446bf64b3e94a5
+                                exitCode: 0
+                                finishedAt: "2023-09-01T21:09:37Z"
+                                reason: Completed
+                                startedAt: "2023-09-01T21:09:37Z"
+                          phase: Running
+                          podIP: 10.244.0.55
+                          podIPs:
+                          - ip: 10.244.0.55
+                          qosClass: Burstable
+                          startTime: "2023-09-01T21:09:36Z"
+                        """;
+
+        final Pod pod = SerializationUtil.readYaml(podYaml, Pod.class);
+        final KubeUtil.PodStatus status = KubeUtil.getPodsStatuses(List.of(pod)).get("pod1");
+        assertEquals(KubeUtil.PodStatus.State.RUNNING, status.getState());
+        assertNull(status.getMessage());
+    }
 }