Change:
What happened: What you expected to happen: How to reproduce it (as minimally and precisely as possible): $oc create job -f $ oc get job jobname -ojson | jq .status { "active": 1, "failed": 1, "startTime": "2020-07-30T06:45:37Z" } { "active": 1, "failed": 2, "startTime": "2020-07-30T06:45:37Z" } { "active": 1, "failed": 3, "startTime": "2020-07-30T06:45:37Z" } { "active": 1, "failed": 4, "startTime": "2020-07-30T06:45:37Z" } { "conditions": [ { "lastProbeTime": "2020-07-30T06:53:10Z", "lastTransitionTime": "2020-07-30T06:53:10Z", "message": "Job has reached the specified backoff limit", "reason": "BackoffLimitExceeded", "status": "True", "type": "Failed" } ], "failed": 5, "startTime": "2020-07-30T06:45:37Z" } $ oc get job jobname -ojson | jq .spec.backoffLimit 6 Anything else we need to know?: Environment: Kubernetes version (use ): Client Version: 4.5.3 Server Version: 4.4.13 Kubernetes Version: v1.17.1+ Cloud provider or hardware configuration: OS (e.g: ): NAME="Red Hat Enterprise Linux CoreOS" VERSIONID="4.4" OPENSHIFTVERSION="4.4" RHELVERSION="8.2" ID="rhcos" IDLIKE="rhel fedora" ANSICOLOR="0;31" CPENAME="cpe:/o:redhat:enterpriselinux:8::coreos" HOMEURL="" BUGREPORTURL="" REDHATBUGZILLAPRODUCT="OpenShift Container Platform" REDHATBUGZILLAPRODUCTVERSION="4.4" REDHATSUPPORTPRODUCT="OpenShift Container Platform" REDHATSUPPORTPRODUCTVERSION="4.4" OSTREEVERSION='44.82.-0' Kernel (e.g. ): Linux ip-10-0-192-232 4.18.0-193.13.2.el82.x8664 SMP Mon Jul 13 23:17:28 UTC 2020 x8664 x8664 x8664 GNU/Linux Install tools: Others: apiVersion: batch/v1 kind: Job metadata: name: sre-build-test-fail namespace: openshift-build-test spec: backoffLimit: 6 completions: 1 parallelism: 1 template: spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: matchExpressions: key: node- operator: In values: infra containers: command: /bin/bash -c | # ensure we fail if something exits non-zero set -o errexit set -o nounset set -o pipefail exit 1 imagePullPolicy: Always name: sre-build-test restartPolicy: Never terminationGracePeriodSeconds: 30 tolerations: effect: NoSchedule key: node- operator: Equal value: infra
/sig apps
not sure but that is the best I can think of at this time
i have noticed this problem also, and sometimes, it seems to work normally.
so I found out that in the test is so this can cause the issue.. it has been there from the beginning of the codes inception.. WDYT?
/assign
Duplicate of . /close
Closing this issue. <detailsIn response to : Duplicate of . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

Discussion:
return true, nil } <del> // retrieve the previous number of retry previousRetry := jm.queue.NumRequeues(key) </del> // Check the expectations of the job before counting active pods, otherwise a new pod can sneak in // and update the expectations after we've retrieved active pods from the store. If a new pod enters // the store after we've checked the expectation, the job sync is just deferred till the next relist.

// is different than parallelism, otherwise the previous controller loop // failed updating status so even if we pick up failure it is not a new one exceedsBackoffLimit := jobHaveNewFailure && (active != *job.Spec.Parallelism) && <del> (int32(previousRetry)+1 > *job.Spec.BackoffLimit) </del> <ins> (failed > *job.Spec.BackoffLimit) </ins> if exceedsBackoffLimit || pastBackoffLimitOnFailure(&job, pods) { // check if the number of pod restart exceeds backoff (for restart OnFailure only)

}) } } <ins> func TestJobBackoffOnRestartPolicyNever(t *testing.T) { jobConditionFailed := batch.JobFailed testCases := map[string]struct { // job setup parallelism  int32 completions  int32 backoffLimit int32 // pod setup activePodsPhase v1.PodPhase activePods      int32 failedPods      int32 // expectations isExpectingAnError      bool jobKeyForget            bool expectedActive          int32 expectedSucceeded       int32 expectedFailed          int32 expectedCondition       *batch.JobConditionType expectedConditionReason string }{ "not enough failures with backoffLimit 0 - single pod": { 1, 1, 0, v1.PodRunning, 1, 0, false, true, 1, 0, 0, nil, "", }, "not enough failures with backoffLimit 1 - single pod": { 1, 1, 1, "", 0, 1, true, false, 1, 0, 1, nil, "", }, "too many failures with backoffLimit 1 - single pod": { 1, 1, 1, "", 0, 2, false, true, 0, 0, 2, &jobConditionFailed, "BackoffLimitExceeded", }, "not enough failures with backoffLimit 6 - multiple pods": { 2, 2, 6, v1.PodRunning, 1, 6, true, false, 2, 0, 6, nil, "", }, "too many failures with backoffLimit 6 - multiple pods": { 2, 2, 6, "", 0, 7, false, true, 0, 0, 7, &jobConditionFailed, "BackoffLimitExceeded", }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { // job manager setup clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: "", ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}}) manager, sharedInformerFactory := newControllerFromClient(clientset, controller.NoResyncPeriodFunc) fakePodControl := controller.FakePodControl{} manager.podControl = &fakePodControl manager.podStoreSynced = alwaysReady manager.jobStoreSynced = alwaysReady var actual *batch.Job manager.updateHandler = func(job *batch.Job) error { actual = job return nil } // job & pods setup job := newJob(tc.parallelism, tc.completions, tc.backoffLimit) job.Spec.Template.Spec.RestartPolicy = v1.RestartPolicyNever sharedInformerFactory.Batch().V1().Jobs().Informer().GetIndexer().Add(job) podIndexer := sharedInformerFactory.Core().V1().Pods().Informer().GetIndexer() for _, pod := range newPodList(tc.failedPods, v1.PodFailed, job) { podIndexer.Add(&pod) } for _, pod := range newPodList(tc.activePods, tc.activePodsPhase, job) { podIndexer.Add(&pod) } // run forget, err := manager.syncJob(testutil.GetKey(job, t)) if (err != nil) != tc.isExpectingAnError { t.Errorf("unexpected error syncing job. Got %#v, isExpectingAnError: %vn", err, tc.isExpectingAnError) } if forget != tc.jobKeyForget { t.Errorf("unexpected forget value. Expected %v, saw %vn", tc.jobKeyForget, forget) } // validate status if actual.Status.Active != tc.expectedActive { t.Errorf("unexpected number of active pods. Expected %d, saw %dn", tc.expectedActive, actual.Status.Active) } if actual.Status.Succeeded != tc.expectedSucceeded { t.Errorf("unexpected number of succeeded pods. Expected %d, saw %dn", tc.expectedSucceeded, actual.Status.Succeeded) } if actual.Status.Failed != tc.expectedFailed { t.Errorf("unexpected number of failed pods. Expected %d, saw %dn", tc.expectedFailed, actual.Status.Failed) } // validate conditions if tc.expectedCondition != nil && !getCondition(actual, *tc.expectedCondition, tc.expectedConditionReason) { t.Errorf("expected completion condition. Got %#v", actual.Status.Conditions) } }) } } </ins>

ginkgo.By(fmt.Sprintf("Checking that %d pod created and status is failed", backoff+1)) pods, err := e2ejob.GetJobPods(f.ClientSet, f.Namespace.Name, job.Name) framework.ExpectNoError(err, "failed to get PodList for job %s in namespace: %s", job.Name, f.Namespace.Name) <del> // gomega.Expect(pods.Items).To(gomega.HaveLen(backoff + 1)) // due to NumRequeus not being stable enough, especially with failed status // updates we need to allow more than backoff+1 // TODO revert this back to above when https://github.com/kubernetes/kubernetes/issues/64787 gets fixed if len(pods.Items) < backoff+1 { framework.Failf("Not enough pod created expected at least %d, got %#v", backoff+1, pods.Items) } </del> <ins> gomega.Expect(pods.Items).To(gomega.HaveLen(backoff + 1)) </ins> for _, pod := range pods.Items { framework.ExpectEqual(pod.Status.Phase, v1.PodFailed) }