Change:
<!-- Please only use this template for submitting reports about flaky tests or jobs (pass or fail with no underlying change in code) in Kubernetes CI --Which jobs are flaking: pull-kubernetes-e2e-gce ci-kubernetes-e2e-gci-gce Which test(s) are flaking: [sig-apps] CronJob should delete successful/failed finished jobs with limit of one job Testgrid link: Reason for failure: Anything else we need to know: this is the top flake for pull-kubernetes-e2e-gce: triage link: specific failures /sig apps /priority important-soon
this is the second flakiest test (failing on ~15% of PR runs)
I'll look into it. /assign
if there's not an obvious immediate fix, I would recommend: the test as to skip the test in merge-blocking job runs a second PR that reverts the change and adds debugging commits and do on that PR to flush out issues this issue open and in the current milestone to track resolution of the test before release (either by fixing the test, finding/fixing an underlying bug, or proving we have test coverage of the function via another method and deleting the test)
marked as in , will leave this issue open to track root causing, fixing, restoring the test before release
To follow up here, this was confirmed as root-caused, fixed and restored by

Discussion:
framework.ExpectNoError(err, "Failed to remove %s cronjob in namespace %s", cronJob.Name, f.Namespace.Name) }) <del> // cleanup of successful/failed finished jobs, with successfulJobsHistoryLimit and failedJobsHistoryLimit ginkgo.It("should delete successful/failed finished jobs with limit of one job [Flaky]", func() { testCases := []struct { description  string command      []string successLimit int32 failedLimit  int32 }{ { description:  "successful-jobs-history-limit", command:      successCommand, successLimit: 1, // keep one successful job failedLimit:  0, // keep none failed job }, { description:  "failed-jobs-history-limit", command:      failureCommand, successLimit: 0, // keep none succcessful job failedLimit:  1, // keep one failed job }, } </del> <ins> // cleanup of successful finished jobs, with limit of one successful job ginkgo.It("should delete successful finished jobs with limit of one successful job", func() { ginkgo.By("Creating an AllowConcurrent cronjob with custom history limit") successLimit := int32(1) failedLimit := int32(0) cronJob := newTestCronJob("successful-jobs-history-limit", "*/1 * * * ?", batchv1beta1.AllowConcurrent, successCommand, &successLimit, &failedLimit) ensureHistoryLimits(f.ClientSet, f.Namespace.Name, cronJob) }) </ins> <del> for _, t := range testCases { ginkgo.By(fmt.Sprintf("Creating a AllowConcurrent cronjob with custom %s", t.description)) cronJob := newTestCronJob(t.description, "*/1 * * * ?", batchv1beta1.AllowConcurrent, t.command, &t.successLimit, &t.failedLimit) cronJob, err := createCronJob(f.ClientSet, f.Namespace.Name, cronJob) framework.ExpectNoError(err, "Failed to create allowconcurrent cronjob with custom history limits in namespace %s", f.Namespace.Name) // Job is going to complete instantly: do not check for an active job // as we are most likely to miss it ginkgo.By("Ensuring a finished job exists") err = waitForAnyFinishedJob(f.ClientSet, f.Namespace.Name) framework.ExpectNoError(err, "Failed to ensure a finished cronjob exists in namespace %s", f.Namespace.Name) ginkgo.By("Ensuring a finished job exists by listing jobs explicitly") jobs, err := f.ClientSet.BatchV1().Jobs(f.Namespace.Name).List(metav1.ListOptions{}) framework.ExpectNoError(err, "Failed to ensure a finished cronjob exists by listing jobs explicitly in namespace %s", f.Namespace.Name) activeJobs, finishedJobs := filterActiveJobs(jobs) if len(finishedJobs) != 1 { framework.Logf("Expected one finished job in namespace %s; activeJobs=%v; finishedJobs=%v", f.Namespace.Name, activeJobs, finishedJobs) framework.ExpectEqual(len(finishedJobs), 1) } </del> <ins> // cleanup of failed finished jobs, with limit of one failed job ginkgo.It("should delete failed finished jobs with limit of one job", func() { ginkgo.By("Creating an AllowConcurrent cronjob with custom history limit") successLimit := int32(0) failedLimit := int32(1) cronJob := newTestCronJob("failed-jobs-history-limit", "*/1 * * * ?", batchv1beta1.AllowConcurrent, failureCommand, &successLimit, &failedLimit) </ins> <del> // Job should get deleted when the next job finishes the next minute ginkgo.By("Ensuring this job and its pods does not exist anymore") err = waitForJobToDisappear(f.ClientSet, f.Namespace.Name, finishedJobs[0]) framework.ExpectNoError(err, "Failed to ensure that job does not exists anymore in namespace %s", f.Namespace.Name) err = waitForJobsPodToDisappear(f.ClientSet, f.Namespace.Name, finishedJobs[0]) framework.ExpectNoError(err, "Failed to ensure that pods for job does not exists anymore in namespace %s", f.Namespace.Name) ginkgo.By("Ensuring there is 1 finished job by listing jobs explicitly") jobs, err = f.ClientSet.BatchV1().Jobs(f.Namespace.Name).List(metav1.ListOptions{}) framework.ExpectNoError(err, "Failed to ensure there is one finished job by listing job explicitly in namespace %s", f.Namespace.Name) _, finishedJobs = filterActiveJobs(jobs) framework.ExpectEqual(len(finishedJobs), 1) ginkgo.By("Removing cronjob") err = deleteCronJob(f.ClientSet, f.Namespace.Name, cronJob.Name) framework.ExpectNoError(err, "Failed to remove the %s cronjob in namespace %s", cronJob.Name, f.Namespace.Name) } </del> <ins> ensureHistoryLimits(f.ClientSet, f.Namespace.Name, cronJob) </ins> }) }) <ins> func ensureHistoryLimits(c clientset.Interface, ns string, cronJob *batchv1beta1.CronJob) { cronJob, err := createCronJob(c, ns, cronJob) framework.ExpectNoError(err, "Failed to create allowconcurrent cronjob with custom history limits in namespace %s", ns) // Job is going to complete instantly: do not check for an active job // as we are most likely to miss it ginkgo.By("Ensuring a finished job exists") err = waitForAnyFinishedJob(c, ns) framework.ExpectNoError(err, "Failed to ensure a finished cronjob exists in namespace %s", ns) ginkgo.By("Ensuring a finished job exists by listing jobs explicitly") jobs, err := c.BatchV1().Jobs(ns).List(metav1.ListOptions{}) framework.ExpectNoError(err, "Failed to ensure a finished cronjob exists by listing jobs explicitly in namespace %s", ns) activeJobs, finishedJobs := filterActiveJobs(jobs) if len(finishedJobs) != 1 { framework.Logf("Expected one finished job in namespace %s; activeJobs=%v; finishedJobs=%v", ns, activeJobs, finishedJobs) framework.ExpectEqual(len(finishedJobs), 1) } // Job should get deleted when the next job finishes the next minute ginkgo.By("Ensuring this job and its pods does not exist anymore") err = waitForJobToDisappear(c, ns, finishedJobs[0]) framework.ExpectNoError(err, "Failed to ensure that job does not exists anymore in namespace %s", ns) err = waitForJobsPodToDisappear(c, ns, finishedJobs[0]) framework.ExpectNoError(err, "Failed to ensure that pods for job does not exists anymore in namespace %s", ns) ginkgo.By("Ensuring there is 1 finished job by listing jobs explicitly") jobs, err = c.BatchV1().Jobs(ns).List(metav1.ListOptions{}) framework.ExpectNoError(err, "Failed to ensure there is one finished job by listing job explicitly in namespace %s", ns) activeJobs, finishedJobs = filterActiveJobs(jobs) if len(finishedJobs) != 1 { framework.Logf("Expected one finished job in namespace %s; activeJobs=%v; finishedJobs=%v", ns, activeJobs, finishedJobs) framework.ExpectEqual(len(finishedJobs), 1) } ginkgo.By("Removing cronjob") err = deleteCronJob(c, ns, cronJob.Name) framework.ExpectNoError(err, "Failed to remove the %s cronjob in namespace %s", cronJob.Name, ns) } </ins> // newTestCronJob returns a cronjob which does one of several testing behaviors. func newTestCronJob(name, schedule string, concurrencyPolicy batchv1beta1.ConcurrencyPolicy, command []string, successfulJobsHistoryLimit *int32, failedJobsHistoryLimit *int32) *batchv1beta1.CronJob {