Change:
Which jobs are failing: Which test(s) are failing: Since when has it been failing: Testgrid link: Reason for failure: Anything else we need to know: /sig api-machinery /assign
/cc
want to take a look at the audit event issue first? I can dig in after that if issues remain.
yes, I'm looking at the audit event issue now
actually the failure message suggests that the missing audit log is caused by unexpected admission invocation: I think it's unrelated to auditing Note that there are two tests named "no reinvocation of webhook B when in-tree or prior webhook mutations" :( and from the audit log (), it also suggests that the test is hitting a wrong endpoint
Thanks I'll dig in.

Discussion:
"net/http/httptest" "os" "reflect" <ins> "strconv" </ins> "strings" "sync" "testing"

} } <del> _, err = client.CoreV1().Pods("default").Create(reinvocationMarkerFixture) if err != nil { t.Fatal(err) } </del> for i, tt := range testCases { t.Run(tt.name, func(t *testing.T) { upCh := recorder.Reset() <del> ns := fmt.Sprintf("reinvoke-%d", i) _, err = client.CoreV1().Namespaces().Create(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}) </del> <ins> testCaseID := strconv.Itoa(i) ns := "reinvoke-" + testCaseID nsLabels := map[string]string{"test-case": testCaseID} _, err = client.CoreV1().Namespaces().Create(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns, Labels: nsLabels}}) if err != nil { t.Fatal(err) } // Write markers to a separate namespace to avoid cross-talk markerNs := ns + "-markers" markerNsLabels := map[string]string{"test-markers": testCaseID} _, err = client.CoreV1().Namespaces().Create(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: markerNs, Labels: markerNsLabels}}) </ins> if err != nil { t.Fatal(err) } <ins> // Create a maker object to use to check for the webhook configurations to be ready. marker, err := client.CoreV1().Pods(markerNs).Create(newReinvocationMarkerFixture(markerNs)) if err != nil { t.Fatal(err) } fail := admissionv1beta1.Fail </ins> webhooks := []admissionv1beta1.MutatingWebhook{} for j, webhook := range tt.webhooks { <del> name := fmt.Sprintf("admission.integration.test.%d.%s", j, strings.TrimPrefix(webhook.path, "/")) fail := admissionv1beta1.Fail </del> endpoint := webhookServer.URL + webhook.path <ins> name := fmt.Sprintf("admission.integration.test.%d.%s", j, strings.TrimPrefix(webhook.path, "/")) </ins> webhooks = append(webhooks, admissionv1beta1.MutatingWebhook{ Name: name, ClientConfig: admissionv1beta1.WebhookClientConfig{

Rule:       admissionv1beta1.Rule{APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"pods"}}, }}, ObjectSelector:          webhook.objectSelector, <ins> NamespaceSelector:       &metav1.LabelSelector{MatchLabels: nsLabels}, </ins> FailurePolicy:           &fail, ReinvocationPolicy:      webhook.policy, AdmissionReviewVersions: []string{"v1beta1"}, }) } <ins> // Register a marker checking webhook with each set of webhook configurations markerEndpoint := webhookServer.URL + "/marker" webhooks = append(webhooks, admissionv1beta1.MutatingWebhook{ Name: "admission.integration.test.marker", ClientConfig: admissionv1beta1.WebhookClientConfig{ URL:      &markerEndpoint, CABundle: localhostCert, }, Rules: []admissionv1beta1.RuleWithOperations{{ Operations: []admissionv1beta1.OperationType{admissionv1beta1.OperationAll}, Rule:       admissionv1beta1.Rule{APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"pods"}}, }}, NamespaceSelector:       &metav1.LabelSelector{MatchLabels: markerNsLabels}, ObjectSelector:          &metav1.LabelSelector{MatchLabels: map[string]string{"marker": "true"}}, AdmissionReviewVersions: []string{"v1beta1"}, }) </ins> cfg, err := client.AdmissionregistrationV1beta1().MutatingWebhookConfigurations().Create(&admissionv1beta1.MutatingWebhookConfiguration{ ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("admission.integration.test-%d", i)},

// wait until new webhook is called the first time if err := wait.PollImmediate(time.Millisecond*5, wait.ForeverTestTimeout, func() (bool, error) { <del> _, err = client.CoreV1().Pods("default").Patch(reinvocationMarkerFixture.Name, types.JSONPatchType, []byte("[]")) </del> <ins> _, err = client.CoreV1().Pods(markerNs).Patch(marker.Name, types.JSONPatchType, []byte("[]")) </ins> select { case <-upCh: return true, nil

http.Error(w, err.Error(), 400) } <del> // When resetting between tests, a marker object is patched until this webhook // observes it, at which point it is considered ready. if pod.Namespace == reinvocationMarkerFixture.Namespace && pod.Name == reinvocationMarkerFixture.Name { recorder.MarkerReceived() allow(w) return } </del> recorder.IncrementCount(r.URL.Path) switch r.URL.Path { <ins> case "/marker": // When resetting between tests, a marker object is patched until this webhook // observes it, at which point it is considered ready. recorder.MarkerReceived() allow(w) return </ins> case "/noop": allow(w) case "/settrue":

} } <del> var reinvocationMarkerFixture = &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name:      "marker", }, Spec: corev1.PodSpec{ Containers: []v1.Container{{ Name:  "fake-name", Image: "fakeimage", }}, }, </del> <ins> func newReinvocationMarkerFixture(namespace string) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name:      "marker", Labels: map[string]string{ "marker": "true", }, }, Spec: corev1.PodSpec{ Containers: []v1.Container{{ Name:  "fake-name", Image: "fakeimage", }}, }, } </ins> }