Change:
What would you like to be : The api server should retry sending audit events to configured webhook sinks regardless of the type of HTTP error code it might receive from the webhooks. Why is this needed: While some components may live with failed data transmissions, this is a particularly sensitive topic for the audit area. The API server audit mechanisms must ensure that no other reasonable option exists before dropping an audit event that may be the only trace of malicious behavior in the cluster. It seems that the current implementation may be improved further in this direction. Currently, when the API server is configured for non-blocking sending of audit events to a webhook audit sink backend, it will retry up to 5 times with exponential backoff, should there be an error in communicating with the webhook, as implemented . However, not all errors will trigger a retry. Some are considered transient and yield retry. Others simply stop the retry cycle. For example, it retries on receiving HTTP code , but not on receiving . The list of error codes that will yield a retry is actually quite small - three HTTP errors and one network error. Any other error will not trigger a retry and the audit events will be lost. The code comments explain that the retry will consider only recoverable failures, which may make sense in other cases, but as far as sending audit events to an audit backend webhook is concerned, it is not sufficient policy. A more robust approach would be to consider any error and give the backend service a chance to recover instead of relying on the error codes semantics and the webhook implementations interpretation of it. That is more inline with having the transmission of audit events as top priority. /sig auth /sig api-machinery

Discussion:
// timeout of the HTTP request, including reading the response body. const defaultRequestTimeout = 30 * time.Second <ins> // GenericWebhook defines a generic client for webhooks with commonly used capabilities, // such as retry requests. </ins> type GenericWebhook struct { RestClient     *rest.RESTClient InitialBackoff time.Duration <ins> ShouldRetry    func(error) bool } // DefaultShouldRetry is a default implementation for the GenericWebhook ShouldRetry function property. // If the error reason is one of: networking (connection reset) or http (InternalServerError (500), GatewayTimeout (504), TooManyRequests (429)), // or apierrors.SuggestsClientDelay() returns true, then the function advises a retry. // Otherwise it returns false for an immediate fail. func DefaultShouldRetry(err error) bool { // these errors indicate a transient error that should be retried. if net.IsConnectionReset(err) || apierrors.IsInternalError(err) || apierrors.IsTimeout(err) || apierrors.IsTooManyRequests(err) { return true } // if the error sends the Retry-After header, we respect it as an explicit confirmation we should retry. if _, shouldRetry := apierrors.SuggestsClientDelay(err); shouldRetry { return true } return false </ins> } // NewGenericWebhook creates a new GenericWebhook from the provided kubeconfig file.

return nil, err } <del> return &GenericWebhook{restClient, initialBackoff}, nil </del> <ins> return &GenericWebhook{restClient, initialBackoff, DefaultShouldRetry}, nil </ins> } // WithExponentialBackoff will retry webhookFn() up to 5 times with exponentially increasing backoff when <del> // it returns an error for which apierrors.SuggestsClientDelay() or apierrors.IsInternalError() returns true. </del> <ins> // it returns an error for which this GenericWebhook's ShouldRetry function returns true, confirming it to // be retriable. If no ShouldRetry has been defined for the webhook, then the default one is used (DefaultShouldRetry). </ins> func (g *GenericWebhook) WithExponentialBackoff(ctx context.Context, webhookFn func() rest.Result) rest.Result { var result rest.Result <ins> shouldRetry := g.ShouldRetry if shouldRetry == nil { shouldRetry = DefaultShouldRetry } </ins> WithExponentialBackoff(ctx, g.InitialBackoff, func() error { result = webhookFn() return result.Error() <del> }) </del> <ins> }, shouldRetry) </ins> return result } <del> // WithExponentialBackoff will retry webhookFn() up to 5 times with exponentially increasing backoff when // it returns an error for which apierrors.SuggestsClientDelay() or apierrors.IsInternalError() returns true. func WithExponentialBackoff(ctx context.Context, initialBackoff time.Duration, webhookFn func() error) error { </del> <ins> // WithExponentialBackoff will retry webhookFn up to 5 times with exponentially increasing backoff when // it returns an error for which shouldRetry returns true, confirming it to be retriable. func WithExponentialBackoff(ctx context.Context, initialBackoff time.Duration, webhookFn func() error, shouldRetry func(error) bool) error { </ins> backoff := wait.Backoff{ Duration: initialBackoff, Factor:   1.5,

var err error wait.ExponentialBackoff(backoff, func() (bool, error) { err = webhookFn() if ctx.Err() != nil { // we timed out or were cancelled, we should not retry return true, err } <del> // these errors indicate a transient error that should be retried. if net.IsConnectionReset(err) || apierrors.IsInternalError(err) || apierrors.IsTimeout(err) || apierrors.IsTooManyRequests(err) { return false, nil } // if the error sends the Retry-After header, we respect it as an explicit confirmation we should retry. if _, shouldRetry := apierrors.SuggestsClientDelay(err); shouldRetry { </del> <ins> if shouldRetry(err) { </ins> return false, nil } if err != nil {

install.Install(audit.Scheme) } <ins> // retryOnError enforces the webhook client to retry requests // on error regardless of its nature. // The default implementation considers a very limited set of // 'retriable' errors, assuming correct use of HTTP codes by // external webhooks. // That may easily lead to dropped audit events. In fact, there is // hardly any error that could be a justified reason NOT to retry // sending audit events if there is even a slight chance that the // receiving service gets back to normal at some point. func retryOnError(err error) bool { if err != nil { return true } return false } </ins> func loadWebhook(configFile string, groupVersion schema.GroupVersion, initialBackoff time.Duration) (*webhook.GenericWebhook, error) { <del> return webhook.NewGenericWebhook(audit.Scheme, audit.Codecs, configFile, </del> <ins> w, err := webhook.NewGenericWebhook(audit.Scheme, audit.Codecs, configFile, </ins> []schema.GroupVersion{groupVersion}, initialBackoff) <ins> w.ShouldRetry = retryOnError return w, err </ins> } type backend struct {

w: &webhook.GenericWebhook{ RestClient:     rc, InitialBackoff: initialBackoff, <ins> ShouldRetry:    retryOnError, </ins> }, name: fmt.Sprintf("dynamic_%s", PluginName), }

webhook.WithExponentialBackoff(ctx, w.initialBackoff, func() error { result, err = w.tokenReview.CreateContext(ctx, r) return err <del> }) </del> <ins> }, webhook.DefaultShouldRetry) </ins> if err != nil { // An error here indicates bad configuration or an outage. Log for debugging. klog.Errorf("Failed to make webhook authenticator request: %v", err)

webhook.WithExponentialBackoff(ctx, w.initialBackoff, func() error { result, err = w.subjectAccessReview.CreateContext(ctx, r) return err <del> }) </del> <ins> }, webhook.DefaultShouldRetry) </ins> if err != nil { // An error here indicates bad configuration or an outage. Log for debugging. klog.Errorf("Failed to make webhook authorizer request: %v", err)