Repository: knative/eventing (Go)

Issue:
Large amount of SubjectAccessReview for ApiServerSource
<!-- 
Are you using Knative? If you do, we would love to know!
https://github.com/knative/community/issues/new?template=ADOPTERS.yaml&title=%5BADOPTERS%5D%3A+%24%7BCOMPANY+NAME+HERE%7D
-->

**Describe the bug**

When creating an ApiServerSource that needs to monitor a large number of namespaces, Knative Eventing creates a large amount of SubjectAccessReview that impact the performance of the Kubernetes API. Source: https://github.com/knative/eventing/blob/main/pkg/reconciler/apiserversource/apiserversource.go#L294

 **Expected behavior**

The ApiServerSource does not flood the Kubernetes API with SubjectAccessReview requests.

 **To Reproduce**

```
kind create cluster

kubectl apply -f https://github.com/knative/eventing/releases/download/knative-v1.18.2/eventing-crds.yaml
kubectl apply -f https://github.com/knative/eventing/releases/download/knative-v1.18.2/eventing-core.yaml

for i in {1..10};
do
    kubectl create ns test-$i
    kubectl label ns test-$i knative=enabled
    kubectl create role get-events --namespace test-$i --verb get,list,watch --resource Events
    kubectl create rolebinding get-events --namespace test-$i --role get-events --serviceaccount default:default
done


# userAgent=controller/v0.0.0 identifies the requests made by the Knative Eventing Controller
# this should be 0
kubectl logs -n kube-system kube-apiserver-kind-control-plane | grep subjectaccessreviews | grep v0.0.0 | wc -l

kubectl apply -f ass.yaml


# this should be 150
kubectl logs -n kube-system kube-apiserver-kind-control-plane | grep subjectaccessreviews | grep v0.0.0 | wc -l
```

The complete example is in this Gist: https://gist.github.com/rh-hemartin/3971acd757abb7757610342e525c7b0c


 **Knative release version**

v1.18.2

 **Additional context**

My case is 5 resources and 1000 namespaces.



Discussion: I like the idea of having an option on the spec of the ApiServerSource that allows the user to skip the check. This means that the pod could fail setting up watches, but with good reporting on the pod and good documentation warning users this could be a great option.
I'll start working on this, could someone assign it to me? Thanks!
I'm able to confirm this works by using:

```
helm upgrade --install --wait --create-namespace --namespace prom-operator \
	prometheus prometheus-community/kube-prometheus-stack \
	--set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false

kubectl port-forward -n prom-operator svc/prometheus-kube-prometheus-prometheus 9090:9090 &
curl --silent -g 'localhost:9090/api/v1/query?query=apiserver_request_total{resource="subjectaccessreviews"}' | jq '.data.result[].value[1]'
```

After creating 50 namespaces as stated in the gist, the number of requests is  below 40. After applying the APiServerSource with the skip-permissions set to true I dont see an increment in the number of requests. After reapplying the ApiServerSource with the skip-permissions set to false (default and current behaviour) the number immediately spikes to 351 and then to 489. Sadly the metrics do not give insight into these requests.

After creating the namespace 51 the requests jump to 848 and then 1098. After creating the namespace 52 the requests jump to 1719.

The numbers do not match the theoretical formula (n resources * namespaces * 3 verbs) which in this case should be ~150 requests (events, ~50 namespaces, 3 verbs). That is strange... but the increase is there. I can try to get information from the K8s API logs.



Fix (patch):
diff --git a/pkg/adapter/apiserver/adapter.go b/pkg/adapter/apiserver/adapter.go
index 9f8d35208c7..bb9f79886f6 100644
--- a/pkg/adapter/apiserver/adapter.go
+++ b/pkg/adapter/apiserver/adapter.go
@@ -19,14 +19,17 @@ package apiserver
 import (
 	"context"
 	"fmt"
+	"sync"
 	"time"
 
 	cloudevents "github.com/cloudevents/sdk-go/v2"
 	"go.uber.org/zap"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/watch"
+
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/tools/cache"
@@ -55,16 +58,137 @@ type apiServerAdapter struct {
 	name     string // TODO: who dis?
 }
 
+type resourceWatchMatch struct {
+	// resourceWatch is the target configuration for the resource watch.
+	resourceWatch *ResourceWatch
+	// apiResource is the API resource information matched to the resource watch.
+	apiResource *metav1.APIResource
+	// resourceInterfaces are the dynamic interfaces that are matched to the api resource and namespace configuration in the resource watch.
+	resourceInterfaces []dynamic.ResourceInterface
+}
+
 func (a *apiServerAdapter) Start(ctx context.Context) error {
 	return a.start(ctx, ctx.Done())
 }
 
 func (a *apiServerAdapter) start(ctx context.Context, stopCh <-chan struct{}) error {
+	delegate := a.setupDelegate()
+
+	a.logger.Infof("STARTING -- %#v", a.config)
+
+	matches, err := a.collectResourceMatches()
+	if err != nil {
+		return fmt.Errorf("failed to collect resource matches: %v", err)
+	}
+
+	// we have two modes of operation for the ApiServerSource adapter:
+	// 1. Resilient Mode (Default): The adapter uses `reflector.Run()` to continuously retry establishing watches
+	//    on resources, making it resilient to transient errors or delayed permission grants.
+	// 2. Fail-Fast Mode: In this mode, the adapter uses `reflector.ListAndWatchWithContext()`. If any resource watch
+	//    fails to be established on the first attempt, the entire adapter will fail immediately. This provides faster
+	//    feedback and a clearer failure state in environments where permissions are expected to be correct at startup.
+
+	if a.config.FailFast {
+		a.logger.Info("Starting in fail-fast mode. Any single watch failure will stop the adapter.")
+		return a.startFailFast(ctx, stopCh, delegate, matches)
+	}
+	a.logger.Info("Starting in resilient mode. Watch failures will be retried.")
+	return a.startResilient(ctx, stopCh, delegate, matches)
+}
+
+func (a *apiServerAdapter) startResilient(ctx context.Context, stopCh <-chan struct{}, delegate cache.Store, matches []resourceWatchMatch) error {
 	// Local stop channel.
 	stop := make(chan struct{})
 
 	resyncPeriod := 10 * time.Hour
 
+	a.logger.Infof("STARTING -- %#v", a.config)
+
+	for _, match := range matches {
+		if match.apiResource == nil {
+			a.logger.Errorf("could not retrieve information about resource %s: it doesn't exist. skipping...", match.resourceWatch.GVR.String())
+			// This was already logged as a warning in collectResourceMatches.
+			// In resilient mode, we just skip it and continue.
+			continue
+		}
+		for _, res := range match.resourceInterfaces {
+			lw := &cache.ListWatch{
+				ListFunc:  asUnstructuredLister(ctx, res.List, match.resourceWatch.LabelSelector),
+				WatchFunc: asUnstructuredWatcher(ctx, res.Watch, match.resourceWatch.LabelSelector),
+			}
+
+			reflector := cache.NewReflector(lw, &unstructured.Unstructured{}, delegate, resyncPeriod)
+			go reflector.Run(stop)
+		}
+	}
+
+	<-stopCh
+	close(stop)
+	return nil
+}
+
+// startFailFast starts the ApiServerSource in fail fast mode, where it attempts to create watches only once.
+// If any single watch fails, it will return an error and stop all watches.
+// This would result in the APIServerSource status marked as failed.
+func (a *apiServerAdapter) startFailFast(ctx context.Context, stopCh <-chan struct{}, delegate cache.Store, matches []resourceWatchMatch) error {
+	resyncPeriod := 10 * time.Hour
+
+	a.logger.Infof("STARTING -- %#v", a.config)
+	a.logger.Info("ApiServerSource is in fail fast mode, so watch creation will be attempted only once")
+
+	watchCtx, cancelWatchers := context.WithCancel(ctx)
+	defer cancelWatchers()
+
+	var wg sync.WaitGroup
+	errorChan := make(chan error, 1)
+
+	for _, match := range matches {
+		if match.apiResource == nil {
+			a.logger.Errorf("could not retrieve information about resource %s: it doesn't exist.", match.resourceWatch.GVR.String())
+			return fmt.Errorf("resource %s does not exist", match.resourceWatch.GVR.String())
+		}
+		for _, res := range match.resourceInterfaces {
+			lw := &cache.ListWatch{
+				ListFunc:  asUnstructuredLister(watchCtx, res.List, match.resourceWatch.LabelSelector),
+				WatchFunc: asUnstructuredWatcher(watchCtx, res.Watch, match.resourceWatch.LabelSelector),
+			}
+
+			reflector := cache.NewReflector(lw, &unstructured.Unstructured{}, delegate, resyncPeriod)
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				if err := reflector.ListAndWatchWithContext(watchCtx); err != nil {
+					a.logger.Errorf("reflector failed: %v", err)
+					select {
+					case errorChan <- fmt.Errorf("failed to watch: %v", err):
+					default:
+					}
+					cancelWatchers()
+				}
+			}()
+		}
+	}
+
+	go func() {
+		wg.Wait()
+		close(errorChan)
+	}()
+
+	select {
+	case err := <-errorChan:
+		if err != nil {
+			a.logger.Errorf("ApiServerSource failed: %v", err)
+			return err
+		}
+	case <-stopCh:
+		cancelWatchers()
+		return nil
+	}
+
+	return nil
+}
+
+func (a *apiServerAdapter) setupDelegate() cache.Store {
 	var delegate cache.Store = &resourceDelegate{
 		ce:                  a.ce,
 		source:              a.source,
@@ -83,51 +207,47 @@ func (a *apiServerAdapter) start(ctx context.Context, stopCh <-chan struct{}) er
 			delegate:   delegate,
 		}
 	}
+	return delegate
+}
 
-	a.logger.Infof("STARTING -- %#v", a.config)
+func (a *apiServerAdapter) collectResourceMatches() ([]resourceWatchMatch, error) {
+	matches := make([]resourceWatchMatch, 0, len(a.config.Resources))
 
 	for _, configRes := range a.config.Resources {
-
 		resources, err := a.discover.ServerResourcesForGroupVersion(configRes.GVR.GroupVersion().String())
 		if err != nil {
-			return fmt.Errorf("failed to retrieve information about resource %s: %v", configRes.GVR.String(), err)
+			return nil, fmt.Errorf("failed to retrieve information about resource %s: %v", configRes.GVR.String(), err)
 		}
 
-		exists := false
+		match := resourceWatchMatch{
+			resourceWatch:      &configRes,
+			apiResource:        nil,
+			resourceInterfaces: make([]dynamic.ResourceInterface, 0),
+		}
 		for _, apires := range resources.APIResources {
 			if apires.Name == configRes.GVR.Resource {
-				var resources []dynamic.ResourceInterface
+				match.apiResource = &apires
+
 				if apires.Namespaced && !a.config.AllNamespaces {
 					for _, ns := range a.config.Namespaces {
-						resources = append(resources, a.k8s.Resource(configRes.GVR).Namespace(ns))
+						match.resourceInterfaces = append(match.resourceInterfaces, a.k8s.Resource(configRes.GVR).Namespace(ns))
 					}
 				} else {
-					resources = append(resources, a.k8s.Resource(configRes.GVR))
+					match.resourceInterfaces = append(match.resourceInterfaces, a.k8s.Resource(configRes.GVR))
 				}
 
-				for _, res := range resources {
-					lw := &cache.ListWatch{
-						ListFunc:  asUnstructuredLister(ctx, res.List, configRes.LabelSelector),
-						WatchFunc: asUnstructuredWatcher(ctx, res.Watch, configRes.LabelSelector),
-					}
-
-					reflector := cache.NewReflector(lw, &unstructured.Unstructured{}, delegate, resyncPeriod)
-					go reflector.Run(stop)
-				}
-
-				exists = true
 				break
 			}
 		}
 
-		if !exists {
+		matches = append(matches, match)
+
+		if match.apiResource == nil {
 			a.logger.Errorf("could not retrieve information about resource %s: it doesn't exist", configRes.GVR.String())
 		}
 	}
 
-	<-stopCh
-	close(stop)
-	return nil
+	return matches, nil
 }
 
 type unstructuredLister func(context.Context, metav1.ListOptions) (*unstructured.UnstructuredList, error)
diff --git a/pkg/adapter/apiserver/config.go b/pkg/adapter/apiserver/config.go
index c85f7f4de85..58831190a53 100644
--- a/pkg/adapter/apiserver/config.go
+++ b/pkg/adapter/apiserver/config.go
@@ -68,4 +68,10 @@ type Config struct {
 	//
 	// +optional
 	Filters []eventingv1.SubscriptionsAPIFilter `json:"filters,omitempty"`
+
+	// FailFast is a field that communicates that the ApiServerSource adapter should not retry failed watches.
+	// This is useful, when for example, the `skip permissions check` is set to true
+	// (via the features.knative.dev/apiserversource-skip-permissions-check annotation), and the ApiServerSource
+	// adapter should not keep trying to establish watches on resources that it perhaps does not have permissions for.
+	FailFast bool `json:"failFast,omitempty"`
 }
diff --git a/pkg/reconciler/apiserversource/apiserversource.go b/pkg/reconciler/apiserversource/apiserversource.go
index bfddada12d7..b51467033fb 100644
--- a/pkg/reconciler/apiserversource/apiserversource.go
+++ b/pkg/reconciler/apiserversource/apiserversource.go
@@ -60,7 +60,8 @@ const (
 	apiserversourceDeploymentCreated = "ApiServerSourceDeploymentCreated"
 	apiserversourceDeploymentUpdated = "ApiServerSourceDeploymentUpdated"
 
-	component = "apiserversource"
+	component                 = "apiserversource"
+	skipPermissionsAnnotation = "features.knative.dev/apiserversource-skip-permissions-check"
 )
 
 func newWarningSinkNotFound(sink *duckv1.Destination) pkgreconciler.Event {
@@ -146,10 +147,19 @@ func (r *Reconciler) ReconcileKind(ctx context.Context, source *v1.ApiServerSour
 	}
 	source.Status.Namespaces = namespaces
 
-	err = r.runAccessCheck(ctx, source, namespaces)
-	if err != nil {
-		logging.FromContext(ctx).Errorw("Not enough permission", zap.Error(err))
-		return err
+	// We don't check if it really exists because in case it does not exist, the value is an empty string
+	// which also serves our purposes as by default we will check permissions
+	annotations := source.GetAnnotations()
+	skipPermissions := annotations[skipPermissionsAnnotation]
+	if skipPermissions == "true" {
+		// If skip permissions, mark enough permissions directly
+		source.Status.MarkSufficientPermissions()
+	} else {
+		err = r.runAccessCheck(ctx, source, namespaces)
+		if err != nil {
+			logging.FromContext(ctx).Errorw("Not enough permission", zap.Error(err))
+			return err
+		}
 	}
 
 	if err := r.propagateTrustBundles(ctx, source); err != nil {
@@ -224,6 +234,9 @@ func (r *Reconciler) createReceiveAdapter(ctx context.Context, src *v1.ApiServer
 	// 	return nil, err
 	// }
 
+	annotations := src.GetAnnotations()
+	skipPermissions := annotations[skipPermissionsAnnotation]
+
 	featureFlags := feature.FromContext(ctx)
 
 	adapterArgs := resources.ReceiveAdapterArgs{
@@ -237,6 +250,7 @@ func (r *Reconciler) createReceiveAdapter(ctx context.Context, src *v1.ApiServer
 		Namespaces:    namespaces,
 		AllNamespaces: allNamespaces,
 		NodeSelector:  featureFlags.NodeSelector(),
+		FailFast:      skipPermissions == "true",
 	}
 
 	expected, err := resources.MakeReceiveAdapter(&adapterArgs)
diff --git a/pkg/reconciler/apiserversource/resources/receive_adapter.go b/pkg/reconciler/apiserversource/resources/receive_adapter.go
index 4bb823647b2..fc640aa7ced 100644
--- a/pkg/reconciler/apiserversource/resources/receive_adapter.go
+++ b/pkg/reconciler/apiserversource/resources/receive_adapter.go
@@ -50,6 +50,7 @@ type ReceiveAdapterArgs struct {
 	Namespaces    []string
 	AllNamespaces bool
 	NodeSelector  map[string]string
+	FailFast      bool
 }
 
 // MakeReceiveAdapter generates (but does not insert into K8s) the Receive Adapter Deployment for
@@ -138,6 +139,7 @@ func makeEnv(args *ReceiveAdapterArgs) ([]corev1.EnvVar, error) {
 		EventMode:     args.Source.Spec.EventMode,
 		AllNamespaces: args.AllNamespaces,
 		Filters:       args.Source.Spec.Filters,
+		FailFast:      args.FailFast,
 	}
 
 	for _, r := range args.Source.Spec.Resources {