Change:
The "AppArmor load AppArmor profiles should enforce an AppArmor profile" e2e test in failing on our 5k-node gce clusters - https://k8s- (run no. 8) due to timeout: Seems like the test is trying to load the profiles across all the nodes using a daemonset and then running a test pod to check that. It's passing on our 2k-node clusters (https://k8s-) but failing for 5k - suggesting that it's likely due to size of the cluster. I think it's the ds creation taking longer on large cluster. cc
Seems like you're the most relevant here. Could one of you take a look into it? And does increasing the timeout for the test-pod based on no. of nodes sound reasonable?
Assigning it you since is OOO. Feel free to reassign as apt.
This is part of the correctness suite and large cluster tests are release-blocking.
FYI
Reassigning to who is back on Tuesday. He wrote the AppArmor stuff and should be able to debug this quickly. /assign
Ref
Could you manage to take a look into this?
Sorry, I meant to reply to this! I have a fix, just need to confirm it works. I think you're right about the DaemonSet timing out. There's no reason this needs to load the profiles on every node, so the fix just runs a single loader, and uses pod affinity to co-schedule the test pod.
Smooth!
[MILESTONENOTIFIER] Milestone Labels Complete Issue label settings: sig/node sig/scalability: Issue will be escalated to these SIGs if needed. priority/important-soon: Escalate to the issue owners and SIG owner; move out of milestone after several unsuccessful escalation attempts. kind/bug: Fixes a bug discovered during the current release. <detailsAdditional instructions available <a href=""here</aThe commands available for adding these labels are documented <a href=""here</a</details

Discussion:
"//vendor/golang.org/x/net/websocket:go_default_library", "//vendor/k8s.io/api/autoscaling/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", <del> "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", </del> "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",

"fmt" api "k8s.io/api/core/v1" <del> extensions "k8s.io/api/extensions/v1beta1" </del> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <ins> "k8s.io/apimachinery/pkg/labels" </ins> "k8s.io/kubernetes/pkg/security/apparmor" "k8s.io/kubernetes/test/e2e/framework" <ins> . "github.com/onsi/gomega" </ins> ) const ( appArmorProfilePrefix = "e2e-apparmor-test-" appArmorAllowedPath   = "/expect_allowed_write" appArmorDeniedPath    = "/expect_permission_denied" <ins> loaderLabelKey   = "name" loaderLabelValue = "e2e-apparmor-loader" </ins> ) // AppArmorDistros are distros with AppArmor support

} func LoadAppArmorProfiles(f *framework.Framework) { <del> _, err := createAppArmorProfileCM(f) framework.ExpectNoError(err) _, err = createAppArmorProfileLoader(f) framework.ExpectNoError(err) </del> <ins> createAppArmorProfileCM(f) createAppArmorProfileLoader(f) </ins> } // CreateAppArmorTestPod creates a pod that tests apparmor profile enforcement. The pod exits with

done`, testCmd) } <ins> loaderAffinity := &api.Affinity{ PodAffinity: &api.PodAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{{ Namespaces: []string{f.Namespace.Name}, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{loaderLabelKey: loaderLabelValue}, }, TopologyKey: "kubernetes.io/hostname", }}, }, } </ins> pod := &api.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-apparmor-",

}, }, Spec: api.PodSpec{ <ins> Affinity: loaderAffinity, </ins> Containers: []api.Container{{ Name:    "test", Image:   busyboxImage,

pod = f.PodClient().Create(pod) framework.ExpectNoError(framework.WaitForPodSuccessInNamespace( f.ClientSet, pod.Name, f.Namespace.Name)) <ins> var err error pod, err = f.PodClient().Get(pod.Name, metav1.GetOptions{}) framework.ExpectNoError(err) </ins> } else { pod = f.PodClient().CreateSync(pod) framework.ExpectNoError(f.WaitForPodReady(pod.Name)) } <ins> // Verify Pod affinity colocated the Pods. loader := getRunningLoaderPod(f) Expect(pod.Spec.NodeName).To(Equal(loader.Spec.NodeName)) </ins> return pod } <del> func createAppArmorProfileCM(f *framework.Framework) (*api.ConfigMap, error) { </del> <ins> func createAppArmorProfileCM(f *framework.Framework) { </ins> profileName := appArmorProfilePrefix + f.Namespace.Name profile := fmt.Sprintf(`#include <tunables/global> profile %s flags=(attach_disconnected) {

profileName: profile, }, } <del> return f.ClientSet.Core().ConfigMaps(f.Namespace.Name).Create(cm) </del> <ins> _, err := f.ClientSet.Core().ConfigMaps(f.Namespace.Name).Create(cm) framework.ExpectNoError(err, "Failed to create apparmor-profiles ConfigMap") </ins> } <del> func createAppArmorProfileLoader(f *framework.Framework) (*extensions.DaemonSet, error) { </del> <ins> func createAppArmorProfileLoader(f *framework.Framework) { </ins> True := true <del> // Copied from https://github.com/kubernetes/contrib/blob/master/apparmor/loader/example-configmap.yaml loader := &extensions.DaemonSet{ </del> <ins> One := int32(1) loader := &api.ReplicationController{ </ins> ObjectMeta: metav1.ObjectMeta{ Name:      "apparmor-loader", Namespace: f.Namespace.Name, }, <del> Spec: extensions.DaemonSetSpec{ Template: api.PodTemplateSpec{ </del> <ins> Spec: api.ReplicationControllerSpec{ Replicas: &One, Template: &api.PodTemplateSpec{ </ins> ObjectMeta: metav1.ObjectMeta{ <del> Labels: map[string]string{"name": "apparmor-loader"}, </del> <ins> Labels: map[string]string{loaderLabelKey: loaderLabelValue}, </ins> }, Spec: api.PodSpec{ Containers: []api.Container{{

}, }, } <del> return f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Create(loader) </del> <ins> _, err := f.ClientSet.Core().ReplicationControllers(f.Namespace.Name).Create(loader) framework.ExpectNoError(err, "Failed to create apparmor-loader ReplicationController") // Wait for loader to be ready. getRunningLoaderPod(f) } func getRunningLoaderPod(f *framework.Framework) *api.Pod { label := labels.SelectorFromSet(labels.Set(map[string]string{loaderLabelKey: loaderLabelValue})) pods, err := framework.WaitForPodsWithLabelScheduled(f.ClientSet, f.Namespace.Name, label) framework.ExpectNoError(err, "Failed to schedule apparmor-loader Pod") pod := &pods.Items[0] framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.ClientSet, pod), "Failed to run apparmor-loader Pod") return pod </ins> }