Repository: Azure/ARO-RP
Pull request #1182 (2020-11-18 22:06:19)

make pod storage test less flaky
Stop using a naked pod for the pod storage test - we hit node rotations and the test flakes.  Use a stateful set and up the timeout.


Patch:
diff --git a/pkg/cluster/deploystorage.go b/pkg/cluster/deploystorage.go
index 3f6181cd3..a6929930d 100644
--- a/pkg/cluster/deploystorage.go
+++ b/pkg/cluster/deploystorage.go
@@ -236,6 +236,7 @@ func (m *manager) deployStorageTemplate(ctx context.Context, installConfig *inst
 var extraDenyAssignmentExclusions = map[string][]string{
 	"Microsoft.RedHatOpenShift/RedHatEngineering": {
 		"Microsoft.Network/networkInterfaces/effectiveRouteTable/action",
+		"Microsoft.Resources/subscriptions/resourceGroups/write", // enable resource group tagging
 	},
 }
 
diff --git a/pkg/util/ready/ready.go b/pkg/util/ready/ready.go
index a6241ee6a..01b595a43 100644
--- a/pkg/util/ready/ready.go
+++ b/pkg/util/ready/ready.go
@@ -115,3 +115,31 @@ func CheckPodIsRunning(ctx context.Context, cli corev1client.PodInterface, name
 		return PodIsRunning(p), nil
 	}
 }
+
+// StatefulSetIsReady returns true if a StatefulSet is considered ready
+func StatefulSetIsReady(s *appsv1.StatefulSet) bool {
+	specReplicas := int32(1)
+	if s.Spec.Replicas != nil {
+		specReplicas = *s.Spec.Replicas
+	}
+
+	return specReplicas == s.Status.ReadyReplicas &&
+		specReplicas == s.Status.UpdatedReplicas &&
+		s.Generation == s.Status.ObservedGeneration
+}
+
+// CheckStatefulSetIsReady returns a function which polls a StatefulSet and
+// returns its readiness
+func CheckStatefulSetIsReady(ctx context.Context, cli appsv1client.StatefulSetInterface, name string) func() (bool, error) {
+	return func() (bool, error) {
+		s, err := cli.Get(ctx, name, metav1.GetOptions{})
+		switch {
+		case errors.IsNotFound(err):
+			return false, nil
+		case err != nil:
+			return false, err
+		}
+
+		return StatefulSetIsReady(s), nil
+	}
+}


Test patch:
diff --git a/pkg/cluster/deploystorage_test.go b/pkg/cluster/deploystorage_test.go
index 837f0754e..39d2bcd28 100644
--- a/pkg/cluster/deploystorage_test.go
+++ b/pkg/cluster/deploystorage_test.go
@@ -44,6 +44,7 @@ var daTestCases = []struct {
 			"Microsoft.Compute/snapshots/write",
 			"Microsoft.Compute/snapshots/delete",
 			"Microsoft.Network/networkInterfaces/effectiveRouteTable/action",
+			"Microsoft.Resources/subscriptions/resourceGroups/write",
 		},
 	},
 }
diff --git a/test/e2e/cluster.go b/test/e2e/cluster.go
index d82e8c29b..6a43d895d 100644
--- a/test/e2e/cluster.go
+++ b/test/e2e/cluster.go
@@ -5,25 +5,25 @@ package e2e
 
 import (
 	"context"
-	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	"github.com/Azure/go-autorest/autorest/to"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 
-	ready "github.com/Azure/ARO-RP/pkg/util/ready"
+	"github.com/Azure/ARO-RP/pkg/util/ready"
 	proj "github.com/Azure/ARO-RP/test/util/project"
 )
 
 const (
 	testNamespace = "test-e2e"
-	testPodName   = "busybox"
 )
 
 var _ = Describe("Cluster smoke test", func() {
@@ -48,17 +48,13 @@ var _ = Describe("Cluster smoke test", func() {
 		}, 5*time.Minute, 10*time.Second).Should(BeNil())
 	})
 
-	Specify("Can run a pod which is using Azure Disk storage", func() {
+	Specify("Can run a stateful set which is using Azure Disk storage", func() {
 		ctx := context.Background()
-		err := createPVC(ctx, clients.Kubernetes)
+		err := createStatefulSet(ctx, clients.Kubernetes)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = createPodWithPVC(ctx, clients.Kubernetes)
+		err = wait.PollImmediate(10*time.Second, 15*time.Minute, ready.CheckStatefulSetIsReady(ctx, clients.Kubernetes.AppsV1().StatefulSets(testNamespace), "busybox"))
 		Expect(err).NotTo(HaveOccurred())
-
-		Eventually(func() error {
-			return verifyPodSucceeded(clients.Kubernetes)
-		}).Should(BeNil())
 	})
 
 	Specify("Can create load balancer services", func() {
@@ -89,87 +85,68 @@ var _ = Describe("Cluster smoke test", func() {
 	})
 })
 
-func createPVC(ctx context.Context, cli kubernetes.Interface) error {
+func createStatefulSet(ctx context.Context, cli kubernetes.Interface) error {
 	pvcStorage, err := resource.ParseQuantity("2Gi")
 	if err != nil {
 		return err
 	}
-	pvcName := fmt.Sprintf("%s-pvc", testPodName)
-	storageClass := "managed-premium"
-	_, err = cli.CoreV1().PersistentVolumeClaims(testNamespace).Create(context.Background(), &corev1.PersistentVolumeClaim{
+
+	_, err = cli.AppsV1().StatefulSets(testNamespace).Create(context.Background(), &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: pvcName,
+			Name: "busybox",
 		},
-		Spec: corev1.PersistentVolumeClaimSpec{
-			AccessModes: []corev1.PersistentVolumeAccessMode{
-				corev1.PersistentVolumeAccessMode("ReadWriteOnce"),
+		Spec: appsv1.StatefulSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"app": "busybox"},
 			},
-			StorageClassName: to.StringPtr(storageClass),
-			Resources: corev1.ResourceRequirements{
-				Requests: corev1.ResourceList{
-					corev1.ResourceStorage: pvcStorage,
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"app": "busybox"},
 				},
-			},
-		},
-	}, metav1.CreateOptions{})
-	return err
-}
-
-func createPodWithPVC(ctx context.Context, cli kubernetes.Interface) error {
-	pvcName := fmt.Sprintf("%s-pvc", testPodName)
-	volumeName := fmt.Sprintf("%s-vol", pvcName)
-	_, err := cli.CoreV1().Pods(testNamespace).Create(context.Background(), &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: testPodName,
-		},
-		Spec: corev1.PodSpec{
-			Containers: []corev1.Container{
-				{
-					Name:  testPodName,
-					Image: testPodName,
-					Command: []string{
-						"/bin/dd",
-						"if=/dev/urandom",
-						fmt.Sprintf("of=/data/%s.bin", testNamespace),
-						"bs=1M",
-						"count=100",
-					},
-					VolumeMounts: []corev1.VolumeMount{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
 						{
-							Name:      volumeName,
-							MountPath: "/data",
-							ReadOnly:  false,
+							Name:  "busybox",
+							Image: "busybox",
+							Command: []string{
+								"/bin/sh",
+								"-c",
+								"while true; do sleep 1; done",
+							},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "busybox",
+									MountPath: "/data",
+									ReadOnly:  false,
+								},
+							},
 						},
 					},
 				},
 			},
-			Volumes: []corev1.Volume{
+			VolumeClaimTemplates: []corev1.PersistentVolumeClaim{
 				{
-					Name: volumeName,
-					VolumeSource: corev1.VolumeSource{
-						PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-							ClaimName: pvcName,
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "busybox",
+					},
+					Spec: corev1.PersistentVolumeClaimSpec{
+						AccessModes: []corev1.PersistentVolumeAccessMode{
+							corev1.ReadWriteOnce,
+						},
+						StorageClassName: to.StringPtr("managed-premium"),
+						Resources: corev1.ResourceRequirements{
+							Requests: corev1.ResourceList{
+								corev1.ResourceStorage: pvcStorage,
+							},
 						},
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}, metav1.CreateOptions{})
 	return err
 }
 
-func verifyPodSucceeded(cli kubernetes.Interface) error {
-	pod, err := cli.CoreV1().Pods(testNamespace).Get(context.Background(), testPodName, metav1.GetOptions{})
-	if err != nil {
-		return err
-	}
-	if pod.Status.Phase != corev1.PodSucceeded {
-		return fmt.Errorf("Pod '%s' is not running: '%+v'", "busybox", pod.Status.Conditions)
-	}
-	return nil
-}
-
 func createLoadBalancerService(ctx context.Context, cli kubernetes.Interface, name string, annotations map[string]string) error {
 	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{