Repository: Azure/ARO-RP
Pull request #1069 (2020-10-14 19:09:40)

Move pkg/routefix to ARO operator in it's own controller
### Which issue this PR addresses:

<!--
Please include a link to the VSTS work item as well as any GitHub issues.

Usage: `Fixes #<GitHub issue number>`, or `Fixes (paste link of issue)`.
-->
Fixes #[7816439](https://dev.azure.com/msazure/AzureRedHatOpenShift/_workitems/edit/7816439/)

### What this PR does / why we need it:

<!--
Include a brief summary of what the PR is intended to accomplish and how the PR
does it. (2-3 sentences)
-->
This moves the `pkg/routefix` to be controlled by the ARO operator.  This will help with the mitigation efforts if cluster components get deleted to reapply the necessary k8s objects.  This has a target unknown until the fix can be applied upstream. 

### Test plan for issue:

<!--
How did you test that this PR works?

- Are there unit tests?
- Are there integration/e2e tests?
- If it is not possible to write automated tests, explain why and document how
  to manually test and verify the feature.
-->
No tests are associated with the changes.

### Is there any documentation that needs to be updated for this PR?

<!--
- If yes and the docs are in GitHub, include doc updates in the PR.
- If yes and the docs are not in GitHub (i.e. VSTS wiki), include a link to the
  docs.
- If no, explain why (e.g. "tech debt cleanup, N/A").
-->
~~No, we are just moving it from one package to the other.  The workaround controller will handle the reapplication of the k8s objects when necessary.~~
This will be moved to its own controller and implementation will reside around adding a e2e test.

troy0820: New operator image is not published to accomplish E2E test _(not yet written)_ to prove resources respawn on reconciliation.  

mjudeikis: > New operator image is not published to accomplish E2E test _(not yet written)_ to prove resources respawn on reconciliation.

Lets have a call tomorrow your morning. Please send an invite :) 

Patch:
diff --git a/Makefile b/Makefile
index 05dd3cfd1..de4314b70 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,9 @@ SHELL = /bin/bash
 COMMIT = $(shell git rev-parse --short HEAD)$(shell [[ $$(git status --porcelain) = "" ]] || echo -dirty)
 ARO_IMAGE ?= ${RP_IMAGE_ACR}.azurecr.io/aro:$(COMMIT)
 
-export CGO_CFLAGS=-Dgpgme_off_t=off_t
+ifneq ($(shell uname -s),Darwin)
+    export CGO_CFLAGS=-Dgpgme_off_t=off_t
+endif
 
 aro: generate
 	go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(COMMIT)" ./cmd/aro
diff --git a/README.md b/README.md
index 4f49d0554..e0207a824 100644
--- a/README.md
+++ b/README.md
@@ -102,6 +102,9 @@ questions or comments.
 
     * workaround: Applies a set of temporay workarounds to the ARO cluster.
 
+    * routefix: Ensures all the routefix resources in the namespace
+      `openshift-azure-routefix` remain on the cluster.
+
   * pkg/swagger: Swagger specification generation code.
 
   * pkg/util: Utility libraries.
diff --git a/cmd/aro/operator.go b/cmd/aro/operator.go
index aa9c6b136..e0750dd71 100644
--- a/cmd/aro/operator.go
+++ b/cmd/aro/operator.go
@@ -23,6 +23,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/checker"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/genevalogging"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/pullsecret"
+	"github.com/Azure/ARO-RP/pkg/operator/controllers/routefix"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/workaround"
 	"github.com/Azure/ARO-RP/pkg/util/deployment"
 	utillog "github.com/Azure/ARO-RP/pkg/util/log"
@@ -101,6 +102,11 @@ func operator(ctx context.Context, log *logrus.Entry) error {
 			kubernetescli, configcli, mcocli, arocli, restConfig)).SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("unable to create controller Workaround: %v", err)
 		}
+		if err = (routefix.NewReconciler(
+			log.WithField("controller", controllers.RouteFixControllerName),
+			kubernetescli, securitycli, arocli, restConfig)).SetupWithManager(mgr); err != nil {
+			return fmt.Errorf("unable to create controller RouteFix: %v", err)
+		}
 	}
 
 	if err = (checker.NewReconciler(
@@ -108,6 +114,7 @@ func operator(ctx context.Context, log *logrus.Entry) error {
 		clustercli, arocli, role, deploymentMode)).SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("unable to create controller InternetChecker: %v", err)
 	}
+
 	// +kubebuilder:scaffold:builder
 
 	log.Info("starting manager")
diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go
index 5ed6b716a..29f21f5e5 100644
--- a/pkg/cluster/install.go
+++ b/pkg/cluster/install.go
@@ -34,7 +34,6 @@ func (m *manager) AdminUpgrade(ctx context.Context) error {
 		steps.Condition(m.apiServersReady, 30*time.Minute),
 		steps.Action(m.ensureBillingRecord), // belt and braces
 		steps.Action(m.fixPullSecret),       // TODO(mj): Remove when operator deployed
-		steps.Action(m.ensureRouteFix),
 		steps.Action(m.ensureAROOperator),
 		steps.Condition(m.aroDeploymentReady, 20*time.Minute),
 		steps.Action(m.upgradeCertificates),
@@ -62,7 +61,6 @@ func (m *manager) Install(ctx context.Context, installConfig *installconfig.Inst
 			steps.Action(m.createCertificates),
 			steps.Action(m.initializeKubernetesClients),
 			steps.Condition(m.bootstrapConfigMapReady, 30*time.Minute),
-			steps.Action(m.ensureRouteFix),
 			steps.Action(m.ensureAROOperator),
 			steps.Action(m.incrInstallPhase),
 		},
diff --git a/pkg/cluster/routefix.go b/pkg/cluster/routefix.go
deleted file mode 100644
index f047dfeef..000000000
--- a/pkg/cluster/routefix.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package cluster
-
-// Copyright (c) Microsoft Corporation.
-// Licensed under the Apache License 2.0.
-
-import (
-	"context"
-
-	"github.com/Azure/ARO-RP/pkg/routefix"
-)
-
-func (m *manager) ensureRouteFix(ctx context.Context) error {
-	rf := routefix.New(m.log, m.env, m.kubernetescli, m.securitycli)
-	return rf.CreateOrUpdate(ctx)
-}
diff --git a/pkg/operator/controllers/const.go b/pkg/operator/controllers/const.go
index 05a7372d2..5d38f7c3a 100644
--- a/pkg/operator/controllers/const.go
+++ b/pkg/operator/controllers/const.go
@@ -9,4 +9,5 @@ const (
 	PullSecretControllerName    = "PullSecret"
 	WorkaroundControllerName    = "Workaround"
 	CheckerControllerName       = "Checker"
+	RouteFixControllerName      = "RouteFix"
 )
diff --git a/pkg/operator/controllers/routefix/routefix.go b/pkg/operator/controllers/routefix/routefix.go
new file mode 100644
index 000000000..020f92fdf
--- /dev/null
+++ b/pkg/operator/controllers/routefix/routefix.go
@@ -0,0 +1,106 @@
+package routefix
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/go-autorest/autorest/to"
+	projectv1 "github.com/openshift/api/project/v1"
+	securityv1 "github.com/openshift/api/security/v1"
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
+	"github.com/Azure/ARO-RP/pkg/util/version"
+)
+
+const (
+	kubeName           = "routefix"
+	kubeNamespace      = "openshift-azure-routefix"
+	kubeServiceAccount = "system:serviceaccount:" + kubeNamespace + ":default"
+	shellScript        = `for ((;;))
+do
+  if ip route show cache | grep -q 'mtu 1450'; then
+    ip route show cache
+    ip route flush cache
+  fi
+  sleep 60
+done`
+)
+
+func (r *RouteFixReconciler) securityContextConstraints(ctx context.Context, name, serviceAccountName string) (*securityv1.SecurityContextConstraints, error) {
+	scc, err := r.securitycli.SecurityV1().SecurityContextConstraints().Get(ctx, "privileged", metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	scc.ObjectMeta = metav1.ObjectMeta{
+		Name: name,
+	}
+	scc.Groups = []string{}
+	scc.Users = []string{serviceAccountName}
+	return scc, nil
+}
+
+func (r *RouteFixReconciler) resources(ctx context.Context, cluster *arov1alpha1.Cluster) ([]runtime.Object, error) {
+	scc, err := r.securityContextConstraints(ctx, "privileged-routefix", kubeServiceAccount)
+	if err != nil {
+		return nil, err
+	}
+	return []runtime.Object{
+		&v1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:        kubeNamespace,
+				Annotations: map[string]string{projectv1.ProjectNodeSelector: ""},
+			},
+		},
+		scc,
+		&appsv1.DaemonSet{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      kubeName,
+				Namespace: kubeNamespace,
+			},
+			Spec: appsv1.DaemonSetSpec{
+				Selector: &metav1.LabelSelector{
+					MatchLabels: map[string]string{"app": "routefix"},
+				},
+				Template: v1.PodTemplateSpec{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: map[string]string{"app": "routefix"},
+					},
+					Spec: v1.PodSpec{
+						Containers: []v1.Container{
+							{
+								Name:  kubeName,
+								Image: version.RouteFixImage(cluster.Spec.ACRName),
+								Args: []string{
+									"sh",
+									"-c",
+									shellScript,
+								},
+								// TODO: specify requests/limits
+								SecurityContext: &v1.SecurityContext{
+									Privileged: to.BoolPtr(true),
+								},
+							},
+						},
+						HostNetwork: true,
+						Tolerations: []v1.Toleration{
+							{
+								Effect:   v1.TaintEffectNoExecute,
+								Operator: v1.TolerationOpExists,
+							},
+							{
+								Effect:   v1.TaintEffectNoSchedule,
+								Operator: v1.TolerationOpExists,
+							},
+						},
+					},
+				},
+			},
+		},
+	}, nil
+}
diff --git a/pkg/operator/controllers/routefix/routefix_controller.go b/pkg/operator/controllers/routefix/routefix_controller.go
new file mode 100644
index 000000000..88770b735
--- /dev/null
+++ b/pkg/operator/controllers/routefix/routefix_controller.go
@@ -0,0 +1,131 @@
+package routefix
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"sort"
+
+	securityclient "github.com/openshift/client-go/security/clientset/versioned"
+	"github.com/sirupsen/logrus"
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
+	"github.com/Azure/ARO-RP/pkg/operator/clientset/versioned/scheme"
+	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned/typed/aro.openshift.io/v1alpha1"
+	"github.com/Azure/ARO-RP/pkg/operator/controllers"
+	"github.com/Azure/ARO-RP/pkg/util/dynamichelper"
+)
+
+//RouteFixReconciler is the controller struct
+type RouteFixReconciler struct {
+	kubernetescli kubernetes.Interface
+	securitycli   securityclient.Interface
+	arocli        aroclient.AroV1alpha1Interface
+	restConfig    *rest.Config
+	log           *logrus.Entry
+}
+
+//NewReconciler creates a new Reconciler
+func NewReconciler(log *logrus.Entry, kubernetescli kubernetes.Interface, securitycli securityclient.Interface, arocli aroclient.AroV1alpha1Interface, restConfig *rest.Config) *RouteFixReconciler {
+	return &RouteFixReconciler{
+		securitycli:   securitycli,
+		kubernetescli: kubernetescli,
+		arocli:        arocli,
+		restConfig:    restConfig,
+		log:           log,
+	}
+}
+
+//Reconcile fixes the daemonset Routefix
+func (r *RouteFixReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
+	ctx := context.Background()
+	if request.Name != arov1alpha1.SingletonClusterName {
+		return reconcile.Result{}, nil
+	}
+
+	instance, err := r.arocli.Clusters().Get(ctx, request.Name, metav1.GetOptions{})
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	// TODO: dh should be a field in r, but the fact that it is initialised here
+	// each time currently saves us in the case that the controller runs before
+	// the SCC API is registered.
+	dh, err := dynamichelper.New(r.log, r.restConfig)
+	if err != nil {
+		r.log.Error(err)
+		return reconcile.Result{}, err
+	}
+
+	resources, err := r.resources(ctx, instance)
+	if err != nil {
+		r.log.Error(err)
+		return reconcile.Result{}, err
+	}
+
+	for _, res := range resources {
+		o, err := meta.Accessor(res)
+		if err != nil {
+			r.log.Errorf("Accessor %s/%s: %v", o.GetNamespace(), o.GetName(), err)
+			return reconcile.Result{}, err
+		}
+
+		// This sets the reference on all objects that we create
+		// to our cluster instance. This causes the Owns() below to work and
+		// to get Reconcile events when anything happens to our objects.
+		err = controllerutil.SetControllerReference(instance, o, scheme.Scheme)
+		if err != nil {
+			r.log.Errorf("SetControllerReference %s/%s: %v", o.GetNamespace(), o.GetName(), err)
+			return reconcile.Result{}, err
+		}
+	}
+
+	err = dynamichelper.HashWorkloadConfigs(resources)
+	if err != nil {
+		r.log.Errorf("HashWorkloadConfigs %v", err)
+		return reconcile.Result{}, err
+	}
+
+	uns := make([]*unstructured.Unstructured, 0, len(resources))
+	for _, res := range resources {
+		un := &unstructured.Unstructured{}
+		err = scheme.Scheme.Convert(res, un, nil)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+		uns = append(uns, un)
+	}
+
+	sort.Slice(uns, func(i, j int) bool {
+		return dynamichelper.CreateOrder(uns[i], uns[j])
+	})
+
+	for _, un := range uns {
+		err = dh.Ensure(ctx, un)
+		if err != nil {
+			r.log.Error(err)
+			return reconcile.Result{}, err
+		}
+	}
+
+	return reconcile.Result{}, nil
+}
+
+//SetupWithManager creates the controller
+func (r *RouteFixReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&arov1alpha1.Cluster{}).
+		Owns(&appsv1.DaemonSet{}).
+		Named(controllers.RouteFixControllerName).
+		Complete(r)
+}
diff --git a/pkg/routefix/routefix.go b/pkg/routefix/routefix.go
deleted file mode 100644
index fda1353c0..000000000
--- a/pkg/routefix/routefix.go
+++ /dev/null
@@ -1,191 +0,0 @@
-package routefix
-
-// Copyright (c) Microsoft Corporation.
-// Licensed under the Apache License 2.0.
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/Azure/go-autorest/autorest/to"
-	projectv1 "github.com/openshift/api/project/v1"
-	securityv1 "github.com/openshift/api/security/v1"
-	securityclient "github.com/openshift/client-go/security/clientset/versioned"
-	"github.com/sirupsen/logrus"
-	appsv1 "k8s.io/api/apps/v1"
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/util/retry"
-
-	"github.com/Azure/ARO-RP/pkg/env"
-)
-
-const (
-	kubeNamespace       = "openshift-azure-routefix"
-	kubeServiceAccount  = "system:serviceaccount:" + kubeNamespace + ":default"
-	routefixImageFormat = "%s.azurecr.io/routefix:c5c4a5db"
-	shellScript         = `for ((;;))
-do
-  if ip route show cache | grep -q 'mtu 1450'; then
-    ip route show cache
-    ip route flush cache
-  fi
-
-  sleep 60
-done`
-)
-
-type RouteFix interface {
-	CreateOrUpdate(ctx context.Context) error
-}
-
-type routeFix struct {
-	log *logrus.Entry
-	env env.Interface
-
-	cli    kubernetes.Interface
-	seccli securityclient.Interface
-}
-
-func New(log *logrus.Entry, e env.Interface, cli kubernetes.Interface, seccli securityclient.Interface) RouteFix {
-	return &routeFix{
-		log: log,
-		env: e,
-
-		cli:    cli,
-		seccli: seccli,
-	}
-}
-
-func (i *routeFix) routefixImage() string {
-	return fmt.Sprintf(routefixImageFormat, i.env.ACRName())
-}
-
-func (i *routeFix) ensureNamespace(ctx context.Context, ns string) error {
-	_, err := i.cli.CoreV1().Namespaces().Create(ctx, &v1.Namespace{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        ns,
-			Annotations: map[string]string{projectv1.ProjectNodeSelector: ""},
-		},
-	}, metav1.CreateOptions{})
-	if !errors.IsAlreadyExists(err) {
-		return err
-	}
-
-	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		_ns, err := i.cli.CoreV1().Namespaces().Get(ctx, ns, metav1.GetOptions{})
-		if err != nil {
-			return err
-		}
-
-		if _ns.Annotations == nil {
-			_ns.Annotations = map[string]string{}
-		}
-
-		if annotation, ok := _ns.Annotations[projectv1.ProjectNodeSelector]; annotation == "" && ok {
-			return nil
-		}
-
-		_ns.Annotations[projectv1.ProjectNodeSelector] = ""
-
-		_, err = i.cli.CoreV1().Namespaces().Update(ctx, _ns, metav1.UpdateOptions{})
-		return err
-	})
-}
-
-func (i *routeFix) applyDaemonSet(ctx context.Context, ds *appsv1.DaemonSet) error {
-	_, err := i.cli.AppsV1().DaemonSets(ds.Namespace).Create(ctx, ds, metav1.CreateOptions{})
-	if !errors.IsAlreadyExists(err) {
-		return err
-	}
-
-	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		_ds, err := i.cli.AppsV1().DaemonSets(ds.Namespace).Get(ctx, ds.Name, metav1.GetOptions{})
-		if err != nil {
-			return err
-		}
-
-		ds.ResourceVersion = _ds.ResourceVersion
-		_, err = i.cli.AppsV1().DaemonSets(ds.Namespace).Update(ctx, ds, metav1.UpdateOptions{})
-		return err
-	})
-}
-
-func (i *routeFix) CreateOrUpdate(ctx context.Context) error {
-	err := i.ensureNamespace(ctx, kubeNamespace)
-	if err != nil {
-		return err
-	}
-
-	i.log.Print("waiting for privileged security context constraint")
-	timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Minute)
-	defer cancel()
-	var scc *securityv1.SecurityContextConstraints
-	err = wait.PollImmediateUntil(10*time.Second, func() (bool, error) {
-		scc, err = i.seccli.SecurityV1().SecurityContextConstraints().Get(ctx, "privileged", metav1.GetOptions{})
-		return err == nil, nil
-	}, timeoutCtx.Done())
-	if err != nil {
-		return err
-	}
-
-	scc.ObjectMeta = metav1.ObjectMeta{
-		Name: "privileged-routefix",
-	}
-	scc.Groups = nil
-	scc.Users = []string{kubeServiceAccount}
-
-	_, err = i.seccli.SecurityV1().SecurityContextConstraints().Create(ctx, scc, metav1.CreateOptions{})
-	if err != nil && !errors.IsAlreadyExists(err) {
-		return err
-	}
-
-	return i.applyDaemonSet(ctx, &appsv1.DaemonSet{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "routefix",
-			Namespace: kubeNamespace,
-		},
-		Spec: appsv1.DaemonSetSpec{
-			Selector: &metav1.LabelSelector{
-				MatchLabels: map[string]string{"app": "routefix"},
-			},
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{"app": "routefix"},
-				},
-				Spec: v1.PodSpec{
-					Containers: []v1.Container{
-						{
-							Name:  "routefix",
-							Image: i.routefixImage(),
-							Args: []string{
-								"sh",
-								"-c",
-								shellScript,
-							},
-							// TODO: specify requests/limits
-							SecurityContext: &v1.SecurityContext{
-								Privileged: to.BoolPtr(true),
-							},
-						},
-					},
-					HostNetwork: true,
-					Tolerations: []v1.Toleration{
-						{
-							Effect:   v1.TaintEffectNoExecute,
-							Operator: v1.TolerationOpExists,
-						},
-						{
-							Effect:   v1.TaintEffectNoSchedule,
-							Operator: v1.TolerationOpExists,
-						},
-					},
-				},
-			},
-		},
-	})
-}
diff --git a/pkg/util/version/const.go b/pkg/util/version/const.go
index 2135f6363..70f0ad59b 100644
--- a/pkg/util/version/const.go
+++ b/pkg/util/version/const.go
@@ -46,3 +46,7 @@ func MdmImage(acr string) string {
 func MdsdImage(acr string) string {
 	return acr + ".azurecr.io/genevamdsd:master_330"
 }
+
+func RouteFixImage(acr string) string {
+	return acr + ".azurecr.io/routefix:c5c4a5db"
+}


Test patch:
diff --git a/test/e2e/operator.go b/test/e2e/operator.go
index 79a73c95e..1f4fbc6b0 100644
--- a/test/e2e/operator.go
+++ b/test/e2e/operator.go
@@ -148,3 +148,28 @@ var _ = Describe("ARO Operator - Geneva Logging", func() {
 		Expect(len(final) - len(initial)).To(Equal(1))
 	})
 })
+
+var _ = Describe("ARO Operator - Routefix Daemonset", func() {
+	BeforeEach(func() {
+		_, err := clients.AROClusters.Clusters().Get(context.Background(), "cluster", metav1.GetOptions{})
+		if errors.IsNotFound(err) {
+			Skip("skipping tests as aro-operator is not deployed")
+		}
+		Specify("routefix must be repaired if daemonset is deleted", func() {
+			dsReady := ready.CheckDaemonSetIsReady(context.Background(), clients.Kubernetes.AppsV1().DaemonSets("openshift-azure-routefix"), "routefix")
+			err := wait.PollImmediate(30*time.Second, 15*time.Minute, dsReady)
+			Expect(err).NotTo(HaveOccurred())
+
+			//delete the routefix daemonset
+			err = clients.Kubernetes.AppsV1().DaemonSets("openshift-azure-routefix").Delete(context.Background(), "routefix", metav1.DeleteOptions{})
+			Expect(err).NotTo(HaveOccurred())
+
+			//wait for it to be fixed
+			err = wait.PollImmediate(30*time.Second, 15*time.Minute, dsReady)
+			Expect(err).NotTo(HaveOccurred())
+
+			_, err = clients.Kubernetes.AppsV1().DaemonSets("openshift-azure-routefix").Get(context.Background(), "routefix", metav1.GetOptions{})
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+})