Repository: kubermatic/machine-controller
Pull request #705 (2020-03-03 09:46:08)

Implement the NodeCSRApprover controller
**What this PR does / why we need it**:

This PR implements the NodeCSRApprover controller, which automatically approves the node **serving** certificates.

In #667, we have enabled the [Kubelet TLS bootstrapping](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping), so Kubelet serving and client certificates are requested using CertificateSigningRequests (CSRs). The client certificates are approved automatically by `kube-controller-manager`, however, serving certificates are not. The Kubernetes documentation advises that certificates should be approved manually or using a custom controller.

**Optional Release Note**:
```release-note
Implement the NodeCSRApprover controller for automatically approving node serving certificates
```

/assign @irozzo-1A @kron4eg 

kubermatic-bot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubermatic/machine-controller/pull/705#" title="Author self-approved">xmudrii</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubermatic%2Fmachine-controller).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/kubermatic/machine-controller/blob/master/OWNERS)~~ [xmudrii]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
kubermatic-bot: @xmudrii: The following test **failed**, say `/retest` to rerun all failed tests:

Test name | Commit | Details | Rerun command
--- | --- | --- | ---
pull-machine-controller-lint | 89407bbd7a0e294afda4b24e327ed0eb1b29997c | [link](https://public-prow.loodse.com/view/gcs/prow-dev-public-data/pr-logs/pull/kubermatic_machine-controller/705/pull-machine-controller-lint/1234777131586686976) | `/test pull-machine-controller-lint`

[Full PR test history](https://public-prow.loodse.com/pr-history?org=kubermatic&repo=machine-controller&pr=705)

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository. I understand the commands that are listed [here](https://go.k8s.io/bot-commands).
</details>
<!-- test report -->
kubermatic-bot: @xmudrii: The following tests **failed**, say `/retest` to rerun all failed tests:

Test name | Commit | Details | Rerun command
--- | --- | --- | ---
pull-machine-controller-lint | 89407bbd7a0e294afda4b24e327ed0eb1b29997c | [link](https://public-prow.loodse.com/view/gcs/prow-dev-public-data/pr-logs/pull/kubermatic_machine-controller/705/pull-machine-controller-lint/1234777131586686976) | `/test pull-machine-controller-lint`
pull-machine-controller-verify-boilerplate | 89407bbd7a0e294afda4b24e327ed0eb1b29997c | [link](https://public-prow.loodse.com/view/gcs/prow-dev-public-data/pr-logs/pull/kubermatic_machine-controller/705/pull-machine-controller-verify-boilerplate/1234777131607658497) | `/test pull-machine-controller-verify-boilerplate`

[Full PR test history](https://public-prow.loodse.com/pr-history?org=kubermatic&repo=machine-controller&pr=705)

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository. I understand the commands that are listed [here](https://go.k8s.io/bot-commands).
</details>
<!-- test report -->

Patch:
diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go
index 074fc571..9c3dcfbe 100644
--- a/cmd/machine-controller/main.go
+++ b/cmd/machine-controller/main.go
@@ -51,6 +51,7 @@ import (
 	machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine"
 	machinedeploymentcontroller "github.com/kubermatic/machine-controller/pkg/controller/machinedeployment"
 	machinesetcontroller "github.com/kubermatic/machine-controller/pkg/controller/machineset"
+	"github.com/kubermatic/machine-controller/pkg/controller/nodecsrapprover"
 	machinehealth "github.com/kubermatic/machine-controller/pkg/health"
 	machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1"
 	"github.com/kubermatic/machine-controller/pkg/signals"
@@ -73,6 +74,7 @@ var (
 	externalCloudProvider            bool
 	bootstrapTokenServiceAccountName string
 	skipEvictionAfter                time.Duration
+	nodeCSRApprover                  bool
 
 	nodeHTTPProxy          string
 	nodeNoProxy            string
@@ -132,6 +134,9 @@ type controllerRunOptions struct {
 	// Will instruct the machine-controller to skip the eviction if the machine deletion is older than skipEvictionAfter
 	skipEvictionAfter time.Duration
 
+	// Enable NodeCSRApprover controller to automatically approve node serving certificate requests.
+	nodeCSRApprover bool
+
 	node machinecontroller.NodeSettings
 }
 
@@ -161,6 +166,7 @@ func main() {
 	flag.StringVar(&nodeRegistryMirrors, "node-registry-mirrors", "", "Comma separated list of Docker image mirrors")
 	flag.StringVar(&nodePauseImage, "node-pause-image", "", "Image for the pause container including tag. If not set, the kubelet default will be used: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/")
 	flag.StringVar(&nodeHyperkubeImage, "node-hyperkube-image", "k8s.gcr.io/hyperkube-amd64", "Image for the hyperkube container excluding tag.")
+	flag.BoolVar(&nodeCSRApprover, "node-csr-approver", false, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests.")
 
 	flag.Parse()
 	kubeconfig = flag.Lookup("kubeconfig").Value.(flag.Getter).Get().(string)
@@ -238,6 +244,7 @@ func main() {
 		cfg:                   machineCfg,
 		externalCloudProvider: externalCloudProvider,
 		skipEvictionAfter:     skipEvictionAfter,
+		nodeCSRApprover:       nodeCSRApprover,
 		node: machinecontroller.NodeSettings{
 			ClusterDNSIPs:  clusterDNSIPs,
 			HTTPProxy:      nodeHTTPProxy,
@@ -420,6 +427,13 @@ func startControllerViaLeaderElection(runOptions controllerRunOptions) error {
 			runOptions.parentCtxDone()
 			return
 		}
+		if runOptions.nodeCSRApprover {
+			if err := nodecsrapprover.Add(mgr); err != nil {
+				klog.Errorf("failed to add NodeCSRApprover controller to manager: %v", err)
+				runOptions.parentCtxDone()
+				return
+			}
+		}
 
 		klog.Info("machine controller startup complete")
 	}
diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go
index 3c80bfca..6382f5b3 100644
--- a/pkg/cloudprovider/provider/aws/provider.go
+++ b/pkg/cloudprovider/provider/aws/provider.go
@@ -615,7 +615,7 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, _ *cloudprovidertypes.Prov
 	}
 
 	if pc.OperatingSystem == providerconfigtypes.OperatingSystemRHEL && config.manager != nil {
-		if err := config.manager.UnregisterInstance(machine.Name); err != nil {
+		if err := config.manager.UnregisterInstance(*instance.instance.PrivateDnsName); err != nil {
 			return false, fmt.Errorf("failed delete machine %s subscription: %v", machine.Name, err)
 		}
 	}
diff --git a/pkg/controller/nodecsrapprover/doc.go b/pkg/controller/nodecsrapprover/doc.go
new file mode 100644
index 00000000..5a3a45c4
--- /dev/null
+++ b/pkg/controller/nodecsrapprover/doc.go
@@ -0,0 +1,21 @@
+/*
+Copyright 2020 The Machine Controller Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+Package nodecsrapprover contains a controller responsible for autoapproving CSRs created by nodes
+for serving certificates.
+*/
+package nodecsrapprover
diff --git a/pkg/controller/nodecsrapprover/node_csr_approver.go b/pkg/controller/nodecsrapprover/node_csr_approver.go
new file mode 100644
index 00000000..5e253953
--- /dev/null
+++ b/pkg/controller/nodecsrapprover/node_csr_approver.go
@@ -0,0 +1,169 @@
+/*
+Copyright 2020 The Machine Controller Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodecsrapprover
+
+import (
+	"context"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+
+	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
+	certificatesv1beta1client "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
+	"k8s.io/klog"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+const ControllerName = "node_csr_autoapprover"
+
+type reconciler struct {
+	client.Client
+	// Have to use the typed client because csr approval is a subresource
+	// the dynamic client does not approve
+	certClient certificatesv1beta1client.CertificateSigningRequestInterface
+}
+
+func Add(mgr manager.Manager) error {
+	certClient, err := certificatesv1beta1client.NewForConfig(mgr.GetConfig())
+	if err != nil {
+		return fmt.Errorf("failed to create certificate client: %v", err)
+	}
+
+	r := &reconciler{Client: mgr.GetClient(), certClient: certClient.CertificateSigningRequests()}
+	c, err := controller.New(ControllerName, mgr, controller.Options{Reconciler: r})
+	if err != nil {
+		return fmt.Errorf("failed to construct controller: %v", err)
+	}
+	return c.Watch(&source.Kind{Type: &certificatesv1beta1.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{})
+}
+
+func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	err := r.reconcile(ctx, request)
+	if err != nil {
+		klog.Errorf("Reconciliation of request %s failed: %v", request.NamespacedName.String(), err)
+	}
+	return reconcile.Result{}, err
+}
+
+var allowedUsages = []certificatesv1beta1.KeyUsage{certificatesv1beta1.UsageDigitalSignature,
+	certificatesv1beta1.UsageKeyEncipherment,
+	certificatesv1beta1.UsageServerAuth}
+
+func (r *reconciler) reconcile(ctx context.Context, request reconcile.Request) error {
+	// Get the CSR object
+	csr := &certificatesv1beta1.CertificateSigningRequest{}
+	if err := r.Get(ctx, request.NamespacedName, csr); err != nil {
+		if kerrors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+	klog.V(4).Infof("Reconciling CSR %s", csr.ObjectMeta.Name)
+
+	// If CSR is approved, skip it
+	for _, condition := range csr.Status.Conditions {
+		if condition.Type == certificatesv1beta1.CertificateApproved {
+			klog.V(4).Infof("CSR %s already approved, skipping reconciling", csr.ObjectMeta.Name)
+			return nil
+		}
+	}
+
+	// Ensure system:nodes is in groups
+	if !sets.NewString(csr.Spec.Groups...).Has("system:nodes") {
+		klog.V(4).Infof("Skipping reconciling CSR '%s' because 'system:nodes' is not in its groups", csr.ObjectMeta.Name)
+		return nil
+	}
+
+	// Check are present usages matching allowed usages
+	if len(csr.Spec.Usages) != 3 {
+		klog.V(4).Infof("Skipping reconciling CSR '%s' because it has not exactly three usages defined", csr.ObjectMeta.Name)
+		return nil
+	}
+	for _, usage := range csr.Spec.Usages {
+		if !isUsageInUsageList(usage, allowedUsages) {
+			klog.V(4).Infof("Skipping reconciling CSR '%s' because its usage (%v) is not in the list of allowed usages (%v)",
+				csr.ObjectMeta.Name, usage, allowedUsages)
+			return nil
+		}
+	}
+	// Validate the CSR request
+	if err := validateCSRRequest(csr); err != nil {
+		klog.V(4).Infof("Skipping reconciling CSR '%s' because CSR request is not valid: %v", csr.ObjectMeta.Name, err)
+		return nil
+	}
+
+	klog.V(4).Infof("Approving CSR %s", csr.ObjectMeta.Name)
+	approvalCondition := certificatesv1beta1.CertificateSigningRequestCondition{
+		Type:   certificatesv1beta1.CertificateApproved,
+		Reason: "machine-controller NodeCSRApprover controller approved node serving cert",
+	}
+	csr.Status.Conditions = append(csr.Status.Conditions, approvalCondition)
+
+	if _, err := r.certClient.UpdateApproval(csr); err != nil {
+		return fmt.Errorf("failed to approve CSR %q: %v", csr.Name, err)
+	}
+
+	klog.Infof("Successfully approved CSR %s", csr.ObjectMeta.Name)
+	return nil
+}
+
+func isUsageInUsageList(usage certificatesv1beta1.KeyUsage, usageList []certificatesv1beta1.KeyUsage) bool {
+	for _, usageListItem := range usageList {
+		if usage == usageListItem {
+			return true
+		}
+	}
+	return false
+}
+
+// validateCSRRequest parses the CSR request and compares certificate request CommonName with CSR username, and
+// certificateRequest Organization with CSR groups.
+func validateCSRRequest(csr *certificatesv1beta1.CertificateSigningRequest) error {
+	csrBlock, rest := pem.Decode(csr.Spec.Request)
+	if csrBlock == nil {
+		return fmt.Errorf("no certificate request found for the given CSR")
+	}
+	if len(rest) != 0 {
+		return fmt.Errorf("found more than one PEM encoded block in the result")
+	}
+	csrRequest, err := x509.ParseCertificateRequest(csrBlock.Bytes)
+	if err != nil {
+		return err
+	}
+	// Validate Subject CommonName
+	if csrRequest.Subject.CommonName != csr.Spec.Username {
+		return fmt.Errorf("commonName '%s' is different then CSR username '%s'", csrRequest.Subject.CommonName, csr.Spec.Username)
+	}
+	// Validate Subject Organization
+	if len(csrRequest.Subject.Organization) != 1 {
+		return fmt.Errorf("error validating subject organizations")
+	}
+	if csrRequest.Subject.Organization[0] != "system:nodes" {
+		return fmt.Errorf("expected 'system:nodes' in organization, but got '%s'", csrRequest.Subject.Organization[0])
+	}
+
+	return nil
+}
diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go
index 2e416886..a7e20270 100644
--- a/pkg/userdata/rhel/provider.go
+++ b/pkg/userdata/rhel/provider.go
@@ -182,14 +182,13 @@ write_files:
 {{- /*  The normal way of setting it via cloud-init is broken, see */}}
 {{- /*  https://bugs.launchpad.net/cloud-init/+bug/1662542 */}}
     hostnamectl set-hostname {{ .MachineSpec.Name }}
-{{- /*  We should create a subscription for all instances in all cloud providers, however for aws this must be done by using the gold images */}}
-{{- /*  https://github.com/kubermatic/kubermatic/issues/5099 */}}
+    {{ end }}
+
     subscription-manager clean
     subscription-manager register --username='{{.OSConfig.RHELSubscriptionManagerUser}}' --password='{{.OSConfig.RHELSubscriptionManagerPassword}}'
     subscription-manager attach --auto
     yum clean all
     yum repolist
-    {{ end }}
 
     yum config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
 


Test patch:
diff --git a/pkg/controller/nodecsrapprover/node_csr_approver_test.go b/pkg/controller/nodecsrapprover/node_csr_approver_test.go
new file mode 100644
index 00000000..a7e87373
--- /dev/null
+++ b/pkg/controller/nodecsrapprover/node_csr_approver_test.go
@@ -0,0 +1,118 @@
+/*
+Copyright 2020 The Machine Controller Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodecsrapprover
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/typed/certificates/v1beta1/fake"
+	fakeclienttest "k8s.io/client-go/testing"
+
+	k8sclientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+const (
+	testCertificateRequest = `-----BEGIN CERTIFICATE REQUEST-----
+MIIBnTCCAUQCAQAwWTEVMBMGA1UEChMMc3lzdGVtOm5vZGVzMUAwPgYDVQQDEzdz
+eXN0ZW06bm9kZTppcC0xNzItMzEtMTE0LTQ4LmV1LXdlc3QtMy5jb21wdXRlLmlu
+dGVybmFsMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAvwSO5A7Thh4Dw1KejVf
+JwvVeTXCbe5i/AzQr1DRO2g9Hsd/3iwFc27OX29PJ6tXt9wNVG9dn+D7fLuvRIdY
+4KCBiDCBhQYJKoZIhvcNAQkOMXgwdjB0BgNVHREEbTBrgjBlYzItNTItNDctOTct
+MTExLmV1LXdlc3QtMy5jb21wdXRlLmFtYXpvbmF3cy5jb22CK2lwLTE3Mi0zMS0x
+MTQtNDguZXUtd2VzdC0zLmNvbXB1dGUuaW50ZXJuYWyHBKwfcjCHBDQvYW8wCgYI
+KoZIzj0EAwIDRwAwRAIgLdF0Ud9UHmE3Ezxovw5oafCAKYiCE/EirpNXkUHee80C
+IDjKG4ahwgDJQRtpmGqufjFBqrVVI3DxEFvt2RATJ3HA
+-----END CERTIFICATE REQUEST-----`
+)
+
+func TestReconciler_Reconcile(t *testing.T) {
+	reaction := &fakeApproveReaction{}
+	simpleReactor := &fakeclienttest.SimpleReactor{
+		Verb:     "*",
+		Resource: "*",
+		Reaction: reaction.approveReaction,
+	}
+	testCases := []struct {
+		name       string
+		reconciler reconciler
+	}{
+		{
+			name: "test approving a created certificate",
+			reconciler: reconciler{
+				Client: k8sclientfake.NewFakeClient(&certificatesv1beta1.CertificateSigningRequest{
+					ObjectMeta: metav1.ObjectMeta{
+						ResourceVersion: "123456",
+						Name:            "csr",
+						Namespace:       metav1.NamespaceSystem,
+					},
+					Spec: certificatesv1beta1.CertificateSigningRequestSpec{
+						Request: []byte(testCertificateRequest),
+						Usages: []certificatesv1beta1.KeyUsage{
+							certificatesv1beta1.UsageDigitalSignature,
+							certificatesv1beta1.UsageKeyEncipherment,
+							certificatesv1beta1.UsageServerAuth,
+						},
+						Username: "system:node:ip-172-31-114-48.eu-west-3.compute.internal",
+						Groups: []string{
+							"system:nodes",
+						},
+					},
+				}),
+				certClient: &fake.FakeCertificateSigningRequests{
+					Fake: &fake.FakeCertificatesV1beta1{
+						Fake: &fakeclienttest.Fake{
+							RWMutex:       sync.RWMutex{},
+							ReactionChain: []fakeclienttest.Reactor{simpleReactor},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := context.Background()
+			if err := tc.reconciler.reconcile(ctx, reconcile.Request{
+				NamespacedName: types.NamespacedName{Name: "csr", Namespace: metav1.NamespaceSystem}}); err != nil {
+				t.Fatalf("failed executing test: %v", err)
+			}
+
+			for _, cond := range reaction.expectedCSR.Status.Conditions {
+				if cond.Type != certificatesv1beta1.CertificateApproved {
+					t.Fatalf("failed updating csr condition")
+				}
+			}
+		})
+	}
+}
+
+type fakeApproveReaction struct {
+	expectedCSR *certificatesv1beta1.CertificateSigningRequest
+}
+
+func (f *fakeApproveReaction) approveReaction(action fakeclienttest.Action) (bool, runtime.Object, error) {
+	f.expectedCSR = action.(fakeclienttest.UpdateActionImpl).Object.(*certificatesv1beta1.CertificateSigningRequest)
+	return true, f.expectedCSR, nil
+}
diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.10-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.10-aws.yaml
index 6b8e7970..e9ac03d7 100644
--- a/pkg/userdata/rhel/testdata/kubelet-v1.10-aws.yaml
+++ b/pkg/userdata/rhel/testdata/kubelet-v1.10-aws.yaml
@@ -69,6 +69,12 @@ write_files:
     swapoff -a
 
 
+    subscription-manager clean
+    subscription-manager register --username='' --password=''
+    subscription-manager attach --auto
+    yum clean all
+    yum repolist
+
     yum config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
 
     mkdir -p "/etc/docker"
diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.11-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.11-aws.yaml
index 881865d5..f50f6917 100644
--- a/pkg/userdata/rhel/testdata/kubelet-v1.11-aws.yaml
+++ b/pkg/userdata/rhel/testdata/kubelet-v1.11-aws.yaml
@@ -69,6 +69,12 @@ write_files:
     swapoff -a
 
 
+    subscription-manager clean
+    subscription-manager register --username='' --password=''
+    subscription-manager attach --auto
+    yum clean all
+    yum repolist
+
     yum config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
 
     mkdir -p "/etc/docker"
diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.12-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.12-aws-external.yaml
index c2d61892..a14e0592 100644
--- a/pkg/userdata/rhel/testdata/kubelet-v1.12-aws-external.yaml
+++ b/pkg/userdata/rhel/testdata/kubelet-v1.12-aws-external.yaml
@@ -69,6 +69,12 @@ write_files:
     swapoff -a
 
 
+    subscription-manager clean
+    subscription-manager register --username='' --password=''
+    subscription-manager attach --auto
+    yum clean all
+    yum repolist
+
     yum config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
 
     mkdir -p "/etc/docker"
diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.12-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.12-aws.yaml
index f8d1de9c..3de4c87e 100644
--- a/pkg/userdata/rhel/testdata/kubelet-v1.12-aws.yaml
+++ b/pkg/userdata/rhel/testdata/kubelet-v1.12-aws.yaml
@@ -69,6 +69,12 @@ write_files:
     swapoff -a
 
 
+    subscription-manager clean
+    subscription-manager register --username='' --password=''
+    subscription-manager attach --auto
+    yum clean all
+    yum repolist
+
     yum config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
 
     mkdir -p "/etc/docker"
diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.12-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.12-vsphere-mirrors.yaml
index 85424b4f..e08aa779 100644
--- a/pkg/userdata/rhel/testdata/kubelet-v1.12-vsphere-mirrors.yaml
+++ b/pkg/userdata/rhel/testdata/kubelet-v1.12-vsphere-mirrors.yaml
@@ -79,13 +79,14 @@ write_files:
     swapoff -a
 
     hostnamectl set-hostname node1
+
+
     subscription-manager clean
     subscription-manager register --username='' --password=''
     subscription-manager attach --auto
     yum clean all
     yum repolist
 
-
     yum config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
 
     mkdir -p "/etc/docker"
diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.12-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.12-vsphere-proxy.yaml
index ce1a281a..76be8c8b 100644
--- a/pkg/userdata/rhel/testdata/kubelet-v1.12-vsphere-proxy.yaml
+++ b/pkg/userdata/rhel/testdata/kubelet-v1.12-vsphere-proxy.yaml
@@ -79,13 +79,14 @@ write_files:
     swapoff -a
 
     hostnamectl set-hostname node1
+
+
     subscription-manager clean
     subscription-manager register --username='' --password=''
     subscription-manager attach --auto
     yum clean all
     yum repolist
 
-
     yum config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
 
     mkdir -p "/etc/docker"
diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.12-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.12-vsphere.yaml
index 1f781b35..b4eb271f 100644
--- a/pkg/userdata/rhel/testdata/kubelet-v1.12-vsphere.yaml
+++ b/pkg/userdata/rhel/testdata/kubelet-v1.12-vsphere.yaml
@@ -71,13 +71,14 @@ write_files:
     swapoff -a
 
     hostnamectl set-hostname node1
+
+
     subscription-manager clean
     subscription-manager register --username='' --password=''
     subscription-manager attach --auto
     yum clean all
     yum repolist
 
-
     yum config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
 
     mkdir -p "/etc/docker"
diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.9-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.9-aws.yaml
index 2f28b3ff..c93160a6 100644
--- a/pkg/userdata/rhel/testdata/kubelet-v1.9-aws.yaml
+++ b/pkg/userdata/rhel/testdata/kubelet-v1.9-aws.yaml
@@ -69,6 +69,12 @@ write_files:
     swapoff -a
 
 
+    subscription-manager clean
+    subscription-manager register --username='' --password=''
+    subscription-manager attach --auto
+    yum clean all
+    yum repolist
+
     yum config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
 
     mkdir -p "/etc/docker"
diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go
index 4580ebce..39ff12ad 100644
--- a/test/e2e/provisioning/helper.go
+++ b/test/e2e/provisioning/helper.go
@@ -148,7 +148,7 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar
 		scenarioParams = append(scenarioParams, fmt.Sprintf("<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>=%s", rhsmOfflineToken))
 		scenarioParams = append(scenarioParams, fmt.Sprintf("<< DISK_SIZE >>=%v", 50))
 		scenarioParams = append(scenarioParams, fmt.Sprintf("<< CUSTOM-IMAGE >>=%v", "rhel-8-1-custom"))
-		scenarioParams = append(scenarioParams, fmt.Sprintf("<< AMI >>=%s", "ami-0badcc5b522737046"))
+		scenarioParams = append(scenarioParams, fmt.Sprintf("<< AMI >>=%s", "ami-08c04369895785ac4"))
 	} else {
 		scenarioParams = append(scenarioParams, fmt.Sprintf("<< AMI >>=%s", ""))
 		scenarioParams = append(scenarioParams, fmt.Sprintf("<< IMAGE_ID >>=%s", ""))