Repository: syself/cluster-api-provider-hetzner (go)

Issue:
Improve handling of "no available host"
/kind feature

**Describe the solution you'd like**
Currently, the handling is that we reconcile every 30 seconds until we find a host. However, this is usually unnecessary. We can increase the timeout to e.g. 5 minutes, and in this way reduce the load of the controller, and keep most of the functionality. 

We can additionally improve the reactivity by reacting on "HetznerBareMetalHost create" events in the hetznerbaremetalmachine-controller. In this way, the usual case that a new host is added and then used by one of the machines, will be leveraged with perfect reactivity.

Pull request description:
:seedling: Increase requeueAfter No Available Host. And trigger Reconcile of hbmm when hbmh is free.
Fixes #1588


Patch:
diff --git a/controllers/hetznerbaremetalmachine_controller.go b/controllers/hetznerbaremetalmachine_controller.go
index 3268fad0..35938853 100644
--- a/controllers/hetznerbaremetalmachine_controller.go
+++ b/controllers/hetznerbaremetalmachine_controller.go
@@ -230,7 +230,7 @@ func (r *HetznerBareMetalMachineReconciler) SetupWithManager(ctx context.Context
 		).
 		Watches(
 			&infrav1.HetznerBareMetalHost{},
-			handler.EnqueueRequestsFromMapFunc(r.BareMetalHostToBareMetalMachines(log)),
+			handler.EnqueueRequestsFromMapFunc(BareMetalHostToBareMetalMachines(r.Client, log)),
 		).
 		Watches(
 			&clusterv1.Cluster{},
@@ -336,25 +336,69 @@ func (r *HetznerBareMetalMachineReconciler) ClusterToBareMetalMachines(ctx conte
 
 // BareMetalHostToBareMetalMachines will return a reconcile request for a BareMetalMachine if the event is for a
 // BareMetalHost and that BareMetalHost references a BareMetalMachine.
-func (r *HetznerBareMetalMachineReconciler) BareMetalHostToBareMetalMachines(log logr.Logger) handler.MapFunc {
-	return func(_ context.Context, obj client.Object) []reconcile.Request {
-		if host, ok := obj.(*infrav1.HetznerBareMetalHost); ok {
-			if host.Spec.ConsumerRef != nil &&
-				host.Spec.ConsumerRef.Kind == "HetznerBareMetalMachine" &&
-				host.Spec.ConsumerRef.GroupVersionKind().Group == infrav1.GroupVersion.Group {
-				return []reconcile.Request{
-					{
-						NamespacedName: types.NamespacedName{
-							Name:      host.Spec.ConsumerRef.Name,
-							Namespace: host.Spec.ConsumerRef.Namespace,
-						},
-					},
-				}
-			}
-		} else {
+func BareMetalHostToBareMetalMachines(c client.Client, log logr.Logger) handler.MapFunc {
+	return func(ctx context.Context, obj client.Object) []reconcile.Request {
+		host, ok := obj.(*infrav1.HetznerBareMetalHost)
+		if !ok {
 			log.Error(fmt.Errorf("expected a BareMetalHost but got a %T", obj),
 				"failed to get BareMetalMachine for BareMetalHost")
+			return nil
+		}
+
+		// If this host has a consumerRef (hbmm), then reconcile the corresponding hbmm.
+		if host.Spec.ConsumerRef != nil {
+			return []reconcile.Request{
+				{
+					NamespacedName: types.NamespacedName{
+						Name:      host.Spec.ConsumerRef.Name,
+						Namespace: host.Spec.ConsumerRef.Namespace,
+					},
+				},
+			}
+		}
+
+		if host.Spec.Status.ErrorType != "" {
+			return []reconcile.Request{}
+		}
+
+		// We have a free host. Trigger a matching HetznerBareMetalMachine to be reconciled.
+		hbmmList := infrav1.HetznerBareMetalMachineList{}
+		err := c.List(ctx, &hbmmList, client.InNamespace(host.Namespace))
+		if err != nil {
+			log.Error(err, "failed to list HetznerBareMetalMachines")
+			return []reconcile.Request{}
+		}
+
+		// Search for a machines which would like to use this host.
+		var found []reconcile.Request
+		for i := range hbmmList.Items {
+			hbmm := &hbmmList.Items[i]
+
+			// Skip if the hbmm is already in use.
+			if hbmm.HasHostAnnotation() {
+				continue
+			}
+
+			hosts := []infrav1.HetznerBareMetalHost{*host}
+			chosenHost, _, err := baremetal.ChooseHost(hbmm, hosts)
+			if err != nil {
+				log.Error(err, "failed to choose host for HetznerBareMetalMachine")
+				continue
+			}
+
+			// this hbmm does not match the host
+			if chosenHost == nil {
+				continue
+			}
+
+			// We found a matching hbmm for the free host. Trigger Reconcile for the hbmm.
+			found = append(found, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Name:      hbmm.Name,
+					Namespace: hbmm.Namespace,
+				},
+			})
 		}
-		return []reconcile.Request{}
+		return found
 	}
 }
diff --git a/main.go b/main.go
index 5141f903..4382cafa 100644
--- a/main.go
+++ b/main.go
@@ -85,6 +85,7 @@ var (
 	syncPeriod                         time.Duration
 	rateLimitWaitTime                  time.Duration
 	preProvisionCommand                string
+	skipWebhooks                       bool
 )
 
 func main() {
@@ -105,6 +106,7 @@ func main() {
 	fs.DurationVar(&rateLimitWaitTime, "rate-limit", 5*time.Minute, "The rate limiting for HCloud controller (e.g. 5m)")
 	fs.BoolVar(&hcloudclient.DebugAPICalls, "debug-hcloud-api-calls", false, "Debug all calls to the hcloud API.")
 	fs.StringVar(&preProvisionCommand, "pre-provision-command", "", "Command to run (in rescue-system) before installing the image on bare metal servers. You can use that to check if the machine is healthy before installing the image. If the exit value is non-zero, the machine is considered unhealthy. This command must be accessible by the controller pod. You can use an initContainer to copy the command to a shared emptyDir.")
+	fs.BoolVar(&skipWebhooks, "skip-webhooks", false, "Skip setting up of webhooks. Together with --leader-elect=false, you can use `go run main.go` to run CAPH in a cluster connected via KUBECONFIG. You should scale down the caph deployment to 0 before doing that. This is only for testing!")
 	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
 	pflag.Parse()
 
@@ -133,13 +135,10 @@ func main() {
 		}
 	}
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:                 scheme,
-		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
-		HealthProbeBindAddress: probeAddr,
-		WebhookServer: webhook.NewServer(webhook.Options{
-			Port: 9443,
-		}),
+	options := ctrl.Options{
+		Scheme:                        scheme,
+		Metrics:                       metricsserver.Options{BindAddress: metricsAddr},
+		HealthProbeBindAddress:        probeAddr,
 		LeaderElection:                enableLeaderElection,
 		LeaderElectionID:              "hetzner.cluster.x-k8s.io",
 		LeaderElectionNamespace:       leaderElectionNamespace,
@@ -150,7 +149,15 @@ func main() {
 			SyncPeriod:        &syncPeriod,
 			DefaultNamespaces: watchNamespaces,
 		},
-	})
+	}
+
+	if !skipWebhooks {
+		options.WebhookServer = webhook.NewServer(webhook.Options{
+			Port: 9443,
+		})
+	}
+
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options)
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
@@ -245,18 +252,20 @@ func main() {
 		os.Exit(1)
 	}
 
-	setUpWebhookWithManager(mgr)
+	if !skipWebhooks {
+		setUpWebhookWithManager(mgr)
 
-	//+kubebuilder:scaffold:builder
+		//+kubebuilder:scaffold:builder
 
-	if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil {
-		setupLog.Error(err, "unable to create ready check")
-		os.Exit(1)
-	}
+		if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil {
+			setupLog.Error(err, "unable to create ready check")
+			os.Exit(1)
+		}
 
-	if err := mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil {
-		setupLog.Error(err, "unable to create health check")
-		os.Exit(1)
+		if err := mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil {
+			setupLog.Error(err, "unable to create health check")
+			os.Exit(1)
+		}
 	}
 
 	setupLog.Info("starting manager", "version", caphversion.Get().String())
diff --git a/pkg/services/baremetal/baremetal/baremetal.go b/pkg/services/baremetal/baremetal/baremetal.go
index fd05bd11..0117fdc7 100644
--- a/pkg/services/baremetal/baremetal/baremetal.go
+++ b/pkg/services/baremetal/baremetal/baremetal.go
@@ -61,6 +61,8 @@ const (
 	// requeueAfter gives the duration of time until the next reconciliation should be performed.
 	requeueAfter = time.Second * 30
 
+	requeueAfterNoAvailableHost = time.Minute * 7
+
 	// FailureMessageMaintenanceMode indicates that host is in maintenance mode.
 	FailureMessageMaintenanceMode = "host machine in maintenance mode"
 )
@@ -292,7 +294,15 @@ func (s *Service) associate(ctx context.Context) error {
 	}
 
 	// choose new host
-	host, helper, reason, err := s.chooseHost(ctx)
+
+	// get list of hosts scoped to namespace of machine
+	hosts := &infrav1.HetznerBareMetalHostList{}
+	if err := s.scope.Client.List(ctx, hosts,
+		client.InNamespace(s.scope.BareMetalMachine.Namespace)); err != nil {
+		return fmt.Errorf("failed to list hosts: %w", err)
+	}
+
+	host, reason, err := ChooseHost(s.scope.BareMetalMachine, hosts.Items)
 	if err != nil {
 		return fmt.Errorf("failed to choose host: %w", err)
 	}
@@ -307,7 +317,12 @@ func (s *Service) associate(ctx context.Context) error {
 			"%s",
 			fmt.Sprintf("no available host (%s)", reason),
 		)
-		return &scope.RequeueAfterError{RequeueAfter: requeueAfter}
+		return &scope.RequeueAfterError{RequeueAfter: requeueAfterNoAvailableHost}
+	}
+
+	helper, err := patch.NewHelper(host, s.scope.Client)
+	if err != nil {
+		return fmt.Errorf("failed to create patch helper: %w", err)
 	}
 
 	// ensure cluster label on host
@@ -374,40 +389,26 @@ func (s *Service) getAssociatedHost(ctx context.Context) (*infrav1.HetznerBareMe
 	return &host, helper, nil
 }
 
-// chooseHost tries to find a free hbmh.
+// ChooseHost tries to find a free hbmh.
 // If no hbmh was found, then hbmh and err are nil, and the string
 // "reason" contains human readable details.
-func (s *Service) chooseHost(ctx context.Context) (
-	hbmh *infrav1.HetznerBareMetalHost, patchHelper *patch.Helper, reason string, err error,
+func ChooseHost(hbmm *infrav1.HetznerBareMetalMachine, hosts []infrav1.HetznerBareMetalHost) (
+	hbmh *infrav1.HetznerBareMetalHost, reason string, err error,
 ) {
-	// get list of hosts scoped to namespace of machine
-	hosts := infrav1.HetznerBareMetalHostList{}
-	opts := &client.ListOptions{
-		Namespace: s.scope.BareMetalMachine.Namespace,
-	}
-
-	if err := s.scope.Client.List(ctx, &hosts, opts); err != nil {
-		return nil, nil, "", fmt.Errorf("failed to list hosts: %w", err)
-	}
-
-	labelSelector := s.getLabelSelector()
+	labelSelector := getLabelSelector(hbmm)
 
 	// count all hosts that are not in use already
 	unusedHostsCounter := 0
 
 	// hosts are "available" if they are not in use already by some Kubernetes cluster and do not have
 	// another reason to not be chosen (labels that don't match the selector, maintenance mode, error state, etc.)
-	availableHosts := make([]*infrav1.HetznerBareMetalHost, 0, len(hosts.Items))
+	availableHosts := make([]*infrav1.HetznerBareMetalHost, 0, len(hosts))
 
 	mapOfSkipReasons := make(map[string]int)
 
-	for i, host := range hosts.Items {
-		if host.Spec.ConsumerRef != nil && consumerRefMatches(host.Spec.ConsumerRef, s.scope.BareMetalMachine) {
-			helper, err := patch.NewHelper(&hosts.Items[i], s.scope.Client)
-			if err != nil {
-				return nil, nil, "", fmt.Errorf("failed to create patch helper: %w", err)
-			}
-			return &hosts.Items[i], helper, "", nil
+	for i, host := range hosts {
+		if host.Spec.ConsumerRef != nil && consumerRefMatches(host.Spec.ConsumerRef, hbmm) {
+			return &hosts[i], "", nil
 		}
 		if host.Spec.ConsumerRef != nil {
 			continue
@@ -416,22 +417,22 @@ func (s *Service) chooseHost(ctx context.Context) (
 		// from now on each "continue" should add an entry
 		// to mapOfSkipReasons.
 		unusedHostsCounter++
-		if s.skipHost(labelSelector, host, mapOfSkipReasons) {
+		if skipHost(labelSelector, hbmm, host, mapOfSkipReasons) {
 			continue
 		}
 
-		availableHosts = append(availableHosts, &hosts.Items[i])
+		availableHosts = append(availableHosts, &hosts[i])
 	}
 
 	// return if all hosts are in use with a specific message
 	if unusedHostsCounter == 0 {
-		return nil, nil, fmt.Sprintf("all hosts are in use - found %d hosts",
-			len(hosts.Items)), nil
+		return nil, fmt.Sprintf("all hosts are in use - found %d hosts",
+			len(hosts)), nil
 	}
 
 	// found hosts that are not in use, but all of them had some reason to not be chosen
 	if len(availableHosts) == 0 {
-		return nil, nil, reasonString(mapOfSkipReasons, unusedHostsCounter), nil
+		return nil, reasonString(mapOfSkipReasons, unusedHostsCounter), nil
 	}
 
 	// Choose HetznerBareMetalHosts with RootDeviceHints set over those ones without
@@ -449,20 +450,15 @@ func (s *Service) chooseHost(ctx context.Context) (
 	// we found available hosts - choose one
 	randomNumber, err := rand.Int(rand.Reader, big.NewInt(int64(len(availableHosts))))
 	if err != nil {
-		return nil, nil, "", fmt.Errorf("failed to create random number: %w", err)
+		return nil, "", fmt.Errorf("failed to create random number: %w", err)
 	}
 
 	chosenHost := availableHosts[randomNumber.Int64()]
 
-	helper, err := patch.NewHelper(chosenHost, s.scope.Client)
-	if err != nil {
-		return nil, nil, "", fmt.Errorf("failed to create patch helper: %w", err)
-	}
-
-	return chosenHost, helper, "", nil
+	return chosenHost, "", nil
 }
 
-func (s *Service) skipHost(labelSelector labels.Selector, host infrav1.HetznerBareMetalHost, mapOfSkipReasons map[string]int) bool {
+func skipHost(labelSelector labels.Selector, hbmm *infrav1.HetznerBareMetalMachine, host infrav1.HetznerBareMetalHost, mapOfSkipReasons map[string]int) bool {
 	// This comes first, because we should not look too deep into machines
 	// which are not in our scope.
 	if !labelSelector.Matches(labels.Set(host.ObjectMeta.Labels)) {
@@ -509,7 +505,7 @@ func (s *Service) skipHost(labelSelector labels.Selector, host infrav1.HetznerBa
 		return true
 	}
 
-	if s.scope.BareMetalMachine.Spec.InstallImage.Swraid == 1 {
+	if hbmm.Spec.InstallImage.Swraid == 1 {
 		// Machine should have RAID. Skip machines which have less than two WWNs
 		lenOfWwnSlice := len(host.Spec.RootDeviceHints.Raid.WWN)
 		if lenOfWwnSlice < 2 {
@@ -643,17 +639,17 @@ func (s *Service) removeAttachedServerOfLoadBalancer(ctx context.Context, host *
 	return nil
 }
 
-func (s *Service) getLabelSelector() labels.Selector {
+func getLabelSelector(hbmm *infrav1.HetznerBareMetalMachine) labels.Selector {
 	labelSelector := labels.NewSelector()
 	var reqs labels.Requirements
 
-	for labelKey, labelVal := range s.scope.BareMetalMachine.Spec.HostSelector.MatchLabels {
+	for labelKey, labelVal := range hbmm.Spec.HostSelector.MatchLabels {
 		r, err := labels.NewRequirement(labelKey, selection.Equals, []string{labelVal})
 		if err == nil { // ignore invalid host selector
 			reqs = append(reqs, *r)
 		}
 	}
-	for _, req := range s.scope.BareMetalMachine.Spec.HostSelector.MatchExpressions {
+	for _, req := range hbmm.Spec.HostSelector.MatchExpressions {
 		lowercaseOperator := selection.Operator(strings.ToLower(string(req.Operator)))
 		r, err := labels.NewRequirement(req.Key, lowercaseOperator, req.Values)
 		if err == nil { // ignore invalid host selector
diff --git a/pkg/services/baremetal/host/host.go b/pkg/services/baremetal/host/host.go
index d9c1dce6..6c9dcde4 100644
--- a/pkg/services/baremetal/host/host.go
+++ b/pkg/services/baremetal/host/host.go
@@ -1137,12 +1137,11 @@ func (s *Service) actionImageInstalling(ctx context.Context) actionResult {
 	case sshclient.InstallImageStateRunning:
 		s.scope.Logger.Info("installimage is still running. Checking again in some seconds.")
 		return actionContinue{delay: 10 * time.Second}
-
 	case sshclient.InstallImageStateFinished:
 		s.scope.Logger.Info("installimage is finished.")
 		return s.actionImageInstallingFinished(ctx, sshClient)
 	case sshclient.InstallImageStateNotStartedYet:
-		// install-image not started yet. Start it now.
+		s.scope.Logger.Info("installimage is not started yet. Starting it now")
 		return s.actionImageInstallingStartBackgroundProcess(ctx, sshClient)
 	default:
 		panic(fmt.Sprintf("Unknown InstallImageState %+v", state))
@@ -1513,6 +1512,8 @@ func verifyConnectionRefused(sshClient sshclient.Client, port int) bool {
 	return true
 }
 
+// prev: ImageInstalling
+// next: Provisioned
 func (s *Service) actionEnsureProvisioned(_ context.Context) (ar actionResult) {
 	markProvisionPending(s.scope.HetznerBareMetalHost, infrav1.StateEnsureProvisioned)
 	sshClient := s.scope.SSHClientFactory.NewClient(sshclient.Input{
diff --git a/pkg/services/hcloud/network/network.go b/pkg/services/hcloud/network/network.go
index 3db250f0..b3614b7c 100644
--- a/pkg/services/hcloud/network/network.go
+++ b/pkg/services/hcloud/network/network.go
@@ -99,7 +99,7 @@ func (s *Service) createNetwork(ctx context.Context) (*hcloud.Network, error) {
 		return nil, fmt.Errorf("failed to create network: %w", err)
 	}
 
-	record.Eventf(s.scope.HetznerCluster, "NetworkCreated", "Created network with opts %s", opts)
+	record.Eventf(s.scope.HetznerCluster, "NetworkCreated", "Created network with opts %+v", opts)
 	return resp, nil
 }
 


Test patch:
diff --git a/controllers/hetznerbaremetalmachine_controller_test.go b/controllers/hetznerbaremetalmachine_controller_test.go
index 2909ad66..737689c6 100644
--- a/controllers/hetznerbaremetalmachine_controller_test.go
+++ b/controllers/hetznerbaremetalmachine_controller_test.go
@@ -17,20 +17,29 @@ limitations under the License.
 package controllers
 
 import (
+	"context"
+	"testing"
 	"time"
 
+	"github.com/go-logr/logr"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
 	"github.com/syself/hrobot-go/models"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/utils/ptr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	infrav1 "github.com/syself/cluster-api-provider-hetzner/api/v1beta1"
 	"github.com/syself/cluster-api-provider-hetzner/pkg/services/baremetal/baremetal"
@@ -825,4 +834,262 @@ var _ = Describe("HetznerBareMetalMachineReconciler", func() {
 			})
 		})
 	})
+
+	Context("Test EnqueueRequestsFromMapFunc BareMetalHostToBareMetalMachines", func() {
+		It("should enqueue the second HetznerBareMetalMachine, when the first gets deleted", func() {
+			// We test this part in BareMetalHostToBareMetalMachines:
+			// We have a free host. Trigger a matching HetznerBareMetalMachine to be reconciled.
+			capiMachine := &clusterv1.Machine{
+				ObjectMeta: metav1.ObjectMeta{
+					GenerateName: "capi-machine-",
+					Namespace:    testNs.Name,
+					Finalizers:   []string{clusterv1.MachineFinalizer},
+					Labels: map[string]string{
+						clusterv1.ClusterNameLabel: capiCluster.Name,
+					},
+				},
+				Spec: clusterv1.MachineSpec{
+					ClusterName: capiCluster.Name,
+					InfrastructureRef: corev1.ObjectReference{
+						APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1",
+						Kind:       "HetznerBareMetalMachine",
+						Name:       bmMachineName,
+					},
+					FailureDomain: &defaultFailureDomain,
+					Bootstrap: clusterv1.Bootstrap{
+						DataSecretName: ptr.To("bootstrap-secret"),
+					},
+				},
+			}
+			Expect(testEnv.Create(ctx, capiMachine)).To(Succeed())
+
+			bmMachine = &infrav1.HetznerBareMetalMachine{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      bmMachineName,
+					Namespace: testNs.Name,
+					Labels: map[string]string{
+						clusterv1.ClusterNameLabel: capiCluster.Name,
+					},
+					OwnerReferences: []metav1.OwnerReference{
+						{
+							APIVersion: "cluster.x-k8s.io/v1beta1",
+							Kind:       "Machine",
+							Name:       capiMachine.Name,
+							UID:        capiMachine.UID,
+						},
+					},
+				},
+				Spec: getDefaultHetznerBareMetalMachineSpec(),
+			}
+			Expect(testEnv.Create(ctx, bmMachine)).To(Succeed())
+
+			// Create a second Capi- and HetznerBareMetalMachine
+			bmMachineName2 := "hbmm2"
+			capiMachine2 := &clusterv1.Machine{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:       utils.GenerateName(nil, "capimachine-name-"),
+					Namespace:  testNs.Name,
+					Finalizers: []string{clusterv1.MachineFinalizer},
+					Labels: map[string]string{
+						clusterv1.ClusterNameLabel: capiCluster.Name,
+					},
+				},
+				Spec: clusterv1.MachineSpec{
+					ClusterName: capiCluster.Name,
+					InfrastructureRef: corev1.ObjectReference{
+						APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1",
+						Kind:       "HetznerBareMetalMachine",
+						Name:       bmMachineName2,
+					},
+					Bootstrap: clusterv1.Bootstrap{
+						DataSecretName: ptr.To("bootstrap-secret"),
+					},
+					FailureDomain: &defaultFailureDomain,
+				},
+			}
+			Expect(testEnv.Create(ctx, capiMachine2)).To(Succeed())
+
+			bmMachine2 := &infrav1.HetznerBareMetalMachine{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      bmMachineName2,
+					Namespace: testNs.Name,
+					Labels: map[string]string{
+						clusterv1.ClusterNameLabel: capiCluster.Name,
+					},
+					OwnerReferences: []metav1.OwnerReference{
+						{
+							APIVersion: "cluster.x-k8s.io/v1beta1",
+							Kind:       "Machine",
+							Name:       capiMachine2.Name,
+							UID:        capiMachine2.UID,
+						},
+					},
+				},
+				Spec: getDefaultHetznerBareMetalMachineSpec(),
+			}
+			Expect(testEnv.Create(ctx, bmMachine2)).To(Succeed())
+
+			osSSHSecret := helpers.GetDefaultSSHSecret("os-ssh-secret", testNs.Name)
+			Expect(testEnv.Create(ctx, osSSHSecret)).To(Succeed())
+
+			host := helpers.BareMetalHost(
+				hostName,
+				testNs.Name,
+				helpers.WithRootDeviceHintWWN(),
+				helpers.WithHetznerClusterRef(hetznerClusterName),
+			)
+			Expect(testEnv.Create(ctx, host)).To(Succeed())
+
+			// Wait until bmMachine is provisioned.
+			var pickedMachine *infrav1.HetznerBareMetalMachine
+			var waitingMachine *infrav1.HetznerBareMetalMachine
+
+			// The mock should wait until we know which machine is picked.
+			w := make(chan time.Time)
+			osSSHClient.On("GetHostName").WaitUntil(w)
+
+			Eventually(func() bool {
+				if err := testEnv.Get(ctx, client.ObjectKeyFromObject(host), host); err != nil {
+					return false
+				}
+				if host.Spec.ConsumerRef == nil {
+					return false
+				}
+
+				if host.Spec.ConsumerRef.Name == bmMachineName {
+					pickedMachine = bmMachine
+					waitingMachine = bmMachine2
+				} else {
+					pickedMachine = bmMachine2
+					waitingMachine = bmMachine
+				}
+
+				if w != nil {
+					// We need to change the mock on-the-fly. There are two
+					// machines, and we need to adapt the mock to the one that is
+					// picked.
+					osSSHClient.On("GetHostName").Return(sshclient.Output{
+						StdOut: infrav1.BareMetalHostNamePrefix + pickedMachine.Name,
+						StdErr: "",
+						Err:    nil,
+					})
+					// Close the channel, so that mock "GetHostName" does no longer block.
+					close(w)
+
+					// We updated the mock, and closed the channel. Set channel to nil,
+					// so that we don't mock and close again.
+					// Closing a closed channel would panic.
+					w = nil
+				}
+
+				return host.Spec.Status.ProvisioningState != infrav1.StateProvisioned
+			}, timeout).Should(BeTrue())
+
+			// Ensure the second machine is not ready (waiting for a hbmh)
+			Expect(testEnv.Get(ctx, client.ObjectKeyFromObject(waitingMachine),
+				waitingMachine)).To(Succeed())
+			Expect(waitingMachine.Status.Ready).To(BeFalse())
+
+			// Delete the first machine
+			Expect(testEnv.Delete(ctx, pickedMachine)).To(Succeed())
+
+			// Wait until the waiting machine is ready
+			Eventually(func() bool {
+				Expect(testEnv.Get(ctx, client.ObjectKeyFromObject(waitingMachine),
+					waitingMachine)).To(Succeed())
+				return waitingMachine.Status.Ready
+			}, timeout).Should(BeTrue())
+		})
+	})
 })
+
+func Test_BareMetalHostToBareMetalMachines(t *testing.T) {
+	ctx := context.Background()
+	scheme := runtime.NewScheme()
+	utilruntime.Must(infrav1.AddToScheme(scheme))
+	host := &infrav1.HetznerBareMetalHost{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-host",
+			Namespace: "test-ns",
+		},
+	}
+	hbmm := &infrav1.HetznerBareMetalMachine{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-machine-with-label-foo",
+			Namespace: "test-ns",
+		},
+		Spec: infrav1.HetznerBareMetalMachineSpec{
+			HostSelector: infrav1.HostSelector{
+				MatchLabels: map[string]string{
+					"key": "foo",
+				},
+			},
+		},
+	}
+	hbmmWithHostAnnotation := &infrav1.HetznerBareMetalMachine{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-machine-with-host-annotation",
+			Namespace: "test-ns",
+			Annotations: map[string]string{
+				infrav1.HostAnnotation: "test-host",
+			},
+		},
+		Spec: infrav1.HetznerBareMetalMachineSpec{
+			HostSelector: infrav1.HostSelector{
+				MatchLabels: map[string]string{
+					"key": "foo",
+				},
+			},
+		},
+	}
+
+	hbmmWithoutLabel := &infrav1.HetznerBareMetalMachine{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-machine-with-label-bar",
+			Namespace: "test-ns",
+		},
+		Spec: infrav1.HetznerBareMetalMachineSpec{
+			HostSelector: infrav1.HostSelector{
+				MatchLabels: map[string]string{
+					"key": "bar",
+				},
+			},
+		},
+	}
+	hbmmOtherNS := &infrav1.HetznerBareMetalMachine{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-machine-other-namespace",
+			Namespace: "test-other-namespace",
+		},
+		Spec: infrav1.HetznerBareMetalMachineSpec{
+			HostSelector: infrav1.HostSelector{
+				MatchLabels: map[string]string{
+					"key": "foo",
+				},
+			},
+		},
+	}
+	c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(
+		host, hbmm, hbmmOtherNS, hbmmWithoutLabel, hbmmWithHostAnnotation).Build()
+
+	// host does not have a label.
+	f := BareMetalHostToBareMetalMachines(c, logr.Logger{})
+	result := f(ctx, host)
+	require.Nil(t, result)
+	host.Labels = map[string]string{
+		"key": "foo",
+	}
+
+	// host has label "foo"
+	err := c.Update(ctx, host)
+	require.NoError(t, err)
+	result = f(ctx, host)
+	require.Equal(t, []reconcile.Request{
+		{
+			NamespacedName: types.NamespacedName{
+				Name:      "test-machine-with-label-foo",
+				Namespace: "test-ns",
+			},
+		},
+	}, result)
+}
diff --git a/controllers/hetznercluster_controller_test.go b/controllers/hetznercluster_controller_test.go
index 4201c0fe..52d6eaac 100644
--- a/controllers/hetznercluster_controller_test.go
+++ b/controllers/hetznercluster_controller_test.go
@@ -334,7 +334,7 @@ var _ = Describe("Hetzner ClusterReconciler", func() {
 		})
 
 		Context("load balancer", func() {
-			It("should create load balancer and update it accordingly", func() {
+			It("should create load balancer and update it accordingly (flaky)", func() {
 				Expect(testEnv.Create(ctx, instance)).To(Succeed())
 
 				Eventually(func() bool {
@@ -400,7 +400,7 @@ var _ = Describe("Hetzner ClusterReconciler", func() {
 					}
 
 					return true
-				}, timeout, 1*time.Second).Should(BeTrue())
+				}, 2*timeout, 1*time.Second).Should(BeTrue())
 			})
 
 			It("should update extra targets", func() {
diff --git a/pkg/services/baremetal/baremetal/baremetal_test.go b/pkg/services/baremetal/baremetal/baremetal_test.go
index 6773a5a4..6bbb0e20 100644
--- a/pkg/services/baremetal/baremetal/baremetal_test.go
+++ b/pkg/services/baremetal/baremetal/baremetal_test.go
@@ -244,9 +244,14 @@ var _ = Describe("chooseHost", func() {
 			bmMachine.Spec.HostSelector = tc.HostSelector
 			service := newTestService(bmMachine, c)
 
-			host, _, expectedReason, err := service.chooseHost(context.TODO())
+			hosts := &infrav1.HetznerBareMetalHostList{}
+			err := service.scope.Client.List(context.TODO(), hosts,
+				client.InNamespace(service.scope.BareMetalMachine.Namespace))
 			Expect(err).To(Succeed())
-			Expect(expectedReason).To(Equal(""))
+
+			host, reason, err := ChooseHost(bmMachine, hosts.Items)
+			Expect(err).To(Succeed())
+			Expect(reason).To(Equal(""))
 			if tc.ExpectedHostName == "" {
 				Expect(host).To(BeNil())
 			} else {
@@ -328,7 +333,12 @@ var _ = Describe("chooseHost", func() {
 			}
 			service := newTestService(bmMachine, c)
 
-			host, _, reason, err := service.chooseHost(context.TODO())
+			hosts := &infrav1.HetznerBareMetalHostList{}
+			err := service.scope.Client.List(context.TODO(), hosts,
+				client.InNamespace(service.scope.BareMetalMachine.Namespace))
+			Expect(err).To(Succeed())
+			host, reason, err := ChooseHost(bmMachine, hosts.Items)
+
 			Expect(err).To(Succeed())
 			Expect(reason).To(Equal(tc.expectedReason))
 			Expect(host).To(BeNil())
diff --git a/test/helpers/envtest.go b/test/helpers/envtest.go
index d6c22587..49d556cb 100644
--- a/test/helpers/envtest.go
+++ b/test/helpers/envtest.go
@@ -42,6 +42,7 @@ import (
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/log"
 	"sigs.k8s.io/cluster-api/util/kubeconfig"
+	"sigs.k8s.io/cluster-api/util/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -61,7 +62,7 @@ import (
 
 func init() {
 	klog.InitFlags(nil)
-	logger := textlogger.NewLogger(textlogger.NewConfig())
+	logger := textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(4)))
 
 	// use klog as the internal logger for this envtest environment.
 	log.SetLogger(logger)
@@ -149,6 +150,8 @@ func NewTestEnvironment() *TestEnvironment {
 		klog.Fatalf("unable to create manager: %s", err)
 	}
 
+	record.InitFromRecorder(mgr.GetEventRecorderFor("hetzner-controller"))
+
 	if err := (&infrav1.HetznerCluster{}).SetupWebhookWithManager(mgr); err != nil {
 		klog.Fatalf("failed to set up webhook with manager for HetznerCluster: %s", err)
 	}