Repository: syself/cluster-api-provider-hetzner (go)

Issue:
Handle install-image asynchronously
/kind feature

**Describe the solution you'd like**

I want the `Reconcile()` calls in Go to finish quickly, so that the reconciling of other events is not blocked.

At the moment the install-image phase takes roughly 2.5min. During that time the controller does not reconcile
other events.

We want to call install-image asynchronously like this:

```
ssh user@remote_host 'nohup my_command > output.log 2>&1 < /dev/null &'
```

Additionally, we should log a warning, if a reconcile call takes longer than 15 seconds.

Pull request description:
:seedling: Call install-image asynchronously (bare-metal)
Up to now the caph controller blocked for roughly 2 minutes while running install-image. If you run with max reconcile set to one, then the controller did not handle other events for 2 minutes.

This PR starts the install-image in background, and checks for the result every N seconds, which keeps the caph controller responsive.


Fixes #1408

Patch:
diff --git a/controllers/hetznerbaremetalhost_controller.go b/controllers/hetznerbaremetalhost_controller.go
index 0965bc4e..2f45b95e 100644
--- a/controllers/hetznerbaremetalhost_controller.go
+++ b/controllers/hetznerbaremetalhost_controller.go
@@ -66,6 +66,16 @@ type HetznerBareMetalHostReconciler struct {
 // Reconcile implements the reconcilement of HetznerBareMetalHost objects.
 func (r *HetznerBareMetalHostReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, reterr error) {
 	log := ctrl.LoggerFrom(ctx)
+
+	start := time.Now()
+	defer func() {
+		// check duration of reconcile. Warn if it took too long.
+		duration := time.Since(start)
+		if duration > 15*time.Second {
+			log.Info("Reconcile took too long", "duration", duration, "res", res, "reterr", reterr)
+		}
+	}()
+
 	// Fetch the Hetzner bare metal host instance.
 	bmHost := &infrav1.HetznerBareMetalHost{}
 	err := r.Get(ctx, req.NamespacedName, bmHost)
diff --git a/pkg/services/baremetal/client/mocks/ssh/Client.go b/pkg/services/baremetal/client/mocks/ssh/Client.go
index f1590312..213951ce 100644
--- a/pkg/services/baremetal/client/mocks/ssh/Client.go
+++ b/pkg/services/baremetal/client/mocks/ssh/Client.go
@@ -971,47 +971,112 @@ func (_c *Client_GetHostName_Call) RunAndReturn(run func() sshclient.Output) *Cl
 	return _c
 }
 
-// GetRunningInstallImageProcesses provides a mock function with given fields:
-func (_m *Client) GetRunningInstallImageProcesses() sshclient.Output {
+// GetInstallImageState provides a mock function with given fields:
+func (_m *Client) GetInstallImageState() (sshclient.InstallImageState, error) {
 	ret := _m.Called()
 
 	if len(ret) == 0 {
-		panic("no return value specified for GetRunningInstallImageProcesses")
+		panic("no return value specified for GetInstallImageState")
 	}
 
-	var r0 sshclient.Output
-	if rf, ok := ret.Get(0).(func() sshclient.Output); ok {
+	var r0 sshclient.InstallImageState
+	var r1 error
+	if rf, ok := ret.Get(0).(func() (sshclient.InstallImageState, error)); ok {
+		return rf()
+	}
+	if rf, ok := ret.Get(0).(func() sshclient.InstallImageState); ok {
 		r0 = rf()
 	} else {
-		r0 = ret.Get(0).(sshclient.Output)
+		r0 = ret.Get(0).(sshclient.InstallImageState)
 	}
 
-	return r0
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
 }
 
-// Client_GetRunningInstallImageProcesses_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetRunningInstallImageProcesses'
-type Client_GetRunningInstallImageProcesses_Call struct {
+// Client_GetInstallImageState_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetInstallImageState'
+type Client_GetInstallImageState_Call struct {
 	*mock.Call
 }
 
-// GetRunningInstallImageProcesses is a helper method to define mock.On call
-func (_e *Client_Expecter) GetRunningInstallImageProcesses() *Client_GetRunningInstallImageProcesses_Call {
-	return &Client_GetRunningInstallImageProcesses_Call{Call: _e.mock.On("GetRunningInstallImageProcesses")}
+// GetInstallImageState is a helper method to define mock.On call
+func (_e *Client_Expecter) GetInstallImageState() *Client_GetInstallImageState_Call {
+	return &Client_GetInstallImageState_Call{Call: _e.mock.On("GetInstallImageState")}
 }
 
-func (_c *Client_GetRunningInstallImageProcesses_Call) Run(run func()) *Client_GetRunningInstallImageProcesses_Call {
+func (_c *Client_GetInstallImageState_Call) Run(run func()) *Client_GetInstallImageState_Call {
 	_c.Call.Run(func(args mock.Arguments) {
 		run()
 	})
 	return _c
 }
 
-func (_c *Client_GetRunningInstallImageProcesses_Call) Return(_a0 sshclient.Output) *Client_GetRunningInstallImageProcesses_Call {
-	_c.Call.Return(_a0)
+func (_c *Client_GetInstallImageState_Call) Return(_a0 sshclient.InstallImageState, _a1 error) *Client_GetInstallImageState_Call {
+	_c.Call.Return(_a0, _a1)
+	return _c
+}
+
+func (_c *Client_GetInstallImageState_Call) RunAndReturn(run func() (sshclient.InstallImageState, error)) *Client_GetInstallImageState_Call {
+	_c.Call.Return(run)
+	return _c
+}
+
+// GetResultOfInstallImage provides a mock function with given fields:
+func (_m *Client) GetResultOfInstallImage() (string, error) {
+	ret := _m.Called()
+
+	if len(ret) == 0 {
+		panic("no return value specified for GetResultOfInstallImage")
+	}
+
+	var r0 string
+	var r1 error
+	if rf, ok := ret.Get(0).(func() (string, error)); ok {
+		return rf()
+	}
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Client_GetResultOfInstallImage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetResultOfInstallImage'
+type Client_GetResultOfInstallImage_Call struct {
+	*mock.Call
+}
+
+// GetResultOfInstallImage is a helper method to define mock.On call
+func (_e *Client_Expecter) GetResultOfInstallImage() *Client_GetResultOfInstallImage_Call {
+	return &Client_GetResultOfInstallImage_Call{Call: _e.mock.On("GetResultOfInstallImage")}
+}
+
+func (_c *Client_GetResultOfInstallImage_Call) Run(run func()) *Client_GetResultOfInstallImage_Call {
+	_c.Call.Run(func(args mock.Arguments) {
+		run()
+	})
+	return _c
+}
+
+func (_c *Client_GetResultOfInstallImage_Call) Return(_a0 string, _a1 error) *Client_GetResultOfInstallImage_Call {
+	_c.Call.Return(_a0, _a1)
 	return _c
 }
 
-func (_c *Client_GetRunningInstallImageProcesses_Call) RunAndReturn(run func() sshclient.Output) *Client_GetRunningInstallImageProcesses_Call {
+func (_c *Client_GetResultOfInstallImage_Call) RunAndReturn(run func() (string, error)) *Client_GetResultOfInstallImage_Call {
 	_c.Call.Return(run)
 	return _c
 }
diff --git a/pkg/services/baremetal/client/ssh/ssh_client.go b/pkg/services/baremetal/client/ssh/ssh_client.go
index 6e76134c..4cca01dc 100644
--- a/pkg/services/baremetal/client/ssh/ssh_client.go
+++ b/pkg/services/baremetal/client/ssh/ssh_client.go
@@ -176,6 +176,18 @@ type Output struct {
 	Err    error
 }
 
+// InstallImageState defines three states of the process.
+type InstallImageState string
+
+const (
+	// InstallImageStateNotStartedYet means the process has not started yet.
+	InstallImageStateNotStartedYet InstallImageState = "not-started-yet"
+	// InstallImageStateRunning means the process is still running.
+	InstallImageStateRunning InstallImageState = "running"
+	// InstallImageStateFinished has finished.
+	InstallImageStateFinished InstallImageState = "finished"
+)
+
 func (o Output) String() string {
 	s := make([]string, 0, 3)
 	stdout := strings.TrimSpace(o.StdOut)
@@ -200,6 +212,19 @@ func (o Output) String() string {
 	return strings.Join(s, ". ")
 }
 
+// ExitStatus returns the exit status of the remote shell command.
+// There are three case:
+// First case: Remote command finished with exit 0: 0, nil.
+// Second case: Remote command finished with non zero: N, nil.
+// Third case: Remote command was not called successfully (like host not reachable): 0, err.
+func (o Output) ExitStatus() (int, error) {
+	var exitError *ssh.ExitError
+	if errors.As(o.Err, &exitError) {
+		return exitError.ExitStatus(), nil
+	}
+	return 0, o.Err
+}
+
 // Client is the interface defining all functions necessary to talk to a bare metal server via SSH.
 type Client interface {
 	GetHostName() Output
@@ -213,7 +238,8 @@ type Client interface {
 	GetHardwareDetailsCPUThreads() Output
 	GetHardwareDetailsCPUCores() Output
 	GetHardwareDetailsDebug() Output
-	GetRunningInstallImageProcesses() Output
+	GetInstallImageState() (InstallImageState, error)
+	GetResultOfInstallImage() (string, error)
 	GetCloudInitOutput() Output
 	CreateAutoSetup(data string) Output
 	DownloadImage(path, url string) Output
@@ -385,9 +411,26 @@ EOF_VIA_SSH`, data))
 	return c.runSSH(`chmod +x /root/post-install.sh`)
 }
 
-// GetRunningInstallImageProcesses returns the running installimage processes. Output.StdOut is empty if no processes are running.
-func (c *sshClient) GetRunningInstallImageProcesses() Output {
-	return c.runSSH(`ps aux| grep installimage | grep -v grep; true`)
+// GetInstallImageState returns the running installimage processes.
+func (c *sshClient) GetInstallImageState() (InstallImageState, error) {
+	out := c.runSSH(`ps aux| grep installimage | grep -v grep; true`)
+	if out.Err != nil {
+		return "", fmt.Errorf("failed to run `ps aux` to get running installimage process: %w", out.Err)
+	}
+	if out.StdOut != "" {
+		// installimage is running
+		return InstallImageStateRunning, nil
+	}
+
+	out = c.runSSH(`[ -e /root/installimage-wrapper.sh.log ]`)
+	exists, err := out.ExitStatus()
+	if err != nil {
+		return "", fmt.Errorf("failed to check if installimage-wrapper.sh.log exists: %w", err)
+	}
+	if exists == 0 {
+		return InstallImageStateFinished, nil
+	}
+	return InstallImageStateNotStartedYet, nil
 }
 
 // ExecuteInstallImage implements the ExecuteInstallImage method of the SSHClient interface.
@@ -415,32 +458,34 @@ EOF_VIA_SSH`, cmd))
 		return out
 	}
 
-	installImageOut := c.runSSH(`sh /root/installimage-wrapper.sh`)
+	return c.runSSH(`nohup /root/installimage-wrapper.sh >/root/installimage-wrapper.sh.log 2>&1 </dev/null &`)
+}
 
-	debugTxtOut := c.runSSH(`cat /root/debug.txt`)
-	installImageOut.StdOut = fmt.Sprintf(`debug.txt:
-%s
-%s
-######################################
+// GetResultOfInstallImage returns the logs of install-image.
+// Before calling this method be sure that installimage is already terminated.
+func (c *sshClient) GetResultOfInstallImage() (string, error) {
+	out := c.runSSH(`cat /root/debug.txt`)
+	if out.Err != nil {
+		return "", fmt.Errorf("failed to get debug.txt: %w", out.Err)
+	}
+	debugTxt := out.StdOut
 
-/root/installimage-wrapper.sh stdout:
-%s
-######################################
+	out = c.runSSH(`cat /root/installimage-wrapper.sh.log`)
+	if out.Err != nil {
+		return "", fmt.Errorf("failed to get installimage-wrapper.sh.log: %w", out.Err)
+	}
+	wrapperLog := out.StdOut
 
-stderr:
-%s`,
-		debugTxtOut.StdOut, debugTxtOut.StdErr, installImageOut.StdOut, installImageOut.StdErr)
+	return fmt.Sprintf(`debug.txt:
+%s
 
-	if installImageOut.Err != nil {
-		return installImageOut
-	}
+######################################
 
-	if debugTxtOut.Err != nil {
-		installImageOut.StdOut += fmt.Sprintf("\nfailed to get /root/debug.txt:\n%s", debugTxtOut.Err.Error())
-	}
+/root/installimage-wrapper.sh stdout+stderr:
 
-	// Ignore StdErr in this command
-	return Output{StdOut: installImageOut.StdOut}
+%s
+`,
+		debugTxt, wrapperLog), nil
 }
 
 // Reboot implements the Reboot method of the SSHClient interface.
@@ -508,12 +553,14 @@ EOF_VIA_SSH
 }
 
 func (c *sshClient) UntarTGZ() Output {
+	// read tgz from container image.
 	fileName := "/installimage.tgz"
 	data, err := os.ReadFile(fileName)
 	if err != nil {
 		return Output{Err: fmt.Errorf("ReadInstallimageTgzFailed %s: %w", fileName, err)}
 	}
 
+	// send base64 encoded binary to machine via ssh.
 	return c.runSSH(fmt.Sprintf("echo %s | base64 -d | tar -xzf-",
 		base64.StdEncoding.EncodeToString(data)))
 }
diff --git a/pkg/services/baremetal/host/host.go b/pkg/services/baremetal/host/host.go
index 9f096fd0..d31b20b6 100644
--- a/pkg/services/baremetal/host/host.go
+++ b/pkg/services/baremetal/host/host.go
@@ -1074,7 +1074,27 @@ func (s *Service) actionImageInstalling(ctx context.Context) actionResult {
 	}
 
 	s.scope.HetznerBareMetalHost.Spec.Status.SSHStatus.OSKey = &sshKey
+	state, err := sshClient.GetInstallImageState()
+	if err != nil {
+		return actionError{err: fmt.Errorf("failed to get state of installimage processes: %w", err)}
+	}
+	switch state {
+	case sshclient.InstallImageStateRunning:
+		s.scope.Logger.Info("installimage is still running. Checking again in some seconds.")
+		return actionContinue{delay: 10 * time.Second}
 
+	case sshclient.InstallImageStateFinished:
+		s.scope.Logger.Info("installimage is finished.")
+		return s.actionImageInstallingFinished(ctx, sshClient)
+	case sshclient.InstallImageStateNotStartedYet:
+		// install-image not started yet. Start it now.
+		return s.actionImageInstallingStartBackgroundProcess(ctx, sshClient)
+	default:
+		panic(fmt.Sprintf("Unknown InstallImageState %+v", state))
+	}
+}
+
+func (s *Service) actionImageInstallingStartBackgroundProcess(ctx context.Context, sshClient sshclient.Client) actionResult {
 	// If there is a Linux OS on an other disk, then the reboot after the provisioning
 	// will likely fail, because the machine boots into the other operating system.
 	// We want detect that early, and not start the provisioning process.
@@ -1115,19 +1135,6 @@ func (s *Service) actionImageInstalling(ctx context.Context) actionResult {
 	}
 	record.Eventf(s.scope.HetznerBareMetalHost, "NoLinuxOnAnotherDisk", "OK, no Linux on another disk:\n%s\n\n%s", out.StdOut, out.StdErr)
 
-	// if the previous reconcile was stopped, then wait until the first
-	// run of installimage was finished.
-	out = sshClient.GetRunningInstallImageProcesses()
-	if out.Err != nil {
-		return actionError{err: fmt.Errorf("failed to get running installimage processes: %q %q %w", out.StdOut, out.StdErr, out.Err)}
-	}
-	if out.StdOut != "" {
-		// This can happen if the controller was restarted.
-		record.Warn(s.scope.HetznerBareMetalHost, "InstallImageAlreadyRunning",
-			"installimage is already running.")
-		return actionError{err: fmt.Errorf("installimage is already running")}
-	}
-
 	record.Event(s.scope.HetznerBareMetalHost, "InstallImagePreflightCheckSuccessful", "Rescue system reachable, disks look good.")
 
 	autoSetupInput, actionRes := s.createAutoSetupInput(sshClient)
@@ -1200,22 +1207,33 @@ echo %q
 		record.Warnf(s.scope.HetznerBareMetalHost, "ExecuteInstallImageFailed", out.String())
 		return actionError{err: fmt.Errorf("failed to execute installimage: %w", out.Err)}
 	}
-	if !strings.Contains(out.StdOut, PostInstallScriptFinished) {
-		record.Warnf(s.scope.HetznerBareMetalHost, "ExecuteInstallImageDidNotFinish", out.StdOut)
-		return actionError{err: fmt.Errorf("did not find marker %q in stdout. Failed to execute installimage",
-			PostInstallScriptFinished)}
+	return actionContinue{delay: 10 * time.Second}
+}
+
+func (s *Service) actionImageInstallingFinished(_ context.Context, sshClient sshclient.Client) actionResult {
+	output, err := sshClient.GetResultOfInstallImage()
+	if err != nil {
+		return actionError{
+			err: fmt.Errorf("GetResultOfInstallImage failed: %w", err),
+		}
+	}
+	if !strings.Contains(output, PostInstallScriptFinished) {
+		record.Warn(s.scope.HetznerBareMetalHost, "InstallImageNotSuccessful", output)
+		return actionError{err: fmt.Errorf("did not find marker %q in stdout. Installimage was not successful: %s",
+			PostInstallScriptFinished, output)}
 	}
-	record.Eventf(s.scope.HetznerBareMetalHost, "ExecuteInstallImageSucceeded", out.StdOut)
-	s.scope.Logger.Info("ExecuteInstallImageSucceeded", "stdout", out.StdOut, "stderr", out.StdErr)
+
+	record.Event(s.scope.HetznerBareMetalHost, "InstallImageOutput", output)
+	s.scope.Logger.Info("InstallImageOutput", "output", output)
 
 	// Update name in robot API
-	if _, err := s.scope.RobotClient.SetBMServerName(s.scope.HetznerBareMetalHost.Spec.ServerID, autoSetupInput.hostName); err != nil {
+	if _, err := s.scope.RobotClient.SetBMServerName(s.scope.HetznerBareMetalHost.Spec.ServerID, s.scope.Hostname()); err != nil {
 		record.Warn(s.scope.HetznerBareMetalHost, "SetBMServerNameFailed", err.Error())
 		s.handleRobotRateLimitExceeded(err, "SetBMServerName")
 		return actionError{err: fmt.Errorf("failed to update name of host in robot API: %w", err)}
 	}
 
-	out = sshClient.Reboot()
+	out := sshClient.Reboot()
 	if err := handleSSHError(out); err != nil {
 		err = fmt.Errorf("failed to reboot server (after install-image): %w", err)
 		record.Warn(s.scope.HetznerBareMetalHost, "RebootFailed", err.Error())
@@ -1369,35 +1387,6 @@ func (s *Service) actionEnsureProvisioned(_ context.Context) (ar actionResult) {
 		IP:         s.scope.HetznerBareMetalHost.Spec.Status.GetIPAddress(),
 	})
 
-	defer func() {
-		// Create an Event which contains the content of /var/log/cloud-init-output.log
-
-		if _, ok := ar.(actionContinue); ok {
-			// don't create an event
-			return
-		}
-		out := sshClient.GetCloudInitOutput()
-		if out.Err != nil || out.StdErr != "" {
-			record.Warnf(s.scope.HetznerBareMetalHost, "GetCloudInitOutputFailed",
-				fmt.Sprintf("GetCloudInitOutput failed to get /var/log/cloud-init-output.log: stdout %q, stderr %q, err %q",
-					out.StdOut, out.StdErr, out.Err.Error()))
-			return
-		}
-		_, ok := ar.(actionComplete)
-		if ok {
-			record.Eventf(s.scope.HetznerBareMetalHost, "CloudInitOutput",
-				"/var/log/cloud-init-output.log: "+out.StdOut)
-		} else {
-			_, err := ar.Result()
-			errString := ""
-			if err != nil {
-				errString = err.Error()
-			}
-			record.Warnf(s.scope.HetznerBareMetalHost, "CloudInitOutput", fmt.Sprintf("cloud init output (%s):\n%s",
-				errString,
-				out.StdOut))
-		}
-	}()
 	// Check hostname with sshClient
 	wantHostName := s.scope.Hostname()
 
@@ -1437,11 +1426,45 @@ func (s *Service) actionEnsureProvisioned(_ context.Context) (ar actionResult) {
 		return actionContinue{delay: 10 * time.Second}
 	}
 
+	// from now on we know that the machine is reachable and
+	// is no longer in the rescue system.
+
+	createEventWithCloudInitOutput := func(ar actionResult) actionResult {
+		// Create an Event which contains the cloud-init-output.
+		var err error
+		errMsg := ""
+		f := record.Warnf
+		switch v := ar.(type) {
+		case actionContinue:
+			// Do not create and event containing the output, wait until finished.
+			return ar
+		case actionComplete:
+			f = record.Eventf
+		case actionError:
+			err = v.err
+			errMsg = fmt.Sprintf(" (%s)", v.err.Error())
+		}
+		out := sshClient.GetCloudInitOutput()
+		if out.Err != nil || out.StdErr != "" {
+			record.Warnf(s.scope.HetznerBareMetalHost, "GetCloudInitOutputFailed",
+				"GetCloudInitOutput failed to get /var/log/cloud-init-output.log: stdout %q, stderr %q, err %q",
+				out.StdOut, out.StdErr, out.Err.Error())
+			if err != nil {
+				return actionError{err: fmt.Errorf("failed to get cloud init output: %w, while handling: %w", out.Err, err)}
+			}
+			return actionError{err: fmt.Errorf("failed to get cloud init output: %w", err)}
+		}
+		f(s.scope.HetznerBareMetalHost, "CloudInitOutput", "cloud init output%s:\n%s",
+			errMsg,
+			out.StdOut)
+		return ar
+	}
+
 	// Check the status of cloud init
 	actResult, msg, _ := s.checkCloudInitStatus(sshClient)
 	if _, complete := actResult.(actionComplete); !complete {
 		record.Event(s.scope.HetznerBareMetalHost, "CloudInitStillRunning", msg)
-		return actResult
+		return createEventWithCloudInitOutput(actResult)
 	}
 
 	// Check whether cloud init did not run successfully even though it shows "done"
@@ -1451,14 +1474,14 @@ func (s *Service) actionEnsureProvisioned(_ context.Context) (ar actionResult) {
 		actResult = s.handleCloudInitNotStarted()
 		if _, complete := actResult.(actionComplete); !complete {
 			s.scope.Logger.Info("ensureProvisioned: handleCloudInitNotStarted", "actResult", actResult)
-			return actResult
+			return createEventWithCloudInitOutput(actResult)
 		}
 	}
 
 	record.Event(s.scope.HetznerBareMetalHost, "ServerProvisioned", "server successfully provisioned")
 	conditions.MarkTrue(s.scope.HetznerBareMetalHost, infrav1.ProvisionSucceededCondition)
 	s.scope.HetznerBareMetalHost.ClearError()
-	return actionComplete{} // next: EnsureProvisioned
+	return createEventWithCloudInitOutput(actionComplete{}) // next: actionProvisioned
 }
 
 // handleConnectionRefused checks cloud init status via ssh to the old ssh port if the new ssh port
diff --git a/pkg/services/baremetal/host/utils.go b/pkg/services/baremetal/host/utils.go
index 9e0edada..172b7b39 100644
--- a/pkg/services/baremetal/host/utils.go
+++ b/pkg/services/baremetal/host/utils.go
@@ -29,9 +29,9 @@ type autoSetupInput struct {
 	image     string
 }
 
-func buildAutoSetup(installImageSpec *infrav1.InstallImage, autoSetupInput autoSetupInput) string {
+func buildAutoSetup(installImageSpec *infrav1.InstallImage, asi autoSetupInput) string {
 	var drives string
-	for i, osDevice := range autoSetupInput.osDevices {
+	for i, osDevice := range asi.osDevices {
 		if i > 0 {
 			drives = fmt.Sprintf(`%s
 DRIVE%v /dev/%s`, drives, i+1, osDevice)
@@ -42,7 +42,7 @@ DRIVE%v /dev/%s`, drives, i+1, osDevice)
 
 	hostName := fmt.Sprintf(`
 HOSTNAME %s
-SWRAID %v`, autoSetupInput.hostName, installImageSpec.Swraid)
+SWRAID %v`, asi.hostName, installImageSpec.Swraid)
 	if installImageSpec.Swraid == 1 {
 		hostName = fmt.Sprintf(`%s
 SWRAIDLEVEL %v`, hostName, installImageSpec.SwraidLevel)
@@ -70,7 +70,7 @@ SUBVOL %s %s %s`, btrfsDefinitions, btrfs.Volume, btrfs.SubVolume, btrfs.Mount)
 	}
 
 	image := fmt.Sprintf(`
-IMAGE %s`, autoSetupInput.image)
+IMAGE %s`, asi.image)
 
 	output := fmt.Sprintf(`%s
 %s


Test patch:
diff --git a/controllers/hcloudmachine_controller_test.go b/controllers/hcloudmachine_controller_test.go
index 37bef3a8..4542cfee 100644
--- a/controllers/hcloudmachine_controller_test.go
+++ b/controllers/hcloudmachine_controller_test.go
@@ -363,7 +363,7 @@ var _ = Describe("HCloudMachineReconciler", func() {
 				}, timeout).Should(BeTrue())
 			})
 
-			It("creates the HCloud machine in Hetzner 1", func() {
+			It("creates the HCloud machine in Hetzner 1 (flaky)", func() {
 				By("checking that no servers exist")
 
 				Eventually(func() bool {
diff --git a/controllers/hetznerbaremetalhost_controller_test.go b/controllers/hetznerbaremetalhost_controller_test.go
index d8d83230..2d9e897b 100644
--- a/controllers/hetznerbaremetalhost_controller_test.go
+++ b/controllers/hetznerbaremetalhost_controller_test.go
@@ -896,5 +896,6 @@ name="eth0" model="Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express
 	sshClient.On("Reboot").Return(sshclient.Output{})
 	sshClient.On("GetCloudInitOutput").Return(sshclient.Output{StdOut: "dummy content of /var/log/cloud-init-output.log"})
 	sshClient.On("DetectLinuxOnAnotherDisk", mock.Anything).Return(sshclient.Output{})
-	sshClient.On("GetRunningInstallImageProcesses").Return(sshclient.Output{})
+	sshClient.On("GetInstallImageState").Return(sshclient.InstallImageStateFinished, nil)
+	sshClient.On("GetResultOfInstallImage").Return(hostpkg.PostInstallScriptFinished, nil)
 }
diff --git a/pkg/services/baremetal/host/utils_test.go b/pkg/services/baremetal/host/utils_test.go
index 0065594e..acbf9572 100644
--- a/pkg/services/baremetal/host/utils_test.go
+++ b/pkg/services/baremetal/host/utils_test.go
@@ -26,12 +26,12 @@ import (
 var _ = Describe("buildAutoSetup", func() {
 	type testCaseBuildAutoSetup struct {
 		installImageSpec *infrav1.InstallImage
-		autoSetupInput   autoSetupInput
+		asi              autoSetupInput
 		expectedOutput   string
 	}
 	DescribeTable("buildAutoSetup",
 		func(tc testCaseBuildAutoSetup) {
-			Expect(buildAutoSetup(tc.installImageSpec, tc.autoSetupInput)).Should(Equal(tc.expectedOutput))
+			Expect(buildAutoSetup(tc.installImageSpec, tc.asi)).Should(Equal(tc.expectedOutput))
 		},
 		Entry("multiple entries", testCaseBuildAutoSetup{
 			installImageSpec: &infrav1.InstallImage{
@@ -78,7 +78,7 @@ var _ = Describe("buildAutoSetup", func() {
 				Swraid:      0,
 				SwraidLevel: 1,
 			},
-			autoSetupInput: autoSetupInput{
+			asi: autoSetupInput{
 				image:     "my-image",
 				osDevices: []string{"device"},
 				hostName:  "my-host",
@@ -127,7 +127,7 @@ IMAGE my-image`,
 				Swraid:      1,
 				SwraidLevel: 1,
 			},
-			autoSetupInput: autoSetupInput{
+			asi: autoSetupInput{
 				image:     "my-image",
 				osDevices: []string{"device"},
 				hostName:  "my-host",
@@ -174,7 +174,7 @@ IMAGE my-image`,
 				Swraid:      0,
 				SwraidLevel: 1,
 			},
-			autoSetupInput: autoSetupInput{
+			asi: autoSetupInput{
 				image:     "my-image",
 				osDevices: []string{"device1", "device2"},
 				hostName:  "my-host",
@@ -207,7 +207,7 @@ IMAGE my-image`,
 				Swraid:           0,
 				SwraidLevel:      1,
 			},
-			autoSetupInput: autoSetupInput{
+			asi: autoSetupInput{
 				image:     "my-image",
 				osDevices: []string{"device"},
 				hostName:  "my-host",