Repository: metal3-io/baremetal-operator
Pull request #1237 (2023-03-16 17:57:30)

✨ Supported forced reboot of preprovisioning images
This change adds a new reboot variant:
```yaml
reboot.metal3.io: '{"force": true}'
```

This reboot mode works in available, provisioning and inspecting states.

Includes:
- https://github.com/metal3-io/baremetal-operator/pull/1240

metal3-io-bot: Skipping CI for Draft Pull Request.
If you want CI signal for your change, please convert it to an actual PR.
You can still manually trigger a test run with `/test all`
metal3-io-bot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by:
To complete the [pull request process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process), please ask for approval from **dtantsur** after the PR has been reviewed.

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=metal3-io%2Fbaremetal-operator).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/metal3-io/baremetal-operator/blob/main/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["dtantsur"]} -->

Patch:
diff --git a/apis/metal3.io/v1alpha1/baremetalhost_types.go b/apis/metal3.io/v1alpha1/baremetalhost_types.go
index e3987554..3ca77052 100644
--- a/apis/metal3.io/v1alpha1/baremetalhost_types.go
+++ b/apis/metal3.io/v1alpha1/baremetalhost_types.go
@@ -657,19 +657,8 @@ const (
 
 // RebootAnnotationArguments defines the arguments of the RebootAnnotation type
 type RebootAnnotationArguments struct {
-	Mode RebootMode `json:"mode"`
-}
-
-type DetachedDeleteAction string
-
-const (
-	DetachedDeleteActionDelay  = "delay"
-	DetachedDeleteActionDelete = "delete"
-)
-
-type DetachedAnnotationArguments struct {
-	// DeleteAction indicates the desired delete logic when the detached annotation is present
-	DeleteAction DetachedDeleteAction `json:"deleteAction,omitempty"`
+	Mode  RebootMode `json:"mode"`
+	Force bool       `json:"force"`
 }
 
 // Match compares the saved status information with the name and
diff --git a/apis/metal3.io/v1alpha1/zz_generated.deepcopy.go b/apis/metal3.io/v1alpha1/zz_generated.deepcopy.go
index 0dcd05ba..031d6294 100644
--- a/apis/metal3.io/v1alpha1/zz_generated.deepcopy.go
+++ b/apis/metal3.io/v1alpha1/zz_generated.deepcopy.go
@@ -382,21 +382,6 @@ func (in DesiredSettingsMap) DeepCopy() DesiredSettingsMap {
 	return *out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DetachedAnnotationArguments) DeepCopyInto(out *DetachedAnnotationArguments) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DetachedAnnotationArguments.
-func (in *DetachedAnnotationArguments) DeepCopy() *DetachedAnnotationArguments {
-	if in == nil {
-		return nil
-	}
-	out := new(DetachedAnnotationArguments)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Firmware) DeepCopyInto(out *Firmware) {
 	*out = *in
diff --git a/cmd/make-bm-worker/templates/templates.go b/cmd/make-bm-worker/templates/templates.go
index cb3c8a62..c4bc6832 100644
--- a/cmd/make-bm-worker/templates/templates.go
+++ b/cmd/make-bm-worker/templates/templates.go
@@ -3,8 +3,7 @@ package templates
 import (
 	"bytes"
 	"encoding/base64"
-
-	"github.com/google/safetext/yamltemplate"
+	"text/template"
 )
 
 var templateBody = `---
@@ -103,7 +102,7 @@ func encodeToSecret(input string) string {
 // was a problem rendering it.
 func (t Template) Render() (string, error) {
 	buf := new(bytes.Buffer)
-	tmpl := yamltemplate.Must(yamltemplate.New("yaml_out").Parse(templateBody))
+	tmpl := template.Must(template.New("yaml_out").Parse(templateBody))
 	err := tmpl.Execute(buf, t)
 	return buf.String(), err
 }
diff --git a/cmd/make-virt-host/main.go b/cmd/make-virt-host/main.go
index b1fc2baa..14ed576f 100644
--- a/cmd/make-virt-host/main.go
+++ b/cmd/make-virt-host/main.go
@@ -9,8 +9,7 @@ import (
 	"os"
 	"os/exec"
 	"strings"
-
-	"github.com/google/safetext/yamltemplate"
+	"text/template"
 )
 
 const (
@@ -213,7 +212,7 @@ func main() {
 		Consumer:          strings.TrimSpace(*consumer),
 		ConsumerNamespace: strings.TrimSpace(*consumerNamespace),
 	}
-	t := yamltemplate.Must(yamltemplate.New("yaml_out").Parse(templateBody))
+	t := template.Must(template.New("yaml_out").Parse(templateBody))
 	err = t.Execute(os.Stdout, args)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
diff --git a/controllers/metal3.io/baremetalhost_controller.go b/controllers/metal3.io/baremetalhost_controller.go
index 0bad0678..ff3023a1 100644
--- a/controllers/metal3.io/baremetalhost_controller.go
+++ b/controllers/metal3.io/baremetalhost_controller.go
@@ -385,18 +385,23 @@ func (r *BareMetalHostReconciler) credentialsErrorResult(err error, request ctrl
 }
 
 // hasRebootAnnotation checks for existence of reboot annotations and returns true if at least one exist
-func hasRebootAnnotation(info *reconcileInfo) (hasReboot bool, rebootMode metal3v1alpha1.RebootMode) {
+func hasRebootAnnotation(info *reconcileInfo, expectForce bool) (hasReboot bool, rebootMode metal3v1alpha1.RebootMode) {
 	rebootMode = metal3v1alpha1.RebootModeSoft
 
 	for annotation, value := range info.host.GetAnnotations() {
 		if isRebootAnnotation(annotation) {
+			newReboot := getRebootAnnotationArguments(value, info)
+			if expectForce && !newReboot.Force {
+				continue
+			}
+
 			hasReboot = true
-			newRebootMode := getRebootMode(value, info)
 			// If any annotation has asked for a hard reboot, that
 			// mode takes precedence.
-			if newRebootMode == metal3v1alpha1.RebootModeHard {
-				rebootMode = newRebootMode
+			if newReboot.Mode == metal3v1alpha1.RebootModeHard {
+				rebootMode = newReboot.Mode
 			}
+
 			// Don't use a break here as we may have multiple clients setting
 			// reboot annotations and we always want hard requests honoured
 		}
@@ -404,21 +409,20 @@ func hasRebootAnnotation(info *reconcileInfo) (hasReboot bool, rebootMode metal3
 	return
 }
 
-func getRebootMode(annotation string, info *reconcileInfo) metal3v1alpha1.RebootMode {
-
+func getRebootAnnotationArguments(annotation string, info *reconcileInfo) (result metal3v1alpha1.RebootAnnotationArguments) {
+	result.Mode = metal3v1alpha1.RebootModeSoft
 	if annotation == "" {
 		info.log.Info("No reboot annotation value specified, assuming soft-reboot.")
-		return metal3v1alpha1.RebootModeSoft
+		return
 	}
 
-	annotations := metal3v1alpha1.RebootAnnotationArguments{}
-	err := json.Unmarshal([]byte(annotation), &annotations)
+	err := json.Unmarshal([]byte(annotation), &result)
 	if err != nil {
 		info.publishEvent("InvalidAnnotationValue", fmt.Sprintf("could not parse reboot annotation (%s) - invalid json, assuming soft-reboot", annotation))
 		info.log.Info(fmt.Sprintf("Could not parse reboot annotation (%q) - invalid json, assuming soft-reboot", annotation))
-		return metal3v1alpha1.RebootModeSoft
+		return
 	}
-	return annotations.Mode
+	return
 }
 
 // isRebootAnnotation returns true if the provided annotation is a reboot annotation (either suffixed or not)
@@ -905,12 +909,15 @@ func (r *BareMetalHostReconciler) actionInspecting(prov provisioner.Provisioner,
 	info.log.Info("inspecting hardware")
 
 	refresh := hasInspectAnnotation(info.host)
+	forceReboot, _ := hasRebootAnnotation(info, true)
+
 	provResult, started, details, err := prov.InspectHardware(
 		provisioner.InspectData{
 			BootMode: info.host.Status.Provisioning.BootMode,
 		},
 		info.host.Status.ErrorType == metal3v1alpha1.InspectionError,
-		refresh)
+		refresh,
+		forceReboot)
 	if err != nil {
 		return actionError{errors.Wrap(err, "hardware inspection failed")}
 	}
@@ -919,13 +926,26 @@ func (r *BareMetalHostReconciler) actionInspecting(prov provisioner.Provisioner,
 		return recordActionFailure(info, metal3v1alpha1.InspectionError, provResult.ErrorMessage)
 	}
 
-	// Delete inspect annotation if exists
-	if started && hasInspectAnnotation(info.host) {
-		delete(info.host.Annotations, inspectAnnotationPrefix)
-		if err := r.Update(context.TODO(), info.host); err != nil {
-			return actionError{errors.Wrap(err, "failed to remove inspect annotation from host")}
+	if started {
+		dirty := false
+
+		// Delete inspect annotation if exists
+		if hasInspectAnnotation(info.host) {
+			delete(info.host.Annotations, inspectAnnotationPrefix)
+			dirty = true
+		}
+
+		// Inspection is either freshly started or was aborted. Either way, remove the reboot annotation.
+		if clearRebootAnnotations(info.host) {
+			dirty = true
+		}
+
+		if dirty {
+			if err := r.Update(context.TODO(), info.host); err != nil {
+				return actionError{errors.Wrap(err, "failed to update the host after inspection start")}
+			}
+			return actionContinue{}
 		}
-		return actionContinue{}
 	}
 
 	if provResult.Dirty || details == nil {
@@ -1126,12 +1146,7 @@ func (r *BareMetalHostReconciler) actionProvisioning(prov provisioner.Provisione
 				info.host.HardwareProfile()))}
 	}
 
-	if clearRebootAnnotations(info.host) {
-		if err := r.Update(context.TODO(), info.host); err != nil {
-			return actionError{errors.Wrap(err, "failed to remove reboot annotations from host")}
-		}
-		return actionContinue{}
-	}
+	forceReboot, _ := hasRebootAnnotation(info, true)
 
 	var image metal3v1alpha1.Image
 	if info.host.Spec.Image != nil {
@@ -1145,7 +1160,7 @@ func (r *BareMetalHostReconciler) actionProvisioning(prov provisioner.Provisione
 		BootMode:        info.host.Status.Provisioning.BootMode,
 		HardwareProfile: hwProf,
 		RootDeviceHints: info.host.Status.Provisioning.RootDeviceHints.DeepCopy(),
-	})
+	}, forceReboot)
 	if err != nil {
 		return actionError{errors.Wrap(err, "failed to provision")}
 	}
@@ -1155,6 +1170,13 @@ func (r *BareMetalHostReconciler) actionProvisioning(prov provisioner.Provisione
 		return recordActionFailure(info, metal3v1alpha1.ProvisioningError, provResult.ErrorMessage)
 	}
 
+	if clearRebootAnnotations(info.host) {
+		if err := r.Update(context.TODO(), info.host); err != nil {
+			return actionError{errors.Wrap(err, "failed to remove reboot annotations from host")}
+		}
+		return actionContinue{}
+	}
+
 	if provResult.Dirty {
 		// Go back into the queue and wait for the Provision() method
 		// to return false, indicating that it has no more work to
@@ -1283,10 +1305,12 @@ func (r *BareMetalHostReconciler) manageHostPower(prov provisioner.Provisioner,
 	}
 
 	provState := info.host.Status.Provisioning.State
+	// Normal reboots only work in provisioned states, changing online is also possible for available hosts.
 	isProvisioned := provState == metal3v1alpha1.StateProvisioned || provState == metal3v1alpha1.StateExternallyProvisioned
 
-	desiredReboot, desiredRebootMode := hasRebootAnnotation(info)
-	if desiredReboot && isProvisioned {
+	desiredReboot, desiredRebootMode := hasRebootAnnotation(info, !isProvisioned)
+
+	if desiredReboot {
 		desiredPowerOnState = false
 	}
 
diff --git a/controllers/metal3.io/host_state_machine.go b/controllers/metal3.io/host_state_machine.go
index 518c017e..e97317ba 100644
--- a/controllers/metal3.io/host_state_machine.go
+++ b/controllers/metal3.io/host_state_machine.go
@@ -1,13 +1,11 @@
 package controllers
 
 import (
-	"encoding/json"
 	"fmt"
 
 	metal3v1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
 	"github.com/metal3-io/baremetal-operator/pkg/provisioner"
 
-	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -183,7 +181,7 @@ func (hsm *hostStateMachine) ReconcileState(info *reconcileInfo) (actionRes acti
 		return delayedResult
 	}
 
-	if hsm.checkInitiateDelete(info.log) {
+	if hsm.checkInitiateDelete() {
 		info.log.Info("Initiating host deletion")
 		return actionComplete{}
 	}
@@ -215,7 +213,7 @@ func updateBootModeStatus(host *metal3v1alpha1.BareMetalHost) bool {
 	return true
 }
 
-func (hsm *hostStateMachine) checkInitiateDelete(log logr.Logger) bool {
+func (hsm *hostStateMachine) checkInitiateDelete() bool {
 	if hsm.Host.DeletionTimestamp.IsZero() {
 		// Delete not requested
 		return false
@@ -225,12 +223,11 @@ func (hsm *hostStateMachine) checkInitiateDelete(log logr.Logger) bool {
 	default:
 		hsm.NextState = metal3v1alpha1.StateDeleting
 	case metal3v1alpha1.StateProvisioning, metal3v1alpha1.StateProvisioned:
+		if len(hsm.Host.Finalizers) > 1 {
+			// Allow other finalizers to run before changing state
+			return false
+		}
 		if hsm.Host.OperationalStatus() == metal3v1alpha1.OperationalStatusDetached {
-			if delayDeleteForDetachedHost(hsm.Host) {
-				log.Info("Delaying detached host deletion")
-				deleteDelayedForDetached.Inc()
-				return false
-			}
 			hsm.NextState = metal3v1alpha1.StateDeleting
 		} else {
 			hsm.NextState = metal3v1alpha1.StateDeprovisioning
@@ -256,27 +253,6 @@ func hasDetachedAnnotation(host *metal3v1alpha1.BareMetalHost) bool {
 	return false
 }
 
-func delayDeleteForDetachedHost(host *metal3v1alpha1.BareMetalHost) bool {
-	annotations := host.GetAnnotations()
-	args := metal3v1alpha1.DetachedAnnotationArguments{}
-	val, present := annotations[metal3v1alpha1.DetachedAnnotation]
-
-	// if the host is detached, but missing the annotation, also delay delete
-	// to allow for the host to be re-attached
-	if !present && host.OperationalStatus() == metal3v1alpha1.OperationalStatusDetached {
-		return true
-	}
-
-	if present {
-		if err := json.Unmarshal([]byte(val), &args); err != nil {
-			// default behavior if these are missing or not json is to not delay
-			return false
-		}
-	}
-
-	return args.DeleteAction == metal3v1alpha1.DetachedDeleteActionDelay
-}
-
 func (hsm *hostStateMachine) checkDetachedHost(info *reconcileInfo) (result actionResult) {
 	// If the detached annotation is set we remove any host from the
 	// provisioner and take no further action
diff --git a/controllers/metal3.io/metrics.go b/controllers/metal3.io/metrics.go
index 66611d7f..b9d02197 100644
--- a/controllers/metal3.io/metrics.go
+++ b/controllers/metal3.io/metrics.go
@@ -123,11 +123,6 @@ var provisionerNotReady = prometheus.NewCounter(prometheus.CounterOpts{
 	Help: "Number of times a host is not provision ready",
 })
 
-var deleteDelayedForDetached = prometheus.NewCounter(prometheus.CounterOpts{
-	Name: "metal3_delete_delayed_for_detached_total",
-	Help: "Number of times a host delete action was delayed due to the detached annotation",
-})
-
 func init() {
 	metrics.Registry.MustRegister(
 		reconcileCounters,
@@ -154,8 +149,7 @@ func init() {
 		hostRegistrationRequired,
 		hostUnmanaged,
 		deleteWithoutDeprov,
-		provisionerNotReady,
-		deleteDelayedForDetached)
+		provisionerNotReady)
 }
 
 func hostMetricLabels(request ctrl.Request) prometheus.Labels {
diff --git a/go.mod b/go.mod
index b4b1a385..a9c5ec27 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,6 @@ go 1.19
 
 require (
 	github.com/go-logr/logr v1.2.3
-	github.com/google/safetext v0.0.0-20230106111101-7156a760e523
 	github.com/gophercloud/gophercloud v1.2.0
 	github.com/metal3-io/baremetal-operator/apis v0.1.2
 	github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.1.2
diff --git a/go.sum b/go.sum
index c41ba22c..568fab5c 100644
--- a/go.sum
+++ b/go.sum
@@ -205,8 +205,6 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/safetext v0.0.0-20230106111101-7156a760e523 h1:i4NsbmB9pD5+Ggp5GZKyvYY6MkjvPE8CIMlkvXFF8gA=
-github.com/google/safetext v0.0.0-20230106111101-7156a760e523/go.mod h1:mJNEy0r5YPHC7ChQffpOszlGB4L1iqjXWpIEKcFpr9s=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -738,8 +736,8 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
diff --git a/ironic-deployment/base/ironic.yaml b/ironic-deployment/base/ironic.yaml
index 31609189..ea5bdf35 100644
--- a/ironic-deployment/base/ironic.yaml
+++ b/ironic-deployment/base/ironic.yaml
@@ -19,190 +19,134 @@ spec:
     spec:
       hostNetwork: true
       containers:
-      - name: ironic-dnsmasq
-        image: quay.io/metal3-io/ironic
-        imagePullPolicy: Always
-        securityContext:
-          # Must be true so dnsmasq may get the capabilities via file caps
-          # KEP: https://github.com/kubernetes/enhancements/blob/master/keps/sig-security/2763-ambient-capabilities/README.md
-          allowPrivilegeEscalation: true
-          capabilities:
-            drop:
-            - ALL
-            add:
-            - NET_ADMIN
-            - NET_BIND_SERVICE
-            - NET_RAW
-          privileged: false
-          runAsUser: 997 # ironic
-          runAsGroup: 994 # ironic
-        command:
-        - /bin/rundnsmasq
-        livenessProbe:
-          exec:
-            command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"]
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 10
-          successThreshold: 1
-          failureThreshold: 10
-        readinessProbe:
-          exec:
-            command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"]
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 10
-          successThreshold: 1
-          failureThreshold: 10
-        volumeMounts:
-        - mountPath: /shared
-          name: ironic-data-volume
-        envFrom:
-        - configMapRef:
-            name: ironic-bmo-configmap
-      - name: ironic
-        image: quay.io/metal3-io/ironic
-        imagePullPolicy: Always
-        command:
-        - /bin/runironic
-        livenessProbe:
-          exec:
-            command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"]
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 10
-          successThreshold: 1
-          failureThreshold: 10
-        readinessProbe:
-          exec:
-            command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"]
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 10
-          successThreshold: 1
-          failureThreshold: 10
-        volumeMounts:
-        - mountPath: /shared
-          name: ironic-data-volume
-        envFrom:
-        - configMapRef:
-            name: ironic-bmo-configmap
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          privileged: false
-          runAsUser: 997 # ironic
-          runAsGroup: 994 # ironic
-      - name: ironic-log-watch
-        image: quay.io/metal3-io/ironic
-        imagePullPolicy: Always
-        command:
-        - /bin/runlogwatch.sh
-        volumeMounts:
-        - mountPath: /shared
-          name: ironic-data-volume
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          privileged: false
-          runAsUser: 997 # ironic
-          runAsGroup: 994 # ironic
-      - name: ironic-inspector
-        image: quay.io/metal3-io/ironic
-        imagePullPolicy: Always
-        readinessProbe:
-          exec:
-            command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"]
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 10
-          successThreshold: 1
-          failureThreshold: 10
-        livenessProbe:
-          exec:
-            command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"]
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 10
-          successThreshold: 1
-          failureThreshold: 10
-        command:
-        - /bin/runironic-inspector
-        envFrom:
-        - configMapRef:
-            name: ironic-bmo-configmap
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          privileged: false
-          runAsUser: 996 # ironic-inspector
-          runAsGroup: 993 # ironicinspector
-      - name: ironic-httpd
-        image: quay.io/metal3-io/ironic
-        imagePullPolicy: Always
-        command:
-        - /bin/runhttpd
-        livenessProbe:
-          exec:
-            command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"]
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 10
-          successThreshold: 1
-          failureThreshold: 10
-        readinessProbe:
-          exec:
-            command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"]
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 10
-          successThreshold: 1
-          failureThreshold: 10
-        volumeMounts:
-        - mountPath: /shared
-          name: ironic-data-volume
-        envFrom:
-        - configMapRef:
-            name: ironic-bmo-configmap
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          privileged: false
-          runAsUser: 997 # ironic
-          runAsGroup: 994 # ironic
+        - name: ironic-dnsmasq
+          image: quay.io/metal3-io/ironic
+          imagePullPolicy: Always
+          securityContext:
+             capabilities:
+               add: ["NET_ADMIN", "NET_RAW"]
+          command:
+            - /bin/rundnsmasq
+          livenessProbe:
+           exec:
+             command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"]
+           initialDelaySeconds: 30
+           periodSeconds: 30
+           timeoutSeconds: 10
+           successThreshold: 1
+           failureThreshold: 10
+          readinessProbe:
+           exec:
+             command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"]
+           initialDelaySeconds: 30
+           periodSeconds: 30
+           timeoutSeconds: 10
+           successThreshold: 1
+           failureThreshold: 10
+          volumeMounts:
+            - mountPath: /shared
+              name: ironic-data-volume
+          envFrom:
+            - configMapRef:
+                name: ironic-bmo-configmap
+        - name: ironic
+          image: quay.io/metal3-io/ironic
+          imagePullPolicy: Always
+          command:
+            - /bin/runironic
+          livenessProbe:
+           exec:
+             command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"]
+           initialDelaySeconds: 30
+           periodSeconds: 30
+           timeoutSeconds: 10
+           successThreshold: 1
+           failureThreshold: 10
+          readinessProbe:
+           exec:
+             command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"]
+           initialDelaySeconds: 30
+           periodSeconds: 30
+           timeoutSeconds: 10
+           successThreshold: 1
+           failureThreshold: 10
+          volumeMounts:
+            - mountPath: /shared
+              name: ironic-data-volume
+          envFrom:
+            - configMapRef:
+                name: ironic-bmo-configmap
+        - name: ironic-log-watch
+          image: quay.io/metal3-io/ironic
+          imagePullPolicy: Always
+          command:
+            - /bin/runlogwatch.sh
+          volumeMounts:
+            - mountPath: /shared
+              name: ironic-data-volume
+        - name: ironic-inspector
+          image: quay.io/metal3-io/ironic
+          imagePullPolicy: Always
+          readinessProbe:
+            exec:
+              command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"]
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 10
+            successThreshold: 1
+            failureThreshold: 10
+          livenessProbe:
+            exec:
+              command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"]
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 10
+            successThreshold: 1
+            failureThreshold: 10
+          command:
+            - /bin/runironic-inspector
+          envFrom:
+            - configMapRef:
+                name: ironic-bmo-configmap
+        - name: ironic-httpd
+          image: quay.io/metal3-io/ironic
+          imagePullPolicy: Always
+          command:
+            - /bin/runhttpd
+          livenessProbe:
+           exec:
+             command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"]
+           initialDelaySeconds: 30
+           periodSeconds: 30
+           timeoutSeconds: 10
+           successThreshold: 1
+           failureThreshold: 10
+          readinessProbe:
+           exec:
+             command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"]
+           initialDelaySeconds: 30
+           periodSeconds: 30
+           timeoutSeconds: 10
+           successThreshold: 1
+           failureThreshold: 10
+          volumeMounts:
+            - mountPath: /shared
+              name: ironic-data-volume
+          envFrom:
+            - configMapRef:
+                name: ironic-bmo-configmap
       initContainers:
-      - name: ironic-ipa-downloader
-        image: quay.io/metal3-io/ironic-ipa-downloader
-        imagePullPolicy: Always
-        command:
-        - /usr/local/bin/get-resource.sh
-        envFrom:
-        - configMapRef:
-            name: ironic-bmo-configmap
-        volumeMounts:
-        - mountPath: /shared
-          name: ironic-data-volume
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          privileged: false
-          runAsUser: 997 # ironic
-          runAsGroup: 994 # ironic
+        - name: ironic-ipa-downloader
+          image: quay.io/metal3-io/ironic-ipa-downloader
+          imagePullPolicy: Always
+          command:
+            - /usr/local/bin/get-resource.sh
+          envFrom:
+            - configMapRef:
+                name: ironic-bmo-configmap
+          volumeMounts:
+            - mountPath: /shared
+              name: ironic-data-volume
       volumes:
-      - name: ironic-data-volume
-        emptyDir: {}
-      securityContext:
-        runAsNonRoot: true
-        seccompProfile:
-          type: RuntimeDefault
-        fsGroup: 994
+        - name: ironic-data-volume
+          emptyDir: {}
diff --git a/ironic-deployment/components/keepalived/keepalived_patch.yaml b/ironic-deployment/components/keepalived/keepalived_patch.yaml
index a6a03cdf..5d246882 100644
--- a/ironic-deployment/components/keepalived/keepalived_patch.yaml
+++ b/ironic-deployment/components/keepalived/keepalived_patch.yaml
@@ -10,19 +10,8 @@ spec:
       - image: quay.io/metal3-io/keepalived
         name: ironic-endpoint-keepalived
         securityContext:
-          # Must be true so dnsmasq may get the capabilities via file caps
-          # KEP: https://github.com/kubernetes/enhancements/blob/master/keps/sig-security/2763-ambient-capabilities/README.md
-          allowPrivilegeEscalation: true
-          capabilities:
-            drop:
-            - ALL
-            add:
-            - NET_ADMIN
-            - NET_BROADCAST
-            - NET_RAW
-          privileged: false
-          runAsUser: 65532
-          runAsGroup: 65532
+              capabilities:
+                add: ["NET_ADMIN", "NET_RAW"]
         envFrom:
-        - configMapRef:
-            name: ironic-bmo-configmap
+          - configMapRef:
+              name: ironic-bmo-configmap
diff --git a/ironic-deployment/components/mariadb/mariadb_patch.yaml b/ironic-deployment/components/mariadb/mariadb_patch.yaml
index acffad24..ca75d16d 100644
--- a/ironic-deployment/components/mariadb/mariadb_patch.yaml
+++ b/ironic-deployment/components/mariadb/mariadb_patch.yaml
@@ -6,66 +6,58 @@ spec:
   template:
     spec:
       containers:
-      - name: mariadb
-        image: quay.io/metal3-io/mariadb
-        imagePullPolicy: Always
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          privileged: false
-          runAsUser: 27
-          runAsGroup: 27
-        livenessProbe:
-          exec:
-            command: ["sh", "-c", "mysqladmin status -uironic -p$(printenv MARIADB_PASSWORD)"]
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 10
-          successThreshold: 1
-          failureThreshold: 10
-        readinessProbe:
-          exec:
-            command: ["sh", "-c", "mysqladmin status -uironic -p$(printenv MARIADB_PASSWORD)"]
-          initialDelaySeconds: 30
-          periodSeconds: 30
-          timeoutSeconds: 10
-          successThreshold: 1
-          failureThreshold: 10
-        volumeMounts:
-        - mountPath: /shared
-          name: ironic-data-volume
-        - name: cert-mariadb
-          mountPath: "/certs/mariadb"
-          readOnly: true
-        - name: cert-mariadb-ca
-          mountPath: "/certs/ca/mariadb"
-          readOnly: true
-        env:
-        - name: MARIADB_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: mariadb-password
-              key: password
-        - name: RESTART_CONTAINER_CERTIFICATE_UPDATED
-          valueFrom:
-            configMapKeyRef:
-              name: ironic-bmo-configmap
-              key: RESTART_CONTAINER_CERTIFICATE_UPDATED
-      - name: ironic
-        env:
-        - name: MARIADB_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: mariadb-password
-              key: password
-        - name: IRONIC_USE_MARIADB
-          value: "true"
-        volumeMounts:
-        - name: cert-mariadb-ca
-          mountPath: "/certs/ca/mariadb"
-          readOnly: true
+        - name: mariadb
+          image: quay.io/metal3-io/mariadb
+          imagePullPolicy: Always
+          livenessProbe:
+           exec:
+             command: ["sh", "-c", "mysqladmin status -uironic -p$(printenv MARIADB_PASSWORD)"]
+           initialDelaySeconds: 30
+           periodSeconds: 30
+           timeoutSeconds: 10
+           successThreshold: 1
+           failureThreshold: 10
+          readinessProbe:
+           exec:
+             command: ["sh", "-c", "mysqladmin status -uironic -p$(printenv MARIADB_PASSWORD)"]
+           initialDelaySeconds: 30
+           periodSeconds: 30
+           timeoutSeconds: 10
+           successThreshold: 1
+           failureThreshold: 10
+          volumeMounts:
+            - mountPath: /shared
+              name: ironic-data-volume
+            - name: cert-mariadb
+              mountPath: "/certs/mariadb"
+              readOnly: true
+            - name: cert-mariadb-ca
+              mountPath: "/certs/ca/mariadb"
+              readOnly: true
+          env:
+            - name: MARIADB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mariadb-password
+                  key: password
+            - name: RESTART_CONTAINER_CERTIFICATE_UPDATED
+              valueFrom:
+                 configMapKeyRef:
+                  name: ironic-bmo-configmap
+                  key: RESTART_CONTAINER_CERTIFICATE_UPDATED
+        - name: ironic
+          env:
+            - name: MARIADB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mariadb-password
+                  key: password
+            - name: IRONIC_USE_MARIADB
+              value: "true"
+          volumeMounts:
+            - name: cert-mariadb-ca
+              mountPath: "/certs/ca/mariadb"
+              readOnly: true
       volumes:
       - name: cert-mariadb
         secret:
diff --git a/pkg/provisioner/demo/demo.go b/pkg/provisioner/demo/demo.go
index be39cfd4..1f05aafd 100644
--- a/pkg/provisioner/demo/demo.go
+++ b/pkg/provisioner/demo/demo.go
@@ -85,7 +85,7 @@ func (p *demoProvisioner) HasCapacity() (result bool, err error) {
 
 // ValidateManagementAccess tests the connection information for the
 // host to verify that the location and credentials work.
-func (p *demoProvisioner) ValidateManagementAccess(data provisioner.ManagementAccessData, credentialsChanged, force bool) (result provisioner.Result, provID string, err error) {
+func (p *demoProvisioner) ValidateManagementAccess(data provisioner.ManagementAccessData, credentialsChanged, restartOnFailure bool) (result provisioner.Result, provID string, err error) {
 	p.log.Info("testing management access")
 
 	hostName := p.objectMeta.Name
@@ -122,7 +122,7 @@ func (p *demoProvisioner) PreprovisioningImageFormats() ([]metal3v1alpha1.ImageF
 // details of devices discovered on the hardware. It may be called
 // multiple times, and should return true for its dirty flag until the
 // inspection is completed.
-func (p *demoProvisioner) InspectHardware(data provisioner.InspectData, force, refresh bool) (result provisioner.Result, started bool, details *metal3v1alpha1.HardwareDetails, err error) {
+func (p *demoProvisioner) InspectHardware(data provisioner.InspectData, restartOnFailure, refresh, forceReboot bool) (result provisioner.Result, started bool, details *metal3v1alpha1.HardwareDetails, err error) {
 	started = true
 	hostName := p.objectMeta.Name
 
@@ -193,7 +193,7 @@ func (p *demoProvisioner) UpdateHardwareState() (hwState provisioner.HardwareSta
 }
 
 // Prepare remove existing configuration and set new configuration
-func (p *demoProvisioner) Prepare(data provisioner.PrepareData, unprepared bool, force bool) (result provisioner.Result, started bool, err error) {
+func (p *demoProvisioner) Prepare(data provisioner.PrepareData, unprepared bool, restartOnFailure bool) (result provisioner.Result, started bool, err error) {
 	hostName := p.objectMeta.Name
 
 	switch hostName {
@@ -218,7 +218,7 @@ func (p *demoProvisioner) Prepare(data provisioner.PrepareData, unprepared bool,
 
 // Adopt notifies the provisioner that the state machine believes the host
 // to be currently provisioned, and that it should be managed as such.
-func (p *demoProvisioner) Adopt(data provisioner.AdoptData, force bool) (result provisioner.Result, err error) {
+func (p *demoProvisioner) Adopt(data provisioner.AdoptData, restartOnFailure bool) (result provisioner.Result, err error) {
 	p.log.Info("adopting host")
 	result.Dirty = false
 	return
@@ -227,7 +227,7 @@ func (p *demoProvisioner) Adopt(data provisioner.AdoptData, force bool) (result
 // Provision writes the image from the host spec to the host. It may
 // be called multiple times, and should return true for its dirty flag
 // until the provisioning operation is completed.
-func (p *demoProvisioner) Provision(data provisioner.ProvisionData) (result provisioner.Result, err error) {
+func (p *demoProvisioner) Provision(data provisioner.ProvisionData, forceReboot bool) (result provisioner.Result, err error) {
 
 	hostName := p.objectMeta.Name
 	p.log.Info("provisioning image to host")
@@ -253,7 +253,7 @@ func (p *demoProvisioner) Provision(data provisioner.ProvisionData) (result prov
 // Deprovision removes the host from the image. It may be called
 // multiple times, and should return true for its dirty flag until the
 // deprovisioning operation is completed.
-func (p *demoProvisioner) Deprovision(force bool) (result provisioner.Result, err error) {
+func (p *demoProvisioner) Deprovision(restartOnFailure bool) (result provisioner.Result, err error) {
 
 	hostName := p.objectMeta.Name
 	switch hostName {
diff --git a/pkg/provisioner/fixture/fixture.go b/pkg/provisioner/fixture/fixture.go
index 6b60ac0a..22287487 100644
--- a/pkg/provisioner/fixture/fixture.go
+++ b/pkg/provisioner/fixture/fixture.go
@@ -97,7 +97,7 @@ func (p *fixtureProvisioner) HasCapacity() (result bool, err error) {
 
 // ValidateManagementAccess tests the connection information for the
 // host to verify that the location and credentials work.
-func (p *fixtureProvisioner) ValidateManagementAccess(data provisioner.ManagementAccessData, credentialsChanged, force bool) (result provisioner.Result, provID string, err error) {
+func (p *fixtureProvisioner) ValidateManagementAccess(data provisioner.ManagementAccessData, credentialsChanged, restartOnFailure bool) (result provisioner.Result, provID string, err error) {
 	p.log.Info("testing management access")
 
 	if p.state.validateError != "" {
@@ -125,7 +125,7 @@ func (p *fixtureProvisioner) PreprovisioningImageFormats() ([]metal3v1alpha1.Ima
 // details of devices discovered on the hardware. It may be called
 // multiple times, and should return true for its dirty flag until the
 // inspection is completed.
-func (p *fixtureProvisioner) InspectHardware(data provisioner.InspectData, force, refresh bool) (result provisioner.Result, started bool, details *metal3v1alpha1.HardwareDetails, err error) {
+func (p *fixtureProvisioner) InspectHardware(data provisioner.InspectData, restartOnFailure, refresh, forceReboot bool) (result provisioner.Result, started bool, details *metal3v1alpha1.HardwareDetails, err error) {
 	// The inspection is ongoing. We'll need to check the fixture
 	// status for the server here until it is ready for us to get the
 	// inspection details. Simulate that for now by creating the
@@ -191,7 +191,7 @@ func (p *fixtureProvisioner) UpdateHardwareState() (hwState provisioner.Hardware
 }
 
 // Prepare remove existing configuration and set new configuration
-func (p *fixtureProvisioner) Prepare(data provisioner.PrepareData, unprepared bool, force bool) (result provisioner.Result, started bool, err error) {
+func (p *fixtureProvisioner) Prepare(data provisioner.PrepareData, unprepared bool, restartOnFailure bool) (result provisioner.Result, started bool, err error) {
 	p.log.Info("preparing host")
 	started = unprepared
 	return
@@ -199,7 +199,7 @@ func (p *fixtureProvisioner) Prepare(data provisioner.PrepareData, unprepared bo
 
 // Adopt notifies the provisioner that the state machine believes the host
 // to be currently provisioned, and that it should be managed as such.
-func (p *fixtureProvisioner) Adopt(data provisioner.AdoptData, force bool) (result provisioner.Result, err error) {
+func (p *fixtureProvisioner) Adopt(data provisioner.AdoptData, restartOnFailure bool) (result provisioner.Result, err error) {
 	p.log.Info("adopting host")
 	if !p.state.adopted {
 		p.state.adopted = true
@@ -212,7 +212,7 @@ func (p *fixtureProvisioner) Adopt(data provisioner.AdoptData, force bool) (resu
 // Provision writes the image from the host spec to the host. It may
 // be called multiple times, and should return true for its dirty flag
 // until the provisioning operation is completed.
-func (p *fixtureProvisioner) Provision(data provisioner.ProvisionData) (result provisioner.Result, err error) {
+func (p *fixtureProvisioner) Provision(data provisioner.ProvisionData, forceReboot bool) (result provisioner.Result, err error) {
 	p.log.Info("provisioning image to host")
 
 	if data.CustomDeploy != nil && p.state.customDeploy == nil {
@@ -238,7 +238,7 @@ func (p *fixtureProvisioner) Provision(data provisioner.ProvisionData) (result p
 // Deprovision removes the host from the image. It may be called
 // multiple times, and should return true for its dirty flag until the
 // deprovisioning operation is completed.
-func (p *fixtureProvisioner) Deprovision(force bool) (result provisioner.Result, err error) {
+func (p *fixtureProvisioner) Deprovision(restartOnFailure bool) (result provisioner.Result, err error) {
 	p.log.Info("ensuring host is deprovisioned")
 
 	result.RequeueAfter = deprovisionRequeueDelay
diff --git a/pkg/provisioner/ironic/ironic.go b/pkg/provisioner/ironic/ironic.go
index e81b855c..482b46c6 100644
--- a/pkg/provisioner/ironic/ironic.go
+++ b/pkg/provisioner/ironic/ironic.go
@@ -321,7 +321,7 @@ func (p *ironicProvisioner) createPXEEnabledNodePort(uuid, macAddress string) er
 //
 // FIXME(dhellmann): We should rename this method to describe what it
 // actually does.
-func (p *ironicProvisioner) ValidateManagementAccess(data provisioner.ManagementAccessData, credentialsChanged, force bool) (result provisioner.Result, provID string, err error) {
+func (p *ironicProvisioner) ValidateManagementAccess(data provisioner.ManagementAccessData, credentialsChanged, restartOnFailure bool) (result provisioner.Result, provID string, err error) {
 	bmcAccess, err := p.bmcAccess()
 	if err != nil {
 		result, err = operationFailed(err.Error())
@@ -373,7 +373,7 @@ func (p *ironicProvisioner) ValidateManagementAccess(data provisioner.Management
 				Driver:              bmcAccess.Driver(),
 				BIOSInterface:       bmcAccess.BIOSInterface(),
 				BootInterface:       bmcAccess.BootInterface(),
-				Name:                ironicNodeName(p.objectMeta),
+				Name:                p.objectMeta.Name,
 				DriverInfo:          driverInfo,
 				DeployInterface:     p.deployInterface(data),
 				InspectInterface:    "inspector",
@@ -479,7 +479,7 @@ func (p *ironicProvisioner) ValidateManagementAccess(data provisioner.Management
 	case nodes.Enroll:
 
 		// If ironic is reporting an error, stop working on the node.
-		if ironicNode.LastError != "" && !(credentialsChanged || force) {
+		if ironicNode.LastError != "" && !(credentialsChanged || restartOnFailure) {
 			result, err = operationFailed(ironicNode.LastError)
 			return
 		}
@@ -750,11 +750,21 @@ func (p *ironicProvisioner) changeNodeProvisionState(ironicNode *nodes.Node, opt
 	return
 }
 
+func (p *ironicProvisioner) abortInspection(ironicNode *nodes.Node) (result provisioner.Result, started bool, details *metal3v1alpha1.HardwareDetails, err error) {
+	// Set started to let the controller know about the change
+	p.log.Info("aborting inspection to force reboot of preprovisioning image")
+	started, result, err = p.tryChangeNodeProvisionState(
+		ironicNode,
+		nodes.ProvisionStateOpts{Target: nodes.TargetAbort},
+	)
+	return
+}
+
 // InspectHardware updates the HardwareDetails field of the host with
 // details of devices discovered on the hardware. It may be called
 // multiple times, and should return true for its dirty flag until the
 // inspection is completed.
-func (p *ironicProvisioner) InspectHardware(data provisioner.InspectData, force, refresh bool) (result provisioner.Result, started bool, details *metal3v1alpha1.HardwareDetails, err error) {
+func (p *ironicProvisioner) InspectHardware(data provisioner.InspectData, restartOnFailure, refresh, forceReboot bool) (result provisioner.Result, started bool, details *metal3v1alpha1.HardwareDetails, err error) {
 	p.log.Info("inspecting hardware")
 
 	ironicNode, err := p.getNode()
@@ -764,6 +774,12 @@ func (p *ironicProvisioner) InspectHardware(data provisioner.InspectData, force,
 	}
 
 	status, err := introspection.GetIntrospectionStatus(p.inspector, ironicNode.UUID).Extract()
+	if status != nil && strings.Contains(status.Error, "Canceled") {
+		// Inspection gets canceled when we detect a new preprovisioning image, not need to report an error, just restart.
+		refresh = true
+		restartOnFailure = true
+	}
+
 	if err != nil || refresh {
 		if _, isNotFound := err.(gophercloud.ErrDefault404); isNotFound || refresh {
 			switch nodes.ProvisionState(ironicNode.ProvisionState) {
@@ -773,12 +789,18 @@ func (p *ironicProvisioner) InspectHardware(data provisioner.InspectData, force,
 					nodes.ProvisionStateOpts{Target: nodes.TargetManage},
 				)
 				return
-			case nodes.Inspecting, nodes.InspectWait:
+			case nodes.InspectWait:
+				if forceReboot {
+					return p.abortInspection(ironicNode)
+				}
+
+				fallthrough
+			case nodes.Inspecting:
 				p.log.Info("inspection already started")
 				result, err = operationContinuing(introspectionRequeueDelay)
 				return
 			case nodes.InspectFail:
-				if !force {
+				if !restartOnFailure {
 					p.log.Info("starting inspection failed", "error", status.Error)
 					failure := ironicNode.LastError
 					if failure == "" {
@@ -819,6 +841,9 @@ func (p *ironicProvisioner) InspectHardware(data provisioner.InspectData, force,
 		result, err = operationFailed(status.Error)
 		return
 	}
+	if !status.Finished && forceReboot && nodes.ProvisionState(ironicNode.ProvisionState) == nodes.InspectWait {
+		return p.abortInspection(ironicNode)
+	}
 	if !status.Finished || (nodes.ProvisionState(ironicNode.ProvisionState) == nodes.Inspecting || nodes.ProvisionState(ironicNode.ProvisionState) == nodes.InspectWait) {
 		p.log.Info("inspection in progress", "started_at", status.StartedAt)
 		result, err = operationContinuing(introspectionRequeueDelay)
@@ -1150,7 +1175,7 @@ func (p *ironicProvisioner) deployInterface(data provisioner.ManagementAccessDat
 
 // Adopt notifies the provisioner that the state machine believes the host
 // to be currently provisioned, and that it should be managed as such.
-func (p *ironicProvisioner) Adopt(data provisioner.AdoptData, force bool) (result provisioner.Result, err error) {
+func (p *ironicProvisioner) Adopt(data provisioner.AdoptData, restartOnFailure bool) (result provisioner.Result, err error) {
 	ironicNode, err := p.getNode()
 	if err != nil {
 		return transientError(err)
@@ -1183,7 +1208,7 @@ func (p *ironicProvisioner) Adopt(data provisioner.AdoptData, force bool) (resul
 	case nodes.Adopting:
 		return operationContinuing(provisionRequeueDelay)
 	case nodes.AdoptFail:
-		if force {
+		if restartOnFailure {
 			return p.changeNodeProvisionState(
 				ironicNode,
 				nodes.ProvisionStateOpts{
@@ -1349,7 +1374,7 @@ func (p *ironicProvisioner) startManualCleaning(bmcAccess bmc.AccessDetails, iro
 
 // Prepare remove existing configuration and set new configuration.
 // If `started` is true,  it means that we successfully executed `tryChangeNodeProvisionState`.
-func (p *ironicProvisioner) Prepare(data provisioner.PrepareData, unprepared bool, force bool) (result provisioner.Result, started bool, err error) {
+func (p *ironicProvisioner) Prepare(data provisioner.PrepareData, unprepared bool, restartOnFailure bool) (result provisioner.Result, started bool, err error) {
 	bmcAccess, err := p.bmcAccess()
 	if err != nil {
 		result, err = transientError(err)
@@ -1400,9 +1425,9 @@ func (p *ironicProvisioner) Prepare(data provisioner.PrepareData, unprepared boo
 
 	case nodes.CleanFail:
 		// When clean failed, we need to clean host provisioning settings.
-		// If force is false, it means the settings aren't cleared.
+		// If restartOnFailure is false, it means the settings aren't cleared.
 		// So we can't set the node's state to manageable, until the settings are cleared.
-		if !force {
+		if !restartOnFailure {
 			result, err = operationFailed(ironicNode.LastError)
 			return
 		}
@@ -1498,7 +1523,7 @@ func (p *ironicProvisioner) getCustomDeploySteps(customDeploy *metal3v1alpha1.Cu
 // Provision writes the image from the host spec to the host. It may
 // be called multiple times, and should return true for its dirty flag
 // until the provisioning operation is completed.
-func (p *ironicProvisioner) Provision(data provisioner.ProvisionData) (result provisioner.Result, err error) {
+func (p *ironicProvisioner) Provision(data provisioner.ProvisionData, forceReboot bool) (result provisioner.Result, err error) {
 	ironicNode, err := p.getNode()
 	if err != nil {
 		return transientError(err)
@@ -1591,6 +1616,18 @@ func (p *ironicProvisioner) Provision(data provisioner.ProvisionData) (result pr
 		p.log.Info("finished provisioning")
 		return operationComplete()
 
+	case nodes.DeployWait:
+		if forceReboot {
+			p.log.Info("aborting provisioning to force reboot of preprovisioning image")
+			_, result, err = p.tryChangeNodeProvisionState(
+				ironicNode,
+				nodes.ProvisionStateOpts{Target: nodes.TargetDeleted},
+			)
+			return
+		}
+
+		fallthrough
+
 	default:
 		// wait states like cleaning and clean wait
 		p.log.Info("waiting for host to become available",
@@ -1624,7 +1661,7 @@ func (p *ironicProvisioner) setMaintenanceFlag(ironicNode *nodes.Node, value boo
 // Deprovision removes the host from the image. It may be called
 // multiple times, and should return true for its dirty flag until the
 // deprovisioning operation is completed.
-func (p *ironicProvisioner) Deprovision(force bool) (result provisioner.Result, err error) {
+func (p *ironicProvisioner) Deprovision(restartOnFailure bool) (result provisioner.Result, err error) {
 	p.log.Info("deprovisioning")
 
 	ironicNode, err := p.getNode()
@@ -1643,7 +1680,7 @@ func (p *ironicProvisioner) Deprovision(force bool) (result provisioner.Result,
 
 	switch nodes.ProvisionState(ironicNode.ProvisionState) {
 	case nodes.Error:
-		if !force {
+		if !restartOnFailure {
 			p.log.Info("deprovisioning failed")
 			if ironicNode.LastError == "" {
 				result.ErrorMessage = "Deprovisioning failed"
@@ -1660,7 +1697,7 @@ func (p *ironicProvisioner) Deprovision(force bool) (result provisioner.Result,
 		)
 
 	case nodes.CleanFail:
-		if !force {
+		if !restartOnFailure {
 			p.log.Info("cleaning failed", "lastError", ironicNode.LastError)
 			return operationFailed(fmt.Sprintf("Cleaning failed: %s", ironicNode.LastError))
 		}
diff --git a/pkg/provisioner/provisioner.go b/pkg/provisioner/provisioner.go
index 74b671bd..411bb46a 100644
--- a/pkg/provisioner/provisioner.go
+++ b/pkg/provisioner/provisioner.go
@@ -125,7 +125,7 @@ type Provisioner interface {
 	// of credentials it has are different from the credentials it has
 	// previously been using, without implying that either set of
 	// credentials is correct.
-	ValidateManagementAccess(data ManagementAccessData, credentialsChanged, force bool) (result Result, provID string, err error)
+	ValidateManagementAccess(data ManagementAccessData, credentialsChanged, restartOnFailure bool) (result Result, provID string, err error)
 
 	// PreprovisioningImageFormats returns a list of acceptable formats for a
 	// pre-provisioning image to be built by a PreprovisioningImage object. The
@@ -136,7 +136,7 @@ type Provisioner interface {
 	// details of devices discovered on the hardware. It may be called
 	// multiple times, and should return true for its dirty flag until the
 	// inspection is completed.
-	InspectHardware(data InspectData, force, refresh bool) (result Result, started bool, details *metal3v1alpha1.HardwareDetails, err error)
+	InspectHardware(data InspectData, restartOnFailure, refresh, forceReboot bool) (result Result, started bool, details *metal3v1alpha1.HardwareDetails, err error)
 
 	// UpdateHardwareState fetches the latest hardware state of the
 	// server and updates the HardwareDetails field of the host with
@@ -146,20 +146,20 @@ type Provisioner interface {
 
 	// Adopt brings an externally-provisioned host under management by
 	// the provisioner.
-	Adopt(data AdoptData, force bool) (result Result, err error)
+	Adopt(data AdoptData, restartOnFailure bool) (result Result, err error)
 
 	// Prepare remove existing configuration and set new configuration
-	Prepare(data PrepareData, unprepared bool, force bool) (result Result, started bool, err error)
+	Prepare(data PrepareData, unprepared bool, restartOnFailure bool) (result Result, started bool, err error)
 
 	// Provision writes the image from the host spec to the host. It
 	// may be called multiple times, and should return true for its
 	// dirty flag until the provisioning operation is completed.
-	Provision(data ProvisionData) (result Result, err error)
+	Provision(data ProvisionData, forceReboot bool) (result Result, err error)
 
 	// Deprovision removes the host from the image. It may be called
 	// multiple times, and should return true for its dirty flag until
 	// the deprovisioning operation is completed.
-	Deprovision(force bool) (result Result, err error)
+	Deprovision(restartOnFailure bool) (result Result, err error)
 
 	// Delete removes the host from the provisioning system. It may be
 	// called multiple times, and should return true for its dirty
diff --git a/resources/keepalived-docker/Dockerfile b/resources/keepalived-docker/Dockerfile
index f7c752cd..a8a48456 100644
--- a/resources/keepalived-docker/Dockerfile
+++ b/resources/keepalived-docker/Dockerfile
@@ -4,13 +4,11 @@ ARG BASE_IMAGE=ubuntu:22.04
 FROM $BASE_IMAGE
 ARG DEBIAN_FRONTEND=noninteractive
 
+COPY sample.keepalived.conf /etc/keepalived/keepalived.conf
+COPY manage-keepalived.sh manage-keepalived.sh
+
 RUN apt-get -y update && \
     apt-get -y install keepalived && \
     apt-get -y clean
 
-COPY sample.keepalived.conf /etc/keepalived/keepalived.conf
-COPY manage-keepalived.sh configure-nonroot.sh /bin/
-
-RUN /bin/configure-nonroot.sh && rm /bin/configure-nonroot.sh
-
-CMD ["/bin/bash", "/bin/manage-keepalived.sh"]
+ENTRYPOINT ["/bin/bash", "manage-keepalived.sh"]
diff --git a/resources/keepalived-docker/configure-nonroot.sh b/resources/keepalived-docker/configure-nonroot.sh
deleted file mode 100755
index 6bebd789..00000000
--- a/resources/keepalived-docker/configure-nonroot.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/bash
-
-set -eux
-
-# create nonroot image matching the keepalived manifest
-NONROOT_USER="nonroot"
-NONROOT_GROUP="nonroot"
-NONROOT_UID=65532
-NONROOT_GID=65532
-
-# run as non-root, allow editing the keepalived.conf during startup
-groupadd -g "${NONROOT_GID}" "${NONROOT_GROUP}"
-useradd -u "${NONROOT_UID}" -g "${NONROOT_GID}" -m "${NONROOT_USER}"
-
-mkdir -p /run/keepalived
-chown -R root:"${NONROOT_GROUP}" /etc/keepalived /run/keepalived
-chmod 2775 /etc/keepalived /run/keepalived
-chmod 664 /etc/keepalived/keepalived.conf
-
-setcap "cap_net_raw,cap_net_broadcast,cap_net_admin=+eip" /usr/sbin/keepalived
diff --git a/resources/keepalived-docker/manage-keepalived.sh b/resources/keepalived-docker/manage-keepalived.sh
index 392077ba..bf4841c4 100644
--- a/resources/keepalived-docker/manage-keepalived.sh
+++ b/resources/keepalived-docker/manage-keepalived.sh
@@ -1,11 +1,8 @@
-#!/usr/bin/bash
+#!/bin/bash
+export assignedIP="$PROVISIONING_IP/32"
+export interface=$PROVISIONING_INTERFACE
 
-set -eux
+sed  -i "s~INTERFACE~$interface~g" /etc/keepalived/keepalived.conf
+sed  -i "s~CHANGEIP~$assignedIP~g" /etc/keepalived/keepalived.conf
 
-export assignedIP="${PROVISIONING_IP}/32"
-export interface="${PROVISIONING_INTERFACE}"
-
-sed -i "s~INTERFACE~${interface}~g" /etc/keepalived/keepalived.conf
-sed -i "s~CHANGEIP~${assignedIP}~g" /etc/keepalived/keepalived.conf
-
-exec /usr/sbin/keepalived -n -l -p /run/keepalived/keepalived.pid -r /run/keepalived/vrrp.pid
+exec /usr/sbin/keepalived -n -l
\ No newline at end of file


Test patch:
diff --git a/controllers/metal3.io/baremetalhost_controller_test.go b/controllers/metal3.io/baremetalhost_controller_test.go
index f264bd8a..e613e470 100644
--- a/controllers/metal3.io/baremetalhost_controller_test.go
+++ b/controllers/metal3.io/baremetalhost_controller_test.go
@@ -598,88 +598,104 @@ func makeReconcileInfo(host *metal3v1alpha1.BareMetalHost) *reconcileInfo {
 }
 
 func TestHasRebootAnnotation(t *testing.T) {
-	host := newDefaultHost(t)
-	info := makeReconcileInfo(host)
-	host.Annotations = make(map[string]string)
-
-	hasReboot, rebootMode := hasRebootAnnotation(info)
-	assert.False(t, hasReboot)
-	assert.Equal(t, metal3v1alpha1.RebootModeSoft, rebootMode)
-
-	host.Annotations = make(map[string]string)
-	suffixedAnnotation := rebootAnnotationPrefix + "/foo"
-	host.Annotations[suffixedAnnotation] = ""
-
-	hasReboot, rebootMode = hasRebootAnnotation(info)
-	assert.True(t, hasReboot)
-	assert.Equal(t, metal3v1alpha1.RebootModeSoft, rebootMode)
-
-	delete(host.Annotations, suffixedAnnotation)
-	host.Annotations[rebootAnnotationPrefix] = ""
-
-	hasReboot, rebootMode = hasRebootAnnotation(info)
-	assert.True(t, hasReboot)
-	assert.Equal(t, metal3v1alpha1.RebootModeSoft, rebootMode)
 
-	host.Annotations[suffixedAnnotation] = ""
-
-	hasReboot, rebootMode = hasRebootAnnotation(info)
-	assert.True(t, hasReboot)
-	assert.Equal(t, metal3v1alpha1.RebootModeSoft, rebootMode)
-
-	//two suffixed annotations to simulate multiple clients
-
-	host.Annotations[suffixedAnnotation+"bar"] = ""
-
-	hasReboot, rebootMode = hasRebootAnnotation(info)
-	assert.True(t, hasReboot)
-	assert.Equal(t, metal3v1alpha1.RebootModeSoft, rebootMode)
-}
-
-func TestHasHardRebootAnnotation(t *testing.T) {
-	host := newDefaultHost(t)
-	info := makeReconcileInfo(host)
-	host.Annotations = make(map[string]string)
-
-	hasReboot, rebootMode := hasRebootAnnotation(info)
-	assert.False(t, hasReboot)
-	assert.Equal(t, metal3v1alpha1.RebootModeSoft, rebootMode)
-
-	host.Annotations = make(map[string]string)
-	suffixedAnnotation := rebootAnnotationPrefix + "/foo/"
-	host.Annotations[suffixedAnnotation] = "{\"mode\": \"hard\"}"
-
-	hasReboot, rebootMode = hasRebootAnnotation(info)
-	assert.True(t, hasReboot)
-	assert.Equal(t, metal3v1alpha1.RebootModeHard, rebootMode)
-
-	delete(host.Annotations, suffixedAnnotation)
-	host.Annotations[rebootAnnotationPrefix] = "{\"mode\": \"soft\"}"
-
-	hasReboot, rebootMode = hasRebootAnnotation(info)
-	assert.True(t, hasReboot)
-	assert.Equal(t, metal3v1alpha1.RebootModeSoft, rebootMode)
-
-	delete(host.Annotations, suffixedAnnotation)
-	host.Annotations[rebootAnnotationPrefix] = "{\"bad\"= \"json\"]"
-
-	hasReboot, rebootMode = hasRebootAnnotation(info)
-	assert.True(t, hasReboot)
-	assert.Equal(t, metal3v1alpha1.RebootModeSoft, rebootMode)
-
-	host.Annotations[suffixedAnnotation] = ""
-
-	hasReboot, rebootMode = hasRebootAnnotation(info)
-	assert.True(t, hasReboot)
-	assert.Equal(t, metal3v1alpha1.RebootModeSoft, rebootMode)
+	testCases := []struct {
+		Scenario       string
+		Annotations    map[string]string
+		expectForce    bool
+		expectedReboot bool
+		expectedMode   metal3v1alpha1.RebootMode
+	}{
+		{
+			Scenario: "No annotations",
+		},
+		{
+			Scenario: "Simple with empty value",
+			Annotations: map[string]string{
+				rebootAnnotationPrefix: "",
+			},
+			expectedReboot: true,
+		},
+		{
+			Scenario: "Suffixed with empty value",
+			Annotations: map[string]string{
+				rebootAnnotationPrefix + "/foo": "",
+			},
+			expectedReboot: true,
+		},
+		{
+			Scenario: "Two suffixed with empty value",
+			Annotations: map[string]string{
+				rebootAnnotationPrefix + "/foo": "",
+				rebootAnnotationPrefix + "/bar": "",
+			},
+			expectedReboot: true,
+		},
+		{
+			Scenario: "Suffixed with soft reboot",
+			Annotations: map[string]string{
+				rebootAnnotationPrefix + "/foo": "{\"mode\": \"soft\"}",
+			},
+			expectedReboot: true,
+			expectedMode:   metal3v1alpha1.RebootModeSoft,
+		},
+		{
+			Scenario: "Suffixed with hard reboot",
+			Annotations: map[string]string{
+				rebootAnnotationPrefix + "/foo": "{\"mode\": \"hard\"}",
+			},
+			expectedReboot: true,
+			expectedMode:   metal3v1alpha1.RebootModeHard,
+		},
+		{
+			Scenario: "Suffixed with bad JSON",
+			Annotations: map[string]string{
+				rebootAnnotationPrefix + "/foo": "{\"bad\"= \"json\"]",
+			},
+			expectedReboot: true,
+		},
+		{
+			Scenario: "Two suffixed with different values",
+			Annotations: map[string]string{
+				rebootAnnotationPrefix + "/foo": "{\"mode\": \"hard\"}",
+				rebootAnnotationPrefix + "/bar": "{\"mode\": \"soft\"}",
+			},
+			expectedReboot: true,
+			expectedMode:   metal3v1alpha1.RebootModeHard,
+		},
+		{
+			Scenario: "Suffixed with force",
+			Annotations: map[string]string{
+				rebootAnnotationPrefix + "/foo": "{\"force\": true}",
+			},
+			expectForce:    true,
+			expectedReboot: true,
+		},
+		{
+			Scenario: "Expect force",
+			Annotations: map[string]string{
+				rebootAnnotationPrefix + "/foo": "{\"mode\": \"hard\"}",
+			},
+			expectForce:    true,
+			expectedReboot: false,
+		},
+	}
 
-	//two suffixed annotations to simulate multiple clients
+	for _, tc := range testCases {
+		t.Run(tc.Scenario, func(t *testing.T) {
+			host := newDefaultHost(t)
+			info := makeReconcileInfo(host)
+			host.Annotations = tc.Annotations
 
-	host.Annotations[suffixedAnnotation+"bar"] = "{\"mode\": \"hard\"}"
+			if tc.expectedMode == "" {
+				tc.expectedMode = metal3v1alpha1.RebootModeSoft
+			}
 
-	hasReboot, rebootMode = hasRebootAnnotation(info)
-	assert.True(t, hasReboot)
-	assert.Equal(t, metal3v1alpha1.RebootModeHard, rebootMode)
+			hasReboot, rebootMode := hasRebootAnnotation(info, tc.expectForce)
+			assert.Equal(t, tc.expectedReboot, hasReboot)
+			assert.Equal(t, tc.expectedMode, rebootMode)
+		})
+	}
 }
 
 // TestRebootWithSuffixlessAnnotation tests full reboot cycle with suffixless
diff --git a/controllers/metal3.io/host_state_machine_test.go b/controllers/metal3.io/host_state_machine_test.go
index 9a700a21..298e8ae4 100644
--- a/controllers/metal3.io/host_state_machine_test.go
+++ b/controllers/metal3.io/host_state_machine_test.go
@@ -1075,59 +1075,43 @@ func TestErrorClean(t *testing.T) {
 	}
 }
 
-func TestDeleteWaitsForDetach(t *testing.T) {
+func TestDeleteWaitsForFinalizers(t *testing.T) {
 	tests := []struct {
-		Scenario                  string
-		Host                      *metal3v1alpha1.BareMetalHost
-		ExpectedState             metal3v1alpha1.ProvisioningState
-		ExpectedOperationalStatus metal3v1alpha1.OperationalStatus
+		Scenario      string
+		Host          *metal3v1alpha1.BareMetalHost
+		ExpectedState metal3v1alpha1.ProvisioningState
 	}{
 		{
-			Scenario: "detached-delay",
-			Host: host(metal3v1alpha1.StateProvisioned).
-				SetOperationalStatus(metal3v1alpha1.OperationalStatusDetached).
+			Scenario: "provisioning-2-finalizers",
+			Host: host(metal3v1alpha1.StateProvisioning).
 				setDeletion().
-				setDetached("{\"deleteAction\": \"delay\"}").
+				setFinalizers([]string{metal3v1alpha1.BareMetalHostFinalizer, "other-finalizer"}).
 				build(),
-			ExpectedState:             metal3v1alpha1.StateProvisioned,
-			ExpectedOperationalStatus: metal3v1alpha1.OperationalStatusDetached,
+			ExpectedState: metal3v1alpha1.StateProvisioned,
 		},
 		{
-			Scenario: "detached-delete",
+			Scenario: "provisioned-2-finalizers",
 			Host: host(metal3v1alpha1.StateProvisioned).
-				SetOperationalStatus(metal3v1alpha1.OperationalStatusDetached).
 				setDeletion().
-				setDetached("{\"deleteAction\": \"delete\"}").
+				setFinalizers([]string{metal3v1alpha1.BareMetalHostFinalizer, "other-finalizer"}).
 				build(),
-			ExpectedState:             metal3v1alpha1.StateDeleting,
-			ExpectedOperationalStatus: metal3v1alpha1.OperationalStatusDetached,
+			ExpectedState: metal3v1alpha1.StateProvisioned,
 		},
 		{
-			Scenario: "detached-not-json",
-			Host: host(metal3v1alpha1.StateProvisioned).
-				SetOperationalStatus(metal3v1alpha1.OperationalStatusDetached).
+			Scenario: "provisioning-1-finalizer",
+			Host: host(metal3v1alpha1.StateProvisioning).
 				setDeletion().
-				setDetached("true").
+				setFinalizers([]string{metal3v1alpha1.BareMetalHostFinalizer}).
 				build(),
-			ExpectedState:             metal3v1alpha1.StateDeleting,
-			ExpectedOperationalStatus: metal3v1alpha1.OperationalStatusDetached,
+			ExpectedState: metal3v1alpha1.StateDeprovisioning,
 		},
 		{
-			Scenario: "detached-no-annotation",
+			Scenario: "provisioned-1-finalizer",
 			Host: host(metal3v1alpha1.StateProvisioned).
-				SetOperationalStatus(metal3v1alpha1.OperationalStatusDetached).
 				setDeletion().
+				setFinalizers([]string{metal3v1alpha1.BareMetalHostFinalizer}).
 				build(),
-			ExpectedState:             metal3v1alpha1.StateProvisioned,
-			ExpectedOperationalStatus: metal3v1alpha1.OperationalStatusOK,
-		},
-		{
-			Scenario: "attached",
-			Host: host(metal3v1alpha1.StateProvisioned).
-				setDeletion().
-				build(),
-			ExpectedState:             metal3v1alpha1.StateDeprovisioning,
-			ExpectedOperationalStatus: metal3v1alpha1.OperationalStatusOK,
+			ExpectedState: metal3v1alpha1.StateDeprovisioning,
 		},
 	}
 	for _, tt := range tests {
@@ -1141,7 +1125,6 @@ func TestDeleteWaitsForDetach(t *testing.T) {
 			hsm.ReconcileState(info)
 
 			assert.Equal(t, tt.ExpectedState, tt.Host.Status.Provisioning.State)
-			assert.Equal(t, tt.ExpectedOperationalStatus, tt.Host.Status.OperationalStatus)
 		})
 	}
 }
@@ -1265,11 +1248,8 @@ func (hb *hostBuilder) setDeletion() *hostBuilder {
 	return hb
 }
 
-func (hb *hostBuilder) setDetached(val string) *hostBuilder {
-	if hb.Annotations == nil {
-		hb.Annotations = make(map[string]string, 1)
-	}
-	hb.Annotations[metal3v1alpha1.DetachedAnnotation] = val
+func (hb *hostBuilder) setFinalizers(finalizers []string) *hostBuilder {
+	hb.Finalizers = finalizers
 	return hb
 }
 
@@ -1341,7 +1321,7 @@ func (m *mockProvisioner) PreprovisioningImageFormats() ([]metal3v1alpha1.ImageF
 	return nil, nil
 }
 
-func (m *mockProvisioner) InspectHardware(data provisioner.InspectData, force, refresh bool) (result provisioner.Result, started bool, details *metal3v1alpha1.HardwareDetails, err error) {
+func (m *mockProvisioner) InspectHardware(data provisioner.InspectData, restartOnFailure, refresh, forceReboot bool) (result provisioner.Result, started bool, details *metal3v1alpha1.HardwareDetails, err error) {
 	details = &metal3v1alpha1.HardwareDetails{}
 	return m.getNextResultByMethod("InspectHardware"), true, details, err
 }
@@ -1358,7 +1338,7 @@ func (m *mockProvisioner) Adopt(data provisioner.AdoptData, force bool) (result
 	return m.getNextResultByMethod("Adopt"), err
 }
 
-func (m *mockProvisioner) Provision(data provisioner.ProvisionData) (result provisioner.Result, err error) {
+func (m *mockProvisioner) Provision(data provisioner.ProvisionData, forceReboot bool) (result provisioner.Result, err error) {
 	return m.getNextResultByMethod("Provision"), err
 }
 
diff --git a/controllers/metal3.io/hostfirmwaresettings_test.go b/controllers/metal3.io/hostfirmwaresettings_test.go
index 3ecd510f..bcedc167 100644
--- a/controllers/metal3.io/hostfirmwaresettings_test.go
+++ b/controllers/metal3.io/hostfirmwaresettings_test.go
@@ -83,7 +83,7 @@ func (m *hsfMockProvisioner) Adopt(data provisioner.AdoptData, force bool) (resu
 	return
 }
 
-func (m *hsfMockProvisioner) Provision(data provisioner.ProvisionData) (result provisioner.Result, err error) {
+func (m *hsfMockProvisioner) Provision(data provisioner.ProvisionData, forceReboot bool) (result provisioner.Result, err error) {
 	return
 }
 
diff --git a/pkg/provisioner/ironic/inspecthardware_test.go b/pkg/provisioner/ironic/inspecthardware_test.go
index fd5a44a7..2db436e4 100644
--- a/pkg/provisioner/ironic/inspecthardware_test.go
+++ b/pkg/provisioner/ironic/inspecthardware_test.go
@@ -25,7 +25,8 @@ func TestInspectHardware(t *testing.T) {
 		ironic    *testserver.IronicMock
 		inspector *testserver.InspectorMock
 
-		force bool
+		restartOnFailure bool
+		forceReboot      bool
 
 		expectedStarted      bool
 		expectedDirty        bool
@@ -105,16 +106,34 @@ func TestInspectHardware(t *testing.T) {
 			expectedRequestAfter: 15,
 		},
 		{
-			name: "introspection-aborted",
+			name: "introspection-failed",
 			ironic: testserver.NewIronic(t).Ready().Node(nodes.Node{
 				UUID: nodeUUID,
 			}),
+			inspector: testserver.NewInspector(t).Ready().WithIntrospection(nodeUUID, introspection.Introspection{
+				Finished: true,
+				Error:    "Timeout",
+			}),
+
+			expectedResultError: "Timeout",
+		},
+		{
+			name: "introspection-aborted",
+			ironic: testserver.NewIronic(t).Ready().Node(nodes.Node{
+				UUID: nodeUUID,
+			}).NodeUpdate(nodes.Node{
+				UUID:           nodeUUID,
+				ProvisionState: string(nodes.InspectFail),
+			}).WithNodeStatesProvisionUpdate(nodeUUID),
 			inspector: testserver.NewInspector(t).Ready().WithIntrospection(nodeUUID, introspection.Introspection{
 				Finished: true,
 				Error:    "Canceled by operator",
 			}),
 
-			expectedResultError: "Canceled by operator",
+			expectedStarted:      true,
+			expectedDirty:        true,
+			expectedRequestAfter: 10,
+			expectedPublish:      "InspectionStarted Hardware inspection started",
 		},
 		{
 			name: "inspection-in-progress (not yet finished)",
@@ -128,6 +147,21 @@ func TestInspectHardware(t *testing.T) {
 			expectedDirty:        true,
 			expectedRequestAfter: 15,
 		},
+		{
+			name: "inspection-in-progress - forceReboot",
+			ironic: testserver.NewIronic(t).Ready().Node(nodes.Node{
+				UUID:           nodeUUID,
+				ProvisionState: string(nodes.InspectWait),
+			}).WithNodeStatesProvisionUpdate(nodeUUID),
+			inspector: testserver.NewInspector(t).Ready().WithIntrospection(nodeUUID, introspection.Introspection{
+				Finished: false,
+			}),
+			forceReboot: true,
+
+			expectedStarted:      true,
+			expectedDirty:        true,
+			expectedRequestAfter: 10,
+		},
 		{
 			name: "inspection-in-progress (but node still in InspectWait)",
 			ironic: testserver.NewIronic(t).Ready().Node(nodes.Node{
@@ -166,14 +200,27 @@ func TestInspectHardware(t *testing.T) {
 				UUID:           nodeUUID,
 				ProvisionState: string(nodes.InspectFail),
 			}).WithNodeStatesProvisionUpdate(nodeUUID),
-			inspector: testserver.NewInspector(t).Ready().WithIntrospectionFailed(nodeUUID, http.StatusNotFound),
-			force:     true,
+			inspector:        testserver.NewInspector(t).Ready().WithIntrospectionFailed(nodeUUID, http.StatusNotFound),
+			restartOnFailure: true,
 
 			expectedStarted:      true,
 			expectedDirty:        true,
 			expectedRequestAfter: 10,
 			expectedPublish:      "InspectionStarted Hardware inspection started",
 		},
+		{
+			name: "inspection-forceReboot",
+			ironic: testserver.NewIronic(t).Ready().Node(nodes.Node{
+				UUID:           nodeUUID,
+				ProvisionState: string(nodes.InspectWait),
+			}).WithNodeStatesProvisionUpdate(nodeUUID),
+			inspector:   testserver.NewInspector(t).Ready().WithIntrospectionFailed(nodeUUID, http.StatusNotFound),
+			forceReboot: true,
+
+			expectedStarted:      true,
+			expectedDirty:        true,
+			expectedRequestAfter: 10,
+		},
 		{
 			name: "inspection-complete",
 			ironic: testserver.NewIronic(t).Ready().Node(nodes.Node{
@@ -224,7 +271,7 @@ func TestInspectHardware(t *testing.T) {
 
 			result, started, details, err := prov.InspectHardware(
 				provisioner.InspectData{BootMode: metal3v1alpha1.DefaultBootMode},
-				tc.force, false)
+				tc.restartOnFailure, false, tc.forceReboot)
 
 			assert.Equal(t, tc.expectedStarted, started)
 			assert.Equal(t, tc.expectedDirty, result.Dirty)
diff --git a/pkg/provisioner/ironic/provision_test.go b/pkg/provisioner/ironic/provision_test.go
index bdd9683a..70921747 100644
--- a/pkg/provisioner/ironic/provision_test.go
+++ b/pkg/provisioner/ironic/provision_test.go
@@ -24,13 +24,21 @@ import (
 func TestProvision(t *testing.T) {
 
 	nodeUUID := "33ce8659-7400-4c68-9535-d10766f07a58"
+	testImage := v1alpha1.Image{
+		URL:          "http://test-image",
+		Checksum:     "abcd",
+		ChecksumType: v1alpha1.SHA256,
+	}
+
 	cases := []struct {
-		name                 string
-		ironic               *testserver.IronicMock
-		expectedDirty        bool
-		expectedError        bool
-		expectedErrorMessage bool
-		expectedRequestAfter int
+		name                   string
+		ironic                 *testserver.IronicMock
+		forceReboot            bool
+		expectedDirty          bool
+		expectedError          bool
+		expectedErrorMessage   string
+		expectedRequestAfter   int
+		expectedProvisionState nodes.TargetProvisionState
 	}{
 		{
 			name: "deployFail state",
@@ -38,9 +46,25 @@ func TestProvision(t *testing.T) {
 				ProvisionState: string(nodes.DeployFail),
 				UUID:           nodeUUID,
 			}),
+			expectedRequestAfter:   10,
+			expectedDirty:          true,
+			expectedProvisionState: nodes.TargetActive,
+		},
+		{
+			name: "deployFail state - same image",
+			ironic: testserver.NewIronic(t).WithDefaultResponses().Node(nodes.Node{
+				ProvisionState: string(nodes.DeployFail),
+				UUID:           nodeUUID,
+				InstanceInfo: map[string]interface{}{
+					"image_source":        testImage.URL,
+					"image_os_hash_algo":  string(testImage.ChecksumType),
+					"image_os_hash_value": testImage.Checksum,
+				},
+				LastError: "no work today",
+			}),
 			expectedRequestAfter: 0,
 			expectedDirty:        false,
-			expectedErrorMessage: true,
+			expectedErrorMessage: "Image provisioning failed: no work today",
 		},
 		{
 			name: "cleanFail state",
@@ -48,8 +72,9 @@ func TestProvision(t *testing.T) {
 				ProvisionState: string(nodes.CleanFail),
 				UUID:           nodeUUID,
 			}),
-			expectedRequestAfter: 10,
-			expectedDirty:        true,
+			expectedRequestAfter:   10,
+			expectedDirty:          true,
+			expectedProvisionState: nodes.TargetManage,
 		},
 		{
 			name: "manageable state",
@@ -57,8 +82,9 @@ func TestProvision(t *testing.T) {
 				ProvisionState: string(nodes.Manageable),
 				UUID:           nodeUUID,
 			}),
-			expectedRequestAfter: 10,
-			expectedDirty:        true,
+			expectedRequestAfter:   10,
+			expectedDirty:          true,
+			expectedProvisionState: nodes.TargetProvide,
 		},
 		{
 			name: "available state",
@@ -66,9 +92,9 @@ func TestProvision(t *testing.T) {
 				ProvisionState: string(nodes.Available),
 				UUID:           nodeUUID,
 			}),
-			expectedRequestAfter: 0,
-			expectedDirty:        false,
-			expectedErrorMessage: true,
+			expectedRequestAfter:   10,
+			expectedDirty:          true,
+			expectedProvisionState: nodes.TargetActive,
 		},
 		{
 			name: "active state",
@@ -88,6 +114,26 @@ func TestProvision(t *testing.T) {
 			expectedRequestAfter: 10,
 			expectedDirty:        true,
 		},
+		{
+			name: "other state: Deploy Wait",
+			ironic: testserver.NewIronic(t).Ready().Node(nodes.Node{
+				ProvisionState: string(nodes.DeployWait),
+				UUID:           nodeUUID,
+			}),
+			expectedRequestAfter: 10,
+			expectedDirty:        true,
+		},
+		{
+			name: "forceReboot: Deploy Wait",
+			ironic: testserver.NewIronic(t).WithDefaultResponses().Node(nodes.Node{
+				ProvisionState: string(nodes.DeployWait),
+				UUID:           nodeUUID,
+			}),
+			forceReboot:            true,
+			expectedRequestAfter:   10,
+			expectedDirty:          true,
+			expectedProvisionState: nodes.TargetDeleted,
+		},
 		{
 			name: "fault state",
 			ironic: testserver.NewIronic(t).WithDefaultResponses().Node(nodes.Node{
@@ -138,22 +184,22 @@ func TestProvision(t *testing.T) {
 			}
 
 			result, err := prov.Provision(provisioner.ProvisionData{
+				Image:      testImage,
 				HostConfig: fixture.NewHostConfigData("testUserData", "test: NetworkData", "test: Meta"),
 				BootMode:   v1alpha1.DefaultBootMode,
-			})
+			}, tc.forceReboot)
 
 			assert.Equal(t, tc.expectedDirty, result.Dirty)
 			assert.Equal(t, time.Second*time.Duration(tc.expectedRequestAfter), result.RequeueAfter)
-			if !tc.expectedErrorMessage {
-				assert.Equal(t, "", result.ErrorMessage)
-			} else {
-				assert.NotEqual(t, "", result.ErrorMessage)
-			}
+			assert.Equal(t, tc.expectedErrorMessage, result.ErrorMessage)
 			if !tc.expectedError {
 				assert.NoError(t, err)
 			} else {
 				assert.Error(t, err)
 			}
+
+			lastProvOp := tc.ironic.GetLastNodeStatesProvisionUpdateRequestFor(nodeUUID)
+			assert.Equal(t, tc.expectedProvisionState, lastProvOp.Target)
 		})
 	}
 }
diff --git a/pkg/provisioner/ironic/testserver/ironic.go b/pkg/provisioner/ironic/testserver/ironic.go
index 46af5927..504be4ac 100644
--- a/pkg/provisioner/ironic/testserver/ironic.go
+++ b/pkg/provisioner/ironic/testserver/ironic.go
@@ -27,6 +27,8 @@ func NewIronic(t *testing.T) *IronicMock {
 	}
 }
 
+const validateResult = `{"boot": {"result": true}, "deploy": {"result": true}, "power": {"result": true}}`
+
 // WithDefaultResponses sets a valid answer for all the API calls
 func (m *IronicMock) WithDefaultResponses() *IronicMock {
 	m.AddDefaultResponseJSON("/v1/nodes/{id}", "", http.StatusOK, nodes.Node{
@@ -35,7 +37,7 @@ func (m *IronicMock) WithDefaultResponses() *IronicMock {
 	m.AddDefaultResponse("/v1/nodes/{id}/states/provision", "", http.StatusAccepted, "{}")
 	m.AddDefaultResponse("/v1/nodes/{id}/states/power", "", http.StatusAccepted, "{}")
 	m.AddDefaultResponse("/v1/nodes/{id}/states/raid", "", http.StatusNoContent, "{}")
-	m.AddDefaultResponse("/v1/nodes/{id}/validate", "", http.StatusOK, "{}")
+	m.AddDefaultResponse("/v1/nodes/{id}/validate", "", http.StatusOK, validateResult)
 	m.Ready()
 
 	return m
@@ -142,6 +144,16 @@ func (m *IronicMock) GetLastNodeUpdateRequestFor(id string) (updates []nodes.Upd
 	return
 }
 
+// GetLastNodeStatesProvisionUpdateRequestFor returns the content of the last provisioning request for the specified node
+func (m *IronicMock) GetLastNodeStatesProvisionUpdateRequestFor(id string) (update nodes.ProvisionStateOpts) {
+
+	if bodyRaw, ok := m.GetLastRequestFor("/v1/nodes/"+id+"/states/provision", http.MethodPut); ok {
+		json.Unmarshal([]byte(bodyRaw), &update)
+	}
+
+	return
+}
+
 // NodeMaintenanceError configures configures the server with an error response for [PUT] /v1/nodes/{id}/maintenance
 func (m *IronicMock) NodeMaintenanceError(id string, errorCode int) *IronicMock {
 	m.ResponseWithCode(m.buildURL("/v1/nodes/"+id+"/maintenance", http.MethodPut), "", errorCode)
@@ -258,7 +270,7 @@ func (m *IronicMock) WithNodeStatesPowerUpdate(nodeUUID string, code int) *Ironi
 
 // WithNodeValidate configures the server with a valid response for /v1/nodes/<node>/validate
 func (m *IronicMock) WithNodeValidate(nodeUUID string) *IronicMock {
-	m.ResponseWithCode("/v1/nodes/"+nodeUUID+"/validate", "{}", http.StatusOK)
+	m.ResponseWithCode("/v1/nodes/"+nodeUUID+"/validate", validateResult, http.StatusOK)
 	return m
 }