Repository: keikoproj/upgrade-manager
Pull request #132 (2020-11-05 00:24:45)

Extract script runner to a separate type; fix work with env. variables
Fixes https://github.com/keikoproj/upgrade-manager/issues/129

1. Extract work with scripts to a separate type.
2. Fix work with env. variables.
3. Simplify API by not requiring kubectl binary.

codecov[bot]: # [Codecov](https://codecov.io/gh/keikoproj/upgrade-manager/pull/132?src=pr&el=h1) Report
> Merging [#132](https://codecov.io/gh/keikoproj/upgrade-manager/pull/132?src=pr&el=desc) into [master](https://codecov.io/gh/keikoproj/upgrade-manager/commit/9289198b15479db1c48bf43876f62e2500446342?el=desc) will **increase** coverage by `1.50%`.
> The diff coverage is `91.48%`.

[![Impacted file tree graph](https://codecov.io/gh/keikoproj/upgrade-manager/pull/132/graphs/tree.svg?width=650&height=150&src=pr&token=PbUYkNX6wz)](https://codecov.io/gh/keikoproj/upgrade-manager/pull/132?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #132      +/-   ##
==========================================
+ Coverage   66.97%   68.47%   +1.50%     
==========================================
  Files          10       11       +1     
  Lines         872      885      +13     
==========================================
+ Hits          584      606      +22     
+ Misses        257      252       -5     
+ Partials       31       27       -4     
```

| Flag | Coverage Δ | |
|---|---|---|
| unittests | `68.47% <91.48%> (+1.50%)` | :arrow_up: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags#carryforward-flags-in-the-pull-request-comment) to find out more.

| [Impacted Files](https://codecov.io/gh/keikoproj/upgrade-manager/pull/132?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [controllers/rollingupgrade\_controller.go](https://codecov.io/gh/keikoproj/upgrade-manager/pull/132/diff?src=pr&el=tree#diff-Y29udHJvbGxlcnMvcm9sbGluZ3VwZ3JhZGVfY29udHJvbGxlci5nbw==) | `65.94% <90.47%> (-0.89%)` | :arrow_down: |
| [controllers/script\_runner.go](https://codecov.io/gh/keikoproj/upgrade-manager/pull/132/diff?src=pr&el=tree#diff-Y29udHJvbGxlcnMvc2NyaXB0X3J1bm5lci5nbw==) | `91.78% <91.78%> (ø)` | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/keikoproj/upgrade-manager/pull/132?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/keikoproj/upgrade-manager/pull/132?src=pr&el=footer). Last update [9289198...2ddc029](https://codecov.io/gh/keikoproj/upgrade-manager/pull/132?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).

uthark: @eytan-avisror and/or @shrinandj please take a look.
shrinandj: Looks great. Thanks for fixing these @uthark 

@eytan-avisror Feel free to merge once you've reviewed this as well.
shrinandj: @uthark , sorry we merged few other PRs recently that has resulted in a conflict here. Can you rebased and push again? We can merge and do a new build.
uthark: @shrinandj rebased.

Patch:
diff --git a/controllers/rollingupgrade_controller.go b/controllers/rollingupgrade_controller.go
index 2ab2d4e..6377881 100644
--- a/controllers/rollingupgrade_controller.go
+++ b/controllers/rollingupgrade_controller.go
@@ -18,8 +18,6 @@ package controllers
 import (
 	"context"
 	"fmt"
-	"os"
-	"os/exec"
 	"strconv"
 	"strings"
 	"sync"
@@ -70,11 +68,6 @@ const (
 	instanceIDKey   = "INSTANCE_ID"
 	instanceNameKey = "INSTANCE_NAME"
 
-	// KubeCtlBinary is the path to the kubectl executable
-	KubeCtlBinary = "/usr/local/bin/kubectl"
-	// ShellBinary is the path to the shell executable
-	ShellBinary = "/bin/sh"
-
 	// InService is a state of an instance
 	InService = "InService"
 )
@@ -108,6 +101,7 @@ type RollingUpgradeReconciler struct {
 	ClusterState    ClusterState
 	maxParallel     int
 	CacheConfig     *cache.Config
+	ScriptRunner    ScriptRunner
 }
 
 func (r *RollingUpgradeReconciler) SetMaxParallel(max int) {
@@ -117,87 +111,33 @@ func (r *RollingUpgradeReconciler) SetMaxParallel(max int) {
 	}
 }
 
-func (r *RollingUpgradeReconciler) runScript(script string, background bool, ruObj *upgrademgrv1alpha1.RollingUpgrade) (string, error) {
-	r.info(ruObj, "Running script", "script", script)
-	if background {
-		r.info(ruObj, "Running script in background. Logs not available.")
-		err := exec.Command(ShellBinary, "-c", script).Run()
-		if err != nil {
-			r.info(ruObj, fmt.Sprintf("Script finished with error: %s", err))
-		}
-
-		return "", nil
-	}
-
-	out, err := exec.Command(ShellBinary, "-c", script).CombinedOutput()
-	if err != nil {
-		r.error(ruObj, err, "Script finished", "output", string(out))
-	} else {
-		r.info(ruObj, "Script finished", "output", string(out))
-	}
-	return string(out), err
-}
-
-func (r *RollingUpgradeReconciler) preDrainHelper(ruObj *upgrademgrv1alpha1.RollingUpgrade) error {
-	if ruObj.Spec.PreDrain.Script != "" {
-		script := ruObj.Spec.PreDrain.Script
-		_, err := r.runScript(script, false, ruObj)
-		if err != nil {
-			msg := "Failed to run preDrain script"
-			r.error(ruObj, err, msg)
-			return errors.Wrap(err, msg)
-		}
-	}
-	return nil
+func (r *RollingUpgradeReconciler) preDrainHelper(instanceID, nodeName string, ruObj *upgrademgrv1alpha1.RollingUpgrade) error {
+	return r.ScriptRunner.PreDrain(instanceID, nodeName, ruObj)
 }
 
 // Operates on any scripts that were provided after the draining of the node.
 // kubeCtlCall is provided as an argument to decouple the method from the actual kubectl call
-func (r *RollingUpgradeReconciler) postDrainHelper(ruObj *upgrademgrv1alpha1.RollingUpgrade, nodeName string, kubeCtlCall string) error {
-	if ruObj.Spec.PostDrain.Script != "" {
-		_, err := r.runScript(ruObj.Spec.PostDrain.Script, false, ruObj)
-		if err != nil {
-			msg := "Failed to run postDrain script: "
-			r.error(ruObj, err, msg)
-			result := errors.Wrap(err, msg)
-
-			r.info(ruObj, "Uncordoning the node %s since it failed to run postDrain Script", "nodeName", nodeName)
-			_, err = r.runScript(kubeCtlCall+" uncordon "+nodeName, false, ruObj)
-			if err != nil {
-				r.error(ruObj, err, "Failed to uncordon", "nodeName", nodeName)
-			}
-			return result
-		}
+func (r *RollingUpgradeReconciler) postDrainHelper(instanceID, nodeName string, ruObj *upgrademgrv1alpha1.RollingUpgrade) error {
+	err := r.ScriptRunner.PostDrain(instanceID, nodeName, ruObj)
+	if err != nil {
+		return err
 	}
+
 	r.info(ruObj, "Waiting for postDrainDelay", "postDrainDelay", ruObj.Spec.PostDrainDelaySeconds)
 	time.Sleep(time.Duration(ruObj.Spec.PostDrainDelaySeconds) * time.Second)
 
-	if ruObj.Spec.PostDrain.PostWaitScript != "" {
-		_, err := r.runScript(ruObj.Spec.PostDrain.PostWaitScript, false, ruObj)
-		if err != nil {
-			msg := "Failed to run postDrainWait script: " + err.Error()
-			r.error(ruObj, err, msg)
-			result := errors.Wrap(err, msg)
-
-			r.info(ruObj, "Uncordoning the node %s since it failed to run postDrain Script", "nodeName", nodeName)
-			_, err = r.runScript(kubeCtlCall+" uncordon "+nodeName, false, ruObj)
-			if err != nil {
-				r.error(ruObj, err, "Failed to uncordon", "nodeName", nodeName)
-			}
-			return result
-		}
-	}
-	return nil
+	return r.ScriptRunner.PostWait(instanceID, nodeName, ruObj)
+
 }
 
 // DrainNode runs "kubectl drain" on the given node
 // kubeCtlCall is provided as an argument to decouple the method from the actual kubectl call
 func (r *RollingUpgradeReconciler) DrainNode(ruObj *upgrademgrv1alpha1.RollingUpgrade,
 	nodeName string,
-	kubeCtlCall string,
+	instanceID string,
 	drainTimeout int) error {
 	// Running kubectl drain node.
-	err := r.preDrainHelper(ruObj)
+	err := r.preDrainHelper(instanceID, nodeName, ruObj)
 	if err != nil {
 		return errors.New(ruObj.Name + ": Predrain script failed: " + err.Error())
 	}
@@ -218,7 +158,7 @@ func (r *RollingUpgradeReconciler) DrainNode(ruObj *upgrademgrv1alpha1.RollingUp
 	}
 
 	r.info(ruObj, "Invoking kubectl drain for the node", "nodeName", nodeName)
-	go r.CallKubectlDrain(ctx, nodeName, kubeCtlCall, ruObj, errChan)
+	go r.CallKubectlDrain(nodeName, ruObj, errChan)
 
 	// Listening to signals from the CallKubectlDrain go routine
 	select {
@@ -232,24 +172,21 @@ func (r *RollingUpgradeReconciler) DrainNode(ruObj *upgrademgrv1alpha1.RollingUp
 		r.info(ruObj, "Kubectl drain completed for node", "nodeName", nodeName)
 	}
 
-	return r.postDrainHelper(ruObj, nodeName, kubeCtlCall)
+	return r.postDrainHelper(instanceID, nodeName, ruObj)
 }
 
 // CallKubectlDrain runs the "kubectl drain" for a given node
 // Node will be terminated even if pod eviction is not completed when the drain timeout is exceeded
-func (r *RollingUpgradeReconciler) CallKubectlDrain(ctx context.Context, nodeName, kubeCtlCall string, ruObj *upgrademgrv1alpha1.RollingUpgrade, errChan chan error) {
+func (r *RollingUpgradeReconciler) CallKubectlDrain(nodeName string, ruObj *upgrademgrv1alpha1.RollingUpgrade, errChan chan error) {
 
-	// kops behavior implements the same behavior by using these flags when draining nodes
-	// https://github.com/kubernetes/kops/blob/7a629c77431dda02d02aadf00beb0bed87518cbf/pkg/instancegroups/instancegroups.go lines 337-340
-	script := fmt.Sprintf("%s drain %s --ignore-daemonsets=true --delete-local-data=true --force --grace-period=-1", kubeCtlCall, nodeName)
-	out, err := r.runScript(script, false, ruObj)
+	out, err := r.ScriptRunner.drainNode(nodeName, ruObj)
 	if err != nil {
 		if strings.HasPrefix(out, "Error from server (NotFound)") {
 			r.error(ruObj, err, "Not executing postDrainHelper. Node not found.", "output", out)
 			errChan <- nil
 			return
 		}
-		errChan <- errors.New(ruObj.Name + " :Failed to drain: " + err.Error())
+		errChan <- errors.New(ruObj.Name + ": Failed to drain: " + err.Error())
 		return
 	}
 	errChan <- nil
@@ -392,7 +329,7 @@ func (r *RollingUpgradeReconciler) SetStandby(ruObj *upgrademgrv1alpha1.RollingU
 }
 
 // TerminateNode actually terminates the given node.
-func (r *RollingUpgradeReconciler) TerminateNode(ruObj *upgrademgrv1alpha1.RollingUpgrade, instanceID string) error {
+func (r *RollingUpgradeReconciler) TerminateNode(ruObj *upgrademgrv1alpha1.RollingUpgrade, instanceID string, nodeName string) error {
 
 	input := &autoscaling.TerminateInstanceInAutoScalingGroupInput{
 		InstanceId:                     aws.String(instanceID),
@@ -427,19 +364,7 @@ func (r *RollingUpgradeReconciler) TerminateNode(ruObj *upgrademgrv1alpha1.Rolli
 	r.info(ruObj, "Instance terminated.", "instanceID", instanceID)
 	r.info(ruObj, "starting post termination sleep", "instanceID", instanceID, "nodeIntervalSeconds", ruObj.Spec.NodeIntervalSeconds)
 	time.Sleep(time.Duration(ruObj.Spec.NodeIntervalSeconds) * time.Second)
-	if ruObj.Spec.PostTerminate.Script != "" {
-		out, err := r.runScript(ruObj.Spec.PostTerminate.Script, false, ruObj)
-		if err != nil {
-			if strings.HasPrefix(out, "Error from server (NotFound)") {
-				r.error(ruObj, err, "Node not found when running postTerminate. Ignoring ...", "output", out, "instanceID", instanceID)
-				return nil
-			}
-			msg := "Failed to run postTerminate script"
-			r.error(ruObj, err, msg, "instanceID", instanceID)
-			return errors.Wrap(err, msg)
-		}
-	}
-	return nil
+	return r.ScriptRunner.PostTerminate(instanceID, nodeName, ruObj)
 }
 
 func (r *RollingUpgradeReconciler) getNodeName(i *autoscaling.Instance, nodeList *corev1.NodeList, ruObj *upgrademgrv1alpha1.RollingUpgrade) string {
@@ -502,21 +427,6 @@ func (r *RollingUpgradeReconciler) populateNodeList(ruObj *upgrademgrv1alpha1.Ro
 	return nil
 }
 
-// Loads specific environment variables for scripts to use
-// on a given rollingUpgrade and autoscaling instance
-func loadEnvironmentVariables(ruObj *upgrademgrv1alpha1.RollingUpgrade, instanceID string, nodeName string) error {
-	if err := os.Setenv(asgNameKey, ruObj.Spec.AsgName); err != nil {
-		return errors.New(ruObj.Name + ": Could not load " + asgNameKey + ": " + err.Error())
-	}
-	if err := os.Setenv(instanceIDKey, instanceID); err != nil {
-		return errors.New(ruObj.Name + ": Could not load " + instanceIDKey + ": " + err.Error())
-	}
-	if err := os.Setenv(instanceNameKey, nodeName); err != nil {
-		return errors.New(ruObj.Name + ": Could not load " + instanceNameKey + ": " + err.Error())
-	}
-	return nil
-}
-
 func (r *RollingUpgradeReconciler) getInProgressInstances(instances []*autoscaling.Instance) ([]*autoscaling.Instance, error) {
 	var inProgressInstances []*autoscaling.Instance
 	taggedInstances, err := getTaggedInstances(EC2StateTagKey, "in-progress", r.EC2Client)
@@ -531,7 +441,8 @@ func (r *RollingUpgradeReconciler) getInProgressInstances(instances []*autoscali
 	return inProgressInstances, nil
 }
 
-func (r *RollingUpgradeReconciler) runRestack(ctx *context.Context, ruObj *upgrademgrv1alpha1.RollingUpgrade, KubeCtlCall string) (int, error) {
+func (r *RollingUpgradeReconciler) runRestack(ctx *context.Context, ruObj *upgrademgrv1alpha1.RollingUpgrade) (int, error) {
+
 	asg, err := r.GetAutoScalingGroup(ruObj.Name)
 	if err != nil {
 		return 0, fmt.Errorf("Unable to load ASG with name: %s", ruObj.Name)
@@ -586,7 +497,7 @@ func (r *RollingUpgradeReconciler) runRestack(ctx *context.Context, ruObj *upgra
 		}
 
 		// update the instances
-		err := r.UpdateInstances(ctx, ruObj, instances, launchDefinition, KubeCtlCall)
+		err := r.UpdateInstances(ctx, ruObj, instances, launchDefinition)
 		processedInstances += len(instances)
 		if err != nil {
 			return processedInstances, err
@@ -701,7 +612,7 @@ func (r *RollingUpgradeReconciler) Process(ctx *context.Context,
 	}
 
 	// Run the restack that actually performs the rolling update.
-	nodesProcessed, err := r.runRestack(ctx, ruObj, KubeCtlBinary)
+	nodesProcessed, err := r.runRestack(ctx, ruObj)
 	if err != nil {
 		r.error(ruObj, err, "Failed to runRestack")
 		r.finishExecution(StatusError, nodesProcessed, ctx, ruObj)
@@ -942,8 +853,7 @@ func NewUpdateInstancesError(instanceUpdateErrors []error) *UpdateInstancesError
 func (r *RollingUpgradeReconciler) UpdateInstances(ctx *context.Context,
 	ruObj *upgrademgrv1alpha1.RollingUpgrade,
 	instances []*autoscaling.Instance,
-	launchDefinition *launchDefinition,
-	KubeCtlCall string) error {
+	launchDefinition *launchDefinition) error {
 
 	totalNodes := len(instances)
 	if totalNodes == 0 {
@@ -960,7 +870,7 @@ func (r *RollingUpgradeReconciler) UpdateInstances(ctx *context.Context,
 			"strategy": string(ruObj.Spec.Strategy.Type),
 			"msg":      fmt.Sprintf("Started Updating Instance %s, in AZ: %s", *instance.InstanceId, *instance.AvailabilityZone),
 		})
-		go r.UpdateInstance(ctx, ruObj, instance, launchDefinition, KubeCtlCall, ch)
+		go r.UpdateInstance(ctx, ruObj, instance, launchDefinition, ch)
 	}
 
 	// wait for upgrades to complete
@@ -997,8 +907,7 @@ func (r *RollingUpgradeReconciler) UpdateInstances(ctx *context.Context,
 func (r *RollingUpgradeReconciler) UpdateInstanceEager(
 	ruObj *upgrademgrv1alpha1.RollingUpgrade,
 	nodeName,
-	targetInstanceID,
-	KubeCtlCall string,
+	targetInstanceID string,
 	ch chan error) {
 
 	// Set instance to standby
@@ -1023,19 +932,19 @@ func (r *RollingUpgradeReconciler) UpdateInstanceEager(
 	}
 
 	// Drain and wait for draining node.
-	r.DrainTerminate(ruObj, nodeName, targetInstanceID, KubeCtlCall, ch)
+	r.DrainTerminate(ruObj, nodeName, targetInstanceID, ch)
+
 }
 
 func (r *RollingUpgradeReconciler) DrainTerminate(
 	ruObj *upgrademgrv1alpha1.RollingUpgrade,
 	nodeName,
-	targetInstanceID,
-	KubeCtlCall string,
+	targetInstanceID string,
 	ch chan error) {
 
 	// Drain and wait for draining node.
 	if nodeName != "" {
-		err := r.DrainNode(ruObj, nodeName, KubeCtlCall, ruObj.Spec.Strategy.DrainTimeout)
+		err := r.DrainNode(ruObj, nodeName, targetInstanceID, ruObj.Spec.Strategy.DrainTimeout)
 		if err != nil && !ruObj.Spec.IgnoreDrainFailures {
 			ch <- err
 			return
@@ -1043,7 +952,7 @@ func (r *RollingUpgradeReconciler) DrainTerminate(
 	}
 
 	// Terminate instance.
-	err := r.TerminateNode(ruObj, targetInstanceID)
+	err := r.TerminateNode(ruObj, targetInstanceID, nodeName)
 	if err != nil {
 		ch <- err
 		return
@@ -1055,7 +964,6 @@ func (r *RollingUpgradeReconciler) UpdateInstance(ctx *context.Context,
 	ruObj *upgrademgrv1alpha1.RollingUpgrade,
 	i *autoscaling.Instance,
 	launchDefinition *launchDefinition,
-	KubeCtlCall string,
 	ch chan error) {
 	targetInstanceID := aws.StringValue(i.InstanceId)
 	// If an instance was marked as "in-progress" in ClusterState, it has to be marked
@@ -1084,15 +992,8 @@ func (r *RollingUpgradeReconciler) UpdateInstance(ctx *context.Context,
 
 	nodeName := r.getNodeName(i, r.NodeList, ruObj)
 
-	// Load the environment variables for scripts to run
-	err := loadEnvironmentVariables(ruObj, targetInstanceID, nodeName)
-	if err != nil {
-		ch <- err
-		return
-	}
-
 	// set the EC2 tag indicating the state to in-progress
-	err = r.setStateTag(ruObj, targetInstanceID, "in-progress")
+	err := r.setStateTag(ruObj, targetInstanceID, "in-progress")
 	if err != nil {
 		ch <- err
 		return
@@ -1101,10 +1002,10 @@ func (r *RollingUpgradeReconciler) UpdateInstance(ctx *context.Context,
 	mode := ruObj.Spec.Strategy.Mode.String()
 	if strings.ToLower(mode) == upgrademgrv1alpha1.UpdateStrategyModeEager.String() {
 		r.info(ruObj, "starting replacement with eager mode", "mode", mode)
-		r.UpdateInstanceEager(ruObj, nodeName, targetInstanceID, KubeCtlCall, ch)
+		r.UpdateInstanceEager(ruObj, nodeName, targetInstanceID, ch)
 	} else if strings.ToLower(mode) == upgrademgrv1alpha1.UpdateStrategyModeLazy.String() {
 		r.info(ruObj, "starting replacement with lazy mode", "mode", mode)
-		r.DrainTerminate(ruObj, nodeName, targetInstanceID, KubeCtlCall, ch)
+		r.DrainTerminate(ruObj, nodeName, targetInstanceID, ch)
 	}
 
 	unjoined, err := r.WaitForTermination(ruObj, nodeName, r.generatedClient.CoreV1().Nodes())
diff --git a/controllers/script_runner.go b/controllers/script_runner.go
new file mode 100644
index 0000000..835e7c8
--- /dev/null
+++ b/controllers/script_runner.go
@@ -0,0 +1,180 @@
+/*
+Copyright 2019 Intuit, Inc..
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"fmt"
+	"github.com/go-logr/logr"
+	upgrademgrv1alpha1 "github.com/keikoproj/upgrade-manager/api/v1alpha1"
+	"github.com/pkg/errors"
+	"os/exec"
+	"strings"
+)
+
+const (
+	// KubeCtlBinary is the path to the kubectl executable
+	KubeCtlBinary = "/usr/local/bin/kubectl"
+	// ShellBinary is the path to the shell executable
+	ShellBinary = "/bin/sh"
+)
+
+type ScriptRunner struct {
+	Log         logr.Logger
+	KubectlCall string
+}
+
+func NewScriptRunner(logger logr.Logger) ScriptRunner {
+	return ScriptRunner{
+		Log:         logger,
+		KubectlCall: KubeCtlBinary,
+	}
+}
+
+func (r *ScriptRunner) uncordonNode(nodeName string, ruObj *upgrademgrv1alpha1.RollingUpgrade) (string, error) {
+	script := fmt.Sprintf("%s uncordon %s", r.KubectlCall, nodeName)
+	return r.runScript(script, false, ruObj)
+}
+
+func (r *ScriptRunner) drainNode(nodeName string, ruObj *upgrademgrv1alpha1.RollingUpgrade) (string, error) {
+	// kops behavior implements the same behavior by using these flags when draining nodes
+	// https://github.com/kubernetes/kops/blob/7a629c77431dda02d02aadf00beb0bed87518cbf/pkg/instancegroups/instancegroups.go lines 337-340
+	script := fmt.Sprintf("%s drain %s --ignore-daemonsets=true --delete-local-data=true --force --grace-period=-1", r.KubectlCall, nodeName)
+	return r.runScript(script, false, ruObj)
+}
+
+func (r *ScriptRunner) runScriptWithEnv(script string, background bool, ruObj *upgrademgrv1alpha1.RollingUpgrade, env []string) (string, error) {
+	r.info(ruObj, "Running script", "script", script)
+	command := exec.Command(ShellBinary, "-c", script)
+	if env != nil {
+		command.Env = env
+	}
+
+	if background {
+		r.info(ruObj, "Running script in background. Logs not available.")
+		err := command.Run()
+		if err != nil {
+			r.info(ruObj, fmt.Sprintf("Script finished with error: %s", err))
+		}
+
+		return "", nil
+	}
+
+	out, err := command.CombinedOutput()
+	if err != nil {
+		r.error(ruObj, err, "Script finished", "output", string(out))
+	} else {
+		r.info(ruObj, "Script finished", "output", string(out))
+	}
+	return string(out), err
+}
+
+func (r *ScriptRunner) runScript(script string, background bool, ruObj *upgrademgrv1alpha1.RollingUpgrade) (string, error) {
+	return r.runScriptWithEnv(script, background, ruObj, nil)
+
+}
+
+// logger creates logger for rolling upgrade.
+func (r *ScriptRunner) logger(ruObj *upgrademgrv1alpha1.RollingUpgrade) logr.Logger {
+	return r.Log.WithValues("rollingupgrade", ruObj.Name)
+}
+
+// info logs message with Info level for the specified rolling upgrade.
+func (r *ScriptRunner) info(ruObj *upgrademgrv1alpha1.RollingUpgrade, msg string, keysAndValues ...interface{}) {
+	r.logger(ruObj).Info(msg, keysAndValues...)
+}
+
+// error logs message with Error level for the specified rolling upgrade.
+func (r *ScriptRunner) error(ruObj *upgrademgrv1alpha1.RollingUpgrade, err error, msg string, keysAndValues ...interface{}) {
+	r.logger(ruObj).Error(err, msg, keysAndValues...)
+}
+
+func (r *ScriptRunner) PostTerminate(instanceID string, nodeName string, ruObj *upgrademgrv1alpha1.RollingUpgrade) error {
+	if ruObj.Spec.PostTerminate.Script != "" {
+
+		out, err := r.runScriptWithEnv(ruObj.Spec.PostTerminate.Script, false, ruObj, r.buildEnv(ruObj, instanceID, nodeName))
+		if err != nil {
+			if strings.HasPrefix(out, "Error from server (NotFound)") {
+				r.error(ruObj, err, "Node not found when running postTerminate. Ignoring ...", "output", out, "instanceID", instanceID)
+				return nil
+			}
+			msg := "Failed to run postTerminate script"
+			r.error(ruObj, err, msg, "instanceID", instanceID)
+			return errors.Wrap(err, msg)
+		}
+	}
+	return nil
+
+}
+
+func (r *ScriptRunner) PreDrain(instanceID string, nodeName string, ruObj *upgrademgrv1alpha1.RollingUpgrade) error {
+	if ruObj.Spec.PreDrain.Script != "" {
+		script := ruObj.Spec.PreDrain.Script
+		_, err := r.runScriptWithEnv(script, false, ruObj, r.buildEnv(ruObj, instanceID, nodeName))
+		if err != nil {
+			msg := "Failed to run preDrain script"
+			r.error(ruObj, err, msg)
+			return errors.Wrap(err, msg)
+		}
+	}
+	return nil
+
+}
+func (r *ScriptRunner) PostWait(instanceID string, nodeName string, ruObj *upgrademgrv1alpha1.RollingUpgrade) error {
+	if ruObj.Spec.PostDrain.PostWaitScript != "" {
+		_, err := r.runScriptWithEnv(ruObj.Spec.PostDrain.PostWaitScript, false, ruObj, r.buildEnv(ruObj, instanceID, nodeName))
+		if err != nil {
+			msg := "Failed to run postDrainWait script: " + err.Error()
+			r.error(ruObj, err, msg)
+			result := errors.Wrap(err, msg)
+
+			r.info(ruObj, "Uncordoning the node %s since it failed to run postDrain Script", "nodeName", nodeName)
+			_, err = r.uncordonNode(nodeName, ruObj)
+			if err != nil {
+				r.error(ruObj, err, "Failed to uncordon", "nodeName", nodeName)
+			}
+			return result
+		}
+	}
+	return nil
+}
+
+func (r *ScriptRunner) PostDrain(instanceID string, nodeName string, ruObj *upgrademgrv1alpha1.RollingUpgrade) error {
+	if ruObj.Spec.PostDrain.Script != "" {
+		_, err := r.runScriptWithEnv(ruObj.Spec.PostDrain.Script, false, ruObj, r.buildEnv(ruObj, instanceID, nodeName))
+		if err != nil {
+			msg := "Failed to run postDrain script: "
+			r.error(ruObj, err, msg)
+			result := errors.Wrap(err, msg)
+
+			r.info(ruObj, "Uncordoning the node %s since it failed to run postDrain Script", "nodeName", nodeName)
+			_, err = r.uncordonNode(nodeName, ruObj)
+			if err != nil {
+				r.error(ruObj, err, "Failed to uncordon", "nodeName", nodeName)
+			}
+			return result
+		}
+	}
+	return nil
+}
+
+func (r *ScriptRunner) buildEnv(ruObj *upgrademgrv1alpha1.RollingUpgrade, instanceID string, nodeName string) []string {
+	return []string{
+		fmt.Sprintf("%s=%s", asgNameKey, ruObj.Spec.AsgName),
+		fmt.Sprintf("%s=%s", instanceIDKey, instanceID),
+		fmt.Sprintf("%s=%s", instanceNameKey, nodeName),
+	}
+}
diff --git a/main.go b/main.go
index 883217d..b557b88 100644
--- a/main.go
+++ b/main.go
@@ -142,13 +142,15 @@ func main() {
 		)
 	})
 
+	logger := ctrl.Log.WithName("controllers").WithName("RollingUpgrade")
 	reconciler := &controllers.RollingUpgradeReconciler{
 		Client:       mgr.GetClient(),
-		Log:          ctrl.Log.WithName("controllers").WithName("RollingUpgrade"),
+		Log:          logger,
 		ClusterState: controllers.NewClusterState(),
 		ASGClient:    autoscaling.New(sess),
 		EC2Client:    ec2.New(sess),
 		CacheConfig:  cacheCfg,
+		ScriptRunner: controllers.NewScriptRunner(logger),
 	}
 
 	reconciler.SetMaxParallel(maxParallel)


Test patch:
diff --git a/controllers/events_test.go b/controllers/events_test.go
index dc2f46e..c517e8e 100644
--- a/controllers/events_test.go
+++ b/controllers/events_test.go
@@ -19,7 +19,6 @@ func Test_createK8sV1Event(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	rcRollingUpgrade := &RollingUpgradeReconciler{
 		Client:          mgr.GetClient(),
@@ -31,6 +30,7 @@ func Test_createK8sV1Event(t *testing.T) {
 		ruObjNameToASG:  sync.Map{},
 		ClusterState:    NewClusterState(),
 		CacheConfig:     cache.NewConfig(0*time.Second, 0, 0),
+		ScriptRunner:    NewScriptRunner(log2.NullLogger{}),
 	}
 
 	event := rcRollingUpgrade.createK8sV1Event(ruObj, EventReasonRUStarted, EventLevelNormal, map[string]string{})
diff --git a/controllers/rollingupgrade_controller_test.go b/controllers/rollingupgrade_controller_test.go
index 74ea7d7..e44acaa 100644
--- a/controllers/rollingupgrade_controller_test.go
+++ b/controllers/rollingupgrade_controller_test.go
@@ -34,15 +34,12 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
 	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
 	upgrademgrv1alpha1 "github.com/keikoproj/upgrade-manager/api/v1alpha1"
 )
 
-var c client.Client
-
 func TestMain(m *testing.M) {
 	testEnv = &envtest.Environment{
 		CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
@@ -52,36 +49,6 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func TestEchoScript(t *testing.T) {
-	g := gomega.NewGomegaWithT(t)
-	ru := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}
-	r := &RollingUpgradeReconciler{Log: log2.NullLogger{}}
-	out, err := r.runScript("echo hello", false, ru)
-
-	g.Expect(err).To(gomega.BeNil())
-	g.Expect(out).To(gomega.Equal("hello\n"))
-}
-
-func TestEchoBackgroundScript(t *testing.T) {
-	g := gomega.NewGomegaWithT(t)
-	ru := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}
-	r := &RollingUpgradeReconciler{Log: log2.NullLogger{}}
-	out, err := r.runScript("echo background", true, ru)
-
-	g.Expect(err).To(gomega.BeNil())
-	g.Expect(out).To(gomega.Equal(""))
-}
-
-func TestRunScriptFailure(t *testing.T) {
-	g := gomega.NewGomegaWithT(t)
-	ru := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}
-	r := &RollingUpgradeReconciler{Log: log2.NullLogger{}}
-	out, err := r.runScript("echo this will fail; exit 1", false, ru)
-
-	g.Expect(err).To(gomega.Not(gomega.BeNil()))
-	g.Expect(out).To(gomega.Not(gomega.Equal("")))
-}
-
 func TestErrorStatusMarkJanitor(t *testing.T) {
 	g := gomega.NewGomegaWithT(t)
 	someAsg := "someAsg"
@@ -92,8 +59,6 @@ func TestErrorStatusMarkJanitor(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
-
 	rcRollingUpgrade := &RollingUpgradeReconciler{Client: mgr.GetClient(),
 		generatedClient: kubernetes.NewForConfigOrDie(mgr.GetConfig()),
 		Log:             log2.NullLogger{},
@@ -101,6 +66,7 @@ func TestErrorStatusMarkJanitor(t *testing.T) {
 		ruObjNameToASG:  sync.Map{},
 		inProcessASGs:   sync.Map{},
 		ClusterState:    NewClusterState(),
+		ScriptRunner:    NewScriptRunner(log2.NullLogger{}),
 	}
 
 	ctx := context.TODO()
@@ -148,7 +114,7 @@ func TestPreDrainScriptSuccess(t *testing.T) {
 	instance.Spec.PreDrain.Script = "echo 'Predrain script ran without error'"
 
 	rcRollingUpgrade := createReconciler()
-	err := rcRollingUpgrade.preDrainHelper(instance)
+	err := rcRollingUpgrade.preDrainHelper("test-instance-id", "test", instance)
 	g.Expect(err).To(gomega.BeNil())
 }
 
@@ -159,7 +125,7 @@ func TestPreDrainScriptError(t *testing.T) {
 	instance.Spec.PreDrain.Script = "exit 1"
 
 	rcRollingUpgrade := createReconciler()
-	err := rcRollingUpgrade.preDrainHelper(instance)
+	err := rcRollingUpgrade.preDrainHelper("test-instance-id", "test", instance)
 	g.Expect(err.Error()).To(gomega.HavePrefix("Failed to run preDrain script"))
 }
 
@@ -172,7 +138,8 @@ func TestPostDrainHelperPostDrainScriptSuccess(t *testing.T) {
 	ruObj.Spec.PostDrain.Script = "echo Hello, postDrainScript!"
 
 	rcRollingUpgrade := createReconciler()
-	err := rcRollingUpgrade.postDrainHelper(ruObj, mockNode, mockKubeCtlCall)
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
+	err := rcRollingUpgrade.postDrainHelper("test-instance-id", mockNode, ruObj)
 
 	g.Expect(err).To(gomega.BeNil())
 }
@@ -186,7 +153,8 @@ func TestPostDrainHelperPostDrainScriptError(t *testing.T) {
 	ruObj.Spec.PostDrain.Script = "exit 1"
 
 	rcRollingUpgrade := createReconciler()
-	err := rcRollingUpgrade.postDrainHelper(ruObj, mockNode, mockKubeCtlCall)
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
+	err := rcRollingUpgrade.postDrainHelper("test-instance-id", mockNode, ruObj)
 
 	g.Expect(err).To(gomega.Not(gomega.BeNil()))
 }
@@ -201,7 +169,8 @@ func TestPostDrainHelperPostDrainWaitScriptSuccess(t *testing.T) {
 	ruObj.Spec.PostDrainDelaySeconds = 0
 
 	rcRollingUpgrade := createReconciler()
-	err := rcRollingUpgrade.postDrainHelper(ruObj, mockNode, mockKubeCtlCall)
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
+	err := rcRollingUpgrade.postDrainHelper("test-instance-id", mockNode, ruObj)
 
 	g.Expect(err).To(gomega.BeNil())
 }
@@ -216,7 +185,8 @@ func TestPostDrainHelperPostDrainWaitScriptError(t *testing.T) {
 	ruObj.Spec.PostDrainDelaySeconds = 0
 
 	rcRollingUpgrade := createReconciler()
-	err := rcRollingUpgrade.postDrainHelper(ruObj, mockNode, mockKubeCtlCall)
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
+	err := rcRollingUpgrade.postDrainHelper("test-instance-id", mockNode, ruObj)
 
 	g.Expect(err).To(gomega.Not(gomega.BeNil()))
 }
@@ -228,8 +198,9 @@ func TestDrainNodeSuccess(t *testing.T) {
 
 	ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}
 	rcRollingUpgrade := createReconciler()
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
 
-	err := rcRollingUpgrade.DrainNode(ruObj, mockNode, mockKubeCtlCall, ruObj.Spec.Strategy.DrainTimeout)
+	err := rcRollingUpgrade.DrainNode(ruObj, mockNode, "test-id", ruObj.Spec.Strategy.DrainTimeout)
 	g.Expect(err).To(gomega.BeNil())
 }
 
@@ -241,8 +212,9 @@ func TestDrainNodePreDrainError(t *testing.T) {
 	ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}
 	ruObj.Spec.PreDrain.Script = "exit 1"
 	rcRollingUpgrade := createReconciler()
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
 
-	err := rcRollingUpgrade.DrainNode(ruObj, mockNode, mockKubeCtlCall, ruObj.Spec.Strategy.DrainTimeout)
+	err := rcRollingUpgrade.DrainNode(ruObj, mockNode, "test-id", ruObj.Spec.Strategy.DrainTimeout)
 	g.Expect(err).To(gomega.Not(gomega.BeNil()))
 }
 
@@ -254,8 +226,9 @@ func TestDrainNodePostDrainScriptError(t *testing.T) {
 	ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}
 	ruObj.Spec.PostDrain.Script = "exit 1"
 	rcRollingUpgrade := createReconciler()
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
 
-	err := rcRollingUpgrade.DrainNode(ruObj, mockNode, mockKubeCtlCall, ruObj.Spec.Strategy.DrainTimeout)
+	err := rcRollingUpgrade.DrainNode(ruObj, mockNode, "test-id", ruObj.Spec.Strategy.DrainTimeout)
 	g.Expect(err).To(gomega.Not(gomega.BeNil()))
 }
 
@@ -267,8 +240,9 @@ func TestDrainNodePostDrainWaitScriptError(t *testing.T) {
 	ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}
 	ruObj.Spec.PostDrain.PostWaitScript = "exit 1"
 	rcRollingUpgrade := createReconciler()
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
 
-	err := rcRollingUpgrade.DrainNode(ruObj, mockNode, mockKubeCtlCall, ruObj.Spec.Strategy.DrainTimeout)
+	err := rcRollingUpgrade.DrainNode(ruObj, mockNode, "test-id", ruObj.Spec.Strategy.DrainTimeout)
 	g.Expect(err).To(gomega.Not(gomega.BeNil()))
 }
 
@@ -281,8 +255,9 @@ func TestDrainNodePostDrainFailureToDrainNotFound(t *testing.T) {
 
 	ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}
 	rcRollingUpgrade := createReconciler()
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
 
-	err := rcRollingUpgrade.DrainNode(ruObj, mockNode, mockKubeCtlCall, ruObj.Spec.Strategy.DrainTimeout)
+	err := rcRollingUpgrade.DrainNode(ruObj, mockNode, "test-id", ruObj.Spec.Strategy.DrainTimeout)
 	g.Expect(err).To(gomega.BeNil())
 }
 
@@ -300,8 +275,9 @@ func TestDrainNodePostDrainFailureToDrain(t *testing.T) {
 		},
 	}
 	rcRollingUpgrade := createReconciler()
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
 
-	err := rcRollingUpgrade.DrainNode(ruObj, mockNode, mockKubeCtlCall, ruObj.Spec.Strategy.DrainTimeout)
+	err := rcRollingUpgrade.DrainNode(ruObj, mockNode, "test-id", ruObj.Spec.Strategy.DrainTimeout)
 	g.Expect(err).To(gomega.Not(gomega.BeNil()))
 }
 
@@ -309,6 +285,7 @@ func createReconciler() *RollingUpgradeReconciler {
 	return &RollingUpgradeReconciler{
 		ClusterState: NewClusterState(),
 		Log:          log2.NullLogger{},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
 }
 
@@ -417,6 +394,7 @@ func TestGetInProgressInstances(t *testing.T) {
 			errorFlag: false,
 			awsErr:    nil,
 		},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
 	inProgressInstances, err := reconciler.getInProgressInstances(mockInstances)
 	g.Expect(err).To(gomega.BeNil())
@@ -437,9 +415,10 @@ func TestTerminateNodeSuccess(t *testing.T) {
 			errorFlag: false,
 			awsErr:    nil,
 		},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
 
-	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode)
+	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode, "")
 	g.Expect(err).To(gomega.BeNil())
 }
 
@@ -456,9 +435,10 @@ func TestTerminateNodeErrorNotFound(t *testing.T) {
 		Log:          log2.NullLogger{},
 		ASGClient:    mockAutoscalingGroup,
 		EC2Client:    MockEC2{},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
 
-	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode)
+	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode, "")
 	g.Expect(err).To(gomega.BeNil())
 }
 
@@ -481,6 +461,7 @@ func TestTerminateNodeErrorScalingActivityInProgressWithRetry(t *testing.T) {
 		Log:          log2.NullLogger{},
 		ASGClient:    mockAutoscalingGroup,
 		EC2Client:    MockEC2{},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
 	go func() {
 		time.Sleep(WaiterMaxDelay)
@@ -489,7 +470,7 @@ func TestTerminateNodeErrorScalingActivityInProgressWithRetry(t *testing.T) {
 			awsErr:    nil,
 		}
 	}()
-	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode)
+	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode, "")
 	g.Expect(err).To(gomega.BeNil())
 }
 
@@ -506,8 +487,9 @@ func TestTerminateNodeErrorScalingActivityInProgress(t *testing.T) {
 		Log:          log2.NullLogger{},
 		ASGClient:    mockAutoscalingGroup,
 		EC2Client:    MockEC2{},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
-	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode)
+	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode, "")
 	g.Expect(err.Error()).To(gomega.ContainSubstring("No more retries left"))
 }
 
@@ -524,9 +506,10 @@ func TestTerminateNodeErrorResourceContention(t *testing.T) {
 		Log:          log2.NullLogger{},
 		ASGClient:    mockAutoscalingGroup,
 		EC2Client:    MockEC2{},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
 
-	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode)
+	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode, "")
 	g.Expect(err.Error()).To(gomega.ContainSubstring("No more retries left"))
 }
 
@@ -544,8 +527,9 @@ func TestTerminateNodeErrorOtherError(t *testing.T) {
 		Log:          log2.NullLogger{},
 		ASGClient:    mockAutoscalingGroup,
 		EC2Client:    MockEC2{},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
-	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode)
+	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode, "")
 	g.Expect(err.Error()).To(gomega.ContainSubstring("some error"))
 }
 
@@ -561,8 +545,9 @@ func TestTerminateNodePostTerminateScriptSuccess(t *testing.T) {
 		Log:          log2.NullLogger{},
 		ASGClient:    mockAutoscalingGroup,
 		EC2Client:    MockEC2{},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
-	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode)
+	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode, "")
 	g.Expect(err).To(gomega.BeNil())
 }
 
@@ -578,8 +563,9 @@ func TestTerminateNodePostTerminateScriptErrorNotFoundFromServer(t *testing.T) {
 		Log:          log2.NullLogger{},
 		ASGClient:    mockAutoscalingGroup,
 		EC2Client:    MockEC2{},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
-	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode)
+	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode, "")
 	g.Expect(err).To(gomega.BeNil())
 }
 
@@ -595,8 +581,9 @@ func TestTerminateNodePostTerminateScriptErrorOtherError(t *testing.T) {
 		Log:          log2.NullLogger{},
 		ASGClient:    mockAutoscalingGroup,
 		EC2Client:    MockEC2{},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
-	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode)
+	err := rcRollingUpgrade.TerminateNode(ruObj, mockNode, "")
 	g.Expect(err).To(gomega.Not(gomega.BeNil()))
 	g.Expect(err.Error()).To(gomega.HavePrefix("Failed to run postTerminate script: "))
 }
@@ -604,19 +591,18 @@ func TestTerminateNodePostTerminateScriptErrorOtherError(t *testing.T) {
 func TestLoadEnvironmentVariables(t *testing.T) {
 	g := gomega.NewGomegaWithT(t)
 
-	ruInstance := &upgrademgrv1alpha1.RollingUpgrade{
-		ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
-		Spec:       upgrademgrv1alpha1.RollingUpgradeSpec{AsgName: "asg-foo"}}
+	r := &ScriptRunner{}
 
 	mockID := "fake-id-foo"
 	mockName := "instance-name-foo"
 
-	err := loadEnvironmentVariables(ruInstance, mockID, mockName)
-	g.Expect(err).To(gomega.BeNil())
+	env := r.buildEnv(&upgrademgrv1alpha1.RollingUpgrade{
+		Spec: upgrademgrv1alpha1.RollingUpgradeSpec{
+			AsgName: "asg-foo",
+		},
+	}, mockID, mockName)
+	g.Expect(env).To(gomega.HaveLen(3))
 
-	g.Expect(os.Getenv(asgNameKey)).To(gomega.Equal("asg-foo"))
-	g.Expect(os.Getenv(instanceIDKey)).To(gomega.Equal("fake-id-foo"))
-	g.Expect(os.Getenv(instanceNameKey)).To(gomega.Equal("instance-name-foo"))
 }
 
 func TestGetNodeNameFoundNode(t *testing.T) {
@@ -716,6 +702,7 @@ func TestPopulateAsgSuccess(t *testing.T) {
 		ClusterState: NewClusterState(),
 		ASGClient:    mockAsgClient,
 		EC2Client:    MockEC2{},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
 	err := rcRollingUpgrade.populateAsg(ruObj)
 
@@ -749,6 +736,7 @@ func TestPopulateAsgTooMany(t *testing.T) {
 		ClusterState: NewClusterState(),
 		ASGClient:    mockAsgClient,
 		EC2Client:    MockEC2{},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
 	err := rcRollingUpgrade.populateAsg(ruObj)
 
@@ -880,7 +868,6 @@ func TestFinishExecutionCompleted(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	rcRollingUpgrade := &RollingUpgradeReconciler{Client: mgr.GetClient(),
 		generatedClient: kubernetes.NewForConfigOrDie(mgr.GetConfig()),
@@ -915,7 +902,7 @@ func TestFinishExecutionError(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
+
 	rcRollingUpgrade := &RollingUpgradeReconciler{
 		Client:          mgr.GetClient(),
 		Log:             log2.NullLogger{},
@@ -966,7 +953,6 @@ func TestRunRestackSuccessOneNode(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	fooNode1 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/9213851"}}
 	fooNode2 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/1234501"}}
@@ -993,7 +979,7 @@ func TestRunRestackSuccessOneNode(t *testing.T) {
 
 	ctx := context.TODO()
 
-	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj, "exit 0;")
+	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj)
 	g.Expect(nodesProcessed).To(gomega.Equal(1))
 	g.Expect(err).To(gomega.BeNil())
 	_, exists := rcRollingUpgrade.inProcessASGs.Load(someAsg)
@@ -1020,7 +1006,6 @@ func TestRunRestackSuccessMultipleNodes(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	fooNode1 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/9213851"}}
 	fooNode2 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/1234501"}}
@@ -1047,7 +1032,7 @@ func TestRunRestackSuccessMultipleNodes(t *testing.T) {
 
 	ctx := context.TODO()
 
-	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj, "exit 0;")
+	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj)
 	g.Expect(nodesProcessed).To(gomega.Equal(2))
 	g.Expect(err).To(gomega.BeNil())
 }
@@ -1069,7 +1054,6 @@ func TestRunRestackSameLaunchConfig(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	rcRollingUpgrade := &RollingUpgradeReconciler{
 		Client:          mgr.GetClient(),
@@ -1088,7 +1072,7 @@ func TestRunRestackSameLaunchConfig(t *testing.T) {
 	ctx := context.TODO()
 
 	// This execution should not perform drain or termination, but should pass
-	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj, KubeCtlBinary)
+	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj)
 	g.Expect(nodesProcessed).To(gomega.Equal(1))
 	g.Expect(err).To(gomega.BeNil())
 }
@@ -1106,7 +1090,7 @@ func TestRunRestackRollingUpgradeNotInMap(t *testing.T) {
 	ctx := context.TODO()
 
 	g.Expect(rcRollingUpgrade.ruObjNameToASG.Load(ruObj.Name)).To(gomega.BeNil())
-	int, err := rcRollingUpgrade.runRestack(&ctx, ruObj, KubeCtlBinary)
+	int, err := rcRollingUpgrade.runRestack(&ctx, ruObj)
 	g.Expect(int).To(gomega.Equal(0))
 	g.Expect(err).To(gomega.Not(gomega.BeNil()))
 	g.Expect(err.Error()).To(gomega.HavePrefix("Unable to load ASG with name: foo"))
@@ -1130,7 +1114,6 @@ func TestRunRestackRollingUpgradeNodeNameNotFound(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	emptyNodeList := corev1.NodeList{}
 	rcRollingUpgrade := &RollingUpgradeReconciler{
@@ -1151,7 +1134,7 @@ func TestRunRestackRollingUpgradeNodeNameNotFound(t *testing.T) {
 	ctx := context.TODO()
 
 	// This execution gets past the different launch config check, but fails to be found at the node level
-	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj, KubeCtlBinary)
+	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj)
 	g.Expect(nodesProcessed).To(gomega.Equal(1))
 	g.Expect(err).To(gomega.BeNil())
 }
@@ -1174,7 +1157,6 @@ func TestRunRestackNoNodeName(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	fooNode1 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/9213851"}}
 	fooNode2 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/1234501"}}
@@ -1200,7 +1182,7 @@ func TestRunRestackNoNodeName(t *testing.T) {
 	ctx := context.TODO()
 
 	// This execution gets past the different launch config check, but since there is no node name, it is skipped
-	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj, KubeCtlBinary)
+	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj)
 	g.Expect(nodesProcessed).To(gomega.Equal(1))
 	g.Expect(err).To(gomega.BeNil())
 }
@@ -1239,7 +1221,6 @@ func TestRunRestackDrainNodeFail(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	fooNode1 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/9213851"}}
 	fooNode2 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/1234501"}}
@@ -1266,7 +1247,7 @@ func TestRunRestackDrainNodeFail(t *testing.T) {
 	ctx := context.TODO()
 
 	// This execution gets past the different launch config check, but fails to drain the node because of a predrain failing script
-	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj, KubeCtlBinary)
+	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj)
 
 	g.Expect(nodesProcessed).To(gomega.Equal(1))
 	g.Expect(err.Error()).To(gomega.HavePrefix("Error updating instances, ErrorCount: 1, Errors: ["))
@@ -1301,7 +1282,6 @@ func TestRunRestackTerminateNodeFail(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	fooNode1 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/9213851"}}
 	fooNode2 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/1234501"}}
@@ -1321,6 +1301,7 @@ func TestRunRestackTerminateNodeFail(t *testing.T) {
 		NodeList:        &nodeList,
 		ClusterState:    NewClusterState(),
 		CacheConfig:     cache.NewConfig(0*time.Second, 0, 0),
+		ScriptRunner:    NewScriptRunner(log2.NullLogger{}),
 	}
 	rcRollingUpgrade.admissionMap.Store(ruObj.Name, "processing")
 	rcRollingUpgrade.ruObjNameToASG.Store(ruObj.Name, &mockAsg)
@@ -1328,7 +1309,7 @@ func TestRunRestackTerminateNodeFail(t *testing.T) {
 	ctx := context.TODO()
 
 	// This execution gets past the different launch config check, but fails to terminate node
-	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj, "exit 0;")
+	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj)
 	g.Expect(nodesProcessed).To(gomega.Equal(1))
 	g.Expect(err.Error()).To(gomega.HavePrefix("Error updating instances, ErrorCount: 1, Errors: ["))
 	g.Expect(err.Error()).To(gomega.ContainSubstring("some error"))
@@ -1375,7 +1356,6 @@ func TestUniformAcrossAzUpdateSuccessMultipleNodes(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	fooNode1 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/9213851"}}
 	fooNode2 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/1234501"}}
@@ -1421,7 +1401,7 @@ func TestUniformAcrossAzUpdateSuccessMultipleNodes(t *testing.T) {
 
 	ctx := context.TODO()
 
-	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj, "exit 0;")
+	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj)
 	g.Expect(nodesProcessed).To(gomega.Equal(9))
 	g.Expect(err).To(gomega.BeNil())
 }
@@ -1453,7 +1433,6 @@ func TestUpdateInstances(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	fooNode1 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/9213851"}}
 	fooNode2 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/1234501"}}
@@ -1474,6 +1453,7 @@ func TestUpdateInstances(t *testing.T) {
 		NodeList:        &nodeList,
 		ClusterState:    NewClusterState(),
 		CacheConfig:     cache.NewConfig(0*time.Second, 0, 0),
+		ScriptRunner:    NewScriptRunner(log2.NullLogger{}),
 	}
 	rcRollingUpgrade.admissionMap.Store(ruObj.Name, "processing")
 	rcRollingUpgrade.ruObjNameToASG.Store(ruObj.Name, &mockAsg)
@@ -1481,8 +1461,10 @@ func TestUpdateInstances(t *testing.T) {
 	ctx := context.TODO()
 
 	lcName := "A"
+	rcRollingUpgrade.ScriptRunner.KubectlCall = "exit 0;"
+
 	err = rcRollingUpgrade.UpdateInstances(&ctx,
-		ruObj, mockAsg.Instances, &launchDefinition{launchConfigurationName: &lcName}, "exit 0;")
+		ruObj, mockAsg.Instances, &launchDefinition{launchConfigurationName: &lcName})
 	g.Expect(err).ShouldNot(gomega.HaveOccurred())
 }
 
@@ -1519,7 +1501,6 @@ func TestUpdateInstancesError(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	fooNode1 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/9213851"}}
 	fooNode2 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/1234501"}}
@@ -1540,15 +1521,17 @@ func TestUpdateInstancesError(t *testing.T) {
 		NodeList:        &nodeList,
 		ClusterState:    NewClusterState(),
 		CacheConfig:     cache.NewConfig(0*time.Second, 0, 0),
+		ScriptRunner:    NewScriptRunner(log2.NullLogger{}),
 	}
 	rcRollingUpgrade.admissionMap.Store(ruObj.Name, "processing")
 	rcRollingUpgrade.ruObjNameToASG.Store(ruObj.Name, &mockAsg)
+	rcRollingUpgrade.ScriptRunner.KubectlCall = "exit 0;"
 
 	ctx := context.TODO()
 
 	lcName := "A"
 	err = rcRollingUpgrade.UpdateInstances(&ctx,
-		ruObj, mockAsg.Instances, &launchDefinition{launchConfigurationName: &lcName}, "exit 0;")
+		ruObj, mockAsg.Instances, &launchDefinition{launchConfigurationName: &lcName})
 	g.Expect(err).Should(gomega.HaveOccurred())
 	g.Expect(err).Should(gomega.BeAssignableToTypeOf(&UpdateInstancesError{}))
 	if updateInstancesError, ok := err.(*UpdateInstancesError); ok {
@@ -1592,7 +1575,6 @@ func TestUpdateInstancesPartialError(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	fooNode1 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/9213851"}}
 	fooNode2 := corev1.Node{Spec: corev1.NodeSpec{ProviderID: "foo-bar/1234501"}}
@@ -1613,15 +1595,16 @@ func TestUpdateInstancesPartialError(t *testing.T) {
 		NodeList:        &nodeList,
 		ClusterState:    NewClusterState(),
 		CacheConfig:     cache.NewConfig(0*time.Second, 0, 0),
+		ScriptRunner:    NewScriptRunner(log2.NullLogger{}),
 	}
 	rcRollingUpgrade.admissionMap.Store(ruObj.Name, "processing")
 	rcRollingUpgrade.ruObjNameToASG.Store(ruObj.Name, &mockAsg)
-
+	rcRollingUpgrade.ScriptRunner.KubectlCall = "exit 0;"
 	ctx := context.TODO()
 
 	lcName := "A"
 	err = rcRollingUpgrade.UpdateInstances(&ctx,
-		ruObj, mockAsg.Instances, &launchDefinition{launchConfigurationName: &lcName}, "exit 0;")
+		ruObj, mockAsg.Instances, &launchDefinition{launchConfigurationName: &lcName})
 	g.Expect(err).Should(gomega.HaveOccurred())
 	g.Expect(err).Should(gomega.BeAssignableToTypeOf(&UpdateInstancesError{}))
 	if updateInstancesError, ok := err.(*UpdateInstancesError); ok {
@@ -1635,7 +1618,6 @@ func TestUpdateInstancesWithZeroInstances(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	rcRollingUpgrade := &RollingUpgradeReconciler{
 		Client:          mgr.GetClient(),
@@ -1645,13 +1627,15 @@ func TestUpdateInstancesWithZeroInstances(t *testing.T) {
 		ruObjNameToASG:  sync.Map{},
 		ClusterState:    NewClusterState(),
 		CacheConfig:     cache.NewConfig(0*time.Second, 0, 0),
+		ScriptRunner:    NewScriptRunner(log2.NullLogger{}),
 	}
+	rcRollingUpgrade.ScriptRunner.KubectlCall = "exit 0;"
 
 	ctx := context.TODO()
 
 	lcName := "A"
 	err = rcRollingUpgrade.UpdateInstances(&ctx,
-		nil, nil, &launchDefinition{launchConfigurationName: &lcName}, "exit 0;")
+		nil, nil, &launchDefinition{launchConfigurationName: &lcName})
 	g.Expect(err).ShouldNot(gomega.HaveOccurred())
 }
 
@@ -1662,13 +1646,14 @@ func TestTestCallKubectlDrainWithoutDrainTimeout(t *testing.T) {
 	mockNodeName := "some-node-name"
 	mockAsgName := "some-asg"
 	rcRollingUpgrade := createReconciler()
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
 	ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
 		Spec: upgrademgrv1alpha1.RollingUpgradeSpec{AsgName: mockAsgName}}
 
 	errChan := make(chan error)
 	ctx := context.TODO()
 
-	go rcRollingUpgrade.CallKubectlDrain(ctx, mockNodeName, mockKubeCtlCall, ruObj, errChan)
+	go rcRollingUpgrade.CallKubectlDrain(mockNodeName, ruObj, errChan)
 
 	output := ""
 	select {
@@ -1697,6 +1682,7 @@ func TestTestCallKubectlDrainWithDrainTimeout(t *testing.T) {
 	mockKubeCtlCall := "sleep 1; echo"
 	mockNodeName := "some-node-name"
 	rcRollingUpgrade := createReconciler()
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
 
 	mockAsgName := "some-asg"
 	ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
@@ -1707,7 +1693,7 @@ func TestTestCallKubectlDrainWithDrainTimeout(t *testing.T) {
 	ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
 	defer cancel()
 
-	go rcRollingUpgrade.CallKubectlDrain(ctx, mockNodeName, mockKubeCtlCall, ruObj, errChan)
+	go rcRollingUpgrade.CallKubectlDrain(mockNodeName, ruObj, errChan)
 
 	output := ""
 	select {
@@ -1736,6 +1722,7 @@ func TestTestCallKubectlDrainWithZeroDrainTimeout(t *testing.T) {
 	mockKubeCtlCall := "sleep 1; echo"
 	mockNodeName := "some-node-name"
 	rcRollingUpgrade := createReconciler()
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
 
 	mockAsgName := "some-asg"
 	ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
@@ -1746,7 +1733,7 @@ func TestTestCallKubectlDrainWithZeroDrainTimeout(t *testing.T) {
 	ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
 	defer cancel()
 
-	go rcRollingUpgrade.CallKubectlDrain(ctx, mockNodeName, mockKubeCtlCall, ruObj, errChan)
+	go rcRollingUpgrade.CallKubectlDrain(mockNodeName, ruObj, errChan)
 
 	output := ""
 	select {
@@ -1775,6 +1762,7 @@ func TestTestCallKubectlDrainWithError(t *testing.T) {
 	mockKubeCtlCall := "cat xyz"
 	mockNodeName := "some-node-name"
 	rcRollingUpgrade := createReconciler()
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
 
 	mockAsgName := "some-asg"
 	ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
@@ -1783,7 +1771,7 @@ func TestTestCallKubectlDrainWithError(t *testing.T) {
 	errChan := make(chan error)
 	ctx := context.TODO()
 
-	go rcRollingUpgrade.CallKubectlDrain(ctx, mockNodeName, mockKubeCtlCall, ruObj, errChan)
+	go rcRollingUpgrade.CallKubectlDrain(mockNodeName, ruObj, errChan)
 
 	output := ""
 	select {
@@ -1812,6 +1800,7 @@ func TestTestCallKubectlDrainWithTimeoutOccurring(t *testing.T) {
 	mockKubeCtlCall := "sleep 1; echo"
 	mockNodeName := "some-node-name"
 	rcRollingUpgrade := createReconciler()
+	rcRollingUpgrade.ScriptRunner.KubectlCall = mockKubeCtlCall
 
 	mockAsgName := "some-asg"
 	ruObj := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
@@ -1822,7 +1811,7 @@ func TestTestCallKubectlDrainWithTimeoutOccurring(t *testing.T) {
 	ctx, cancel := context.WithTimeout(ctx, 50*time.Millisecond)
 	defer cancel()
 
-	go rcRollingUpgrade.CallKubectlDrain(ctx, mockNodeName, mockKubeCtlCall, ruObj, errChan)
+	go rcRollingUpgrade.CallKubectlDrain(mockNodeName, ruObj, errChan)
 
 	output := ""
 	select {
@@ -2250,7 +2239,6 @@ func TestRunRestackNoNodeInAsg(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	nodeList := corev1.NodeList{Items: []corev1.Node{}}
 	rcRollingUpgrade := &RollingUpgradeReconciler{
@@ -2270,7 +2258,7 @@ func TestRunRestackNoNodeInAsg(t *testing.T) {
 	ctx := context.TODO()
 
 	// This execution gets past the different launch config check, but since there is no node name, it is skipped
-	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj, KubeCtlBinary)
+	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj)
 	g.Expect(nodesProcessed).To(gomega.Equal(0))
 	g.Expect(err).To(gomega.BeNil())
 }
@@ -2395,7 +2383,6 @@ func TestRunRestackWithNodesLessThanMaxUnavailable(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	rcRollingUpgrade := &RollingUpgradeReconciler{
 		Client:          mgr.GetClient(),
@@ -2413,7 +2400,7 @@ func TestRunRestackWithNodesLessThanMaxUnavailable(t *testing.T) {
 	ctx := context.TODO()
 
 	// This execution should not perform drain or termination, but should pass
-	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj, KubeCtlBinary)
+	nodesProcessed, err := rcRollingUpgrade.runRestack(&ctx, ruObj)
 	g.Expect(err).To(gomega.BeNil())
 	g.Expect(nodesProcessed).To(gomega.Equal(1))
 }
@@ -2662,10 +2649,11 @@ func TestDrainNodeTerminateTerminatesWhenIgnoreDrainFailuresSet(t *testing.T) {
 			errorFlag: false,
 			awsErr:    nil,
 		},
+		ScriptRunner: NewScriptRunner(log2.NullLogger{}),
 	}
 
 	ch := make(chan error, 1)
-	rcRollingUpgrade.DrainTerminate(ruObj, mockNode, mockNode, KubeCtlBinary, ch)
+	rcRollingUpgrade.DrainTerminate(ruObj, mockNode, mockNode, ch)
 
 	select {
 	case err, ok := <-ch:
@@ -2678,7 +2666,7 @@ func TestDrainNodeTerminateTerminatesWhenIgnoreDrainFailuresSet(t *testing.T) {
 
 	// nodeName is empty when node isn't part of the cluster. It must skip drain and terminate.
 	ch = make(chan error, 1)
-	rcRollingUpgrade.DrainTerminate(ruObj, "", mockNode, KubeCtlBinary, ch)
+	rcRollingUpgrade.DrainTerminate(ruObj, "", mockNode, ch)
 
 	select {
 	case err, ok := <-ch:
@@ -2731,6 +2719,8 @@ func TestUpdateInstancesNotExists(t *testing.T) {
 		CacheConfig:     cache.NewConfig(0*time.Second, 0, 0),
 	}
 	rcRollingUpgrade.ruObjNameToASG.Store(ruObj.Name, &mockAsg)
+	rcRollingUpgrade.ScriptRunner.KubectlCall = "date"
+
 	// Intentionally do not populate the admissionMap with the ruObj
 
 	ctx := context.TODO()
@@ -2738,7 +2728,7 @@ func TestUpdateInstancesNotExists(t *testing.T) {
 	instChan := make(chan error)
 	mockInstanceName1 := "foo1"
 	instance1 := autoscaling.Instance{InstanceId: &mockInstanceName1, AvailabilityZone: &az}
-	go rcRollingUpgrade.UpdateInstance(&ctx, ruObj, &instance1, &launchDefinition{launchConfigurationName: &lcName}, "date", instChan)
+	go rcRollingUpgrade.UpdateInstance(&ctx, ruObj, &instance1, &launchDefinition{launchConfigurationName: &lcName}, instChan)
 	processCount := 0
 	select {
 	case <-ctx.Done():
@@ -2770,7 +2760,6 @@ func TestValidateNodesLaunchDefinitionSameLaunchConfig(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	mockAsgClient := MockAutoscalingGroup{
 		autoScalingGroups: []*autoscaling.Group{mockAsg},
@@ -2812,7 +2801,6 @@ func TestValidateNodesLaunchDefinitionDifferentLaunchConfig(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	mockAsgClient := MockAutoscalingGroup{
 		autoScalingGroups: []*autoscaling.Group{mockAsg},
@@ -2852,7 +2840,6 @@ func TestValidateNodesLaunchDefinitionSameLaunchTemplate(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	mockAsgClient := MockAutoscalingGroup{
 		autoScalingGroups: []*autoscaling.Group{mockAsg},
@@ -2893,7 +2880,6 @@ func TestValidateNodesLaunchDefinitionDifferentLaunchTemplate(t *testing.T) {
 
 	mgr, err := buildManager()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	c = mgr.GetClient()
 
 	mockAsgClient := MockAutoscalingGroup{
 		autoScalingGroups: []*autoscaling.Group{mockAsg},
diff --git a/controllers/script_runner_test.go b/controllers/script_runner_test.go
new file mode 100644
index 0000000..a09f433
--- /dev/null
+++ b/controllers/script_runner_test.go
@@ -0,0 +1,52 @@
+package controllers
+
+import (
+	upgrademgrv1alpha1 "github.com/keikoproj/upgrade-manager/api/v1alpha1"
+	"github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtimelog "sigs.k8s.io/controller-runtime/pkg/log"
+	"testing"
+)
+
+func TestEchoScript(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	ru := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}
+	r := &ScriptRunner{Log: runtimelog.NullLogger{}}
+	out, err := r.runScript("echo hello", false, ru)
+
+	g.Expect(err).To(gomega.BeNil())
+	g.Expect(out).To(gomega.Equal("hello\n"))
+}
+
+func TestEchoScriptWithEnv(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	ru := &upgrademgrv1alpha1.RollingUpgrade{
+		Spec: upgrademgrv1alpha1.RollingUpgradeSpec{AsgName: "my-asg"},
+	}
+	r := &ScriptRunner{Log: runtimelog.NullLogger{}}
+	env := r.buildEnv(ru, "testInstanceID", "testNodeName")
+	out, err := r.runScriptWithEnv("echo $INSTANCE_ID:$ASG_NAME:$INSTANCE_NAME", false, ru, env)
+
+	g.Expect(err).To(gomega.BeNil())
+	g.Expect(out).To(gomega.Equal("testInstanceID:my-asg:testNodeName\n"))
+}
+
+func TestEchoBackgroundScript(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	ru := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}
+	r := &ScriptRunner{Log: runtimelog.NullLogger{}}
+	out, err := r.runScript("echo background", true, ru)
+
+	g.Expect(err).To(gomega.BeNil())
+	g.Expect(out).To(gomega.Equal(""))
+}
+
+func TestRunScriptFailure(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	ru := &upgrademgrv1alpha1.RollingUpgrade{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}
+	r := &ScriptRunner{Log: runtimelog.NullLogger{}}
+	out, err := r.runScript("echo this will fail; exit 1", false, ru)
+
+	g.Expect(err).To(gomega.Not(gomega.BeNil()))
+	g.Expect(out).To(gomega.Not(gomega.Equal("")))
+}