Repository: keikoproj/upgrade-manager
Pull request #49 (2020-01-18 20:39:12)

Eager mode: Launch Before Terminate
Contributes to #45 

This implements an addition to `UpdateStrategy` where you can now add `UpdateStrategy.Mode` to either be `Lazy` (default) or `Eager`.
`Lazy` will use the same approach we currently have of terminating first and launching after.
`Eager` will put an instance in standby to cause scale up and only then proceed with termination sequence.

Manual tests this seems to work fine, still need to add unit tests for this code

TBD:
- [x] Unit Tests

Example Spec:
```
apiVersion: upgrademgr.keikoproj.io/v1alpha1
kind: RollingUpgrade
metadata:
  name: my-rollup
  namespace: kube-system
spec:
  asgName: my-asg.cluster.k8s.local
  nodeIntervalSeconds: 300
  postDrainDelaySeconds: 300
  region: us-west-2
  strategy:
    mode: eager
    drainTimeout: 600
    type: randomUpdate
    maxUnavailable: "100%"
```


Patch:
diff --git a/.gitignore b/.gitignore
index e758550..558f504 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,4 @@ bin
 vendor/
 config/default/manager_image_patch.yaml-e
 .vscode/
+cover.html
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 83494c5..f431af2 100644
--- a/Makefile
+++ b/Makefile
@@ -11,6 +11,7 @@ all: manager
 # Run tests
 test: generate fmt vet manifests
 	go test ./api/... ./controllers/... -coverprofile cover.out
+	go tool cover -html=./cover.out -o cover.html
 
 # Build manager binary
 manager: generate fmt vet
diff --git a/README.md b/README.md
index 8f8cca4..3c35322 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,9 @@ RollingUpgrade provides a Kubernetes native mechanism for doing rolling-updates
   - `postDrain.postWaitScript`: The script to run after the node is drained and the waitSeconds have passed. This can be used for ensuring that the drained pods actually were able to start elsewhere.
   - `nodeIntervalSeconds`: The amount of time in seconds to wait after each node in the ASG is terminated.
   - `postTerminate.script`: Optional bash script to execute after the node has terminated.
+  - `strategy.mode`: This field is optional and allows for two possible modes
+    - `lazy` - this is the default mode, upgrade will terminate an instance first.
+    - `eager` - upgrade will launch an instance prior to terminating.
   - `strategy.type`: This field is optional and currently two strategies are supported
     - `randomUpdate` - Default is type is not specified. Picks nodes randomly for updating. Refer to [random_update_strategy.yaml](examples/random_update_strategy.yaml) for sample custom resource definition.
     - `uniformAcrossAzUpdate` - Picks same number of nodes or same percentage of nodes from each AZ for update. Refer to [uniform_across_az_update_strategy.yaml](examples/uniform_across_az_update_strategy.yaml) for sample custom resource definition.
@@ -65,6 +68,7 @@ If you already have an existing cluster created using kops, follow the instructi
     "Action": [
         "ec2:CreateTags",
         "ec2:DescribeInstances",
+        "autoscaling:EnterStandby",
         "autoscaling:DescribeAutoScalingGroups",
         "autoscaling:TerminateInstanceInAutoScalingGroup"
     ],
diff --git a/api/v1alpha1/rollingupgrade_types.go b/api/v1alpha1/rollingupgrade_types.go
index 4fb4468..f4b7207 100644
--- a/api/v1alpha1/rollingupgrade_types.go
+++ b/api/v1alpha1/rollingupgrade_types.go
@@ -91,6 +91,8 @@ func init() {
 // whether to roll the update Az wise or all Azs at once
 type UpdateStrategyType string
 
+type UpdateStrategyMode string
+
 const (
 	// RandomUpdate strategy treats all the Azs as a single unit and picks random nodes for update
 	RandomUpdateStrategy UpdateStrategyType = "randomUpdate"
@@ -98,13 +100,20 @@ const (
 	// RandomUpdate strategy treats all the Azs as a single unit and picks random nodes for update
 	UniformAcrossAzUpdateStrategy UpdateStrategyType = "uniformAcrossAzUpdate"
 
+	UpdateStrategyModeLazy  UpdateStrategyMode = "lazy"
+	UpdateStrategyModeEager UpdateStrategyMode = "eager"
 	// Other update strategies such as rolling update by Az or rolling update with a predifined instance list
 	// can be implemented in future by adding more update strategy types
 )
 
+func (c UpdateStrategyMode) String() string {
+	return string(c)
+}
+
 // UpdateStrategy holds the information needed to perform update based on different update strategies
 type UpdateStrategy struct {
 	Type UpdateStrategyType `json:"type,omitempty"`
+	Mode UpdateStrategyMode `json:"mode,omitempty"`
 	// MaxUnavailable can be specified as number of nodes or the percent of total number of nodes
 	MaxUnavailable intstr.IntOrString `json:"maxUnavailable,omitempty"`
 	// Node will be terminated after drain timeout even if `kubectl drain` has not been completed
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 33befd8..fb0cc32 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// autogenerated by controller-gen object, do not modify manually
+// Code generated by controller-gen. DO NOT EDIT.
 
 package v1alpha1
 
diff --git a/config/crd/bases/upgrademgr.keikoproj.io_rollingupgrades.yaml b/config/crd/bases/upgrademgr.keikoproj.io_rollingupgrades.yaml
index 4430994..5977e55 100644
--- a/config/crd/bases/upgrademgr.keikoproj.io_rollingupgrades.yaml
+++ b/config/crd/bases/upgrademgr.keikoproj.io_rollingupgrades.yaml
@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.4
   creationTimestamp: null
   name: rollingupgrades.upgrademgr.keikoproj.io
 spec:
@@ -22,9 +24,11 @@ spec:
   group: upgrademgr.keikoproj.io
   names:
     kind: RollingUpgrade
+    listKind: RollingUpgradeList
     plural: rollingupgrades
     shortNames:
     - ru
+    singular: rollingupgrade
   scope: Namespaced
   subresources: {}
   validation:
@@ -42,226 +46,17 @@ spec:
             submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
           type: string
         metadata:
-          properties:
-            annotations:
-              additionalProperties:
-                type: string
-              description: 'Annotations is an unstructured key value map stored with
-                a resource that may be set by external tools to store and retrieve
-                arbitrary metadata. They are not queryable and should be preserved
-                when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
-              type: object
-            clusterName:
-              description: The name of the cluster which the object belongs to. This
-                is used to distinguish resources with same name and namespace in different
-                clusters. This field is not set anywhere right now and apiserver is
-                going to ignore it if set in create or update request.
-              type: string
-            creationTimestamp:
-              description: "CreationTimestamp is a timestamp representing the server
-                time when this object was created. It is not guaranteed to be set
-                in happens-before order across separate operations. Clients may not
-                set this value. It is represented in RFC3339 form and is in UTC. \n
-                Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata"
-              format: date-time
-              type: string
-            deletionGracePeriodSeconds:
-              description: Number of seconds allowed for this object to gracefully
-                terminate before it will be removed from the system. Only set when
-                deletionTimestamp is also set. May only be shortened. Read-only.
-              format: int64
-              type: integer
-            deletionTimestamp:
-              description: "DeletionTimestamp is RFC 3339 date and time at which this
-                resource will be deleted. This field is set by the server when a graceful
-                deletion is requested by the user, and is not directly settable by
-                a client. The resource is expected to be deleted (no longer visible
-                from resource lists, and not reachable by name) after the time in
-                this field, once the finalizers list is empty. As long as the finalizers
-                list contains items, deletion is blocked. Once the deletionTimestamp
-                is set, this value may not be unset or be set further into the future,
-                although it may be shortened or the resource may be deleted prior
-                to this time. For example, a user may request that a pod is deleted
-                in 30 seconds. The Kubelet will react by sending a graceful termination
-                signal to the containers in the pod. After that 30 seconds, the Kubelet
-                will send a hard termination signal (SIGKILL) to the container and
-                after cleanup, remove the pod from the API. In the presence of network
-                partitions, this object may still exist after this timestamp, until
-                an administrator or automated process can determine the resource is
-                fully terminated. If not set, graceful deletion of the object has
-                not been requested. \n Populated by the system when a graceful deletion
-                is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata"
-              format: date-time
-              type: string
-            finalizers:
-              description: Must be empty before the object is deleted from the registry.
-                Each entry is an identifier for the responsible component that will
-                remove the entry from the list. If the deletionTimestamp of the object
-                is non-nil, entries in this list can only be removed. Finalizers may
-                be processed and removed in any order.  Order is NOT enforced because
-                it introduces significant risk of stuck finalizers. finalizers is
-                a shared field, any actor with permission can reorder it. If the finalizer
-                list is processed in order, then this can lead to a situation in which
-                the component responsible for the first finalizer in the list is waiting
-                for a signal (field value, external system, or other) produced by
-                a component responsible for a finalizer later in the list, resulting
-                in a deadlock. Without enforced ordering finalizers are free to order
-                amongst themselves and are not vulnerable to ordering changes in the
-                list.
-              items:
-                type: string
-              type: array
-            generateName:
-              description: "GenerateName is an optional prefix, used by the server,
-                to generate a unique name ONLY IF the Name field has not been provided.
-                If this field is used, the name returned to the client will be different
-                than the name passed. This value will also be combined with a unique
-                suffix. The provided value has the same validation rules as the Name
-                field, and may be truncated by the length of the suffix required to
-                make the value unique on the server. \n If this field is specified
-                and the generated name exists, the server will NOT return a 409 -
-                instead, it will either return 201 Created or 500 with Reason ServerTimeout
-                indicating a unique name could not be found in the time allotted,
-                and the client should retry (optionally after the time indicated in
-                the Retry-After header). \n Applied only if Name is not specified.
-                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency"
-              type: string
-            generation:
-              description: A sequence number representing a specific generation of
-                the desired state. Populated by the system. Read-only.
-              format: int64
-              type: integer
-            labels:
-              additionalProperties:
-                type: string
-              description: 'Map of string keys and values that can be used to organize
-                and categorize (scope and select) objects. May match selectors of
-                replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
-              type: object
-            managedFields:
-              description: ManagedFields maps workflow-id and version to the set of
-                fields that are managed by that workflow. This is mostly for internal
-                housekeeping, and users typically shouldn't need to set or understand
-                this field. A workflow can be the user's name, a controller's name,
-                or the name of a specific apply path like "ci-cd". The set of fields
-                is always in the version that the workflow used when modifying the
-                object.
-              items:
-                properties:
-                  apiVersion:
-                    description: APIVersion defines the version of this resource that
-                      this field set applies to. The format is "group/version" just
-                      like the top-level APIVersion field. It is necessary to track
-                      the version of a field set because it cannot be automatically
-                      converted.
-                    type: string
-                  fieldsType:
-                    description: 'FieldsType is the discriminator for the different
-                      fields format and version. There is currently only one possible
-                      value: "FieldsV1"'
-                    type: string
-                  fieldsV1:
-                    description: FieldsV1 holds the first JSON version format as described
-                      in the "FieldsV1" type.
-                    type: object
-                  manager:
-                    description: Manager is an identifier of the workflow managing
-                      these fields.
-                    type: string
-                  operation:
-                    description: Operation is the type of operation which lead to
-                      this ManagedFieldsEntry being created. The only valid values
-                      for this field are 'Apply' and 'Update'.
-                    type: string
-                  time:
-                    description: Time is timestamp of when these fields were set.
-                      It should always be empty if Operation is 'Apply'
-                    format: date-time
-                    type: string
-                type: object
-              type: array
-            name:
-              description: 'Name must be unique within a namespace. Is required when
-                creating resources, although some resources may allow a client to
-                request the generation of an appropriate name automatically. Name
-                is primarily intended for creation idempotence and configuration definition.
-                Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'
-              type: string
-            namespace:
-              description: "Namespace defines the space within each name must be unique.
-                An empty namespace is equivalent to the \"default\" namespace, but
-                \"default\" is the canonical representation. Not all objects are required
-                to be scoped to a namespace - the value of this field for those objects
-                will be empty. \n Must be a DNS_LABEL. Cannot be updated. More info:
-                http://kubernetes.io/docs/user-guide/namespaces"
-              type: string
-            ownerReferences:
-              description: List of objects depended by this object. If ALL objects
-                in the list have been deleted, this object will be garbage collected.
-                If this object is managed by a controller, then an entry in this list
-                will point to this controller, with the controller field set to true.
-                There cannot be more than one managing controller.
-              items:
-                properties:
-                  apiVersion:
-                    description: API version of the referent.
-                    type: string
-                  blockOwnerDeletion:
-                    description: If true, AND if the owner has the "foregroundDeletion"
-                      finalizer, then the owner cannot be deleted from the key-value
-                      store until this reference is removed. Defaults to false. To
-                      set this field, a user needs "delete" permission of the owner,
-                      otherwise 422 (Unprocessable Entity) will be returned.
-                    type: boolean
-                  controller:
-                    description: If true, this reference points to the managing controller.
-                    type: boolean
-                  kind:
-                    description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-                    type: string
-                  name:
-                    description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names'
-                    type: string
-                  uid:
-                    description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids'
-                    type: string
-                required:
-                - apiVersion
-                - kind
-                - name
-                - uid
-                type: object
-              type: array
-            resourceVersion:
-              description: "An opaque value that represents the internal version of
-                this object that can be used by clients to determine when objects
-                have changed. May be used for optimistic concurrency, change detection,
-                and the watch operation on a resource or set of resources. Clients
-                must treat these values as opaque and passed unmodified back to the
-                server. They may only be valid for a particular resource or set of
-                resources. \n Populated by the system. Read-only. Value must be treated
-                as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency"
-              type: string
-            selfLink:
-              description: "SelfLink is a URL representing this object. Populated
-                by the system. Read-only. \n DEPRECATED Kubernetes will stop propagating
-                this field in 1.20 release and the field is planned to be removed
-                in 1.21 release."
-              type: string
-            uid:
-              description: "UID is the unique in time and space value for this object.
-                It is typically generated by the server on successful creation of
-                a resource and is not allowed to change on PUT operations. \n Populated
-                by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids"
-              type: string
           type: object
         spec:
+          description: RollingUpgradeSpec defines the desired state of RollingUpgrade
           properties:
             asgName:
               type: string
             nodeIntervalSeconds:
               type: integer
             postDrain:
+              description: PostDrainSpec contains the fields for actions taken after
+                draining the node.
               properties:
                 postWaitScript:
                   type: string
@@ -274,11 +69,15 @@ spec:
             postDrainDelaySeconds:
               type: integer
             postTerminate:
+              description: PostTerminateSpec contains the fields for actions taken
+                after terminating the node.
               properties:
                 script:
                   type: string
               type: object
             preDrain:
+              description: PreDrainSpec contains the fields for actions taken before
+                draining the node.
               properties:
                 script:
                   type: string
@@ -286,6 +85,8 @@ spec:
             region:
               type: string
             strategy:
+              description: UpdateStrategy holds the information needed to perform
+                update based on different update strategies
               properties:
                 drainTimeout:
                   description: Node will be terminated after drain timeout even if
@@ -294,17 +95,24 @@ spec:
                   type: integer
                 maxUnavailable:
                   anyOf:
-                  - type: string
                   - type: integer
+                  - type: string
                   description: MaxUnavailable can be specified as number of nodes
                     or the percent of total number of nodes
+                  x-kubernetes-int-or-string: true
+                mode:
+                  type: string
                 type:
+                  description: UpdateStrategyType indicates how the update has to
+                    be rolled out whether to roll the update Az wise or all Azs at
+                    once
                   type: string
               required:
               - drainTimeout
               type: object
           type: object
         status:
+          description: RollingUpgradeStatus defines the observed state of RollingUpgrade
           properties:
             currentStatus:
               type: string
@@ -320,6 +128,7 @@ spec:
               type: string
           type: object
       type: object
+  version: v1alpha1
   versions:
   - name: v1alpha1
     served: true
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
index e08b20f..c5467b0 100644
--- a/config/rbac/role.yaml
+++ b/config/rbac/role.yaml
@@ -11,18 +11,18 @@ rules:
   resources:
   - rollingupgrades
   verbs:
+  - create
+  - delete
   - get
   - list
-  - watch
-  - create
-  - update
   - patch
-  - delete
+  - update
+  - watch
 - apiGroups:
   - upgrademgr.keikoproj.io
   resources:
   - rollingupgrades/status
   verbs:
   - get
-  - update
   - patch
+  - update
diff --git a/controllers/helpers.go b/controllers/helpers.go
index 2f49fde..d53eebd 100644
--- a/controllers/helpers.go
+++ b/controllers/helpers.go
@@ -6,6 +6,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
+	corev1 "k8s.io/api/core/v1"
+
 	upgrademgrv1alpha1 "github.com/keikoproj/upgrade-manager/api/v1alpha1"
 	"log"
 	"strconv"
@@ -36,6 +38,37 @@ func getMaxUnavailable(strategy upgrademgrv1alpha1.UpdateStrategy, totalNodes in
 	return maxUnavailable
 }
 
+func isNodeReady(node corev1.Node) bool {
+	for _, condition := range node.Status.Conditions {
+		if condition.Type == corev1.NodeReady {
+			if condition.Status == corev1.ConditionTrue {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func getInServiceCount(instances []*autoscaling.Instance) int64 {
+	var count int64
+	for _, instance := range instances {
+		if aws.StringValue(instance.LifecycleState) == autoscaling.LifecycleStateInService {
+			count++
+		}
+	}
+	return count
+}
+
+func getInServiceIds(instances []*autoscaling.Instance) []string {
+	list := []string{}
+	for _, instance := range instances {
+		if aws.StringValue(instance.LifecycleState) == autoscaling.LifecycleStateInService {
+			list = append(list, aws.StringValue(instance.InstanceId))
+		}
+	}
+	return list
+}
+
 func tagEC2instance(instanceID, tagKey, tagValue string, client ec2iface.EC2API) error {
 	input := &ec2.CreateTagsInput{
 		Resources: aws.StringSlice([]string{instanceID}),
diff --git a/controllers/rollingupgrade_controller.go b/controllers/rollingupgrade_controller.go
index f3a4983..0d5d461 100644
--- a/controllers/rollingupgrade_controller.go
+++ b/controllers/rollingupgrade_controller.go
@@ -17,8 +17,8 @@ package controllers
 
 import (
 	"context"
-	"errors"
 	"fmt"
+	"github.com/pkg/errors"
 	"os"
 	"os/exec"
 	"strconv"
@@ -40,6 +40,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
 	"github.com/go-logr/logr"
+	iebackoff "github.com/keikoproj/inverse-exp-backoff"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	v1errors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -86,6 +87,14 @@ var (
 	TerminationTimeoutSeconds = 3600
 	// TerminationSleepIntervalSeconds is the polling interval for checking if a node object is unjoined
 	TerminationSleepIntervalSeconds = 30
+	// WaiterMaxDelay is the maximum delay for waiters inverse exponential backoff
+	WaiterMaxDelay = time.Second * 90
+	// WaiterMinDelay is the minimum delay for waiters inverse exponential backoff
+	WaiterMinDelay = time.Second * 15
+	// WaiterFactor is the delay reduction factor per retry
+	WaiterFactor = 0.5
+	// WaiterMaxAttempts is the maximum number of retries for waiters
+	WaiterMaxAttempts = uint32(32)
 )
 
 var DefaultRetryer = awsclient.DefaultRetryer{
@@ -244,28 +253,121 @@ func (r *RollingUpgradeReconciler) CallKubectlDrain(ctx context.Context, nodeNam
 	errChan <- nil
 }
 
-func (r *RollingUpgradeReconciler) WaitForTermination(nodeName string, nodeInterface v1.NodeInterface) (bool, error) {
+func (r *RollingUpgradeReconciler) WaitForDesiredInstances(ruObj *upgrademgrv1alpha1.RollingUpgrade) error {
+
+	var err error
+	for ieb, err := iebackoff.NewIEBackoff(WaiterMaxDelay, WaiterMinDelay, 0.5, WaiterMaxAttempts); err == nil; err = ieb.Next() {
+		err = r.populateAsg(ruObj)
+		if err != nil {
+			return err
+		}
+
+		inServiceCount := getInServiceCount(r.Asg.Instances)
+		if inServiceCount == aws.Int64Value(r.Asg.DesiredCapacity) {
+			log.Printf("%v: desired capacity is met, %v instances in service", ruObj.Name, inServiceCount)
+			return nil
+		}
+
+		log.Printf("%v: new instance has not yet joined the scaling group", ruObj.Name)
+	}
+	return errors.Wrapf(err, "%v: WaitForDesiredInstances timed out while waiting for instance to be added", ruObj.Name)
+}
+
+func (r *RollingUpgradeReconciler) WaitForDesiredNodes(ruObj *upgrademgrv1alpha1.RollingUpgrade) error {
+
+	var err error
+	for ieb, err := iebackoff.NewIEBackoff(WaiterMaxDelay, WaiterMinDelay, 0.5, WaiterMaxAttempts); err == nil; err = ieb.Next() {
+		r.populateNodeList(ruObj, r.generatedClient.CoreV1().Nodes())
+
+		// get list of inService instance IDs
+		inServiceInstances := getInServiceIds(r.Asg.Instances)
+		desiredCapacity := aws.Int64Value(r.Asg.DesiredCapacity)
+
+		// check all of them are nodes and are ready
+		var foundCount int64 = 0
+		for _, node := range r.NodeList.Items {
+			tokens := strings.Split(node.Spec.ProviderID, "/")
+			instanceID := tokens[len(tokens)-1]
+			if contains(inServiceInstances, instanceID) && isNodeReady(node) {
+				foundCount++
+			}
+		}
+
+		if foundCount == desiredCapacity {
+			log.Printf("%v: desired capacity is met, %v nodes in service", ruObj.Name, foundCount)
+			return nil
+		}
+
+		log.Printf("%v: new node has not yet joined the cluster", ruObj.Name)
+	}
+	return errors.Wrapf(err, "%v: WaitForDesiredNodes timed out while waiting for nodes to join", ruObj.Name)
+}
+
+func (r *RollingUpgradeReconciler) WaitForTermination(ruObj *upgrademgrv1alpha1.RollingUpgrade, nodeName string, nodeInterface v1.NodeInterface) (bool, error) {
 	var (
 		started = time.Now()
 	)
 	for {
 		if time.Since(started) >= (time.Second * time.Duration(TerminationTimeoutSeconds)) {
-			log.Println("WaitForTermination timed out while waiting for node to unjoin")
+			log.Printf("%v: WaitForTermination timed out while waiting for node to unjoin", ruObj.Name)
 			return false, nil
 		}
 
 		_, err := nodeInterface.Get(nodeName, metav1.GetOptions{})
 		if v1errors.IsNotFound(err) {
-			log.Printf("node %s is unjoined from cluster, upgrade will proceed", nodeName)
+			log.Printf("%v: node %s is unjoined from cluster, upgrade will proceed", ruObj.Name, nodeName)
 			break
 		}
 
-		log.Printf("node %s is still joined to clutster, will wait %vs and retry", nodeName, TerminationSleepIntervalSeconds)
+		log.Printf("%v: node %s is still joined to clutster, will wait %vs and retry", ruObj.Name, nodeName, TerminationSleepIntervalSeconds)
 		time.Sleep(time.Duration(TerminationSleepIntervalSeconds) * time.Second)
 	}
 	return true, nil
 }
 
+// SetStandby sets the autoscaling instance to standby mode.
+func (r *RollingUpgradeReconciler) SetStandby(ruObj *upgrademgrv1alpha1.RollingUpgrade, instanceID string) error {
+
+	log.Infof("%v: Setting %v to stand-by", ruObj.Name, instanceID)
+	input := &autoscaling.EnterStandbyInput{
+		AutoScalingGroupName:           aws.String(ruObj.Spec.AsgName),
+		InstanceIds:                    aws.StringSlice([]string{instanceID}),
+		ShouldDecrementDesiredCapacity: aws.Bool(false),
+	}
+
+	for _, instance := range r.Asg.Instances {
+		if aws.StringValue(instance.InstanceId) == instanceID {
+			if aws.StringValue(instance.LifecycleState) == autoscaling.LifecycleStateInService {
+				break
+			} else if aws.StringValue(instance.LifecycleState) == autoscaling.LifecycleStateStandby {
+				log.Infof("%v: cannot set instance %v to stand-by, instance is already in stand-by", ruObj.Name, instanceID)
+				return nil
+			}
+			log.Warnf("%v: cannot set instance %v to stand-by, instance in state %v", ruObj.Name, instanceID, aws.StringValue(instance.LifecycleState))
+			return nil
+		}
+	}
+	_, err := r.ASGClient.EnterStandby(input)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			if strings.Contains(aerr.Message(), "not found") {
+				log.Printf("%v: Instance %s not found. Moving on\n", ruObj.Name, instanceID)
+				return nil
+			}
+			switch aerr.Code() {
+			case autoscaling.ErrCodeResourceContentionFault:
+				log.Warnf("%v: instance %v failed to enter standby: %v", ruObj.Name, instanceID, aerr.Error())
+				return nil
+			default:
+				log.Errorf("%v: instance %v failed to enter standby: %v", ruObj.Name, instanceID, aerr.Error())
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 // TerminateNode actually terminates the given node.
 func (r *RollingUpgradeReconciler) TerminateNode(ruObj *upgrademgrv1alpha1.RollingUpgrade, instanceID string) error {
 
@@ -274,7 +376,7 @@ func (r *RollingUpgradeReconciler) TerminateNode(ruObj *upgrademgrv1alpha1.Rolli
 		ShouldDecrementDesiredCapacity: aws.Bool(false),
 	}
 
-	result, err := r.ASGClient.TerminateInstanceInAutoScalingGroup(input)
+	_, err := r.ASGClient.TerminateInstanceInAutoScalingGroup(input)
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
 			if strings.Contains(aerr.Message(), "not found") {
@@ -294,9 +396,9 @@ func (r *RollingUpgradeReconciler) TerminateNode(ruObj *upgrademgrv1alpha1.Rolli
 		}
 	}
 
-	log.Printf("%s: Termination output: %v\n", ruObj.Name, result)
+	log.Infof("%v: terminated instance %v", ruObj.Name, instanceID)
 
-	log.Printf("starting post termination sleep for %v seconds", ruObj.Spec.NodeIntervalSeconds)
+	log.Printf("%v: starting post termination sleep for %v seconds", ruObj.Name, ruObj.Spec.NodeIntervalSeconds)
 	time.Sleep(time.Duration(ruObj.Spec.NodeIntervalSeconds) * time.Second)
 	if ruObj.Spec.PostTerminate.Script != "" {
 		out, err := runScript(ruObj.Spec.PostTerminate.Script, false, ruObj.Name)
@@ -366,6 +468,7 @@ func (r *RollingUpgradeReconciler) populateAsg(ruObj *upgrademgrv1alpha1.Rolling
 	}
 
 	asg := result.AutoScalingGroups[0]
+	r.Asg = asg
 	r.ruObjNameToASG.Store(ruObj.Name, asg)
 
 	return nil
@@ -383,7 +486,6 @@ func (r *RollingUpgradeReconciler) populateNodeList(ruObj *upgrademgrv1alpha1.Ro
 	for _, n := range nodeList.Items {
 		nodesFound = n.Name + "," + nodesFound
 	}
-	log.Printf("%s: Kubernetes nodes found: %s", ruObj.Name, nodesFound)
 
 	r.NodeList = nodeList
 	return nil
@@ -665,8 +767,8 @@ func (r *RollingUpgradeReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
-func (r *RollingUpgradeReconciler) setStateTag(instanceID string, state string) error {
-	log.Printf("setting instance %v state to %v", instanceID, state)
+func (r *RollingUpgradeReconciler) setStateTag(ruObj *upgrademgrv1alpha1.RollingUpgrade, instanceID string, state string) error {
+	log.Printf("%v: setting instance %v state to %v", ruObj.Name, instanceID, state)
 	err := tagEC2instance(instanceID, EC2StateTagKey, state, r.EC2Client)
 	if err != nil {
 		return err
@@ -722,6 +824,7 @@ func (r *RollingUpgradeReconciler) setDefaultsForRollingUpdateStrategy(ruObj *up
 		log.Printf("Update strategy not set on the rollup object - %s, setting the default strategy.", ruObj.Name)
 		strategy := upgrademgrv1alpha1.UpdateStrategy{
 			Type:           upgrademgrv1alpha1.RandomUpdateStrategy,
+			Mode:           upgrademgrv1alpha1.UpdateStrategyModeLazy,
 			MaxUnavailable: intstr.IntOrString{IntVal: 1},
 			DrainTimeout:   -1,
 		}
@@ -730,6 +833,10 @@ func (r *RollingUpgradeReconciler) setDefaultsForRollingUpdateStrategy(ruObj *up
 		if ruObj.Spec.Strategy.Type == "" {
 			ruObj.Spec.Strategy.Type = upgrademgrv1alpha1.RandomUpdateStrategy
 		}
+		if ruObj.Spec.Strategy.Mode == "" {
+			// default to lazy mode
+			ruObj.Spec.Strategy.Mode = upgrademgrv1alpha1.UpdateStrategyModeLazy
+		}
 		// intstr.IntOrString has the default value 0 with int types
 		if ruObj.Spec.Strategy.MaxUnavailable.Type == 0 && ruObj.Spec.Strategy.MaxUnavailable.IntVal == 0 {
 			ruObj.Spec.Strategy.MaxUnavailable = intstr.IntOrString{Type: 0, IntVal: 1}
@@ -795,6 +902,62 @@ Loop:
 	return nil
 }
 
+func (r *RollingUpgradeReconciler) UpdateInstanceEager(
+	ruObj *upgrademgrv1alpha1.RollingUpgrade,
+	nodeName,
+	targetInstanceID,
+	KubeCtlCall string,
+	ch chan error) {
+
+	// Set instance to standby.
+	err := r.SetStandby(ruObj, targetInstanceID)
+	if err != nil {
+		ch <- err
+		return
+	}
+
+	// Wait for new instance to be created
+	err = r.WaitForDesiredInstances(ruObj)
+	if err != nil {
+		ch <- err
+		return
+	}
+
+	// Wait for in-service nodes to be ready and match desired
+	err = r.WaitForDesiredNodes(ruObj)
+	if err != nil {
+		ch <- err
+		return
+	}
+
+	// Drain and wait for draining node.
+	r.DrainTerminate(ruObj, nodeName, targetInstanceID, KubeCtlCall, ch)
+
+}
+
+func (r *RollingUpgradeReconciler) DrainTerminate(
+	ruObj *upgrademgrv1alpha1.RollingUpgrade,
+	nodeName,
+	targetInstanceID,
+	KubeCtlCall string,
+	ch chan error) {
+
+	// Drain and wait for draining node.
+	err := r.DrainNode(ruObj, nodeName, KubeCtlCall, ruObj.Spec.Strategy.DrainTimeout)
+	if err != nil {
+		ch <- err
+		return
+	}
+
+	// Terminate instance.
+	err = r.TerminateNode(ruObj, targetInstanceID)
+	if err != nil {
+		ch <- err
+		return
+	}
+
+}
+
 func (r *RollingUpgradeReconciler) UpdateInstance(ctx *context.Context,
 	ruObj *upgrademgrv1alpha1.RollingUpgrade,
 	i *autoscaling.Instance,
@@ -827,38 +990,33 @@ func (r *RollingUpgradeReconciler) UpdateInstance(ctx *context.Context,
 	}
 
 	// set the EC2 tag indicating the state to in-progress
-	err = r.setStateTag(targetInstanceID, "in-progress")
-	if err != nil {
-		ch <- err
-		return
-	}
-
-	// Drain and wait for draining node.
-	err = r.DrainNode(ruObj, nodeName, KubeCtlCall, ruObj.Spec.Strategy.DrainTimeout)
+	err = r.setStateTag(ruObj, targetInstanceID, "in-progress")
 	if err != nil {
 		ch <- err
 		return
 	}
 
-	// Terminate instance.
-	err = r.TerminateNode(ruObj, targetInstanceID)
-	if err != nil {
-		ch <- err
-		return
+	mode := ruObj.Spec.Strategy.Mode.String()
+	if strings.ToLower(mode) == upgrademgrv1alpha1.UpdateStrategyModeEager.String() {
+		log.Printf("%v: starting replacement with eager mode", ruObj.Name)
+		r.UpdateInstanceEager(ruObj, nodeName, targetInstanceID, KubeCtlCall, ch)
+	} else if strings.ToLower(mode) == upgrademgrv1alpha1.UpdateStrategyModeLazy.String() {
+		log.Printf("%v: starting replacement with lazy mode", ruObj.Name)
+		r.DrainTerminate(ruObj, nodeName, targetInstanceID, KubeCtlCall, ch)
 	}
 
-	unjoined, err := r.WaitForTermination(nodeName, r.generatedClient.CoreV1().Nodes())
+	unjoined, err := r.WaitForTermination(ruObj, nodeName, r.generatedClient.CoreV1().Nodes())
 	if err != nil {
 		ch <- err
 		return
 	}
 
 	if !unjoined {
-		log.Warnf("termination waiter completed but %s is still joined, will proceed with upgrade", nodeName)
+		log.Warnf("%v: termination waiter completed but %s is still joined, will proceed with upgrade", ruObj.Name, nodeName)
 	}
 
 	// set the EC2 tag indicating the state to completed
-	err = r.setStateTag(targetInstanceID, "completed")
+	err = r.setStateTag(ruObj, targetInstanceID, "completed")
 	if err != nil {
 		ch <- err
 		return
diff --git a/go.mod b/go.mod
index 6b95280..9e84564 100644
--- a/go.mod
+++ b/go.mod
@@ -6,9 +6,10 @@ require (
 	github.com/aws/aws-sdk-go v1.25.0
 	github.com/go-logr/logr v0.1.0
 	github.com/gogo/protobuf v1.3.1 // indirect
+	github.com/keikoproj/inverse-exp-backoff v0.0.0-20191216014651-04523236b6ca
 	github.com/keikoproj/upgrade-manager/pkg/log v0.0.0
 	github.com/onsi/ginkgo v1.10.1
-	github.com/onsi/gomega v1.7.0
+	github.com/onsi/gomega v1.7.1
 	github.com/pkg/errors v0.8.1
 	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
 	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553
diff --git a/go.sum b/go.sum
index 48f3952..0c249e7 100644
--- a/go.sum
+++ b/go.sum
@@ -196,6 +196,8 @@ github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46O
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/keikoproj/inverse-exp-backoff v0.0.0-20191216014651-04523236b6ca h1:uvqXVaFKbYgjOK+BHvtCMZIa+Mssmq7wuTJM5MPP5Sc=
+github.com/keikoproj/inverse-exp-backoff v0.0.0-20191216014651-04523236b6ca/go.mod h1:ziu/tMrrvs8n+AI+HCZBb6wZS609fcMl5ygR2RttEE4=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -249,6 +251,8 @@ github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=


Test patch:
diff --git a/controllers/helpers_test.go b/controllers/helpers_test.go
index 35177ac..3648279 100644
--- a/controllers/helpers_test.go
+++ b/controllers/helpers_test.go
@@ -3,9 +3,13 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	upgrademgrv1alpha1 "github.com/keikoproj/upgrade-manager/api/v1alpha1"
 	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	"testing"
 )
 
@@ -22,6 +26,93 @@ func TestGetMaxUnavailableWithPercentageValue(t *testing.T) {
 	g.Expect(getMaxUnavailable(strategy, 200)).To(gomega.Equal(150))
 }
 
+func TestIsNodeReady(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	tt := map[corev1.NodeCondition]bool{
+		corev1.NodeCondition{Type: corev1.NodeReady, Status: corev1.ConditionTrue}:  true,
+		corev1.NodeCondition{Type: corev1.NodeReady, Status: corev1.ConditionFalse}: false,
+	}
+
+	for condition, val := range tt {
+		node := corev1.Node{
+			Status: corev1.NodeStatus{
+				Conditions: []corev1.NodeCondition{
+					condition,
+				},
+			},
+		}
+		g.Expect(isNodeReady(node)).To(gomega.Equal(val))
+	}
+}
+
+func TestGetInServiceCount(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	tt := map[*autoscaling.Instance]int64{
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStateInService)}:          1,
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStateDetached)}:           0,
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStateDetaching)}:          0,
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStateEnteringStandby)}:    0,
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStatePending)}:            0,
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStatePendingProceed)}:     0,
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStatePendingWait)}:        0,
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStateQuarantined)}:        0,
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStateStandby)}:            0,
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStateTerminated)}:         0,
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStateTerminating)}:        0,
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStateTerminatingProceed)}: 0,
+		&autoscaling.Instance{LifecycleState: aws.String(autoscaling.LifecycleStateTerminatingWait)}:    0,
+	}
+
+	// test every condition
+	for instance, expectedCount := range tt {
+		instances := []*autoscaling.Instance{
+			instance,
+		}
+		g.Expect(getInServiceCount(instances)).To(gomega.Equal(expectedCount))
+	}
+
+	// test all instances
+	instances := []*autoscaling.Instance{}
+	for instance := range tt {
+		instances = append(instances, instance)
+	}
+	g.Expect(getInServiceCount(instances)).To(gomega.Equal(int64(1)))
+}
+
+func TestGetInServiceIds(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	tt := map[*autoscaling.Instance][]string{
+		&autoscaling.Instance{InstanceId: aws.String("i-1"), LifecycleState: aws.String(autoscaling.LifecycleStateInService)}:           []string{"i-1"},
+		&autoscaling.Instance{InstanceId: aws.String("i-2"), LifecycleState: aws.String(autoscaling.LifecycleStateDetached)}:            []string{},
+		&autoscaling.Instance{InstanceId: aws.String("i-3"), LifecycleState: aws.String(autoscaling.LifecycleStateDetaching)}:           []string{},
+		&autoscaling.Instance{InstanceId: aws.String("i-4"), LifecycleState: aws.String(autoscaling.LifecycleStateEnteringStandby)}:     []string{},
+		&autoscaling.Instance{InstanceId: aws.String("i-5"), LifecycleState: aws.String(autoscaling.LifecycleStatePending)}:             []string{},
+		&autoscaling.Instance{InstanceId: aws.String("i-6"), LifecycleState: aws.String(autoscaling.LifecycleStatePendingProceed)}:      []string{},
+		&autoscaling.Instance{InstanceId: aws.String("i-7"), LifecycleState: aws.String(autoscaling.LifecycleStatePendingWait)}:         []string{},
+		&autoscaling.Instance{InstanceId: aws.String("i-8"), LifecycleState: aws.String(autoscaling.LifecycleStateQuarantined)}:         []string{},
+		&autoscaling.Instance{InstanceId: aws.String("i-9"), LifecycleState: aws.String(autoscaling.LifecycleStateStandby)}:             []string{},
+		&autoscaling.Instance{InstanceId: aws.String("i-10"), LifecycleState: aws.String(autoscaling.LifecycleStateTerminated)}:         []string{},
+		&autoscaling.Instance{InstanceId: aws.String("i-11"), LifecycleState: aws.String(autoscaling.LifecycleStateTerminating)}:        []string{},
+		&autoscaling.Instance{InstanceId: aws.String("i-12"), LifecycleState: aws.String(autoscaling.LifecycleStateTerminatingProceed)}: []string{},
+		&autoscaling.Instance{InstanceId: aws.String("i-13"), LifecycleState: aws.String(autoscaling.LifecycleStateTerminatingWait)}:    []string{},
+	}
+
+	// test every condition
+	for instance, expectedList := range tt {
+		instances := []*autoscaling.Instance{
+			instance,
+		}
+		g.Expect(getInServiceIds(instances)).To(gomega.Equal(expectedList))
+	}
+
+	// test all instances
+	instances := []*autoscaling.Instance{}
+	for instance := range tt {
+		instances = append(instances, instance)
+	}
+	g.Expect(getInServiceIds(instances)).To(gomega.Equal([]string{"i-1"}))
+}
+
 func TestGetMaxUnavailableWithPercentageValue33(t *testing.T) {
 	g := gomega.NewGomegaWithT(t)
 
diff --git a/controllers/rollingupgrade_controller_test.go b/controllers/rollingupgrade_controller_test.go
index b9107cb..a902d3f 100644
--- a/controllers/rollingupgrade_controller_test.go
+++ b/controllers/rollingupgrade_controller_test.go
@@ -317,6 +317,36 @@ func (m MockEC2) DescribeInstancesPages(input *ec2.DescribeInstancesInput, callb
 	return nil
 }
 
+func (mockAutoscalingGroup MockAutoscalingGroup) EnterStandby(input *autoscaling.EnterStandbyInput) (*autoscaling.EnterStandbyOutput, error) {
+	output := &autoscaling.EnterStandbyOutput{}
+	return output, nil
+}
+
+func (mockAutoscalingGroup MockAutoscalingGroup) DescribeAutoScalingGroups(input *autoscaling.DescribeAutoScalingGroupsInput) (*autoscaling.DescribeAutoScalingGroupsOutput, error) {
+	output := autoscaling.DescribeAutoScalingGroupsOutput{
+		AutoScalingGroups: []*autoscaling.Group{},
+	}
+
+	correctAsg := "correct-asg"
+	tooMany := "too-many"
+
+	switch *input.AutoScalingGroupNames[0] {
+	case correctAsg:
+		output.AutoScalingGroups = []*autoscaling.Group{
+			{AutoScalingGroupName: &correctAsg},
+		}
+	case tooMany:
+		output.AutoScalingGroups = []*autoscaling.Group{
+			{AutoScalingGroupName: &tooMany},
+			{AutoScalingGroupName: &tooMany},
+		}
+	default:
+		output.AutoScalingGroups = []*autoscaling.Group{}
+	}
+
+	return &output, nil
+}
+
 func (mockAutoscalingGroup MockAutoscalingGroup) TerminateInstanceInAutoScalingGroup(input *autoscaling.TerminateInstanceInAutoScalingGroupInput) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) {
 	output := &autoscaling.TerminateInstanceInAutoScalingGroupOutput{}
 	if mockAutoscalingGroup.errorFlag {
@@ -579,35 +609,6 @@ func TestGetNodeFromAsgMissingNode(t *testing.T) {
 	g.Expect(node).To(gomega.BeNil())
 }
 
-type MockAutoScalingAPI struct {
-	autoscalingiface.AutoScalingAPI
-}
-
-func (mockAutoScalingInstance *MockAutoScalingAPI) DescribeAutoScalingGroups(input *autoscaling.DescribeAutoScalingGroupsInput) (*autoscaling.DescribeAutoScalingGroupsOutput, error) {
-	output := autoscaling.DescribeAutoScalingGroupsOutput{
-		AutoScalingGroups: []*autoscaling.Group{},
-	}
-
-	correctAsg := "correct-asg"
-	tooMany := "too-many"
-
-	switch *input.AutoScalingGroupNames[0] {
-	case correctAsg:
-		output.AutoScalingGroups = []*autoscaling.Group{
-			{AutoScalingGroupName: &correctAsg},
-		}
-	case tooMany:
-		output.AutoScalingGroups = []*autoscaling.Group{
-			{AutoScalingGroupName: &tooMany},
-			{AutoScalingGroupName: &tooMany},
-		}
-	default:
-		output.AutoScalingGroups = []*autoscaling.Group{}
-	}
-
-	return &output, nil
-}
-
 func TestPopulateAsgSuccess(t *testing.T) {
 	g := gomega.NewGomegaWithT(t)
 
@@ -617,7 +618,7 @@ func TestPopulateAsgSuccess(t *testing.T) {
 
 	rcRollingUpgrade := &RollingUpgradeReconciler{
 		ClusterState: NewClusterState(),
-		ASGClient:    &MockAutoScalingAPI{},
+		ASGClient:    &MockAutoscalingGroup{},
 		EC2Client:    MockEC2{},
 	}
 	err := rcRollingUpgrade.populateAsg(ruObj)
@@ -641,7 +642,7 @@ func TestPopulateAsgTooMany(t *testing.T) {
 
 	rcRollingUpgrade := &RollingUpgradeReconciler{
 		ClusterState: NewClusterState(),
-		ASGClient:    &MockAutoScalingAPI{},
+		ASGClient:    &MockAutoscalingGroup{},
 		EC2Client:    MockEC2{},
 	}
 	err := rcRollingUpgrade.populateAsg(ruObj)
@@ -659,7 +660,7 @@ func TestPopulateAsgNone(t *testing.T) {
 
 	rcRollingUpgrade := &RollingUpgradeReconciler{
 		ClusterState: NewClusterState(),
-		ASGClient:    &MockAutoScalingAPI{},
+		ASGClient:    &MockAutoscalingGroup{},
 		EC2Client:    MockEC2{},
 	}
 	err := rcRollingUpgrade.populateAsg(ruObj)
@@ -2064,14 +2065,20 @@ func TestWaitForTermination(t *testing.T) {
 		ClusterState:    NewClusterState(),
 	}
 	nodeInterface.Create(mockNode)
+	ruObj := &upgrademgrv1alpha1.RollingUpgrade{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "foo",
+			Namespace: "default",
+		},
+	}
 
-	unjoined, err := rcRollingUpgrade.WaitForTermination(mockNodeName, nodeInterface)
+	unjoined, err := rcRollingUpgrade.WaitForTermination(ruObj, mockNodeName, nodeInterface)
 	g.Expect(err).NotTo(gomega.HaveOccurred())
 	g.Expect(unjoined).To(gomega.BeFalse())
 
 	nodeInterface.Delete(mockNodeName, &metav1.DeleteOptions{})
 
-	unjoined, err = rcRollingUpgrade.WaitForTermination(mockNodeName, nodeInterface)
+	unjoined, err = rcRollingUpgrade.WaitForTermination(ruObj, mockNodeName, nodeInterface)
 	g.Expect(err).NotTo(gomega.HaveOccurred())
 	g.Expect(unjoined).To(gomega.BeTrue())
 }