Repository: keikoproj/instance-manager (go)

Issue:
Feature: Recommendation Reconciler
In order to support spot-instances using instance-manager we will need to be able to reconcile based on events.

minion manager (or other recommendation controllers) will emit events with reason `SpotRecommendationGiven` and we should process the message and set spot price on instance groups that have it enabled.



Pull request description:
Support Spot Instances
Fixes #35 

This PR adds support for spot-instances with the following implementation:

- Recommendation reconciler was added to controller to watch all events, and trigger a reconcile for any event that is with reason `SpotRecommendationGiven`, the reconciler will reverse lookup the scaling group name from the event's `involvedObject.Name`, and find the instance group name and namespace in order to trigger it's reconcile.
- aws client (asg) is now needed before context is created (for reverse lookup), we should refactor and move all of the `AwsWorker` object outside of context instead on a later PR.
- `spotPrice` can be set manually in the CR, but will always be overridden by recommendations when it exists
- when recommendations become unavailable instance group will switch back to on-demand
- recommendations must follow the format:
```text
# Event Message
{"apiVersion":"v1alpha1","spotPrice":"0.0067", "useSpot": true}
# Event involvedObject.Name = scaling group name
# Event reason = SpotRecommendationGiven
```

- Also added two additional `status` fields - `usingSpotRecommendations` and `lifecycle`, a printer column was also added for `lifecycle`.

## Testing
- Added unit tests to test recommendations-based and manual setting of spotPrice
- Manually tested create/update in various scenarios


Patch:
diff --git a/api/v1alpha1/instancegroup_types.go b/api/v1alpha1/instancegroup_types.go
index d4b8f05..3453a3a 100644
--- a/api/v1alpha1/instancegroup_types.go
+++ b/api/v1alpha1/instancegroup_types.go
@@ -51,6 +51,7 @@ const (
 // +kubebuilder:printcolumn:name="Group Name",type="string",JSONPath=".status.activeScalingGroupName",description="instancegroup created scalinggroup name"
 // +kubebuilder:printcolumn:name="Provisioner",type="string",JSONPath=".spec.provisioner",description="instance group provisioner"
 // +kubebuilder:printcolumn:name="Strategy",type="string",JSONPath=".spec.strategy.type",description="instance group upgrade strategy"
+// +kubebuilder:printcolumn:name="Lifecycle",type="string",JSONPath=".status.lifecycle",description="instance group lifecycle spot/normal"
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="time passed since instancegroup creation"
 type InstanceGroup struct {
 	metav1.TypeMeta   `json:",inline"`
@@ -106,6 +107,7 @@ type EKSCFConfiguration struct {
 	VolSize            int32               `json:"volSize,omitempty"`
 	Subnets            []string            `json:"subnets,omitempty"`
 	BootstrapArguments string              `json:"bootstrapArguments,omitempty"`
+	SpotPrice          string              `json:"spotPrice,omitempty"`
 	Tags               []map[string]string `json:"tags,omitempty"`
 }
 
@@ -118,6 +120,8 @@ type InstanceGroupStatus struct {
 	ActiveScalingGroupName        string `json:"activeScalingGroupName,omitempty"`
 	NodesArn                      string `json:"nodesInstanceRoleArn,omitempty"`
 	StrategyResourceName          string `json:"strategyResourceName,omitempty"`
+	UsingSpotRecommendation       bool   `json:"usingSpotRecommendation"`
+	Lifecycle                     string `json:"lifecycle,omitempty"`
 }
 
 func (s *AwsUpgradeStrategy) GetCRDType() CRDUpgradeStrategy {
@@ -224,6 +228,22 @@ func (status *InstanceGroupStatus) SetCurrentMax(max int) {
 	status.CurrentMax = max
 }
 
+func (status *InstanceGroupStatus) GetUsingSpotRecommendation() bool {
+	return status.UsingSpotRecommendation
+}
+
+func (status *InstanceGroupStatus) SetUsingSpotRecommendation(condition bool) {
+	status.UsingSpotRecommendation = condition
+}
+
+func (status *InstanceGroupStatus) GetLifecycle() string {
+	return status.Lifecycle
+}
+
+func (status *InstanceGroupStatus) SetLifecycle(phase string) {
+	status.Lifecycle = phase
+}
+
 func (strategy *AwsUpgradeStrategy) GetType() string {
 	return strategy.Type
 }
@@ -256,6 +276,14 @@ func (conf *EKSCFConfiguration) SetKeyName(keypairName string) {
 	conf.KeyPairName = keypairName
 }
 
+func (conf *EKSCFConfiguration) SetSpotPrice(price string) {
+	conf.SpotPrice = price
+}
+
+func (conf *EKSCFConfiguration) GetSpotPrice() string {
+	return conf.SpotPrice
+}
+
 func (conf *EKSCFConfiguration) GetImage() string {
 	return conf.Image
 }
diff --git a/config/crd/bases/instance-manager-configmap.yaml b/config/crd/bases/instance-manager-configmap.yaml
index bec60cf..f991cae 100644
--- a/config/crd/bases/instance-manager-configmap.yaml
+++ b/config/crd/bases/instance-manager-configmap.yaml
@@ -193,6 +193,10 @@ data:
           ImageId: !Ref NodeImageId
           InstanceType: !Ref NodeInstanceType
           KeyName: !Ref KeyName
+          {{$spotPrice := .Spec.EKSCFSpec.EKSCFConfiguration.SpotPrice}}
+          {{if (ne $spotPrice "")}}
+          SpotPrice: {{$spotPrice}}
+          {{end}}
           SecurityGroups: !Ref NodeSecurityGroups
           BlockDeviceMappings:
             - DeviceName: /dev/xvda
diff --git a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml
index c726a85..50bc13e 100644
--- a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml
+++ b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml
@@ -31,6 +31,10 @@ spec:
     description: instance group upgrade strategy
     name: Strategy
     type: string
+  - JSONPath: .status.lifecycle
+    description: instance group lifecycle spot/normal
+    name: Lifecycle
+    type: string
   - JSONPath: .metadata.creationTimestamp
     description: time passed since instancegroup creation
     name: Age
@@ -435,6 +439,8 @@ spec:
                       items:
                         type: string
                       type: array
+                    spotPrice:
+                      type: string
                     subnets:
                       items:
                         type: string
@@ -506,10 +512,16 @@ spec:
               type: integer
             currentState:
               type: string
+            lifecycle:
+              type: string
             nodesInstanceRoleArn:
               type: string
             strategyResourceName:
               type: string
+            usingSpotRecommendation:
+              type: boolean
+          required:
+          - usingSpotRecommendation
           type: object
       required:
       - metadata
diff --git a/controllers/instancegroup_controller.go b/controllers/instancegroup_controller.go
index e0439ea..5235060 100644
--- a/controllers/instancegroup_controller.go
+++ b/controllers/instancegroup_controller.go
@@ -16,31 +16,35 @@ limitations under the License.
 package controllers
 
 import (
-	"bytes"
 	"context"
 	"fmt"
-	"html/template"
 	"os"
 	"strings"
 	"time"
 
-	yaml "gopkg.in/yaml.v2"
-
+	"github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface"
 	"github.com/go-logr/logr"
-	log "github.com/sirupsen/logrus"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
 	v1alpha "github.com/keikoproj/instance-manager/api/v1alpha1"
 	"github.com/keikoproj/instance-manager/controllers/common"
 	"github.com/keikoproj/instance-manager/controllers/providers/aws"
 	"github.com/keikoproj/instance-manager/controllers/provisioners/ekscloudformation"
+	log "github.com/sirupsen/logrus"
+	yaml "gopkg.in/yaml.v2"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
 // InstanceGroupReconciler reconciles an InstanceGroup object
 type InstanceGroupReconciler struct {
 	client.Client
+	ScalingGroups          autoscalingiface.AutoScalingAPI
 	Log                    logr.Logger
 	ControllerConfPath     string
 	ControllerTemplatePath string
@@ -201,7 +205,7 @@ func (r *InstanceGroupReconciler) ReconcileEKSCF(instanceGroup *v1alpha.Instance
 		return err
 	}
 
-	template, err := r.loadCloudformationConfiguration(instanceGroup)
+	template, err := ekscloudformation.LoadCloudformationConfiguration(instanceGroup, r.ControllerTemplatePath)
 	if err != nil {
 		log.Errorf("failed to load cloudformation configuration: %v", err)
 		return err
@@ -222,6 +226,7 @@ func (r *InstanceGroupReconciler) ReconcileEKSCF(instanceGroup *v1alpha.Instance
 
 	ctx.ControllerRegion = awsRegion
 	ctx.DefaultARNList = defaultARNList
+	ctx.TemplatePath = r.ControllerTemplatePath
 
 	// Init State is set when handling reconcile
 	// Reconcile Handler
@@ -310,39 +315,12 @@ func (r *InstanceGroupReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
 func (r *InstanceGroupReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha.InstanceGroup{}).
+		Watches(&source.Kind{Type: &corev1.Event{}}, &handler.EnqueueRequestsFromMapFunc{
+			ToRequests: handler.ToRequestsFunc(r.spotEventReconciler),
+		}).
 		Complete(r)
 }
 
-func (r *InstanceGroupReconciler) loadCloudformationConfiguration(ig *v1alpha.InstanceGroup) (string, error) {
-	var renderBuffer bytes.Buffer
-
-	funcMap := template.FuncMap{
-		"ToLower": strings.ToLower,
-	}
-
-	if _, err := os.Stat(r.ControllerTemplatePath); os.IsNotExist(err) {
-		log.Errorf("controller cloudformation template file not found: %v", err)
-		return "", err
-	}
-
-	rawTemplate, err := common.ReadFile(r.ControllerTemplatePath)
-	if err != nil {
-		return "", err
-	}
-
-	template, err := template.New("InstanceGroup").Funcs(funcMap).Parse(string(rawTemplate))
-	if err != nil {
-		return "", err
-	}
-
-	err = template.Execute(&renderBuffer, ig)
-	if err != nil {
-		return "", err
-	}
-
-	return renderBuffer.String(), nil
-}
-
 func (r *InstanceGroupReconciler) loadControllerConfiguration(ig *v1alpha.InstanceGroup) ([]string, error) {
 	var defaultConfig ekscloudformation.EksCfDefaultConfiguration
 	var specConfig = &ig.Spec.EKSCFSpec.EKSCFConfiguration
@@ -377,3 +355,45 @@ func (r *InstanceGroupReconciler) loadControllerConfiguration(ig *v1alpha.Instan
 
 	return defaultARNs, nil
 }
+
+func (r *InstanceGroupReconciler) spotEventReconciler(obj handler.MapObject) []ctrl.Request {
+	unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj.Object)
+	if err != nil {
+		return nil
+	}
+
+	reason, exists, err := unstructured.NestedString(unstructuredObj, "reason")
+	if !exists || err != nil {
+		return nil
+	}
+	if reason != ekscloudformation.SpotRecommendationReason {
+		return nil
+	}
+
+	ctrl.Log.Info(fmt.Sprintf("spot recommendation %v/%v", obj.Meta.GetNamespace(), obj.Meta.GetName()))
+
+	involvedObjectName, exists, err := unstructured.NestedString(unstructuredObj, "involvedObject", "name")
+	if err != nil || !exists {
+		log.Warnf("failed to process event '%v': %v", obj.Meta.GetName(), err)
+		return nil
+	}
+
+	tags, err := aws.GetScalingGroupTagsByName(involvedObjectName, r.ScalingGroups)
+	if err != nil {
+		log.Warnf("failed to process event '%v': could not find scaling group", obj.Meta.GetName())
+		return nil
+	}
+	instanceGroup := types.NamespacedName{}
+	instanceGroup.Name = aws.GetTagValueByKey(tags, ekscloudformation.TagInstanceGroupName)
+	instanceGroup.Namespace = aws.GetTagValueByKey(tags, ekscloudformation.TagClusterNamespace)
+	if instanceGroup.Name == "" || instanceGroup.Namespace == "" {
+		log.Warnf("failed to process event '%v': could not derive instancegroup", obj.Meta.GetName())
+		return nil
+	}
+
+	return []ctrl.Request{
+		{
+			NamespacedName: instanceGroup,
+		},
+	}
+}
diff --git a/controllers/providers/aws/aws.go b/controllers/providers/aws/aws.go
index 56f6d6d..52b0b6f 100644
--- a/controllers/providers/aws/aws.go
+++ b/controllers/providers/aws/aws.go
@@ -16,6 +16,7 @@ limitations under the License.
 package aws
 
 import (
+	"errors"
 	"fmt"
 	"os"
 
@@ -183,6 +184,35 @@ func (w *AwsWorker) DetectScalingGroupDrift(scalingGroupName string) (bool, erro
 	return false, nil
 }
 
+func GetScalingGroupTagsByName(name string, client autoscalingiface.AutoScalingAPI) ([]*autoscaling.TagDescription, error) {
+	tags := []*autoscaling.TagDescription{}
+	input := &autoscaling.DescribeAutoScalingGroupsInput{
+		AutoScalingGroupNames: aws.StringSlice([]string{name}),
+	}
+	out, err := client.DescribeAutoScalingGroups(input)
+	if err != nil {
+		return tags, err
+	}
+
+	if len(out.AutoScalingGroups) < 1 {
+		err := errors.New("could not find scaling group")
+		return tags, err
+	}
+	tags = out.AutoScalingGroups[0].Tags
+	return tags, nil
+}
+
+func GetTagValueByKey(tags []*autoscaling.TagDescription, key string) string {
+	for _, tag := range tags {
+		k := aws.StringValue(tag.Key)
+		v := aws.StringValue(tag.Value)
+		if key == k {
+			return v
+		}
+	}
+	return ""
+}
+
 func GetRegion() (string, error) {
 	if os.Getenv("AWS_REGION") != "" {
 		return os.Getenv("AWS_REGION"), nil
diff --git a/controllers/provisioners/ekscloudformation/bootstrap.go b/controllers/provisioners/ekscloudformation/bootstrap.go
index 94151ac..7f0fb64 100644
--- a/controllers/provisioners/ekscloudformation/bootstrap.go
+++ b/controllers/provisioners/ekscloudformation/bootstrap.go
@@ -105,7 +105,7 @@ func (ctx *EksCfInstanceGroupContext) updateAuthConfigMap() error {
 		}
 	}
 
-	log.Infof("bootstrapping: %+v\n", newConfigurations)
+	log.Debugf("bootstrapping: %+v\n", newConfigurations)
 
 	d, err := yaml.Marshal(&newConfigurations.MapRoles)
 	if err != nil {
diff --git a/controllers/provisioners/ekscloudformation/ekscloudformation.go b/controllers/provisioners/ekscloudformation/ekscloudformation.go
index 72ac8a9..db084f2 100644
--- a/controllers/provisioners/ekscloudformation/ekscloudformation.go
+++ b/controllers/provisioners/ekscloudformation/ekscloudformation.go
@@ -32,9 +32,9 @@ import (
 
 var (
 	log                       = logrus.New()
-	tagClusterName            = "instancegroups.keikoproj.io/ClusterName"
-	tagInstanceGroupName      = "instancegroups.keikoproj.io/InstanceGroup"
-	tagClusterNamespace       = "instancegroups.keikoproj.io/Namespace"
+	TagClusterName            = "instancegroups.keikoproj.io/ClusterName"
+	TagInstanceGroupName      = "instancegroups.keikoproj.io/InstanceGroup"
+	TagClusterNamespace       = "instancegroups.keikoproj.io/Namespace"
 	outputLaunchConfiguration = "LaunchConfigName"
 	outputScalingGroupName    = "AsgName"
 	outputGroupARN            = "NodeInstanceRole"
@@ -92,7 +92,7 @@ func (ctx *EksCfInstanceGroupContext) Create() error {
 		return err
 	}
 	instanceGroup.SetState(v1alpha1.ReconcileModifying)
-	ctx.CloudDiscovery()
+	ctx.reloadDiscoveryCache()
 	return nil
 }
 
@@ -114,7 +114,7 @@ func (ctx *EksCfInstanceGroupContext) Update() error {
 			instanceGroup.SetState(v1alpha1.ReconcileModified)
 		}
 	}
-	ctx.CloudDiscovery()
+	ctx.reloadDiscoveryCache()
 	return nil
 }
 
@@ -154,7 +154,7 @@ func (ctx *EksCfInstanceGroupContext) Delete() error {
 		return err
 	}
 	instanceGroup.SetState(v1alpha1.ReconcileDeleting)
-	ctx.CloudDiscovery()
+	ctx.reloadDiscoveryCache()
 	return nil
 }
 
@@ -255,19 +255,18 @@ func (ctx *EksCfInstanceGroupContext) discoverInstanceGroups() {
 	for _, stack := range stacks {
 		var group DiscoveredInstanceGroup
 		group.StackName = aws.StringValue(stack.StackName)
-
 		for _, tag := range stack.Tags {
 			key := aws.StringValue(tag.Key)
 			value := aws.StringValue(tag.Value)
 			switch key {
-			case tagClusterName:
+			case TagClusterName:
 				group.ClusterName = value
 				if value == specConfig.GetClusterName() {
 					group.IsClusterMember = true
 				}
-			case tagClusterNamespace:
+			case TagClusterNamespace:
 				group.Namespace = value
-			case tagInstanceGroupName:
+			case TagInstanceGroupName:
 				group.Name = value
 			}
 		}
@@ -311,10 +310,66 @@ func (ctx *EksCfInstanceGroupContext) discoverInstanceGroups() {
 }
 
 func (ctx *EksCfInstanceGroupContext) CloudDiscovery() error {
+	err := ctx.reloadDiscoveryCache()
+	if err != nil {
+		return err
+	}
+	ctx.discoverSpotPrice()
+	return nil
+
+}
+
+func (ctx *EksCfInstanceGroupContext) BootstrapNodes() error {
+	err := ctx.updateAuthConfigMap()
+	if err != nil {
+		log.Errorln("failed to update bootstrap config map")
+		return err
+	}
+	return nil
+}
+
+func (ctx *EksCfInstanceGroupContext) parseTags() {
+	var tags map[string]string
+	var cloudformationTags []*cloudformation.Tag
+
+	instanceGroup := ctx.GetInstanceGroup()
+	spec := &instanceGroup.Spec
+	meta := &instanceGroup.ObjectMeta
+	provisionerConfig := &spec.EKSCFSpec
+	specConfig := &provisionerConfig.EKSCFConfiguration
+
+	tags = map[string]string{
+		TagClusterName:       specConfig.GetClusterName(),
+		TagInstanceGroupName: meta.GetName(),
+		TagClusterNamespace:  meta.GetNamespace(),
+	}
+
+	for _, tagSet := range specConfig.GetTags() {
+		var resourceTagKey, resourceTagValue string
+		for k, v := range tagSet {
+			if strings.ToLower(k) == "key" {
+				resourceTagKey = v
+			} else if strings.ToLower(k) == "value" {
+				resourceTagValue = v
+			}
+		}
+		tags[resourceTagKey] = resourceTagValue
+	}
+
+	for k, v := range tags {
+		tag := &cloudformation.Tag{
+			Key:   aws.String(k),
+			Value: aws.String(v),
+		}
+		cloudformationTags = append(cloudformationTags, tag)
+	}
+	ctx.AwsWorker.StackTags = cloudformationTags
+}
+
+func (ctx *EksCfInstanceGroupContext) reloadDiscoveryCache() error {
 	discovery := &DiscoveredState{}
 	instanceGroup := ctx.GetInstanceGroup()
 	status := &instanceGroup.Status
-
 	stacksOutput, err := ctx.AwsWorker.DescribeCloudformationStacks()
 	if err != nil {
 		log.Errorf("failed to DescribeStacks: %v", err)
@@ -348,6 +403,19 @@ func (ctx *EksCfInstanceGroupContext) CloudDiscovery() error {
 		}
 	}
 
+	for _, launchConfig := range launchConfigOutput.LaunchConfigurations {
+		launchConfigName := aws.StringValue(launchConfig.LaunchConfigurationName)
+		launchSpotPrice := aws.StringValue(launchConfig.SpotPrice)
+		knownLaunchConfig := status.GetActiveLaunchConfigurationName()
+		if knownLaunchConfig == launchConfigName {
+			if launchSpotPrice != "" {
+				status.SetLifecycle("spot")
+			} else {
+				status.SetLifecycle("normal")
+			}
+		}
+	}
+
 	for _, scalingGroup := range discovery.GetScalingGroups() {
 		if aws.StringValue(scalingGroup.AutoScalingGroupName) == status.GetActiveScalingGroupName() {
 			status.SetCurrentMax(int(aws.Int64Value(scalingGroup.MaxSize)))
@@ -362,57 +430,9 @@ func (ctx *EksCfInstanceGroupContext) CloudDiscovery() error {
 			log.Infof("stack state: %v", *stack.StackStatus)
 		}
 	}
-
-	return nil
-}
-
-func (ctx *EksCfInstanceGroupContext) BootstrapNodes() error {
-	err := ctx.updateAuthConfigMap()
-	if err != nil {
-		log.Errorln("failed to update bootstrap config map")
-		return err
-	}
 	return nil
 }
 
-func (ctx *EksCfInstanceGroupContext) parseTags() {
-	var tags map[string]string
-	var cloudformationTags []*cloudformation.Tag
-
-	instanceGroup := ctx.GetInstanceGroup()
-	spec := &instanceGroup.Spec
-	meta := &instanceGroup.ObjectMeta
-	provisionerConfig := &spec.EKSCFSpec
-	specConfig := &provisionerConfig.EKSCFConfiguration
-
-	tags = map[string]string{
-		tagClusterName:       specConfig.GetClusterName(),
-		tagInstanceGroupName: meta.GetName(),
-		tagClusterNamespace:  meta.GetNamespace(),
-	}
-
-	for _, tagSet := range specConfig.GetTags() {
-		var resourceTagKey, resourceTagValue string
-		for k, v := range tagSet {
-			if strings.ToLower(k) == "key" {
-				resourceTagKey = v
-			} else if strings.ToLower(k) == "value" {
-				resourceTagValue = v
-			}
-		}
-		tags[resourceTagKey] = resourceTagValue
-	}
-
-	for k, v := range tags {
-		tag := &cloudformation.Tag{
-			Key:   aws.String(k),
-			Value: aws.String(v),
-		}
-		cloudformationTags = append(cloudformationTags, tag)
-	}
-	ctx.AwsWorker.StackTags = cloudformationTags
-}
-
 func (ctx *EksCfInstanceGroupContext) processParameters() error {
 	instanceGroup := ctx.GetInstanceGroup()
 	spec := &instanceGroup.Spec
diff --git a/controllers/provisioners/ekscloudformation/helpers.go b/controllers/provisioners/ekscloudformation/helpers.go
index 9ee9aa1..ed73336 100644
--- a/controllers/provisioners/ekscloudformation/helpers.go
+++ b/controllers/provisioners/ekscloudformation/helpers.go
@@ -1,10 +1,15 @@
 package ekscloudformation
 
 import (
+	"bytes"
 	"fmt"
+	"html/template"
+	"os"
 	"strings"
 	"time"
 
+	"github.com/keikoproj/instance-manager/api/v1alpha1"
+	"github.com/keikoproj/instance-manager/controllers/common"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -17,7 +22,6 @@ func createConfigMap(k kubernetes.Interface, cmData *corev1.ConfigMap) error {
 	if err != nil {
 		return err
 	}
-
 	for {
 		fieldSelector := fmt.Sprintf("metadata.name=%v", cmData.Name)
 		configMaps, err := k.CoreV1().ConfigMaps(cmData.Namespace).List(metav1.ListOptions{FieldSelector: fieldSelector})
@@ -94,3 +98,42 @@ func getUnstructuredPath(u *unstructured.Unstructured, jsonPath string) (string,
 	}
 	return value, nil
 }
+
+func (ctx *EksCfInstanceGroupContext) reloadCloudformationConfiguration() error {
+	template, err := LoadCloudformationConfiguration(ctx.GetInstanceGroup(), ctx.TemplatePath)
+	if err != nil {
+		return err
+	}
+	ctx.AwsWorker.TemplateBody = template
+	return nil
+}
+
+func LoadCloudformationConfiguration(ig *v1alpha1.InstanceGroup, path string) (string, error) {
+	var renderBuffer bytes.Buffer
+
+	funcMap := template.FuncMap{
+		"ToLower": strings.ToLower,
+	}
+
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		log.Errorf("controller cloudformation template file not found: %v", err)
+		return "", err
+	}
+
+	rawTemplate, err := common.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+
+	template, err := template.New("InstanceGroup").Funcs(funcMap).Parse(string(rawTemplate))
+	if err != nil {
+		return "", err
+	}
+
+	err = template.Execute(&renderBuffer, ig)
+	if err != nil {
+		return "", err
+	}
+
+	return renderBuffer.String(), nil
+}
diff --git a/controllers/provisioners/ekscloudformation/spot.go b/controllers/provisioners/ekscloudformation/spot.go
new file mode 100644
index 0000000..c01444f
--- /dev/null
+++ b/controllers/provisioners/ekscloudformation/spot.go
@@ -0,0 +1,116 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ekscloudformation
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"sort"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	SpotRecommendationReason  = "SpotRecommendationGiven"
+	SpotRecommendationVersion = "v1alpha1"
+)
+
+func (ctx *EksCfInstanceGroupContext) discoverSpotPrice() {
+	var (
+		instanceGroup     = ctx.GetInstanceGroup()
+		status            = &instanceGroup.Status
+		spec              = &instanceGroup.Spec
+		provisionerConfig = &spec.EKSCFSpec
+		specConfig        = &provisionerConfig.EKSCFConfiguration
+		scalingGroupName  = status.GetActiveScalingGroupName()
+	)
+
+	// get recommendations from events
+	recommendation, err := ctx.getLatestSpotRecommendation(scalingGroupName)
+	if err != nil {
+		specConfig.SetSpotPrice("")
+		return
+	}
+
+	// if no recommendations found
+	if reflect.DeepEqual(*recommendation, SpotRecommendation{}) {
+		// if it was using a recommendation before, set to false and leave price manually set
+		if status.GetUsingSpotRecommendation() {
+			status.SetUsingSpotRecommendation(false)
+		} else {
+			// if spotPrice is set without recommendation in custom resource
+			if specConfig.GetSpotPrice() != "" {
+				// keep custom resource provided value
+				log.Warnf("using spot-price '%v' without recommendations, use a recommendations controller or risk losing all instances", specConfig.GetSpotPrice())
+			}
+		}
+		return
+	}
+
+	if recommendation.APIVersion != SpotRecommendationVersion {
+		err := fmt.Sprintf("apiVersion '%v' is unknown", recommendation.APIVersion)
+		log.Warnf("failed to process spot recommendation, will override to on-demand: %v", err)
+	}
+
+	// process event message
+	status.SetUsingSpotRecommendation(true)
+
+	if recommendation.UseSpot {
+		log.Infof("spot enabled with current bid: %v", recommendation.SpotPrice)
+		specConfig.SetSpotPrice(recommendation.SpotPrice)
+	} else {
+		log.Infoln("use-spot is disabled")
+		specConfig.SetSpotPrice("")
+	}
+	ctx.reloadCloudformationConfiguration()
+}
+
+func (ctx *EksCfInstanceGroupContext) getLatestSpotRecommendation(scalingGroupName string) (*SpotRecommendation, error) {
+	var recommendations SpotReccomendations
+
+	fieldSelector := fmt.Sprintf("reason=%v,involvedObject.name=%v", SpotRecommendationReason, scalingGroupName)
+
+	listOpts := metav1.ListOptions{
+		FieldSelector: fieldSelector,
+	}
+
+	eventList, err := ctx.KubernetesClient.Kubernetes.CoreV1().Events("").List(listOpts)
+	if err != nil {
+		return &SpotRecommendation{}, err
+	}
+
+	for _, event := range eventList.Items {
+		recommendation := &SpotRecommendation{
+			EventTime: event.LastTimestamp.Time,
+		}
+		err := json.Unmarshal([]byte(event.Message), recommendation)
+		if err != nil {
+			return &SpotRecommendation{}, err
+		}
+		recommendations = append(recommendations, *recommendation)
+	}
+	sort.Sort(sort.Reverse(recommendations))
+
+	if len(recommendations) < 1 {
+		return &SpotRecommendation{}, nil
+	}
+
+	latestRecommendation := &recommendations[0]
+
+	return latestRecommendation, nil
+
+}
diff --git a/controllers/provisioners/ekscloudformation/types.go b/controllers/provisioners/ekscloudformation/types.go
index 97ede1a..62717c8 100644
--- a/controllers/provisioners/ekscloudformation/types.go
+++ b/controllers/provisioners/ekscloudformation/types.go
@@ -17,6 +17,7 @@ package ekscloudformation
 
 import (
 	"reflect"
+	"time"
 
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/cloudformation"
@@ -54,6 +55,7 @@ type EksCfInstanceGroupContext struct {
 	KubernetesClient common.KubernetesClientSet
 	AwsWorker        aws.AwsWorker
 	DiscoveredState  *DiscoveredState
+	TemplatePath     string
 	StackExists      bool
 	InstanceArn      string
 	ControllerRegion string
@@ -231,3 +233,24 @@ func (d *DiscoveredInstanceGroup) GetARN() string {
 	}
 	return ""
 }
+
+type SpotRecommendation struct {
+	APIVersion string `yaml:"apiVersion"`
+	SpotPrice  string `yaml:"spotPrice"`
+	UseSpot    bool   `yaml:"useSpot"`
+	EventTime  time.Time
+}
+
+type SpotReccomendations []SpotRecommendation
+
+func (p SpotReccomendations) Len() int {
+	return len(p)
+}
+
+func (p SpotReccomendations) Less(i, j int) bool {
+	return p[i].EventTime.Before(p[j].EventTime)
+}
+
+func (p SpotReccomendations) Swap(i, j int) {
+	p[i], p[j] = p[j], p[i]
+}
diff --git a/docs/04_instance-manager.yaml b/docs/04_instance-manager.yaml
index 70744b1..a858217 100644
--- a/docs/04_instance-manager.yaml
+++ b/docs/04_instance-manager.yaml
@@ -35,6 +35,10 @@ spec:
     description: instance group upgrade strategy
     name: Strategy
     type: string
+  - JSONPath: .status.lifecycle
+    description: instance group lifecycle spot/normal
+    name: Lifecycle
+    type: string
   - JSONPath: .metadata.creationTimestamp
     description: time passed since instancegroup creation
     name: Age
@@ -439,6 +443,8 @@ spec:
                       items:
                         type: string
                       type: array
+                    spotPrice:
+                      type: string
                     subnets:
                       items:
                         type: string
@@ -510,10 +516,16 @@ spec:
               type: integer
             currentState:
               type: string
+            lifecycle:
+              type: string
             nodesInstanceRoleArn:
               type: string
             strategyResourceName:
               type: string
+            usingSpotRecommendation:
+              type: boolean
+          required:
+          - usingSpotRecommendation
           type: object
       required:
       - metadata
@@ -813,6 +825,10 @@ data:
           ImageId: !Ref NodeImageId
           InstanceType: !Ref NodeInstanceType
           KeyName: !Ref KeyName
+          {{$spotPrice := .Spec.EKSCFSpec.EKSCFConfiguration.SpotPrice}}
+          {{if (ne $spotPrice "")}}
+          SpotPrice: {{$spotPrice}}
+          {{end}}
           SecurityGroups: !Ref NodeSecurityGroups
           BlockDeviceMappings:
             - DeviceName: /dev/xvda
diff --git a/docs/README.md b/docs/README.md
index ab27bf1..ef20f83 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -386,6 +386,22 @@ There might be cases where you would want to implement custom behavior as part o
 
 In [this](./examples/crd-argo.yaml) example, we submit an [argo](https://github.com/argoproj/argo) workflow as a response to an upgrade events.
 
+#### Using spot instances
+
+You can switch to spot instances in two ways:
+
+- Manually set the `spec.eks-cf.configuration.spotPrice` to a spot price value, the downside is that if price becomes invalid (due to price changes) and instances gets pulled, you may end up with no instances until you modify the price again.
+
+- Use a spot recommendation controller such as `minion-manager`, instance-manager will look at events with the following message format:
+
+```json
+{"apiVersion":"v1alpha1","spotPrice":"0.0067", "useSpot": true}
+```
+
+In addition, the event `involvedObject.name`, must be the name of the autoscaling group to switch, and the event `.reason` must be `SpotRecommendationGiven`.
+
+When recommendations are not available (no events for an hour / recommendation controller is down), instance-group will retain the last provided configuration, until a human either changes back to on-demand (by setting `spotPrice: ""`) or until recommendation events are found again.
+
 ### Clean up
 
 First, delete the instance group:
diff --git a/main.go b/main.go
index b9aef54..5a55a60 100644
--- a/main.go
+++ b/main.go
@@ -21,10 +21,10 @@ import (
 	runt "runtime"
 
 	instancemgrv1alpha1 "github.com/keikoproj/instance-manager/api/v1alpha1"
-	log "github.com/sirupsen/logrus"
-
 	"github.com/keikoproj/instance-manager/controllers"
-
+	awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws"
+	log "github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -42,6 +42,7 @@ const controllerVersion = "instancemgr-0.3.2"
 func init() {
 
 	instancemgrv1alpha1.AddToScheme(scheme)
+	corev1.AddToScheme(scheme)
 	// +kubebuilder:scaffold:scheme
 }
 
@@ -58,19 +59,28 @@ func main() {
 	})
 	printVersion()
 
-	var metricsAddr string
-	var enableLeaderElection bool
-	var controllerConfPath string
-	var controllerTemplatePath string
+	var (
+		metricsAddr            string
+		enableLeaderElection   bool
+		controllerConfPath     string
+		controllerTemplatePath string
+		region                 string
+		err                    error
+	)
+
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&controllerConfPath, "controller-config", "/etc/config/controller.conf", "The controller config file")
 	flag.StringVar(&controllerTemplatePath, "controller-template", "/etc/config/cloudformation.template", "The controller template file")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	flag.Parse()
-
 	ctrl.SetLogger(zap.Logger(true))
 
+	region, err = awsprovider.GetRegion()
+	if err != nil {
+		log.Fatalf("failed to detect region: %v", err)
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,
@@ -86,6 +96,7 @@ func main() {
 		Log:                    ctrl.Log.WithName("controllers").WithName("InstanceGroup"),
 		ControllerConfPath:     controllerConfPath,
 		ControllerTemplatePath: controllerTemplatePath,
+		ScalingGroups:          awsprovider.GetAwsAsgClient(region),
 	}).SetupWithManager(mgr)
 	if err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "InstanceGroup")


Test patch:
diff --git a/controllers/provisioners/ekscloudformation/ekscloudformation_test.go b/controllers/provisioners/ekscloudformation/ekscloudformation_test.go
index f33fa3e..0d84dd3 100644
--- a/controllers/provisioners/ekscloudformation/ekscloudformation_test.go
+++ b/controllers/provisioners/ekscloudformation/ekscloudformation_test.go
@@ -114,9 +114,11 @@ type EksCfUnitTest struct {
 	StackARN              string
 	ExistingARNs          []string
 	ExistingUnmanagedARNs []string
+	ExistingEvents        []*corev1.Event
 	VpcID                 string
 	ExpectedState         v1alpha1.ReconcileState
 	ExpectedCR            int
+	ExpectedSpotPrice     string
 	ExpectedAuthConfigMap *corev1.ConfigMap
 }
 
@@ -159,6 +161,7 @@ type FakeIG struct {
 
 type FakeStack struct {
 	StackName     string
+	AsgName       string
 	ClusterName   string
 	ResourceName  string
 	NamespaceName string
@@ -202,6 +205,7 @@ func (s *stubCF) DescribeStacks(input *cloudformation.DescribeStacksInput) (*clo
 				StackState:    s.StackState,
 				StackARN:      arn,
 				StackName:     "stack-name",
+				AsgName:       "my-asg",
 			}
 			existingStack := createFakeStack(fakeStackInput)
 			existingStacks = append(existingStacks, existingStack)
@@ -217,6 +221,7 @@ func (s *stubCF) DescribeStacks(input *cloudformation.DescribeStacksInput) (*clo
 			StackState:    s.StackState,
 			StackARN:      s.StackARN,
 			StackName:     "stack-name",
+			AsgName:       "my-asg",
 		}
 		fakeStack = createFakeStack(fakeStackInput)
 		output.Stacks = append(output.Stacks, fakeStack)
@@ -351,7 +356,7 @@ func createFakeStack(f FakeStack) *cloudformation.Stack {
 			"LaunchConfigName": f.StackARN,
 		},
 		{
-			"AsgName": f.StackName,
+			"AsgName": f.AsgName,
 		},
 	}
 
@@ -484,6 +489,24 @@ func (f *FakeIG) getInstanceGroup() *v1alpha1.InstanceGroup {
 	return instanceGroup
 }
 
+func getSpotSuggestionEvent(id, scalingGroup, price string, useSpot bool, ts time.Time) *corev1.Event {
+	message := fmt.Sprintf(`{"apiVersion":"v1alpha1","spotPrice":"%v", "useSpot": %t}`, price, useSpot)
+	event := &corev1.Event{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: fmt.Sprintf("spot-manager-%v.000000000000000", id),
+		},
+		LastTimestamp: metav1.Time{Time: ts},
+		Reason:        "SpotRecommendationGiven",
+		Message:       message,
+		Type:          "Normal",
+		InvolvedObject: corev1.ObjectReference{
+			Namespace: "kube-system",
+			Name:      scalingGroup,
+		},
+	}
+	return event
+}
+
 func getBasicContext(t *testing.T, awsMocker awsprovider.AwsWorker) EksCfInstanceGroupContext {
 	client := fake.NewSimpleClientset()
 	dynScheme := runtime.NewScheme()
@@ -563,6 +586,12 @@ func (u *EksCfUnitTest) Run(t *testing.T) {
 	}
 	u.Provisioner = &provisioner
 
+	if len(u.ExistingEvents) != 0 {
+		for _, event := range u.ExistingEvents {
+			kube.Kubernetes.CoreV1().Events("kube-system").Create(event)
+		}
+	}
+
 	u.Provisioner.CloudDiscovery()
 	u.Provisioner.StateDiscovery()
 
@@ -642,6 +671,11 @@ func (u *EksCfUnitTest) Run(t *testing.T) {
 	if !reflect.DeepEqual(cm, u.ExpectedAuthConfigMap) {
 		t.Fatalf("BootstrapNodes, expected:\n %#v, \ngot:\n %#v", u.ExpectedAuthConfigMap.Data, cm.Data)
 	}
+
+	spotPrice := u.Provisioner.InstanceGroup.Spec.EKSCFSpec.EKSCFConfiguration.SpotPrice
+	if spotPrice != u.ExpectedSpotPrice {
+		t.Fatalf("SpotPrice, expected:\n %#v, \ngot:\n %#v", u.ExpectedSpotPrice, spotPrice)
+	}
 }
 
 func TestStateDiscoveryInitUpdate(t *testing.T) {
@@ -875,6 +909,88 @@ func TestCrdStrategyCRExist(t *testing.T) {
 	testCase.Run(t)
 }
 
+func TestSpotInstancesRecommendationEnable(t *testing.T) {
+	ig := FakeIG{
+		UpgradeStrategyType:             "crd",
+		UpgradeStrategyCRD:              "rollingupgrade",
+		UpgradeStrategyCRDSpec:          getRollupSpec("rollup"),
+		UpgradeStrategyCRDStatusPath:    "status.currentStatus",
+		UpgradeStrategyCRDStatusSuccess: "success",
+	}
+	instanceGroup := ig.getInstanceGroup()
+	instanceGroup.Status.ActiveScalingGroupName = "my-asg"
+	events := []*corev1.Event{
+		getSpotSuggestionEvent("1", "my-asg", "0.005", true, time.Now()),
+		getSpotSuggestionEvent("2", "my-asg", "0.007", true, time.Now().Add(time.Minute*time.Duration(3))),
+		getSpotSuggestionEvent("3", "my-asg", "0.006", true, time.Now().Add(time.Minute*time.Duration(5))),
+	}
+	testCase := EksCfUnitTest{
+		Description:       "Spot Instances - should take latest event's recommendation (enable)",
+		LoadCRD:           "rollingupgrade",
+		InstanceGroup:     instanceGroup,
+		StackExist:        true,
+		StackUpdateNeeded: true,
+		ExistingEvents:    events,
+		ExpectedSpotPrice: "0.006",
+		ExpectedState:     v1alpha1.ReconcileInitUpdate,
+		ExpectedCR:        1,
+	}
+	testCase.Run(t)
+}
+
+func TestSpotInstancesRecommendationDisable(t *testing.T) {
+	ig := FakeIG{
+		UpgradeStrategyType:             "crd",
+		UpgradeStrategyCRD:              "rollingupgrade",
+		UpgradeStrategyCRDSpec:          getRollupSpec("rollup"),
+		UpgradeStrategyCRDStatusPath:    "status.currentStatus",
+		UpgradeStrategyCRDStatusSuccess: "success",
+	}
+	instanceGroup := ig.getInstanceGroup()
+	instanceGroup.Status.ActiveScalingGroupName = "my-asg"
+	events := []*corev1.Event{
+		getSpotSuggestionEvent("1", "my-asg", "0.005", true, time.Now()),
+		getSpotSuggestionEvent("3", "my-asg", "0.006", false, time.Now().Add(time.Minute*time.Duration(6))),
+		getSpotSuggestionEvent("2", "my-asg", "0.007", true, time.Now().Add(time.Minute*time.Duration(5))),
+	}
+	testCase := EksCfUnitTest{
+		Description:       "Spot Instances - should take latest event's recommendation (disable)",
+		LoadCRD:           "rollingupgrade",
+		InstanceGroup:     instanceGroup,
+		StackExist:        true,
+		StackUpdateNeeded: true,
+		ExistingEvents:    events,
+		ExpectedSpotPrice: "",
+		ExpectedState:     v1alpha1.ReconcileInitUpdate,
+		ExpectedCR:        1,
+	}
+	testCase.Run(t)
+}
+
+func TestSpotInstancesManual(t *testing.T) {
+	ig := FakeIG{
+		UpgradeStrategyType:             "crd",
+		UpgradeStrategyCRD:              "rollingupgrade",
+		UpgradeStrategyCRDSpec:          getRollupSpec("rollup"),
+		UpgradeStrategyCRDStatusPath:    "status.currentStatus",
+		UpgradeStrategyCRDStatusSuccess: "success",
+	}
+	instanceGroup := ig.getInstanceGroup()
+	instanceGroup.Status.ActiveScalingGroupName = "my-asg"
+	instanceGroup.Spec.EKSCFSpec.EKSCFConfiguration.SpotPrice = "0.005"
+	testCase := EksCfUnitTest{
+		Description:       "Spot Instances - should take user input from custom resource",
+		LoadCRD:           "rollingupgrade",
+		InstanceGroup:     instanceGroup,
+		StackExist:        true,
+		StackUpdateNeeded: true,
+		ExpectedSpotPrice: "0.005",
+		ExpectedState:     v1alpha1.ReconcileInitUpdate,
+		ExpectedCR:        1,
+	}
+	testCase.Run(t)
+}
+
 func TestCrdStrategyCRLongName(t *testing.T) {
 	ig := FakeIG{
 		UpgradeStrategyType:             "crd",
diff --git a/test-bdd/main_test.go b/test-bdd/main_test.go
index f9b1906..4392dc3 100644
--- a/test-bdd/main_test.go
+++ b/test-bdd/main_test.go
@@ -24,11 +24,11 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
 	"github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface"
+	awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws"
+	"github.com/keikoproj/instance-manager/test-bdd/testutil"
 	. "github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
-	awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws"
-	"github.com/keikoproj/instance-manager/test-bdd/testutil"
 	"github.com/sirupsen/logrus"
 	apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/dynamic"