Repository: keikoproj/instance-manager (go)

Issue:
Event publishing
Provisioners should publish the following events:
- IG creation is successful / failed.
- NodesReady condition goes from true/false.
- Upgrade (rotation) started / successful / failed

Pull request description:
Event Publishing
Fixes #100 

Adds basic event publishing

- [x] BDD

Patch:
diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go
index a42bb6e..4f0ab7d 100644
--- a/api/v1alpha1/groupversion_info.go
+++ b/api/v1alpha1/groupversion_info.go
@@ -24,9 +24,16 @@ import (
 )
 
 var (
-	// GroupVersion is group version used to register these objects
+	// GroupVersion is group version used to register instance groups
 	GroupVersion = schema.GroupVersion{Group: "instancemgr.keikoproj.io", Version: "v1alpha1"}
 
+	// GroupVersionResource is group version resource used to register instance groups
+	GroupVersionResource = schema.GroupVersionResource{
+		Group:    "instancemgr.keikoproj.io",
+		Version:  "v1alpha1",
+		Resource: "instancegroups",
+	}
+
 	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
 	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
 
diff --git a/api/v1alpha1/instancegroup_types.go b/api/v1alpha1/instancegroup_types.go
index 338379a..94df259 100644
--- a/api/v1alpha1/instancegroup_types.go
+++ b/api/v1alpha1/instancegroup_types.go
@@ -22,7 +22,6 @@ import (
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
@@ -66,11 +65,6 @@ var (
 		},
 	}
 
-	GroupVersionResource = schema.GroupVersionResource{
-		Group:    "instancemgr.keikoproj.io",
-		Version:  "v1alpha1",
-		Resource: "instancegroups",
-	}
 	log = ctrl.Log.WithName("v1alpha1")
 )
 
diff --git a/controllers/providers/kubernetes/events.go b/controllers/providers/kubernetes/events.go
new file mode 100644
index 0000000..8cfd2ad
--- /dev/null
+++ b/controllers/providers/kubernetes/events.go
@@ -0,0 +1,136 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubernetes
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"time"
+
+	"github.com/keikoproj/instance-manager/api/v1alpha1"
+	"k8s.io/apimachinery/pkg/types"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+// EventKind defines the kind of an event
+type EventKind string
+
+// EventLevel defines the level of an event
+type EventLevel string
+
+var (
+	// InvolvedObjectKind is the default kind of involved objects
+	InvolvedObjectKind = "InstanceGroup"
+	// EventName is the default name for service events
+	EventControllerName = "instance-manager"
+	// EventLevelNormal is the level of a normal event
+	EventLevelNormal = "Normal"
+	// EventLevelWarning is the level of a warning event
+	EventLevelWarning = "Warning"
+
+	InstanceGroupCreatedEvent       EventKind = "InstanceGroupCreated"
+	InstanceGroupDeletedEvent       EventKind = "InstanceGroupDeleted"
+	NodesReadyEvent                 EventKind = "InstanceGroupNodesReady"
+	NodesNotReadyEvent              EventKind = "InstanceGroupNodesNotReady"
+	InstanceGroupUpgradeFailedEvent EventKind = "InstanceGroupUpgradeFailed"
+
+	EventLevels = map[EventKind]string{
+		InstanceGroupCreatedEvent:       EventLevelNormal,
+		InstanceGroupDeletedEvent:       EventLevelNormal,
+		NodesNotReadyEvent:              EventLevelWarning,
+		NodesReadyEvent:                 EventLevelNormal,
+		InstanceGroupUpgradeFailedEvent: EventLevelWarning,
+	}
+
+	EventMessages = map[EventKind]string{
+		InstanceGroupCreatedEvent:       "instance group has been successfully created",
+		InstanceGroupDeletedEvent:       "instance group has been successfully deleted",
+		InstanceGroupUpgradeFailedEvent: "instance group has failed upgrading",
+		NodesNotReadyEvent:              "instance group nodes are not ready",
+		NodesReadyEvent:                 "instance group nodes are ready",
+	}
+)
+
+type EventPublisher struct {
+	Client          kubernetes.Interface
+	Name            string
+	Namespace       string
+	UID             types.UID
+	ResourceVersion string
+}
+
+func (e *EventPublisher) Publish(kind EventKind, keysAndValues ...interface{}) {
+
+	messageFields := make(map[string]string)
+	messageFields["msg"] = getEventMessage(kind)
+
+	for i := 0; i < len(keysAndValues); i += 2 {
+		key := keysAndValues[i].(string)
+		value := keysAndValues[i+1].(string)
+		messageFields[key] = value
+	}
+
+	payload, err := json.Marshal(messageFields)
+	if err != nil {
+		log.Error(err, "failed to marshal event message fields", "fields", messageFields)
+	}
+
+	now := time.Now()
+
+	eventName := fmt.Sprintf("%v.%v.%v", EventControllerName, time.Now().Unix(), rand.Int())
+	event := &v1.Event{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      eventName,
+			Namespace: e.Namespace,
+		},
+		InvolvedObject: v1.ObjectReference{
+			Kind:            InvolvedObjectKind,
+			Namespace:       e.Namespace,
+			Name:            e.Name,
+			APIVersion:      v1alpha1.GroupVersion.Version,
+			UID:             e.UID,
+			ResourceVersion: e.ResourceVersion,
+		},
+		Reason:         string(kind),
+		Message:        string(payload),
+		Type:           getEventLevel(kind),
+		FirstTimestamp: metav1.NewTime(now),
+		LastTimestamp:  metav1.NewTime(now),
+	}
+
+	_, err = e.Client.CoreV1().Events(e.Namespace).Create(event)
+	if err != nil {
+		log.Error(err, "failed to publish event", "event", event)
+	}
+}
+
+func getEventLevel(kind EventKind) string {
+	if val, ok := EventLevels[kind]; ok {
+		return val
+	}
+	return EventLevelNormal
+}
+
+func getEventMessage(kind EventKind) string {
+	if val, ok := EventMessages[kind]; ok {
+		return val
+	}
+	return EventLevelNormal
+}
diff --git a/controllers/provisioners/eks/cloud.go b/controllers/provisioners/eks/cloud.go
index 6039942..f622560 100644
--- a/controllers/provisioners/eks/cloud.go
+++ b/controllers/provisioners/eks/cloud.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	"github.com/keikoproj/instance-manager/api/v1alpha1"
+	kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes"
 	corev1 "k8s.io/api/core/v1"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -27,6 +28,18 @@ import (
 	"github.com/pkg/errors"
 )
 
+type DiscoveredState struct {
+	Provisioned                   bool
+	NodesReady                    bool
+	OwnedScalingGroups            []*autoscaling.Group
+	ScalingGroup                  *autoscaling.Group
+	LaunchConfiguration           *autoscaling.LaunchConfiguration
+	ActiveLaunchConfigurationName string
+	IAMRole                       *iam.Role
+	InstanceProfile               *iam.InstanceProfile
+	Publisher                     kubeprovider.EventPublisher
+}
+
 func (ctx *EksInstanceGroupContext) CloudDiscovery() error {
 	var (
 		state         = ctx.GetDiscoveredState()
@@ -36,6 +49,14 @@ func (ctx *EksInstanceGroupContext) CloudDiscovery() error {
 		clusterName   = configuration.GetClusterName()
 	)
 
+	state.Publisher = kubeprovider.EventPublisher{
+		Client:          ctx.KubernetesClient.Kubernetes,
+		Namespace:       instanceGroup.GetNamespace(),
+		Name:            instanceGroup.GetName(),
+		UID:             instanceGroup.GetUID(),
+		ResourceVersion: instanceGroup.GetResourceVersion(),
+	}
+
 	var roleName, instanceProfileName string
 	if configuration.HasExistingRole() {
 		roleName = configuration.GetRoleName()
@@ -119,17 +140,6 @@ func (ctx *EksInstanceGroupContext) CloudDiscovery() error {
 	return nil
 }
 
-type DiscoveredState struct {
-	Provisioned                   bool
-	NodesReady                    bool
-	OwnedScalingGroups            []*autoscaling.Group
-	ScalingGroup                  *autoscaling.Group
-	LaunchConfiguration           *autoscaling.LaunchConfiguration
-	ActiveLaunchConfigurationName string
-	IAMRole                       *iam.Role
-	InstanceProfile               *iam.InstanceProfile
-}
-
 func (d *DiscoveredState) SetScalingGroup(asg *autoscaling.Group) {
 	if asg != nil {
 		d.ScalingGroup = asg
diff --git a/controllers/provisioners/eks/create.go b/controllers/provisioners/eks/create.go
index c0503d1..f5be0fa 100644
--- a/controllers/provisioners/eks/create.go
+++ b/controllers/provisioners/eks/create.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	"github.com/keikoproj/instance-manager/api/v1alpha1"
+	kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes"
 	"github.com/pkg/errors"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -86,7 +87,6 @@ func (ctx *EksInstanceGroupContext) CreateScalingGroup() error {
 		return err
 	}
 	ctx.Log.Info("created scaling group", "instancegroup", instanceGroup.GetName(), "scalinggroup", asgName)
-
 	scalingGroup, err := ctx.AwsWorker.GetAutoscalingGroup(asgName)
 	if err != nil {
 		return err
@@ -96,6 +96,7 @@ func (ctx *EksInstanceGroupContext) CreateScalingGroup() error {
 		state.SetScalingGroup(scalingGroup)
 	}
 
+	state.Publisher.Publish(kubeprovider.InstanceGroupCreatedEvent, "instancegroup", instanceGroup.GetName(), "scalinggroup", asgName)
 	return nil
 }
 
diff --git a/controllers/provisioners/eks/delete.go b/controllers/provisioners/eks/delete.go
index 72c1f03..511f715 100644
--- a/controllers/provisioners/eks/delete.go
+++ b/controllers/provisioners/eks/delete.go
@@ -17,6 +17,7 @@ package eks
 
 import (
 	"github.com/keikoproj/instance-manager/api/v1alpha1"
+	kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes"
 	"github.com/pkg/errors"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -74,6 +75,7 @@ func (ctx *EksInstanceGroupContext) DeleteScalingGroup() error {
 		return err
 	}
 	ctx.Log.Info("deleted scaling group", "instancegroup", instanceGroup.GetName(), "scalinggroup", asgName)
+	state.Publisher.Publish(kubeprovider.InstanceGroupDeletedEvent, "instancegroup", instanceGroup.GetName(), "scalinggroup", asgName)
 	return nil
 }
 
diff --git a/controllers/provisioners/eks/helpers.go b/controllers/provisioners/eks/helpers.go
index e0c0dfb..aa9e08f 100644
--- a/controllers/provisioners/eks/helpers.go
+++ b/controllers/provisioners/eks/helpers.go
@@ -297,20 +297,27 @@ func (ctx *EksInstanceGroupContext) UpdateNodeReadyCondition() bool {
 		instanceIds = append(instanceIds, aws.StringValue(instance.InstanceId))
 	}
 
+	instances := strings.Join(instanceIds, ",")
+
 	var conditions []v1alpha1.InstanceGroupCondition
 	ok, err := kubeprovider.IsDesiredNodesReady(ctx.KubernetesClient.Kubernetes, instanceIds, desiredCount)
 	if err != nil {
 		ctx.Log.Error(err, "could not update node conditions", "instancegroup", instanceGroup.GetName())
 		return false
 	}
-
 	if ok {
+		if !state.IsNodesReady() {
+			state.Publisher.Publish(kubeprovider.NodesReadyEvent, "instancegroup", instanceGroup.GetName(), "instances", instances)
+		}
 		state.SetNodesReady(true)
 		conditions = append(conditions, v1alpha1.NewInstanceGroupCondition(v1alpha1.NodesReady, corev1.ConditionTrue))
 		status.SetConditions(conditions)
 		return true
 	}
 
+	if state.IsNodesReady() {
+		state.Publisher.Publish(kubeprovider.NodesNotReadyEvent, "instancegroup", instanceGroup.GetName(), "instances", instances)
+	}
 	state.SetNodesReady(false)
 	conditions = append(conditions, v1alpha1.NewInstanceGroupCondition(v1alpha1.NodesReady, corev1.ConditionFalse))
 	status.SetConditions(conditions)
diff --git a/controllers/provisioners/eks/upgrade.go b/controllers/provisioners/eks/upgrade.go
index b256d8f..5545d01 100644
--- a/controllers/provisioners/eks/upgrade.go
+++ b/controllers/provisioners/eks/upgrade.go
@@ -30,6 +30,7 @@ func (ctx *EksInstanceGroupContext) UpgradeNodes() error {
 	var (
 		instanceGroup = ctx.GetInstanceGroup()
 		strategy      = ctx.GetUpgradeStrategy()
+		state         = ctx.GetDiscoveredState()
 		strategyType  = strings.ToLower(strategy.GetType())
 	)
 
@@ -38,6 +39,7 @@ func (ctx *EksInstanceGroupContext) UpgradeNodes() error {
 	case kubeprovider.CRDStrategyName:
 		ok, err := kubeprovider.ProcessCRDStrategy(ctx.KubernetesClient.KubeDynamic, instanceGroup)
 		if err != nil {
+			state.Publisher.Publish(kubeprovider.InstanceGroupUpgradeFailedEvent, "instancegroup", instanceGroup.GetName(), "type", kubeprovider.CRDStrategyName, "error", err.Error())
 			instanceGroup.SetState(v1alpha1.ReconcileErr)
 			return errors.Wrap(err, "failed to process CRD strategy")
 		}
@@ -49,6 +51,7 @@ func (ctx *EksInstanceGroupContext) UpgradeNodes() error {
 		req := ctx.NewRollingUpdateRequest()
 		ok, err := kubeprovider.ProcessRollingUpgradeStrategy(req)
 		if err != nil {
+			state.Publisher.Publish(kubeprovider.InstanceGroupUpgradeFailedEvent, "instancegroup", instanceGroup.GetName(), "type", kubeprovider.RollingUpdateStrategyName, "error", err)
 			instanceGroup.SetState(v1alpha1.ReconcileErr)
 			return errors.Wrap(err, "failed to process rolling-update strategy")
 		}
diff --git a/go.mod b/go.mod
index 7094318..8ddff68 100644
--- a/go.mod
+++ b/go.mod
@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/aws/aws-sdk-go v1.30.24
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cucumber/godog v0.8.1
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-logr/logr v0.1.0
@@ -12,10 +13,11 @@ require (
 	github.com/imdario/mergo v0.3.9 // indirect
 	github.com/json-iterator/go v1.1.9 // indirect
 	github.com/keikoproj/aws-auth v0.0.0-20200510193503-dcb7aa6350e1
-	github.com/keikoproj/aws-sdk-go-cache v0.0.0-20200124200058-ab3c8c94044a
 	github.com/kr/pretty v0.2.0 // indirect
 	github.com/onsi/gomega v1.9.0
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/common v0.9.1 // indirect
+	github.com/prometheus/procfs v0.0.8 // indirect
 	github.com/sirupsen/logrus v1.4.2
 	go.uber.org/atomic v1.4.0 // indirect
 	go.uber.org/zap v1.10.0 // indirect
diff --git a/go.sum b/go.sum
index c0ed681..9175b92 100644
--- a/go.sum
+++ b/go.sum
@@ -27,7 +27,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.25.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.30.24 h1:y3JPD51VuEmVqN3BEDVm4amGpDma2cKJcDPuAU1OR58=
 github.com/aws/aws-sdk-go v1.30.24/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
@@ -184,7 +183,6 @@ github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
 github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -198,14 +196,8 @@ github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGn
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
-github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
-github.com/karlseguin/expect v1.0.1 h1:z4wy4npwwHSWKjGWH85WNJO42VQhovxTCZDSzhjo8hY=
-github.com/karlseguin/expect v1.0.1/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
 github.com/keikoproj/aws-auth v0.0.0-20200510193503-dcb7aa6350e1 h1:WKfi7DAtxHHoQNQM+oU1Y4R81HLmiSS7DHZe2o058oM=
 github.com/keikoproj/aws-auth v0.0.0-20200510193503-dcb7aa6350e1/go.mod h1:/DQXDbPWiYraLMdLTbmf4JHjxQJkcW6+he0EwBG+snU=
-github.com/keikoproj/aws-sdk-go-cache v0.0.0-20200124200058-ab3c8c94044a h1:T4PGY8X0NbpEPM3U7HxLX616PBgjWxvn2YQNYkdaEeU=
-github.com/keikoproj/aws-sdk-go-cache v0.0.0-20200124200058-ab3c8c94044a/go.mod h1:PqUW3kzHKLscAxysVG6mJz5wmk4JV9GSwX+k4qjqi1c=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
@@ -319,8 +311,6 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
-github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
-github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
 github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -377,7 +367,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f h1:QBjCr1Fz5kw158VqdE9JfI9cJnl/ymnJWAdMuinqL7Y=
@@ -482,8 +471,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/karlseguin/expect.v1 v1.0.1 h1:9u0iUltnhFbJTHaSIH0EP+cuTU5rafIgmcsEsg2JQFw=
-gopkg.in/karlseguin/expect.v1 v1.0.1/go.mod h1:uB7QIJBcclvYbwlUDkSCsGjAOMis3fP280LyhuDEf2I=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=


Test patch:
diff --git a/controllers/provisioners/eks/create_test.go b/controllers/provisioners/eks/create_test.go
index 7f41c0c..d106249 100644
--- a/controllers/provisioners/eks/create_test.go
+++ b/controllers/provisioners/eks/create_test.go
@@ -19,6 +19,8 @@ import (
 	"fmt"
 	"testing"
 
+	kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
@@ -41,6 +43,7 @@ func TestCreateManagedRolePositive(t *testing.T) {
 	w := MockAwsWorker(asgMock, iamMock)
 	ctx := MockContext(ig, k, w)
 	state := ctx.GetDiscoveredState()
+	state.Publisher.Client = k.Kubernetes
 
 	// Mock role/profile do not exist so they are always created
 	iamMock.GetRoleErr = errors.New("not found")
@@ -79,6 +82,9 @@ func TestCreateLaunchConfigurationPositive(t *testing.T) {
 	ig.GetEKSConfiguration().SetRoleName("some-role")
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		InstanceProfile: &iam.InstanceProfile{
 			Arn: aws.String("some-profile-arn"),
 		},
@@ -119,6 +125,9 @@ func TestCreateScalingGroupPositive(t *testing.T) {
 	}
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		LaunchConfiguration: mockLaunchConfiguration,
 	})
 
@@ -162,6 +171,9 @@ func TestCreateNoOp(t *testing.T) {
 	}
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		LaunchConfiguration: mockLaunchConfiguration,
 		ScalingGroup:        mockScalingGroup,
 	})
@@ -234,6 +246,9 @@ func TestCreateLaunchConfigNegative(t *testing.T) {
 	ig.GetEKSConfiguration().SetInstanceProfileName("some-profile")
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		InstanceProfile: &iam.InstanceProfile{
 			Arn: aws.String("arn"),
 		},
@@ -279,6 +294,9 @@ func TestCreateAutoScalingGroupNegative(t *testing.T) {
 	ig.GetEKSConfiguration().SetInstanceProfileName("some-profile")
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		LaunchConfiguration: &autoscaling.LaunchConfiguration{LaunchConfigurationName: aws.String("launch-config")},
 	})
 
diff --git a/controllers/provisioners/eks/delete_test.go b/controllers/provisioners/eks/delete_test.go
index bb3c3eb..ecd5d44 100644
--- a/controllers/provisioners/eks/delete_test.go
+++ b/controllers/provisioners/eks/delete_test.go
@@ -43,6 +43,9 @@ func TestDeletePositive(t *testing.T) {
 	ctx := MockContext(ig, k, w)
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		ScalingGroup:        &autoscaling.Group{},
 		LaunchConfiguration: &autoscaling.LaunchConfiguration{},
 		IAMRole:             &iam.Role{},
@@ -67,6 +70,9 @@ func TestDeleteManagedRoleNegative(t *testing.T) {
 	ctx := MockContext(ig, k, w)
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		IAMRole: &iam.Role{},
 	})
 
@@ -95,6 +101,9 @@ func TestDeleteLaunchConfigurationNegative(t *testing.T) {
 	ctx := MockContext(ig, k, w)
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		LaunchConfiguration: &autoscaling.LaunchConfiguration{},
 	})
 
@@ -118,6 +127,9 @@ func TestDeleteAutoScalingGroupNegative(t *testing.T) {
 	ctx := MockContext(ig, k, w)
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		ScalingGroup: &autoscaling.Group{},
 	})
 
@@ -159,6 +171,9 @@ func TestRemoveAuthRoleNegative(t *testing.T) {
 	g.Expect(err).NotTo(gomega.HaveOccurred())
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		IAMRole: &iam.Role{
 			Arn: aws.String("same-role"),
 		},
diff --git a/controllers/provisioners/eks/update_test.go b/controllers/provisioners/eks/update_test.go
index 270049e..f03fa46 100644
--- a/controllers/provisioners/eks/update_test.go
+++ b/controllers/provisioners/eks/update_test.go
@@ -18,6 +18,8 @@ package eks
 import (
 	"testing"
 
+	kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/iam"
@@ -55,6 +57,9 @@ func TestUpdateScalingGroupPositive(t *testing.T) {
 	asgMock.AutoScalingGroups = []*autoscaling.Group{mockScalingGroup}
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		ScalingGroup:                  mockScalingGroup,
 		ActiveLaunchConfigurationName: "some-launch-config",
 		LaunchConfiguration:           mockLaunchConfig,
@@ -108,6 +113,9 @@ func TestUpdateWithDriftRotationPositive(t *testing.T) {
 
 	// missing launch config causes drift
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		ScalingGroup:                  mockScalingGroup,
 		ActiveLaunchConfigurationName: "some-launch-config",
 		InstanceProfile: &iam.InstanceProfile{
@@ -167,6 +175,9 @@ func TestUpdateWithRotationPositive(t *testing.T) {
 	asgMock.AutoScalingGroups = []*autoscaling.Group{mockScalingGroup}
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		ScalingGroup:                  mockScalingGroup,
 		ActiveLaunchConfigurationName: "some-launch-config",
 		LaunchConfiguration:           mockLaunchConfig,
@@ -245,6 +256,9 @@ func TestLaunchConfigurationDrifted(t *testing.T) {
 
 	for _, tc := range tests {
 		ctx.SetDiscoveredState(&DiscoveredState{
+			Publisher: kubeprovider.EventPublisher{
+				Client: k.Kubernetes,
+			},
 			ActiveLaunchConfigurationName: "some-launch-config",
 			LaunchConfiguration:           tc.input,
 		})
@@ -272,6 +286,9 @@ func TestUpdateScalingGroupNegative(t *testing.T) {
 	}
 
 	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
 		ScalingGroup: mockScalingGroup,
 	})
 
diff --git a/controllers/provisioners/eks/upgrade_test.go b/controllers/provisioners/eks/upgrade_test.go
index 287cac8..0abd9c8 100644
--- a/controllers/provisioners/eks/upgrade_test.go
+++ b/controllers/provisioners/eks/upgrade_test.go
@@ -156,7 +156,11 @@ func TestUpgradeCRDStrategy(t *testing.T) {
 
 	w := MockAwsWorker(asgMock, iamMock)
 	ctx := MockContext(ig, k, w)
-
+	ctx.SetDiscoveredState(&DiscoveredState{
+		Publisher: kubeprovider.EventPublisher{
+			Client: k.Kubernetes,
+		},
+	})
 	// get custom resource yaml
 	crYAML, err := yaml.Marshal(cr.Object)
 	g.Expect(err).NotTo(gomega.HaveOccurred())
@@ -263,6 +267,9 @@ func TestUpgradeRollingUpdateStrategyPositive(t *testing.T) {
 
 		ig.SetUpgradeStrategy(MockAwsRollingUpdateStrategy(&tc.maxUnavailable))
 		ctx.SetDiscoveredState(&DiscoveredState{
+			Publisher: kubeprovider.EventPublisher{
+				Client: k.Kubernetes,
+			},
 			ScalingGroup: &autoscaling.Group{
 				LaunchConfigurationName: aws.String("some-launch-config"),
 				AutoScalingGroupName:    aws.String("some-scaling-group"),