Repository: abahmed/kwatch (go)

Issue:
Since v0.8.1 Slack fails with "failed to send msg with Slack: slack server error: 404 Not Found"
**Describe the bug**
After upgrading from 0.8.0 to 0.8.1, Slack messages are no long sent.
The configmap didn't change.

Downgrading to 0.8.0 solves the problem.

**To Reproduce**

Deploy kwatch 0.8.1 with this config map:

```
data:
  config.yaml: |
    maxRecentLogLines: 50
    ignoreFailedGracefulShutdown: true
    app:
      clusterName: xyz
    alert:
      slack:
        webhook: https://hooks.slack.com/services/XXXXXX/ZZZZZ
```

**Expected behavior**
Message sent to Slack:
```
time="2023-03-08T15:45:12Z" level=info msg=":tada: kwatch@v0.8.0 just started!"
time="2023-03-08T15:45:12Z" level=info msg="initializing slack with webhook url: https://hooks.slack.com/services/XXXXXX/ZZZZZ"
time="2023-03-08T15:45:12Z" level=info msg="sending message: :tada: kwatch@v0.8.0 just started!"
```

**Actual behavior**
No message sent to Slack and errors in logs:
```
time="2023-03-08T14:41:20Z" level=info msg=":tada: kwatch@v0.8.1 just started!"
time="2023-03-08T14:41:20Z" level=info msg="initializing slack with webhook url: https://hooks.slack.com/services/XXXXXX/ZZZZZ"
time="2023-03-08T14:41:20Z" level=info msg="sending message: :tada: kwatch@v0.8.1 just started!"
time="2023-03-08T14:41:21Z" level=error msg="failed to send msg with Slack: slack server error: 404 Not Found"
```

**Version/Commit**
0.8.1

Pull request description:
🐛  use type assertions for strings in alerts
Fixes #203 #204

Changes proposed in this pull request:
- use type assertions for strings in alerts
- fix teams messaging


Patch:
diff --git a/README.md b/README.md
index 4ef0ef6..b3202ba 100644
--- a/README.md
+++ b/README.md
@@ -283,6 +283,17 @@ title
 | `alert.feishu.webhook`   | FeiShu bot webhook URL      |
 | `alert.feishu.title`     | Customized title in message |
 
+#### Custom webhook
+
+If you want to enable custom webhook, provide url with optional headers and
+basic auth
+
+| Parameter                 | Description                     |
+|:--------------------------|:--------------------------------|
+| `alert.webhook.url`       | Webhook URL                     |
+| `alert.webhook.headers`   | optional list of name and value |
+| `alert.webhook.basicAuth` | optional username and password  |
+
 ### Cleanup
 
 ```shell
diff --git a/alertmanager/alertmanager.go b/alertmanager/alertmanager.go
index 75b2fbc..62bc428 100644
--- a/alertmanager/alertmanager.go
+++ b/alertmanager/alertmanager.go
@@ -1,16 +1,13 @@
 package alertmanager
 
 import (
-	"github.com/abahmed/kwatch/alertmanager/webhook"
 	"reflect"
 	"strings"
 
-	"github.com/abahmed/kwatch/alertmanager/feishu"
-	"github.com/abahmed/kwatch/config"
-
 	"github.com/abahmed/kwatch/alertmanager/dingtalk"
 	"github.com/abahmed/kwatch/alertmanager/discord"
 	"github.com/abahmed/kwatch/alertmanager/email"
+	"github.com/abahmed/kwatch/alertmanager/feishu"
 	"github.com/abahmed/kwatch/alertmanager/matrix"
 	"github.com/abahmed/kwatch/alertmanager/mattermost"
 	"github.com/abahmed/kwatch/alertmanager/opsgenie"
@@ -19,6 +16,8 @@ import (
 	"github.com/abahmed/kwatch/alertmanager/slack"
 	"github.com/abahmed/kwatch/alertmanager/teams"
 	"github.com/abahmed/kwatch/alertmanager/telegram"
+	"github.com/abahmed/kwatch/alertmanager/webhook"
+	"github.com/abahmed/kwatch/config"
 	"github.com/abahmed/kwatch/event"
 	"github.com/sirupsen/logrus"
 )
diff --git a/alertmanager/dingtalk/dingtalk.go b/alertmanager/dingtalk/dingtalk.go
index c1a90fa..dce967b 100644
--- a/alertmanager/dingtalk/dingtalk.go
+++ b/alertmanager/dingtalk/dingtalk.go
@@ -39,20 +39,22 @@ type DingTalk struct {
 
 // NewDingTalk returns new DingTalk instance
 func NewDingTalk(config map[string]interface{}, appCfg *config.App) *DingTalk {
-	accessToken, ok := config["accessToken"]
-	accessTokenString := fmt.Sprint(accessToken)
-	if !ok || len(accessTokenString) == 0 {
+	accessToken, ok := config["accessToken"].(string)
+	if !ok || len(accessToken) == 0 {
 		logrus.Warnf("initializing dingtalk with empty access token")
 		return nil
 	}
 
-	logrus.Infof("initializing dingtalk with access token: %s", accessTokenString)
+	logrus.Infof("initializing dingtalk with access token: %s", accessToken)
+
+	title, _ := config["title"].(string)
+	secret, _ := config["secret"].(string)
 
 	return &DingTalk{
-		accessToken: accessTokenString,
+		accessToken: accessToken,
 		url:         dingTalkAPIURL,
-		title:       fmt.Sprint(config["title"]),
-		secret:      fmt.Sprint(config["secret"]),
+		title:       title,
+		secret:      secret,
 		appCfg:      appCfg,
 	}
 }
@@ -71,7 +73,7 @@ func (d *DingTalk) SendEvent(e *event.Event) error {
 		title = constant.DefaultTitle
 	}
 
-	msg := e.FormatMarkdown(d.appCfg.ClusterName, "")
+	msg := e.FormatMarkdown(d.appCfg.ClusterName, "", "")
 
 	body := fmt.Sprintf(`{
 		"msgtype": "markdown",
diff --git a/alertmanager/discord/discord.go b/alertmanager/discord/discord.go
index de88cc1..bde4f5b 100644
--- a/alertmanager/discord/discord.go
+++ b/alertmanager/discord/discord.go
@@ -1,7 +1,6 @@
 package discord
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/abahmed/kwatch/config"
@@ -29,30 +28,32 @@ type Discord struct {
 
 // NewDiscord returns new Discord instance
 func NewDiscord(config map[string]interface{}, appCfg *config.App) *Discord {
-	webhook, ok := config["webhook"]
-	webhookString := fmt.Sprint(webhook)
-	if !ok || len(webhookString) == 0 {
+	webhook, ok := config["webhook"].(string)
+	if !ok || len(webhook) == 0 {
 		logrus.Warnf("initializing discord with empty webhook url")
 		return nil
 	}
 
-	webhookList := strings.Split(webhookString, "/")
+	webhookList := strings.Split(webhook, "/")
 	if len(webhookList) <= 1 {
 		logrus.Warnf("initializing discord with missing id or token")
 		return nil
 	}
-	logrus.Infof("initializing discord with webhook url: %s", webhookString)
+	logrus.Infof("initializing discord with webhook url: %s", webhook)
 
 	webhookToken := webhookList[len(webhookList)-1]
 	webhookID := webhookList[len(webhookList)-2]
 
 	discordClient, _ := discordgo.New("")
 
+	title, _ := config["title"].(string)
+	text, _ := config["text"].(string)
+
 	return &Discord{
 		id:     webhookID,
 		token:  webhookToken,
-		title:  fmt.Sprint(config["title"]),
-		text:   fmt.Sprint(config["text"]),
+		title:  title,
+		text:   text,
 		send:   discordClient.WebhookExecute,
 		appCfg: appCfg,
 	}
diff --git a/alertmanager/email/email.go b/alertmanager/email/email.go
index 738b5f4..2ec2ecc 100644
--- a/alertmanager/email/email.go
+++ b/alertmanager/email/email.go
@@ -24,41 +24,36 @@ type Email struct {
 
 // NewEmail returns new email instance
 func NewEmail(config map[string]interface{}, appCfg *config.App) *Email {
-	from, ok := config["from"]
-	fromString := fmt.Sprint(from)
-	if !ok || len(fromString) == 0 {
+	from, ok := config["from"].(string)
+	if !ok || len(from) == 0 {
 		logrus.Warnf("initializing email with an empty from")
 		return nil
 	}
 
-	to, ok := config["to"]
-	toString := fmt.Sprint(to)
-	if !ok || len(toString) == 0 {
+	to, ok := config["to"].(string)
+	if !ok || len(to) == 0 {
 		logrus.Warnf("initializing email with an empty to")
 		return nil
 	}
 
-	password, ok := config["password"]
-	passwordString := fmt.Sprint(password)
-	if !ok || len(passwordString) == 0 {
+	password, ok := config["password"].(string)
+	if !ok || len(password) == 0 {
 		logrus.Warnf("initializing email with an empty password")
 		return nil
 	}
 
-	host, ok := config["host"]
-	hostString := fmt.Sprint(host)
-	if !ok || len(hostString) == 0 {
+	host, ok := config["host"].(string)
+	if !ok || len(host) == 0 {
 		logrus.Warnf("initializing email with an empty host")
 		return nil
 	}
 
-	port, ok := config["port"]
-	portString := fmt.Sprint(port)
-	if !ok || len(portString) == 0 {
+	port, ok := config["port"].(string)
+	if !ok || len(port) == 0 {
 		logrus.Warnf("initializing email with an empty port number")
 		return nil
 	}
-	portNumber, err := strconv.Atoi(portString)
+	portNumber, err := strconv.Atoi(port)
 	if err != nil {
 		logrus.Warnf("initializing email with an invalid port number: %s", err)
 		return nil
@@ -69,11 +64,11 @@ func NewEmail(config map[string]interface{}, appCfg *config.App) *Email {
 		return nil
 	}
 
-	d := gomail.NewDialer(hostString, portNumber, fromString, passwordString)
+	d := gomail.NewDialer(host, portNumber, from, password)
 
 	return &Email{
-		from:   fromString,
-		to:     toString,
+		from:   from,
+		to:     to,
 		send:   d.DialAndSend,
 		appCfg: appCfg,
 	}
diff --git a/alertmanager/feishu/feishu.go b/alertmanager/feishu/feishu.go
index 4d57d4f..8ba0586 100644
--- a/alertmanager/feishu/feishu.go
+++ b/alertmanager/feishu/feishu.go
@@ -27,24 +27,19 @@ type feiShuWebhookContent struct {
 
 // NewFeiShu returns new feishu web bot instance
 func NewFeiShu(config map[string]interface{}, appCfg *config.App) *FeiShu {
-	webhook, ok := config["webhook"]
-	webhookString := fmt.Sprint(webhook)
-	if !ok || len(webhookString) == 0 {
+	webhook, ok := config["webhook"].(string)
+	if !ok || len(webhook) == 0 {
 		logrus.Warnf("initializing Fei Shu with empty webhook url")
 		return nil
 	}
 
-	logrus.Infof("initializing Fei Shu with webhook url: %s", webhookString)
+	logrus.Infof("initializing Fei Shu with webhook url: %s", webhook)
 
-	title, ok := config["title"]
-	titleFormatted := fmt.Sprint(title)
-	if !ok {
-		titleFormatted = ""
-	}
+	title, _ := config["title"].(string)
 
 	return &FeiShu{
-		webhook: webhookString,
-		title:   titleFormatted,
+		webhook: webhook,
+		title:   title,
 		appCfg:  appCfg,
 	}
 
@@ -57,7 +52,7 @@ func (r *FeiShu) Name() string {
 
 // SendEvent sends event to the provider
 func (r *FeiShu) SendEvent(e *event.Event) error {
-	formattedMsg := e.FormatMarkdown(r.appCfg.ClusterName, "")
+	formattedMsg := e.FormatMarkdown(r.appCfg.ClusterName, "", "")
 	return r.sendByFeiShuApi(r.buildRequestBodyFeiShu(formattedMsg))
 }
 
diff --git a/alertmanager/matrix/matrix.go b/alertmanager/matrix/matrix.go
index 5cee7ca..ae53f9f 100644
--- a/alertmanager/matrix/matrix.go
+++ b/alertmanager/matrix/matrix.go
@@ -27,33 +27,33 @@ type Matrix struct {
 
 // NewMatrix returns new Matrix instance
 func NewMatrix(config map[string]interface{}, appCfg *config.App) *Matrix {
-	homeServer, ok := config["homeServer"]
-	homeServerString := fmt.Sprint(homeServer)
-	if !ok || len(homeServerString) == 0 {
+	homeServer, ok := config["homeServer"].(string)
+	if !ok || len(homeServer) == 0 {
 		logrus.Warnf("initializing slack with empty homeServer")
 		return nil
 	}
 
-	accessToken, ok := config["accessToken"]
-	accessTokenString := fmt.Sprint(accessToken)
-	if !ok || len(accessTokenString) == 0 {
+	accessToken, ok := config["accessToken"].(string)
+	if !ok || len(accessToken) == 0 {
 		logrus.Warnf("initializing slack with empty accessToken")
 		return nil
 	}
 
-	internalRoomID, ok := config["internalRoomId"]
-	internalRoomIDString := fmt.Sprint(internalRoomID)
-	if !ok || len(internalRoomIDString) == 0 {
+	internalRoomID, ok := config["internalRoomId"].(string)
+	if !ok || len(internalRoomID) == 0 {
 		logrus.Warnf("initializing slack with empty internalRoomId")
 		return nil
 	}
 
+	title, _ := config["title"].(string)
+	text, _ := config["text"].(string)
+
 	return &Matrix{
-		homeServer:     homeServerString,
-		accessToken:    accessTokenString,
-		internalRoomID: internalRoomIDString,
-		title:          fmt.Sprint(config["title"]),
-		text:           fmt.Sprint(config["text"]),
+		homeServer:     homeServer,
+		accessToken:    accessToken,
+		internalRoomID: internalRoomID,
+		title:          title,
+		text:           text,
 		appCfg:         appCfg,
 	}
 }
diff --git a/alertmanager/mattermost/mattermost.go b/alertmanager/mattermost/mattermost.go
index 7bdcfb0..5b35d6f 100644
--- a/alertmanager/mattermost/mattermost.go
+++ b/alertmanager/mattermost/mattermost.go
@@ -42,19 +42,21 @@ type mmPayload struct {
 
 // NewMattermost returns new mattermost instance
 func NewMattermost(config map[string]interface{}, appCfg *config.App) *Mattermost {
-	webhook, ok := config["webhook"]
-	webhookString := fmt.Sprint(webhook)
-	if !ok || len(webhookString) == 0 {
+	webhook, ok := config["webhook"].(string)
+	if !ok || len(webhook) == 0 {
 		logrus.Warnf("initializing mattermost with empty webhook url")
 		return nil
 	}
 
-	logrus.Infof("initializing mattermost with webhook url: %s", webhookString)
+	logrus.Infof("initializing mattermost with webhook url: %s", webhook)
+
+	title, _ := config["title"].(string)
+	text, _ := config["text"].(string)
 
 	return &Mattermost{
-		webhook: webhookString,
-		title:   fmt.Sprint(config["title"]),
-		text:    fmt.Sprint(config["text"]),
+		webhook: webhook,
+		title:   title,
+		text:    text,
 		appCfg:  appCfg,
 	}
 }
diff --git a/alertmanager/opsgenie/opsgenie.go b/alertmanager/opsgenie/opsgenie.go
index 73f6f01..720535c 100644
--- a/alertmanager/opsgenie/opsgenie.go
+++ b/alertmanager/opsgenie/opsgenie.go
@@ -38,20 +38,22 @@ type ogPayload struct {
 
 // NewOpsgenie returns new opsgenie instance
 func NewOpsgenie(config map[string]interface{}, appCfg *config.App) *Opsgenie {
-	apiKey, ok := config["apiKey"]
-	apiKeyString := fmt.Sprint(apiKey)
-	if !ok || len(apiKeyString) == 0 {
+	apiKey, ok := config["apiKey"].(string)
+	if !ok || len(apiKey) == 0 {
 		logrus.Warnf("initializing opsgenie with empty webhook url")
 		return nil
 	}
 
 	logrus.Infof("initializing opsgenie with secret apikey")
 
+	title, _ := config["title"].(string)
+	text, _ := config["text"].(string)
+
 	return &Opsgenie{
-		apikey: apiKeyString,
+		apikey: apiKey,
 		url:    opsgenieAPIURL,
-		title:  fmt.Sprint(config["title"]),
-		text:   fmt.Sprint(config["text"]),
+		title:  title,
+		text:   text,
 		appCfg: appCfg,
 	}
 }
diff --git a/alertmanager/pagerduty/pagerduty.go b/alertmanager/pagerduty/pagerduty.go
index f3fe6c3..9af6d08 100644
--- a/alertmanager/pagerduty/pagerduty.go
+++ b/alertmanager/pagerduty/pagerduty.go
@@ -27,9 +27,8 @@ type Pagerduty struct {
 
 // NewPagerDuty returns new PagerDuty instance
 func NewPagerDuty(config map[string]interface{}, appCfg *config.App) *Pagerduty {
-	integrationKey, ok := config["integrationKey"]
-	integrationKeyString := fmt.Sprint(integrationKey)
-	if !ok || len(integrationKeyString) == 0 {
+	integrationKey, ok := config["integrationKey"].(string)
+	if !ok || len(integrationKey) == 0 {
 		logrus.Warnf("initializing pagerduty with an empty integration key")
 		return nil
 	}
@@ -37,7 +36,7 @@ func NewPagerDuty(config map[string]interface{}, appCfg *config.App) *Pagerduty
 	logrus.Infof("initializing pagerduty with the provided integration key")
 
 	return &Pagerduty{
-		integrationKey: integrationKeyString,
+		integrationKey: integrationKey,
 		url:            pagerdutyAPIURL,
 		appCfg:         appCfg,
 	}
diff --git a/alertmanager/rocketchat/rocketchat.go b/alertmanager/rocketchat/rocketchat.go
index e7cd228..f397f51 100644
--- a/alertmanager/rocketchat/rocketchat.go
+++ b/alertmanager/rocketchat/rocketchat.go
@@ -26,18 +26,19 @@ type rocketChatWebhookPayload struct {
 
 // NewRocketChat returns new rocket chat instance
 func NewRocketChat(config map[string]interface{}, appCfg *config.App) *RocketChat {
-	webhook, ok := config["webhook"]
-	webhookString := fmt.Sprint(webhook)
-	if !ok || len(webhookString) == 0 {
+	webhook, ok := config["webhook"].(string)
+	if !ok || len(webhook) == 0 {
 		logrus.Warnf("initializing Rocket Chat with empty webhook url")
 		return nil
 	}
 
-	logrus.Infof("initializing Rocket Chat with webhook url: %s", webhookString)
+	logrus.Infof("initializing Rocket Chat with webhook url: %s", webhook)
+
+	text, _ := config["text"].(string)
 
 	return &RocketChat{
-		webhook: webhookString,
-		text:    fmt.Sprint(config["text"]),
+		webhook: webhook,
+		text:    text,
 		appCfg:  appCfg,
 	}
 }
@@ -49,7 +50,7 @@ func (r *RocketChat) Name() string {
 
 // SendEvent sends event to the provider
 func (r *RocketChat) SendEvent(e *event.Event) error {
-	formattedMsg := e.FormatMarkdown(r.appCfg.ClusterName, r.text)
+	formattedMsg := e.FormatMarkdown(r.appCfg.ClusterName, r.text, "")
 	return r.sendByRocketChatApi(r.buildRequestBodyRocketChat(formattedMsg))
 }
 
diff --git a/alertmanager/slack/slack.go b/alertmanager/slack/slack.go
index b4d2818..f86c168 100644
--- a/alertmanager/slack/slack.go
+++ b/alertmanager/slack/slack.go
@@ -33,20 +33,23 @@ type Slack struct {
 
 // NewSlack returns new Slack instance
 func NewSlack(config map[string]interface{}, appCfg *config.App) *Slack {
-	webhook, ok := config["webhook"]
-	webhookString := fmt.Sprint(webhook)
-	if !ok || len(webhookString) == 0 {
+	webhook, ok := config["webhook"].(string)
+	if !ok || len(webhook) == 0 {
 		logrus.Warnf("initializing slack with empty webhook url")
 		return nil
 	}
 
-	logrus.Infof("initializing slack with webhook url: %s", webhookString)
+	logrus.Infof("initializing slack with webhook url: %s", webhook)
+
+	channel, _ := config["channel"].(string)
+	title, _ := config["title"].(string)
+	text, _ := config["text"].(string)
 
 	return &Slack{
-		webhook: webhookString,
-		channel: fmt.Sprint(config["channel"]),
-		title:   fmt.Sprint(config["title"]),
-		text:    fmt.Sprint(config["text"]),
+		webhook: webhook,
+		channel: channel,
+		title:   title,
+		text:    text,
 		send:    slackClient.PostWebhook,
 		appCfg:  appCfg,
 	}
diff --git a/alertmanager/teams/teams.go b/alertmanager/teams/teams.go
index db5d528..1d50b0f 100644
--- a/alertmanager/teams/teams.go
+++ b/alertmanager/teams/teams.go
@@ -33,20 +33,21 @@ type teamsWebhookPayload struct {
 
 // NewTeams returns new team instance
 func NewTeams(config map[string]interface{}, appCfg *config.App) *Teams {
-	webhook, ok := config["webhook"]
-	webhookString := fmt.Sprint(webhook)
-
-	if !ok || len(webhookString) == 0 {
+	webhook, ok := config["webhook"].(string)
+	if !ok || len(webhook) == 0 {
 		logrus.Warnf("initializing Teams with empty webhook url")
 		return nil
 	}
 
-	logrus.Infof("initializing Teams with webhook url: %s", webhookString)
+	logrus.Infof("initializing Teams with webhook url: %s", webhook)
+
+	title, _ := config["title"].(string)
+	text, _ := config["text"].(string)
 
 	return &Teams{
-		webhook: webhookString,
-		title:   fmt.Sprint(config["title"]),
-		text:    fmt.Sprint(config["text"]),
+		webhook: webhook,
+		title:   title,
+		text:    text,
 		appCfg:  appCfg,
 	}
 }
@@ -58,13 +59,22 @@ func (t *Teams) Name() string {
 
 // SendEvent sends event to the provider
 func (t *Teams) SendEvent(e *event.Event) error {
-	return t.SendMessage(t.buildRequestBodyTeams(e))
+	return t.sendAPI(t.buildRequestBodyTeams(e))
 }
 
 // SendMessage sends text message to the provider
 func (t *Teams) SendMessage(msg string) error {
-	client := &http.Client{}
-	buffer := bytes.NewBuffer([]byte(msg))
+
+	msgPayload := &teamsWebhookPayload{
+		Text: msg,
+	}
+
+	jsonBytes, _ := json.Marshal(msgPayload)
+	return t.sendAPI(jsonBytes)
+}
+
+func (t *Teams) sendAPI(b []byte) error {
+	buffer := bytes.NewBuffer(b)
 	request, err := http.NewRequest(http.MethodPost, t.webhook, buffer)
 	if err != nil {
 		return err
@@ -72,6 +82,7 @@ func (t *Teams) SendMessage(msg string) error {
 
 	request.Header.Set("Content-Type", "application/json")
 
+	client := &http.Client{}
 	response, err := client.Do(request)
 	if err != nil {
 		return err
@@ -90,19 +101,19 @@ func (t *Teams) SendMessage(msg string) error {
 }
 
 // buildRequestBodyTeams builds formatted string from event
-func (t *Teams) buildRequestBodyTeams(e *event.Event) string {
+func (t *Teams) buildRequestBodyTeams(e *event.Event) []byte {
 	// use custom title if it's provided, otherwise use default
 	title := t.title
 	if len(title) == 0 {
 		title = defaultTeamsTitle
 	}
 
-	msg := e.FormatMarkdown(t.appCfg.ClusterName, t.text)
+	msg := e.FormatMarkdown(t.appCfg.ClusterName, t.text, "\n\n")
 	msgPayload := &teamsWebhookPayload{
 		Title: title,
 		Text:  msg,
 	}
 
 	jsonBytes, _ := json.Marshal(msgPayload)
-	return string(jsonBytes)
+	return (jsonBytes)
 }
diff --git a/alertmanager/telegram/telegram.go b/alertmanager/telegram/telegram.go
index 13bd13c..70086b5 100644
--- a/alertmanager/telegram/telegram.go
+++ b/alertmanager/telegram/telegram.go
@@ -26,29 +26,27 @@ type Telegram struct {
 
 // NewTelegram returns a new Telegram object
 func NewTelegram(config map[string]interface{}, appCfg *config.App) *Telegram {
-	token, ok := config["token"]
-	tokenString := fmt.Sprint(token)
-	if !ok || len(tokenString) == 0 {
+	token, ok := config["token"].(string)
+	if !ok || len(token) == 0 {
 		logrus.Warnf("initializing telegram with empty token")
 		return nil
 	}
 
-	chatId, ok := config["chatId"]
-	chatIdString := fmt.Sprint(chatId)
-	if !ok || len(chatIdString) == 0 {
+	chatId, ok := config["chatId"].(string)
+	if !ok || len(chatId) == 0 {
 		logrus.Warnf("initializing telegram with empty chat_id")
 		return nil
 	}
 
 	logrus.Infof(
 		"initializing telegram with token  %s and chat_id %s",
-		tokenString,
-		chatIdString)
+		token,
+		chatId)
 
 	// returns a new telegram object
 	return &Telegram{
-		token:  tokenString,
-		chatId: chatIdString,
+		token:  token,
+		chatId: chatId,
 		url:    telegramAPIURL,
 		appCfg: appCfg,
 	}
diff --git a/alertmanager/webhook/webhook.go b/alertmanager/webhook/webhook.go
index a1ac6d2..2ff7366 100644
--- a/alertmanager/webhook/webhook.go
+++ b/alertmanager/webhook/webhook.go
@@ -4,11 +4,12 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"strings"
+
 	"github.com/abahmed/kwatch/config"
 	"github.com/abahmed/kwatch/event"
 	"github.com/abahmed/kwatch/util"
-	"net/http"
-	"strings"
 
 	"github.com/sirupsen/logrus"
 )
@@ -37,9 +38,8 @@ func (w *Webhook) SendMessage(msg string) error {
 
 // NewSlack returns new Slack instance
 func NewWebhook(config map[string]interface{}, appCfg *config.App) *Webhook {
-	url, ok := config["url"]
-	urlString := fmt.Sprint(url)
-	if !ok || len(urlString) == 0 {
+	url, ok := config["url"].(string)
+	if !ok || len(url) == 0 {
 		logrus.Warnf("initializing  with empty webhook url")
 		return nil
 	}
@@ -61,16 +61,14 @@ func NewWebhook(config map[string]interface{}, appCfg *config.App) *Webhook {
 	var a Authentication
 	json.Unmarshal(basicAuthJson, &a)
 
-	userName := fmt.Sprint(a.UserName)
-	password := fmt.Sprint(a.Password)
-
-	logrus.Infof("initializing  with webhook url: %s with headers: %s and username: %s password: %s", url, headers, userName, password)
+	logrus.Infof("initializing  with webhook url: %s "+
+		"with headers: %s and username: %s", url, headers, a.UserName)
 
 	return &Webhook{
-		webhook:  urlString,
+		webhook:  url,
 		headers:  headers,
-		username: userName,
-		password: password,
+		username: a.UserName,
+		password: a.Password,
 		appCfg:   appCfg,
 	}
 }
diff --git a/deploy/chart/templates/rbac.yaml b/deploy/chart/templates/rbac.yaml
index 3f8334e..b8aec1a 100644
--- a/deploy/chart/templates/rbac.yaml
+++ b/deploy/chart/templates/rbac.yaml
@@ -4,7 +4,7 @@ metadata:
   name: {{ .Release.Name }}
 rules:
 - apiGroups: [""]
-  resources: ["pods", "pods/log", "events"]
+  resources: ["pods", "pods/log", "events", "nodes"]
   verbs: ["get", "watch", "list"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/deploy/deploy.yaml b/deploy/deploy.yaml
index cb4f16c..e48a73c 100644
--- a/deploy/deploy.yaml
+++ b/deploy/deploy.yaml
@@ -9,7 +9,7 @@ metadata:
   name: kwatch
 rules:
 - apiGroups: [""]
-  resources: ["pods", "pods/log", "events"]
+  resources: ["pods", "pods/log", "events", "nodes"]
   verbs: ["get", "watch", "list"]
 ---
 apiVersion: v1
diff --git a/event/format.go b/event/format.go
index f35cdfd..0695dd4 100644
--- a/event/format.go
+++ b/event/format.go
@@ -7,7 +7,7 @@ import (
 	"github.com/abahmed/kwatch/constant"
 )
 
-func (e *Event) FormatMarkdown(clusterName, text string) string {
+func (e *Event) FormatMarkdown(clusterName, text, delimiter string) string {
 	// add events part if it exists
 	eventsText := constant.DefaultEvents
 	events := strings.TrimSpace(e.Events)
@@ -27,14 +27,18 @@ func (e *Event) FormatMarkdown(clusterName, text string) string {
 		text = constant.DefaultText
 	}
 
+	if len(delimiter) == 0 {
+		delimiter = "\n"
+	}
+
 	msg := fmt.Sprintf(
-		"%s\n"+
-			"**Cluster:** %s\n"+
-			"**Pod:** %s\n"+
-			"**Container:** %s\n"+
-			"**Namespace:** %s\n"+
-			"**Reason:** %s\n"+
-			"**Events:**\n```\n%s\n```\n"+
+		"%s"+delimiter+
+			"**Cluster:** %s"+delimiter+
+			"**Pod:** %s"+delimiter+
+			"**Container:** %s"+delimiter+
+			"**Namespace:** %s"+delimiter+
+			"**Reason:** %s"+delimiter+
+			"**Events:**\n```\n%s\n```"+delimiter+
 			"**Logs:**\n```\n%s\n```",
 		text,
 		clusterName, e.Name,


Test patch:
diff --git a/alertmanager/alertmanager_test.go b/alertmanager/alertmanager_test.go
index f1afc14..46afa51 100644
--- a/alertmanager/alertmanager_test.go
+++ b/alertmanager/alertmanager_test.go
@@ -86,6 +86,9 @@ func TestGetProviders(t *testing.T) {
 		"feishu": {
 			"webhook": "test",
 		},
+		"webhook": {
+			"url": "test",
+		},
 	}
 
 	alertmanager := AlertManager{}
diff --git a/alertmanager/feishu/feishu_test.go b/alertmanager/feishu/feishu_test.go
index 0393be3..3d98a61 100644
--- a/alertmanager/feishu/feishu_test.go
+++ b/alertmanager/feishu/feishu_test.go
@@ -51,7 +51,7 @@ func TestBuildRequestBodyFeiShu(t *testing.T) {
 		Logs:      "test\ntestlogs",
 		Events:    "test",
 	}
-	formattedMsg := ev.FormatMarkdown(c.appCfg.ClusterName, "")
+	formattedMsg := ev.FormatMarkdown(c.appCfg.ClusterName, "", "")
 
 	expectMessage := "{\"msg_type\": \"interactive\",\"card\": {\"config\": {\"wide_screen_mode\": true},\"header\": {\"title\": {\"tag\": \"plain_text\",\"content\": \"\"},\"template\": \"blue\"},\"elements\": [{\"tag\":\"markdown\",\"content\":\"There is an issue with container in a pod!\\n**Cluster:** dev\\n**Pod:** test-pod\\n**Container:** test-container\\n**Namespace:** default\\n**Reason:** OOMKILLED\\n**Events:**\\n```\\ntest\\n```\\n**Logs:**\\n```\\ntest\\ntestlogs\\n```\"}]}}"
 
diff --git a/alertmanager/webhook/webhook_test.go b/alertmanager/webhook/webhook_test.go
index f23cefe..1f8ff4c 100644
--- a/alertmanager/webhook/webhook_test.go
+++ b/alertmanager/webhook/webhook_test.go
@@ -22,6 +22,12 @@ func TestWebhook(t *testing.T) {
 
 	configMap := map[string]interface{}{
 		"url": "testtest",
+		"headers": []interface{}{
+			map[string]string{
+				"name":  "test",
+				"value": "test",
+			},
+		},
 	}
 	c := NewWebhook(configMap, &config.App{ClusterName: "dev"})
 	assert.NotNil(c)
@@ -41,6 +47,12 @@ func TestSendMessage(t *testing.T) {
 
 	configMap := map[string]interface{}{
 		"url": s.URL,
+		"headers": []interface{}{
+			map[string]string{
+				"name":  "test",
+				"value": "test",
+			},
+		},
 	}
 	c := NewWebhook(configMap, &config.App{ClusterName: "dev"})
 	assert.NotNil(c)
@@ -70,6 +82,34 @@ func TestSendMessageError(t *testing.T) {
 func TestSendEvent(t *testing.T) {
 	assert := assert.New(t)
 
+	s := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusBadGateway)
+		}))
+
+	defer s.Close()
+
+	configMap := map[string]interface{}{
+		"url": s.URL,
+	}
+	c := NewWebhook(configMap, &config.App{ClusterName: "dev"})
+	assert.NotNil(c)
+
+	ev := event.Event{
+		Name:      "test-pod",
+		Container: "test-container",
+		Namespace: "default",
+		Reason:    "OOMKILLED",
+		Logs:      "test\ntestlogs",
+		Events: "event1-event2-event3-event1-event2-event3-event1-event2-" +
+			"event3\nevent5\nevent6-event8-event11-event12",
+	}
+	assert.Error(c.SendEvent(&ev))
+}
+
+func TestSendEventError(t *testing.T) {
+	assert := assert.New(t)
+
 	s := httptest.NewServer(
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			w.Write([]byte(`{"isOk": true}`))
@@ -79,6 +119,12 @@ func TestSendEvent(t *testing.T) {
 
 	configMap := map[string]interface{}{
 		"url": s.URL,
+		"headers": []interface{}{
+			map[string]string{
+				"name":  "test",
+				"value": "test",
+			},
+		},
 	}
 	c := NewWebhook(configMap, &config.App{ClusterName: "dev"})
 	assert.NotNil(c)