Merge pull request #1137 from kongfei605/notification

add mattermost notification

docker/n9eetc/server.conf

@@ -11,6 +11,11 @@ BusiGroupLabelKey = "busigroup"
 # sleep x seconds, then start judge engine
 EngineDelay = 120
 
+DisableUsageReport = false
+
+# config | database
+ReaderFrom = "config"
+
 [Log]
 # log write dir
 Dir = "logs"
@@ -71,7 +76,7 @@ Batch = 5
 TemplatesDir = "./etc/template"
 NotifyConcurrency = 10
 # use builtin go code notify
-NotifyBuiltinChannels = ["email", "dingtalk", "wecom", "feishu"]
+NotifyBuiltinChannels = ["email", "dingtalk", "wecom", "feishu", "mm"]
 
 [Alerting.CallScript]
 # built in sending capability in go code
@@ -83,7 +88,8 @@ ScriptPath = "./etc/script/notify.py"
 Enable = false
 # use a plugin via `go build -buildmode=plugin -o notify.so`
 PluginPath = "./etc/script/notify.so"
-Caller = "n9eCaller"
+# The first letter must be capitalized to be exported
+Caller = "N9eCaller"
 
 [Alerting.RedisPub]
 Enable = false
@@ -136,7 +142,7 @@ MaxIdleConns = 50
 # table prefix
 TablePrefix = ""
 # enable auto migrate or not
-EnableAutoMigrate = false
+# EnableAutoMigrate = false
 
 [Reader]
 # prometheus base url
@@ -147,15 +153,8 @@ BasicAuthUser = ""
 BasicAuthPass = ""
 # timeout settings, unit: ms
 Timeout = 30000
-DialTimeout = 10000
-TLSHandshakeTimeout = 30000
-ExpectContinueTimeout = 1000
-IdleConnTimeout = 90000
-# time duration, unit: ms
-KeepAlive = 30000
-MaxConnsPerHost = 0
-MaxIdleConns = 100
-MaxIdleConnsPerHost = 10
+DialTimeout = 3000
+MaxIdleConnsPerHost = 100
 
 [WriterOpt]
 # queue channel count
@@ -172,8 +171,8 @@ BasicAuthUser = ""
 # Basic auth password
 BasicAuthPass = ""
 # timeout settings, unit: ms
-Timeout = 30000
-DialTimeout = 10000
+Timeout = 10000
+DialTimeout = 3000
 TLSHandshakeTimeout = 30000
 ExpectContinueTimeout = 1000
 IdleConnTimeout = 90000
@@ -182,6 +181,12 @@ KeepAlive = 30000
 MaxConnsPerHost = 0
 MaxIdleConns = 100
 MaxIdleConnsPerHost = 100
+# [[Writers.WriteRelabels]]
+# Action = "replace"
+# SourceLabels = ["__address__"]
+# Regex = "([^:]+)(?::\\d+)?"
+# Replacement = "$1:80"
+# TargetLabel = "__address__"
 
 # [[Writers]]
 # Url = "http://m3db:7201/api/v1/prom/remote/write"

docker/n9eetc/webapi.conf

@@ -4,12 +4,21 @@ RunMode = "release"
 # # custom i18n dict config
 # I18N = "./etc/i18n.json"
 
+# # custom i18n request header key
+# I18NHeaderKey = "X-Language"
+
 # metrics descriptions
 MetricsYamlFile = "./etc/metrics.yaml"
 
 BuiltinAlertsDir = "./etc/alerts"
 BuiltinDashboardsDir = "./etc/dashboards"
 
+# config | api
+ClustersFrom = "config"
+
+# using when ClustersFrom = "api"
+ClustersFromAPIs = []
+
 [[NotifyChannels]]
 Label = "邮箱"
 # do not change Key
@@ -30,6 +39,11 @@ Label = "飞书机器人"
 # do not change Key
 Key = "feishu"
 
+[[NotifyChannels]]
+Label = "mm bot"
+# do not change Key
+Key = "mm"
+
 [[ContactKeys]]
 Label = "Wecom Robot Token"
 # do not change Key
@@ -45,6 +59,11 @@ Label = "Feishu Robot Token"
 # do not change Key
 Key = "feishu_robot_token"
 
+[[ContactKeys]]
+Label = "MatterMost Webhook URL"
+# do not change Key
+Key = "mm_webhook_url"
+
 [Log]
 # log write dir
 Dir = "logs"
@@ -92,6 +111,13 @@ AccessExpired = 1500
 RefreshExpired = 10080
 RedisKeyPrefix = "/jwt/"
 
+[ProxyAuth]
+# if proxy auth enabled, jwt auth is disabled
+Enable = false
+# username key in http proxy header
+HeaderUserNameKey = "X-User-Name"
+DefaultRoles = ["Standard"]
+
 [BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
@@ -121,6 +147,20 @@ Nickname = "cn"
 Phone = "mobile"
 Email = "mail"
 
+[OIDC]
+Enable = false
+RedirectURL = "http://n9e.com/callback"
+SsoAddr = "http://sso.example.org"
+ClientId = ""
+ClientSecret = ""
+CoverAttributes = true
+DefaultRoles = ["Standard"]
+
+[OIDC.Attributes]
+Nickname = "nickname"
+Phone = "phone_number"
+Email = "email"
+
 [Redis]
 # address, ip:port
 Address = "redis:6379"
@@ -145,7 +185,7 @@ MaxIdleConns = 50
 # table prefix
 TablePrefix = ""
 # enable auto migrate or not
-EnableAutoMigrate = false
+# EnableAutoMigrate = false
 
 [[Clusters]]
 # Prometheus cluster name
@@ -158,14 +198,7 @@ BasicAuthUser = ""
 BasicAuthPass = ""
 # timeout settings, unit: ms
 Timeout = 30000
-DialTimeout = 10000
-TLSHandshakeTimeout = 30000
-ExpectContinueTimeout = 1000
-IdleConnTimeout = 90000
-# time duration, unit: ms
-KeepAlive = 30000
-MaxConnsPerHost = 0
-MaxIdleConns = 100
+DialTimeout = 3000
 MaxIdleConnsPerHost = 100
 
 [Ibex]
@@ -180,4 +213,4 @@ Timeout = 3000
 TargetUp = '''max(max_over_time(target_up{ident=~"(%s)"}[%dm])) by (ident)'''
 LoadPerCore = '''max(max_over_time(system_load_norm_1{ident=~"(%s)"}[%dm])) by (ident)'''
 MemUtil = '''100-max(max_over_time(mem_available_percent{ident=~"(%s)"}[%dm])) by (ident)'''
-DiskUtil = '''max(max_over_time(disk_used_percent{ident=~"(%s)", path="/"}[%dm])) by (ident)'''
+DiskUtil = '''max(max_over_time(disk_used_percent{ident=~"(%s)", path="/"}[%dm])) by (ident)'''

etc/server.conf

@@ -76,7 +76,7 @@ Batch = 5
 TemplatesDir = "./etc/template"
 NotifyConcurrency = 10
 # use builtin go code notify
-NotifyBuiltinChannels = ["email", "dingtalk", "wecom", "feishu"]
+NotifyBuiltinChannels = ["email", "dingtalk", "wecom", "feishu", "mm"]
 
 [Alerting.CallScript]
 # built in sending capability in go code

etc/template/mm.tpl

@@ -0,0 +1,7 @@
+级别状态: S{{.Severity}} {{if .IsRecovered}}Recovered{{else}}Triggered{{end}}
+规则名称: {{.RuleName}}{{if .RuleNote}}
+规则备注: {{.RuleNote}}{{end}}
+监控指标: {{.TagsJSON}}
+{{if .IsRecovered}}恢复时间：{{timeformat .LastEvalTime}}{{else}}触发时间: {{timeformat .TriggerTime}}
+触发时值: {{.TriggerValue}}{{end}}
+发送时间: {{timestamp}}

etc/webapi.conf

@@ -39,6 +39,11 @@ Label = "飞书机器人"
 # do not change Key
 Key = "feishu"
 
+[[NotifyChannels]]
+Label = "mm bot"
+# do not change Key
+Key = "mm"
+
 [[ContactKeys]]
 Label = "Wecom Robot Token"
 # do not change Key
@@ -54,6 +59,11 @@ Label = "Feishu Robot Token"
 # do not change Key
 Key = "feishu_robot_token"
 
+[[ContactKeys]]
+Label = "MatterMost Webhook URL"
+# do not change Key
+Key = "mm_webhook_url"
+
 [Log]
 # log write dir
 Dir = "logs"
@@ -207,4 +217,4 @@ Timeout = 3000
 TargetUp = '''max(max_over_time(target_up{ident=~"(%s)"}[%dm])) by (ident)'''
 LoadPerCore = '''max(max_over_time(system_load_norm_1{ident=~"(%s)"}[%dm])) by (ident)'''
 MemUtil = '''100-max(max_over_time(mem_available_percent{ident=~"(%s)"}[%dm])) by (ident)'''
-DiskUtil = '''max(max_over_time(disk_used_percent{ident=~"(%s)", path="/"}[%dm])) by (ident)'''
+DiskUtil = '''max(max_over_time(disk_used_percent{ident=~"(%s)", path="/"}[%dm])) by (ident)'''

src/server/common/sender/dingtalk.go

@@ -50,6 +50,9 @@ func SendDingtalk(message DingtalkMessage) {
 		}
 
 		ur := "https://oapi.dingtalk.com/robot/send?access_token=" + u.Path
+		if strings.HasPrefix(message.Tokens[i], "https://") {
+			ur = message.Tokens[i]
+		}
 		body := dingtalk{
 			Msgtype: "markdown",
 			Markdown: dingtalkMarkdown{

src/server/common/sender/feishu.go

@@ -1,6 +1,7 @@
 package sender
 
 import (
+	"strings"
 	"time"
 
 	"github.com/didi/nightingale/v5/src/pkg/poster"
@@ -31,6 +32,9 @@ type feishu struct {
 func SendFeishu(message FeishuMessage) {
 	for i := 0; i < len(message.Tokens); i++ {
 		url := "https://open.feishu.cn/open-apis/bot/v2/hook/" + message.Tokens[i]
+		if strings.HasPrefix(message.Tokens[i], "https://") {
+			url = message.Tokens[i]
+		}
 		body := feishu{
 			Msgtype: "text",
 			Content: feishuContent{

src/server/common/sender/mm.go

@@ -0,0 +1,57 @@
+package sender
+
+import (
+	"net/url"
+	"time"
+
+	"github.com/didi/nightingale/v5/src/pkg/poster"
+	"github.com/toolkits/pkg/logger"
+)
+
+type MatterMostMessage struct {
+	Text   string
+	Tokens []string
+}
+
+type mm struct {
+	Channel  string `json:"channel"`
+	Username string `json:"username"`
+	Text     string `json:"text"`
+}
+
+func SendMM(message MatterMostMessage) {
+
+	for i := 0; i < len(message.Tokens); i++ {
+		u, err := url.Parse(message.Tokens[i])
+		if err != nil {
+			logger.Errorf("mm_sender: failed to parse error=%v", err)
+		}
+
+		v, err := url.ParseQuery(u.RawQuery)
+		if err != nil {
+			logger.Errorf("mm_sender: failed to parse query error=%v", err)
+		}
+
+		channels := v["channel"] // do not get
+		username := v.Get("username")
+		if err != nil {
+			logger.Errorf("mm_sender: failed to parse error=%v", err)
+		}
+		// simple concatenating
+		ur := u.Scheme + "://" + u.Host + u.Path
+		for _, channel := range channels {
+			body := mm{
+				Channel:  channel,
+				Username: username,
+				Text:     message.Text,
+			}
+
+			res, code, err := poster.PostJSON(ur, time.Second*5, body, 3)
+			if err != nil {
+				logger.Errorf("mm_sender: result=fail url=%s code=%d error=%v response=%s", ur, code, err, string(res))
+			} else {
+				logger.Infof("mm_sender: result=succ url=%s code=%d response=%s", ur, code, string(res))
+			}
+		}
+	}
+}

src/server/common/sender/wecom.go

@@ -1,6 +1,7 @@
 package sender
 
 import (
+	"strings"
 	"time"
 
 	"github.com/didi/nightingale/v5/src/pkg/poster"
@@ -24,6 +25,9 @@ type wecom struct {
 func SendWecom(message WecomMessage) {
 	for i := 0; i < len(message.Tokens); i++ {
 		url := "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=" + message.Tokens[i]
+		if strings.HasPrefix(message.Tokens[i], "https://") {
+			url = message.Tokens[i]
+		}
 		body := wecom{
 			Msgtype: "markdown",
 			Markdown: wecomMarkdown{

src/server/engine/notify.go

@@ -125,6 +125,7 @@ func handleNotice(notice Notice, bs []byte) {
 	wecomset := make(map[string]struct{})
 	dingtalkset := make(map[string]struct{})
 	feishuset := make(map[string]struct{})
+	mmset := make(map[string]struct{})
 
 	for _, user := range notice.Event.NotifyUsersObj {
 		if user.Email != "" {
@@ -155,6 +156,11 @@ func handleNotice(notice Notice, bs []byte) {
 		if ret.Exists() {
 			feishuset[ret.String()] = struct{}{}
 		}
+
+		ret = gjson.GetBytes(bs, "mm_webhook_url")
+		if ret.Exists() {
+			mmset[ret.String()] = struct{}{}
+		}
 	}
 
 	phones := StringSetKeys(phoneset)
@@ -236,6 +242,23 @@ func handleNotice(notice Notice, bs []byte) {
 				AtMobiles: phones,
 				Tokens:    StringSetKeys(feishuset),
 			})
+		case "mm":
+			if len(mmset) == 0 {
+				continue
+			}
+			if !slice.ContainsString(config.C.Alerting.NotifyBuiltinChannels, "mm") {
+				continue
+			}
+
+			content, has := notice.Tpls["mm.tpl"]
+			if !has {
+				content = "mm.tpl not found"
+			}
+
+			sender.SendMM(sender.MatterMostMessage{
+				Text:   content,
+				Tokens: StringSetKeys(mmset),
+			})
 		}
 	}
 }

src/server/engine/notify_maintainer.go

@@ -64,6 +64,7 @@ func notifyMaintainerWithBuiltin(title, msg, triggerTime string, users []*models
 	wecomset := make(map[string]struct{})
 	dingtalkset := make(map[string]struct{})
 	feishuset := make(map[string]struct{})
+	mmset := make(map[string]struct{})
 
 	for _, user := range users {
 		if user.Email != "" {
@@ -94,6 +95,11 @@ func notifyMaintainerWithBuiltin(title, msg, triggerTime string, users []*models
 		if ret.Exists() {
 			feishuset[ret.String()] = struct{}{}
 		}
+
+		ret = gjson.GetBytes(bs, "mm_webhook_url")
+		if ret.Exists() {
+			mmset[ret.String()] = struct{}{}
+		}
 	}
 
 	phones := StringSetKeys(phoneset)
@@ -137,6 +143,15 @@ func notifyMaintainerWithBuiltin(title, msg, triggerTime string, users []*models
 				AtMobiles: phones,
 				Tokens:    StringSetKeys(feishuset),
 			})
+		case "mm":
+			if len(mmset) == 0 {
+				continue
+			}
+			content := "**Title: **" + title + "\n**Content: **" + msg + "\n**Time: **" + triggerTime
+			sender.SendMM(sender.MatterMostMessage{
+				Text:   content,
+				Tokens: StringSetKeys(mmset),
+			})
 		}
 	}
 }