diff --git a/etc/webapi.conf b/etc/webapi.conf index d30f09b5..a6a077c6 100644 --- a/etc/webapi.conf +++ b/etc/webapi.conf @@ -10,19 +10,39 @@ AdminRole = "Admin" # metrics descriptions MetricsYamlFile = "./etc/metrics.yaml" -# Linkage with notify.py script -NotifyChannels = [ "email", "dingtalk", "wecom", "feishu" ] +[[NotifyChannels]] +Label = "邮箱" +# do not change Key +Key = "email" + +[[NotifyChannels]] +Label = "钉钉机器人" +# do not change Key +Key = "dingtalk" + +[[NotifyChannels]] +Label = "企微机器人" +# do not change Key +Key = "wecom" + +[[NotifyChannels]] +Label = "飞书机器人" +# do not change Key +Key = "feishu" [[ContactKeys]] Label = "Wecom Robot Token" +# do not change Key Key = "wecom_robot_token" [[ContactKeys]] Label = "Dingtalk Robot Token" +# do not change Key Key = "dingtalk_robot_token" [[ContactKeys]] Label = "Feishu Robot Token" +# do not change Key Key = "feishu_robot_token" [Log] diff --git a/go.mod b/go.mod index 8be61aac..a4bb3adf 100644 --- a/go.mod +++ b/go.mod @@ -23,12 +23,15 @@ require ( github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.26.0 github.com/prometheus/prometheus v2.5.0+incompatible + github.com/tidwall/gjson v1.14.0 // indirect github.com/toolkits/pkg v1.2.9 github.com/urfave/cli/v2 v2.3.0 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4 // indirect google.golang.org/grpc v1.41.0 // indirect + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect + gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect gorm.io/driver/mysql v1.1.2 gorm.io/driver/postgres v1.1.1 gorm.io/gorm v1.21.15 diff --git a/go.sum b/go.sum index 9730635b..b724ebed 100644 --- a/go.sum +++ b/go.sum @@ -298,6 +298,12 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= +github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/toolkits/pkg v1.2.9 h1:zGlrJDl+2sMBoxBRIoMtAwvKmW5wctuji2+qHCecMKk= github.com/toolkits/pkg v1.2.9/go.mod h1:ZUsQAOoaR99PSbes+RXSirvwmtd6+XIUvizCmrjfUYc= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= @@ -465,12 +471,16 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/src/models/alert_rule.go b/src/models/alert_rule.go index 68cebe0d..c78e05ca 100644 --- a/src/models/alert_rule.go +++ b/src/models/alert_rule.go @@ -7,7 +7,6 @@ import ( "time" "github.com/pkg/errors" - "github.com/toolkits/pkg/slice" "github.com/toolkits/pkg/str" "github.com/didi/nightingale/v5/src/webapi/config" @@ -101,7 +100,7 @@ func (ar *AlertRule) Verify() error { if len(channels) > 0 { nlst := make([]string, 0, len(channels)) for i := 0; i < len(channels); i++ { - if slice.ContainsString(config.C.NotifyChannels, channels[i]) { + if config.LabelAndKeyHasKey(config.C.NotifyChannels, channels[i]) { nlst = append(nlst, channels[i]) } } diff --git a/src/server/engine/consume.go b/src/server/engine/consume.go index c2eeaa62..e86bb72e 100644 --- a/src/server/engine/consume.go +++ b/src/server/engine/consume.go @@ -155,3 +155,11 @@ func mapKeys(m map[int64]struct{}) []int64 { } return lst } + +func StringSetKeys(m map[string]struct{}) []string { + lst := make([]string, 0, len(m)) + for k := range m { + lst = append(lst, k) + } + return lst +} diff --git a/src/server/engine/engine.go b/src/server/engine/engine.go index d55cf703..c2524c54 100644 --- a/src/server/engine/engine.go +++ b/src/server/engine/engine.go @@ -5,6 +5,7 @@ import ( "time" "github.com/didi/nightingale/v5/src/server/config" + "github.com/didi/nightingale/v5/src/server/sender" promstat "github.com/didi/nightingale/v5/src/server/stat" ) @@ -22,6 +23,8 @@ func Start(ctx context.Context) error { go reportQueueSize() + go sender.StartEmailSender() + return nil } diff --git a/src/server/engine/notify.go b/src/server/engine/notify.go index bba59842..2153c84a 100644 --- a/src/server/engine/notify.go +++ b/src/server/engine/notify.go @@ -13,6 +13,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/tidwall/gjson" "github.com/toolkits/pkg/file" "github.com/toolkits/pkg/logger" "github.com/toolkits/pkg/runner" @@ -21,6 +22,7 @@ import ( "github.com/didi/nightingale/v5/src/pkg/sys" "github.com/didi/nightingale/v5/src/server/config" "github.com/didi/nightingale/v5/src/server/memsto" + "github.com/didi/nightingale/v5/src/server/sender" "github.com/didi/nightingale/v5/src/storage" ) @@ -117,8 +119,110 @@ func alertingRedisPub(bs []byte) { func handleNotice(notice Notice, bs []byte) { alertingCallScript(bs) - // TODO 弄个channel发邮件,学习daemon写法 - // 收集tokens、phones,发呗 + emailset := make(map[string]struct{}) + phoneset := make(map[string]struct{}) + wecomset := make(map[string]struct{}) + dingtalkset := make(map[string]struct{}) + feishuset := make(map[string]struct{}) + + for _, user := range notice.Event.NotifyUsersObj { + if user.Email != "" { + emailset[user.Email] = struct{}{} + } + + if user.Phone != "" { + phoneset[user.Phone] = struct{}{} + } + + bs, err := user.Contacts.MarshalJSON() + if err != nil { + logger.Errorf("handle_notice: failed to marshal contacts: %v", err) + continue + } + + ret := gjson.GetBytes(bs, "dingtalk_robot_token") + if ret.Exists() { + dingtalkset[ret.String()] = struct{}{} + } + + ret = gjson.GetBytes(bs, "wecom_robot_token") + if ret.Exists() { + wecomset[ret.String()] = struct{}{} + } + + ret = gjson.GetBytes(bs, "feishu_robot_token") + if ret.Exists() { + feishuset[ret.String()] = struct{}{} + } + } + + phones := StringSetKeys(phoneset) + + for _, ch := range notice.Event.NotifyChannelsJSON { + switch ch { + case "email": + if len(emailset) == 0 { + continue + } + + subject, has := notice.Tpls["subject.tpl"] + if !has { + subject = "subject.tpl not found" + } + + content, has := notice.Tpls["mailbody.tpl"] + if !has { + content = "mailbody.tpl not found" + } + + sender.WriteEmail(subject, content, StringSetKeys(emailset)) + case "dingtalk": + if len(dingtalkset) == 0 { + continue + } + + content, has := notice.Tpls["dingtalk.tpl"] + if !has { + content = "dingtalk.tpl not found" + } + + sender.SendDingtalk(sender.DingtalkMessage{ + Title: notice.Event.RuleName, + Text: content, + AtMobiles: phones, + Tokens: StringSetKeys(dingtalkset), + }) + case "wecom": + if len(wecomset) == 0 { + continue + } + + content, has := notice.Tpls["wecom.tpl"] + if !has { + content = "wecom.tpl not found" + } + sender.SendWecom(sender.WecomMessage{ + Text: content, + Tokens: StringSetKeys(wecomset), + }) + case "feishu": + if len(feishuset) == 0 { + continue + } + + content, has := notice.Tpls["feishu.tpl"] + if !has { + content = "feishu.tpl not found" + } + sender.SendFeishu(sender.FeishuMessage{ + Text: content, + AtMobiles: phones, + Tokens: StringSetKeys(feishuset), + }) + default: + logger.Info("channel ", ch, " not supported by golang") + } + } } func notify(event *models.AlertCurEvent) { diff --git a/src/server/sender/dingtalk.go b/src/server/sender/dingtalk.go index c8325c85..a2da40dd 100644 --- a/src/server/sender/dingtalk.go +++ b/src/server/sender/dingtalk.go @@ -1,6 +1,7 @@ package sender import ( + "strings" "time" "github.com/didi/nightingale/v5/src/server/poster" @@ -31,13 +32,18 @@ type dingtalk struct { } func SendDingtalk(message DingtalkMessage) { + ats := make([]string, len(message.AtMobiles)) + for i := 0; i < len(message.AtMobiles); i++ { + ats[i] = "@" + message.AtMobiles[i] + } + for i := 0; i < len(message.Tokens); i++ { url := "https://oapi.dingtalk.com/robot/send?access_token=" + message.Tokens[i] body := dingtalk{ Msgtype: "markdown", Markdown: dingtalkMarkdown{ Title: message.Title, - Text: message.Text, + Text: message.Text + " " + strings.Join(ats, " "), }, At: dingtalkAt{ AtMobiles: message.AtMobiles, diff --git a/src/server/sender/email.go b/src/server/sender/email.go new file mode 100644 index 00000000..b0e1c18f --- /dev/null +++ b/src/server/sender/email.go @@ -0,0 +1,71 @@ +package sender + +import ( + "crypto/tls" + "time" + + "github.com/didi/nightingale/v5/src/server/config" + "github.com/toolkits/pkg/logger" + "gopkg.in/gomail.v2" +) + +var mailch = make(chan *gomail.Message, 100000) + +func WriteEmail(subject, content string, tos []string) { + m := gomail.NewMessage() + + m.SetHeader("From", config.C.SMTP.From) + m.SetHeader("To", tos...) + m.SetHeader("Subject", subject) + m.SetBody("text/html", content) + + mailch <- m +} + +func dialSmtp(d *gomail.Dialer) gomail.SendCloser { + for { + if s, err := d.Dial(); err != nil { + logger.Errorf("email_sender: failed to dial smtp: %s", err) + time.Sleep(time.Second) + continue + } else { + return s + } + } +} + +func StartEmailSender() { + conf := config.C.SMTP + + d := gomail.NewDialer(conf.Host, conf.Port, conf.User, conf.Pass) + if conf.InsecureSkipVerify { + d.TLSConfig = &tls.Config{InsecureSkipVerify: true} + } + + var s gomail.SendCloser + open := false + for { + select { + case m, ok := <-mailch: + if !ok { + return + } + if !open { + s = dialSmtp(d) + open = true + } + if err := gomail.Send(s, m); err != nil { + logger.Errorf("email_sender: failed to send: %s", err) + } + // Close the connection to the SMTP server if no email was sent in + // the last 30 seconds. + case <-time.After(30 * time.Second): + if open { + if err := s.Close(); err != nil { + logger.Warningf("email_sender: failed to close smtp connection: %s", err) + } + open = false + } + } + } +} diff --git a/src/webapi/config/config.go b/src/webapi/config/config.go index 1cbdd6f0..42864e86 100644 --- a/src/webapi/config/config.go +++ b/src/webapi/config/config.go @@ -78,8 +78,8 @@ type Config struct { I18N string AdminRole string MetricsYamlFile string - ContactKeys []ContactKey - NotifyChannels []string + ContactKeys []LabelAndKey + NotifyChannels []LabelAndKey Log logx.Config HTTP httpx.Config JWTAuth JWTAuth @@ -94,11 +94,20 @@ type Config struct { Ibex Ibex } -type ContactKey struct { +type LabelAndKey struct { Label string `json:"label"` Key string `json:"key"` } +func LabelAndKeyHasKey(keys []LabelAndKey, key string) bool { + for i := 0; i < len(keys); i++ { + if keys[i].Key == key { + return true + } + } + return false +} + type JWTAuth struct { SigningKey string AccessExpired int64