innov
/
nightingale1
1
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

use golang as sender

This commit is contained in:
Ulric Qin 2022-03-01 11:16:55 +08:00
parent 403cb5a6ad
commit 2ff79c7780
10 changed files with 243 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
etc/webapi.conf
View File

@ -10,19 +10,39 @@ AdminRole = "Admin"
# metrics descriptions
MetricsYamlFile = "./etc/metrics.yaml"
# Linkage with notify.py script
NotifyChannels = [ "email", "dingtalk", "wecom", "feishu" ]
[[NotifyChannels]]
Label = "邮箱"
# do not change Key
Key = "email"
[[NotifyChannels]]
Label = "钉钉机器人"
# do not change Key
Key = "dingtalk"
[[NotifyChannels]]
Label = "企微机器人"
# do not change Key
Key = "wecom"
[[NotifyChannels]]
Label = "飞书机器人"
# do not change Key
Key = "feishu"
[[ContactKeys]]
Label = "Wecom Robot Token"
# do not change Key
Key = "wecom_robot_token"
[[ContactKeys]]
Label = "Dingtalk Robot Token"
# do not change Key
Key = "dingtalk_robot_token"
[[ContactKeys]]
Label = "Feishu Robot Token"
# do not change Key
Key = "feishu_robot_token"
[Log]

3
go.mod
View File

@ -23,12 +23,15 @@ require (
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/common v0.26.0
github.com/prometheus/prometheus v2.5.0+incompatible
github.com/tidwall/gjson v1.14.0 // indirect
github.com/toolkits/pkg v1.2.9
github.com/urfave/cli/v2 v2.3.0
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4 // indirect
google.golang.org/grpc v1.41.0 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect
gorm.io/driver/mysql v1.1.2
gorm.io/driver/postgres v1.1.1
gorm.io/gorm v1.21.15

10
go.sum
View File

@ -298,6 +298,12 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/toolkits/pkg v1.2.9 h1:zGlrJDl+2sMBoxBRIoMtAwvKmW5wctuji2+qHCecMKk=
github.com/toolkits/pkg v1.2.9/go.mod h1:ZUsQAOoaR99PSbes+RXSirvwmtd6+XIUvizCmrjfUYc=
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
@ -465,12 +471,16 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

3
src/models/alert_rule.go
View File

@ -7,7 +7,6 @@ import (
"time"
"github.com/pkg/errors"
"github.com/toolkits/pkg/slice"
"github.com/toolkits/pkg/str"
"github.com/didi/nightingale/v5/src/webapi/config"
@ -101,7 +100,7 @@ func (ar *AlertRule) Verify() error {
if len(channels) > 0 {
nlst := make([]string, 0, len(channels))
for i := 0; i < len(channels); i++ {
if slice.ContainsString(config.C.NotifyChannels, channels[i]) {
if config.LabelAndKeyHasKey(config.C.NotifyChannels, channels[i]) {
nlst = append(nlst, channels[i])
}
}

8
src/server/engine/consume.go
View File

@ -155,3 +155,11 @@ func mapKeys(m map[int64]struct{}) []int64 {
}
return lst
}
func StringSetKeys(m map[string]struct{}) []string {
lst := make([]string, 0, len(m))
for k := range m {
lst = append(lst, k)
}
return lst
}

3
src/server/engine/engine.go
View File

@ -5,6 +5,7 @@ import (
"time"
"github.com/didi/nightingale/v5/src/server/config"
"github.com/didi/nightingale/v5/src/server/sender"
promstat "github.com/didi/nightingale/v5/src/server/stat"
)
@ -22,6 +23,8 @@ func Start(ctx context.Context) error {
go reportQueueSize()
go sender.StartEmailSender()
return nil
}

108
src/server/engine/notify.go
View File

@ -13,6 +13,7 @@ import (
"time"
"github.com/pkg/errors"
"github.com/tidwall/gjson"
"github.com/toolkits/pkg/file"
"github.com/toolkits/pkg/logger"
"github.com/toolkits/pkg/runner"
@ -21,6 +22,7 @@ import (
"github.com/didi/nightingale/v5/src/pkg/sys"
"github.com/didi/nightingale/v5/src/server/config"
"github.com/didi/nightingale/v5/src/server/memsto"
"github.com/didi/nightingale/v5/src/server/sender"
"github.com/didi/nightingale/v5/src/storage"
)
@ -117,8 +119,110 @@ func alertingRedisPub(bs []byte) {
func handleNotice(notice Notice, bs []byte) {
alertingCallScript(bs)
// TODO 弄个channel发邮件，学习daemon写法
// 收集tokens、phones，发呗
emailset := make(map[string]struct{})
phoneset := make(map[string]struct{})
wecomset := make(map[string]struct{})
dingtalkset := make(map[string]struct{})
feishuset := make(map[string]struct{})
for _, user := range notice.Event.NotifyUsersObj {
if user.Email != "" {
emailset[user.Email] = struct{}{}
}
if user.Phone != "" {
phoneset[user.Phone] = struct{}{}
}
bs, err := user.Contacts.MarshalJSON()
if err != nil {
logger.Errorf("handle_notice: failed to marshal contacts: %v", err)
continue
}
ret := gjson.GetBytes(bs, "dingtalk_robot_token")
if ret.Exists() {
dingtalkset[ret.String()] = struct{}{}
}
ret = gjson.GetBytes(bs, "wecom_robot_token")
if ret.Exists() {
wecomset[ret.String()] = struct{}{}
}
ret = gjson.GetBytes(bs, "feishu_robot_token")
if ret.Exists() {
feishuset[ret.String()] = struct{}{}
}
}
phones := StringSetKeys(phoneset)
for _, ch := range notice.Event.NotifyChannelsJSON {
switch ch {
case "email":
if len(emailset) == 0 {
continue
}
subject, has := notice.Tpls["subject.tpl"]
if !has {
subject = "subject.tpl not found"
}
content, has := notice.Tpls["mailbody.tpl"]
if !has {
content = "mailbody.tpl not found"
}
sender.WriteEmail(subject, content, StringSetKeys(emailset))
case "dingtalk":
if len(dingtalkset) == 0 {
continue
}
content, has := notice.Tpls["dingtalk.tpl"]
if !has {
content = "dingtalk.tpl not found"
}
sender.SendDingtalk(sender.DingtalkMessage{
Title: notice.Event.RuleName,
Text: content,
AtMobiles: phones,
Tokens: StringSetKeys(dingtalkset),
})
case "wecom":
if len(wecomset) == 0 {
continue
}
content, has := notice.Tpls["wecom.tpl"]
if !has {
content = "wecom.tpl not found"
}
sender.SendWecom(sender.WecomMessage{
Text: content,
Tokens: StringSetKeys(wecomset),
})
case "feishu":
if len(feishuset) == 0 {
continue
}
content, has := notice.Tpls["feishu.tpl"]
if !has {
content = "feishu.tpl not found"
}
sender.SendFeishu(sender.FeishuMessage{
Text: content,
AtMobiles: phones,
Tokens: StringSetKeys(feishuset),
})
default:
logger.Info("channel ", ch, " not supported by golang")
}
}
}
func notify(event *models.AlertCurEvent) {

8
src/server/sender/dingtalk.go
View File

@ -1,6 +1,7 @@
package sender
import (
"strings"
"time"
"github.com/didi/nightingale/v5/src/server/poster"
@ -31,13 +32,18 @@ type dingtalk struct {
}
func SendDingtalk(message DingtalkMessage) {
ats := make([]string, len(message.AtMobiles))
for i := 0; i < len(message.AtMobiles); i++ {
ats[i] = "@" + message.AtMobiles[i]
}
for i := 0; i < len(message.Tokens); i++ {
url := "https://oapi.dingtalk.com/robot/send?access_token=" + message.Tokens[i]
body := dingtalk{
Msgtype: "markdown",
Markdown: dingtalkMarkdown{
Title: message.Title,
Text: message.Text,
Text: message.Text + " " + strings.Join(ats, " "),
},
At: dingtalkAt{
AtMobiles: message.AtMobiles,

71
src/server/sender/email.go Normal file
View File

@ -0,0 +1,71 @@
package sender
import (
"crypto/tls"
"time"
"github.com/didi/nightingale/v5/src/server/config"
"github.com/toolkits/pkg/logger"
"gopkg.in/gomail.v2"
)
var mailch = make(chan *gomail.Message, 100000)
func WriteEmail(subject, content string, tos []string) {
m := gomail.NewMessage()
m.SetHeader("From", config.C.SMTP.From)
m.SetHeader("To", tos...)
m.SetHeader("Subject", subject)
m.SetBody("text/html", content)
mailch <- m
}
func dialSmtp(d *gomail.Dialer) gomail.SendCloser {
for {
if s, err := d.Dial(); err != nil {
logger.Errorf("email_sender: failed to dial smtp: %s", err)
time.Sleep(time.Second)
continue
} else {
return s
}
}
}
func StartEmailSender() {
conf := config.C.SMTP
d := gomail.NewDialer(conf.Host, conf.Port, conf.User, conf.Pass)
if conf.InsecureSkipVerify {
d.TLSConfig = &tls.Config{InsecureSkipVerify: true}
}
var s gomail.SendCloser
open := false
for {
select {
case m, ok := <-mailch:
if !ok {
return
}
if !open {
s = dialSmtp(d)
open = true
}
if err := gomail.Send(s, m); err != nil {
logger.Errorf("email_sender: failed to send: %s", err)
}
// Close the connection to the SMTP server if no email was sent in
// the last 30 seconds.
case <-time.After(30 * time.Second):
if open {
if err := s.Close(); err != nil {
logger.Warningf("email_sender: failed to close smtp connection: %s", err)
}
open = false
}
}
}
}

15
src/webapi/config/config.go
View File

@ -78,8 +78,8 @@ type Config struct {
I18N string
AdminRole string
MetricsYamlFile string
ContactKeys []ContactKey
NotifyChannels []string
ContactKeys []LabelAndKey
NotifyChannels []LabelAndKey
Log logx.Config
HTTP httpx.Config
JWTAuth JWTAuth
@ -94,11 +94,20 @@ type Config struct {
Ibex Ibex
}
type ContactKey struct {
type LabelAndKey struct {
Label string `json:"label"`
Key string `json:"key"`
}
func LabelAndKeyHasKey(keys []LabelAndKey, key string) bool {
for i := 0; i < len(keys); i++ {
if keys[i].Key == key {
return true
}
}
return false
}
type JWTAuth struct {
SigningKey string
AccessExpired int64