diff --git a/etc/script/notify/notify.go b/etc/script/notify/notify.go index f3bca2e0..86e58412 100644 --- a/etc/script/notify/notify.go +++ b/etc/script/notify/notify.go @@ -41,13 +41,13 @@ func (n *N9EPlugin) Notify(bs []byte) { func (n *N9EPlugin) NotifyMaintainer(bs []byte) { fmt.Println("do something... begin") result := string(bs) - fmt.Println("%T",result) + fmt.Println(result) fmt.Println("do something... end") } // will be loaded for alertingCall , The first letter must be capitalized to be exported var N9eCaller = N9EPlugin{ - Name: "n9e", - Description: "演示告警通过动态链接库方式通知", + Name: "N9EPlugin", + Description: "Notification by lib", BuildAt: time.Now().Local().Format("2006/01/02 15:04:05"), } diff --git a/src/notifier/notifier.go b/src/notifier/notifier.go new file mode 100644 index 00000000..3fdda89e --- /dev/null +++ b/src/notifier/notifier.go @@ -0,0 +1,9 @@ +package notifier + +type Notifier interface { + Descript() string + Notify([]byte) + NotifyMaintainer([]byte) +} + +var Instance Notifier diff --git a/src/server/config/config.go b/src/server/config/config.go index 2980e34b..2cf280d8 100644 --- a/src/server/config/config.go +++ b/src/server/config/config.go @@ -2,8 +2,11 @@ package config import ( "fmt" + "log" "net" "os" + "plugin" + "runtime" "strings" "sync" "time" @@ -11,6 +14,7 @@ import ( "github.com/gin-gonic/gin" "github.com/koding/multiconfig" + "github.com/didi/nightingale/v5/src/notifier" "github.com/didi/nightingale/v5/src/pkg/httpx" "github.com/didi/nightingale/v5/src/pkg/logx" "github.com/didi/nightingale/v5/src/pkg/ormx" @@ -100,6 +104,33 @@ func MustLoad(fpaths ...string) { } } + if C.Alerting.CallPlugin.Enable { + if runtime.GOOS == "windows" { + fmt.Println("notify plugin on unsupported os:", runtime.GOOS) + os.Exit(1) + } + + p, err := plugin.Open(C.Alerting.CallPlugin.PluginPath) + if err != nil { + fmt.Println("failed to load plugin:", err) + os.Exit(1) + } + + caller, err := p.Lookup(C.Alerting.CallPlugin.Caller) + if err != nil { + fmt.Println("failed to lookup plugin Caller:", err) + os.Exit(1) + } + + ins, ok := caller.(notifier.Notifier) + if !ok { + log.Println("notifier interface not implemented") + os.Exit(1) + } + + notifier.Instance = ins + } + if C.WriterOpt.QueueMaxSize <= 0 { C.WriterOpt.QueueMaxSize = 100000 } diff --git a/src/server/engine/notify.go b/src/server/engine/notify.go index 535aacb1..3328289f 100644 --- a/src/server/engine/notify.go +++ b/src/server/engine/notify.go @@ -9,8 +9,6 @@ import ( "net/http" "os/exec" "path" - "plugin" - "runtime" "strings" "time" @@ -22,6 +20,7 @@ import ( "github.com/toolkits/pkg/slice" "github.com/didi/nightingale/v5/src/models" + "github.com/didi/nightingale/v5/src/notifier" "github.com/didi/nightingale/v5/src/pkg/sys" "github.com/didi/nightingale/v5/src/pkg/tplx" "github.com/didi/nightingale/v5/src/server/common/sender" @@ -103,7 +102,6 @@ func alertingRedisPub(bs []byte) { func handleNotice(notice Notice, bs []byte) { alertingCallScript(bs) - alertingCallPlugin(bs) if len(config.C.Alerting.NotifyBuiltinChannels) == 0 { @@ -398,12 +396,6 @@ func alertingCallScript(stdinBytes []byte) { logger.Infof("event_notify: exec %s output: %s", fpath, buf.String()) } -type Notifier interface { - Descript() string - Notify([]byte) - NotifyMaintainer([]byte) -} - // call notify.so via golang plugin build // ig. etc/script/notify/notify.so func alertingCallPlugin(stdinBytes []byte) { @@ -411,26 +403,8 @@ func alertingCallPlugin(stdinBytes []byte) { return } - if runtime.GOOS == "windows" { - logger.Errorf("call notify plugin on unsupported os: %s", runtime.GOOS) - return - } - - p, err := plugin.Open(config.C.Alerting.CallPlugin.PluginPath) - if err != nil { - logger.Errorf("failed to open notify plugin: %v", err) - return - } - caller, err := p.Lookup(config.C.Alerting.CallPlugin.Caller) - if err != nil { - logger.Errorf("failed to load caller: %v", err) - return - } - notifier, ok := caller.(Notifier) - if !ok { - logger.Errorf("notifier interface not implemented): %v", err) - return - } - notifier.Notify(stdinBytes) - logger.Debugf("alertingCallPlugin done. %s", notifier.Descript()) + logger.Debugf("alertingCallPlugin begin") + logger.Debugf("payload:", string(stdinBytes)) + notifier.Instance.Notify(stdinBytes) + logger.Debugf("alertingCallPlugin done") } diff --git a/src/server/engine/notify_maintainer.go b/src/server/engine/notify_maintainer.go index c6788630..0d5a135d 100644 --- a/src/server/engine/notify_maintainer.go +++ b/src/server/engine/notify_maintainer.go @@ -2,11 +2,11 @@ package engine import ( "encoding/json" - "plugin" "runtime" "time" "github.com/didi/nightingale/v5/src/models" + "github.com/didi/nightingale/v5/src/notifier" "github.com/didi/nightingale/v5/src/server/common/sender" "github.com/didi/nightingale/v5/src/server/config" "github.com/didi/nightingale/v5/src/server/memsto" @@ -14,13 +14,13 @@ import ( "github.com/toolkits/pkg/logger" ) -type NoticeMaintainer struct { - NotifyUsersObj []*models.User `json:"notify_user_obj" gorm:"-"` - Title string `json:"title"` - Content string `json:"content"` +type MaintainMessage struct { + Tos []*models.User `json:"tos"` + Title string `json:"title"` + Content string `json:"content"` } -func noticeCallPlugin(stdinBytes []byte) { +func notifyMaintainerWithPlugin(e error, title, triggerTime string, users []*models.User) { if !config.C.Alerting.CallPlugin.Enable { return } @@ -30,56 +30,48 @@ func noticeCallPlugin(stdinBytes []byte) { return } - p, err := plugin.Open(config.C.Alerting.CallPlugin.PluginPath) + stdinBytes, err := json.Marshal(MaintainMessage{ + Tos: users, + Title: title, + Content: "Title: " + title + "\nContent: " + e.Error() + "\nTime: " + triggerTime, + }) + if err != nil { - logger.Errorf("failed to open notify plugin: %v", err) + logger.Error("failed to marshal MaintainMessage:", err) return } - caller, err := p.Lookup(config.C.Alerting.CallPlugin.Caller) - if err != nil { - logger.Errorf("failed to load caller: %v", err) - return - } - notifier, ok := caller.(Notifier) - if !ok { - logger.Errorf("notifier interface not implemented): %v", err) - return - } - notifier.NotifyMaintainer(stdinBytes) - logger.Debugf("noticeCallPlugin done. %s", notifier.Descript()) + + notifier.Instance.NotifyMaintainer(stdinBytes) + logger.Debugf("notify maintainer with plugin done") } // notify to maintainer to handle the error func notifyToMaintainer(e error, title string) { + logger.Errorf("notifyToMaintainer, title:%s, error:%v", title, e) - logger.Errorf("notifyToMaintainer，title:%s, error:%v", title, e) - - var noticeMaintainer NoticeMaintainer - maintainerUsers := memsto.UserCache.GetMaintainerUsers() - if len(maintainerUsers) == 0 { + users := memsto.UserCache.GetMaintainerUsers() + if len(users) == 0 { return } - triggerTime := time.Now().Format("2006/01/02 - 15:04:05") - noticeMaintainer.NotifyUsersObj = maintainerUsers - noticeMaintainer.Content = "【内部处理错误】当前标题: " + title + "\n【内部处理错误】当前异常: " + e.Error() + "\n【内部处理错误】发送时间: " + triggerTime - noticeMaintainer.Title = title - stdinBytes, err := json.Marshal(noticeMaintainer) - if err != nil { - logger.Errorf("notifyToMaintainer: failed to marshal noticeMaintainer: %v", err) - } else { - noticeCallPlugin(stdinBytes) - } + triggerTime := time.Now().Format("2006/01/02 - 15:04:05") + + notifyMaintainerWithPlugin(e, title, triggerTime, users) + notifyMaintainerWithBuiltin(e, title, triggerTime, users) +} + +func notifyMaintainerWithBuiltin(e error, title, triggerTime string, users []*models.User) { if len(config.C.Alerting.NotifyBuiltinChannels) == 0 { return } + emailset := make(map[string]struct{}) phoneset := make(map[string]struct{}) wecomset := make(map[string]struct{}) dingtalkset := make(map[string]struct{}) feishuset := make(map[string]struct{}) - for _, user := range maintainerUsers { + for _, user := range users { if user.Email != "" { emailset[user.Email] = struct{}{} } @@ -118,13 +110,13 @@ func notifyToMaintainer(e error, title string) { if len(emailset) == 0 { continue } - content := "【内部处理错误】当前标题: " + title + "\n【内部处理错误】当前异常: " + e.Error() + "\n【内部处理错误】发送时间: " + triggerTime + content := "Title: " + title + "\nContent: " + e.Error() + "\nTime: " + triggerTime sender.WriteEmail(title, content, StringSetKeys(emailset)) case "dingtalk": if len(dingtalkset) == 0 { continue } - content := "**【内部处理错误】当前标题: **" + title + "\n**【内部处理错误】当前异常: **" + e.Error() + "\n**【内部处理错误】发送时间: **" + triggerTime + content := "**Title: **" + title + "\n**Content: **" + e.Error() + "\n**Time: **" + triggerTime sender.SendDingtalk(sender.DingtalkMessage{ Title: title, Text: content, @@ -135,7 +127,7 @@ func notifyToMaintainer(e error, title string) { if len(wecomset) == 0 { continue } - content := "**【内部处理错误】当前标题: **" + title + "\n**【内部处理错误】当前异常: **" + e.Error() + "\n**【内部处理错误】发送时间: **" + triggerTime + content := "**Title: **" + title + "\n**Content: **" + e.Error() + "\n**Time: **" + triggerTime sender.SendWecom(sender.WecomMessage{ Text: content, Tokens: StringSetKeys(wecomset), @@ -145,7 +137,7 @@ func notifyToMaintainer(e error, title string) { continue } - content := "【内部处理错误】当前标题: " + title + "\n【内部处理错误】当前异常: " + e.Error() + "\n【内部处理错误】发送时间: " + triggerTime + content := "Title: " + title + "\nContent: " + e.Error() + "\nTime: " + triggerTime sender.SendFeishu(sender.FeishuMessage{ Text: content, AtMobiles: phones, diff --git a/src/server/engine/worker.go b/src/server/engine/worker.go index 352b085c..6f254a78 100644 --- a/src/server/engine/worker.go +++ b/src/server/engine/worker.go @@ -116,8 +116,7 @@ func (r RuleEval) Work() { value, warnings, err = reader.Client.Query(context.Background(), promql, time.Now()) if err != nil { logger.Errorf("rule_eval:%d promql:%s, error:%v", r.RuleID(), promql, err) - // 告警查询prometheus逻辑出错，发告警信息给管理员 - notifyToMaintainer(err, "查询prometheus出错") + notifyToMaintainer(err, "failed to query prometheus") return } @@ -190,7 +189,6 @@ func (ws *WorkersType) Build(rids []int64) { elst, err := models.AlertCurEventGetByRule(rules[hash].Id) if err != nil { logger.Errorf("worker_build: AlertCurEventGetByRule failed: %v", err) - notifyToMaintainer(err, "AlertCurEventGetByRule Error，ruleID="+fmt.Sprint(rules[hash].Id)) continue }