innov
/
nightingale1
1
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

code refactor notify plugin (#1065)

This commit is contained in:
ulricqin 2022-07-22 17:56:52 +08:00 committed by GitHub
parent c45cbd02cc
commit 17c7361620
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 81 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
etc/script/notify/notify.go
View File

@ -41,13 +41,13 @@ func (n *N9EPlugin) Notify(bs []byte) {
func (n *N9EPlugin) NotifyMaintainer(bs []byte) {
fmt.Println("do something... begin")
result := string(bs)
fmt.Println("%T",result)
fmt.Println(result)
fmt.Println("do something... end")
}
// will be loaded for alertingCall , The first letter must be capitalized to be exported
var N9eCaller = N9EPlugin{
Name: "n9e",
Description: "演示告警通过动态链接库方式通知",
Name: "N9EPlugin",
Description: "Notification by lib",
BuildAt: time.Now().Local().Format("2006/01/02 15:04:05"),
}

9
src/notifier/notifier.go Normal file
View File

@ -0,0 +1,9 @@
package notifier
type Notifier interface {
Descript() string
Notify([]byte)
NotifyMaintainer([]byte)
}
var Instance Notifier

31
src/server/config/config.go
View File

@ -2,8 +2,11 @@ package config
import (
"fmt"
"log"
"net"
"os"
"plugin"
"runtime"
"strings"
"sync"
"time"
@ -11,6 +14,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/koding/multiconfig"
"github.com/didi/nightingale/v5/src/notifier"
"github.com/didi/nightingale/v5/src/pkg/httpx"
"github.com/didi/nightingale/v5/src/pkg/logx"
"github.com/didi/nightingale/v5/src/pkg/ormx"
@ -100,6 +104,33 @@ func MustLoad(fpaths ...string) {
}
}
if C.Alerting.CallPlugin.Enable {
if runtime.GOOS == "windows" {
fmt.Println("notify plugin on unsupported os:", runtime.GOOS)
os.Exit(1)
}
p, err := plugin.Open(C.Alerting.CallPlugin.PluginPath)
if err != nil {
fmt.Println("failed to load plugin:", err)
os.Exit(1)
}
caller, err := p.Lookup(C.Alerting.CallPlugin.Caller)
if err != nil {
fmt.Println("failed to lookup plugin Caller:", err)
os.Exit(1)
}
ins, ok := caller.(notifier.Notifier)
if !ok {
log.Println("notifier interface not implemented")
os.Exit(1)
}
notifier.Instance = ins
}
if C.WriterOpt.QueueMaxSize <= 0 {
C.WriterOpt.QueueMaxSize = 100000
}

36
src/server/engine/notify.go
View File

@ -9,8 +9,6 @@ import (
"net/http"
"os/exec"
"path"
"plugin"
"runtime"
"strings"
"time"
@ -22,6 +20,7 @@ import (
"github.com/toolkits/pkg/slice"
"github.com/didi/nightingale/v5/src/models"
"github.com/didi/nightingale/v5/src/notifier"
"github.com/didi/nightingale/v5/src/pkg/sys"
"github.com/didi/nightingale/v5/src/pkg/tplx"
"github.com/didi/nightingale/v5/src/server/common/sender"
@ -103,7 +102,6 @@ func alertingRedisPub(bs []byte) {
func handleNotice(notice Notice, bs []byte) {
alertingCallScript(bs)
alertingCallPlugin(bs)
if len(config.C.Alerting.NotifyBuiltinChannels) == 0 {
@ -398,12 +396,6 @@ func alertingCallScript(stdinBytes []byte) {
logger.Infof("event_notify: exec %s output: %s", fpath, buf.String())
}
type Notifier interface {
Descript() string
Notify([]byte)
NotifyMaintainer([]byte)
}
// call notify.so via golang plugin build
// ig. etc/script/notify/notify.so
func alertingCallPlugin(stdinBytes []byte) {
@ -411,26 +403,8 @@ func alertingCallPlugin(stdinBytes []byte) {
return
}
if runtime.GOOS == "windows" {
logger.Errorf("call notify plugin on unsupported os: %s", runtime.GOOS)
return
}
p, err := plugin.Open(config.C.Alerting.CallPlugin.PluginPath)
if err != nil {
logger.Errorf("failed to open notify plugin: %v", err)
return
}
caller, err := p.Lookup(config.C.Alerting.CallPlugin.Caller)
if err != nil {
logger.Errorf("failed to load caller: %v", err)
return
}
notifier, ok := caller.(Notifier)
if !ok {
logger.Errorf("notifier interface not implemented): %v", err)
return
}
notifier.Notify(stdinBytes)
logger.Debugf("alertingCallPlugin done. %s", notifier.Descript())
logger.Debugf("alertingCallPlugin begin")
logger.Debugf("payload:", string(stdinBytes))
notifier.Instance.Notify(stdinBytes)
logger.Debugf("alertingCallPlugin done")
}

68
src/server/engine/notify_maintainer.go
View File

@ -2,11 +2,11 @@ package engine
import (
"encoding/json"
"plugin"
"runtime"
"time"
"github.com/didi/nightingale/v5/src/models"
"github.com/didi/nightingale/v5/src/notifier"
"github.com/didi/nightingale/v5/src/server/common/sender"
"github.com/didi/nightingale/v5/src/server/config"
"github.com/didi/nightingale/v5/src/server/memsto"
@ -14,13 +14,13 @@ import (
"github.com/toolkits/pkg/logger"
)
type NoticeMaintainer struct {
NotifyUsersObj []*models.User `json:"notify_user_obj" gorm:"-"`
type MaintainMessage struct {
Tos []*models.User `json:"tos"`
Title string `json:"title"`
Content string `json:"content"`
}
func noticeCallPlugin(stdinBytes []byte) {
func notifyMaintainerWithPlugin(e error, title, triggerTime string, users []*models.User) {
if !config.C.Alerting.CallPlugin.Enable {
return
}
@ -30,56 +30,48 @@ func noticeCallPlugin(stdinBytes []byte) {
return
}
p, err := plugin.Open(config.C.Alerting.CallPlugin.PluginPath)
stdinBytes, err := json.Marshal(MaintainMessage{
Tos: users,
Title: title,
Content: "Title: " + title + "\nContent: " + e.Error() + "\nTime: " + triggerTime,
})
if err != nil {
logger.Errorf("failed to open notify plugin: %v", err)
logger.Error("failed to marshal MaintainMessage:", err)
return
}
caller, err := p.Lookup(config.C.Alerting.CallPlugin.Caller)
if err != nil {
logger.Errorf("failed to load caller: %v", err)
return
}
notifier, ok := caller.(Notifier)
if !ok {
logger.Errorf("notifier interface not implemented): %v", err)
return
}
notifier.NotifyMaintainer(stdinBytes)
logger.Debugf("noticeCallPlugin done. %s", notifier.Descript())
notifier.Instance.NotifyMaintainer(stdinBytes)
logger.Debugf("notify maintainer with plugin done")
}
// notify to maintainer to handle the error
func notifyToMaintainer(e error, title string) {
logger.Errorf("notifyToMaintainer, title:%s, error:%v", title, e)
logger.Errorf("notifyToMaintainer，title:%s, error:%v", title, e)
var noticeMaintainer NoticeMaintainer
maintainerUsers := memsto.UserCache.GetMaintainerUsers()
if len(maintainerUsers) == 0 {
users := memsto.UserCache.GetMaintainerUsers()
if len(users) == 0 {
return
}
triggerTime := time.Now().Format("2006/01/02 - 15:04:05")
noticeMaintainer.NotifyUsersObj = maintainerUsers
noticeMaintainer.Content = "【内部处理错误】当前标题: " + title + "\n【内部处理错误】当前异常: " + e.Error() + "\n【内部处理错误】发送时间: " + triggerTime
noticeMaintainer.Title = title
stdinBytes, err := json.Marshal(noticeMaintainer)
if err != nil {
logger.Errorf("notifyToMaintainer: failed to marshal noticeMaintainer: %v", err)
} else {
noticeCallPlugin(stdinBytes)
}
triggerTime := time.Now().Format("2006/01/02 - 15:04:05")
notifyMaintainerWithPlugin(e, title, triggerTime, users)
notifyMaintainerWithBuiltin(e, title, triggerTime, users)
}
func notifyMaintainerWithBuiltin(e error, title, triggerTime string, users []*models.User) {
if len(config.C.Alerting.NotifyBuiltinChannels) == 0 {
return
}
emailset := make(map[string]struct{})
phoneset := make(map[string]struct{})
wecomset := make(map[string]struct{})
dingtalkset := make(map[string]struct{})
feishuset := make(map[string]struct{})
for _, user := range maintainerUsers {
for _, user := range users {
if user.Email != "" {
emailset[user.Email] = struct{}{}
}
@ -118,13 +110,13 @@ func notifyToMaintainer(e error, title string) {
if len(emailset) == 0 {
continue
}
content := "【内部处理错误】当前标题: " + title + "\n【内部处理错误】当前异常: " + e.Error() + "\n【内部处理错误】发送时间: " + triggerTime
content := "Title: " + title + "\nContent: " + e.Error() + "\nTime: " + triggerTime
sender.WriteEmail(title, content, StringSetKeys(emailset))
case "dingtalk":
if len(dingtalkset) == 0 {
continue
}
content := "**【内部处理错误】当前标题: **" + title + "\n**【内部处理错误】当前异常: **" + e.Error() + "\n**【内部处理错误】发送时间: **" + triggerTime
content := "**Title: **" + title + "\n**Content: **" + e.Error() + "\n**Time: **" + triggerTime
sender.SendDingtalk(sender.DingtalkMessage{
Title: title,
Text: content,
@ -135,7 +127,7 @@ func notifyToMaintainer(e error, title string) {
if len(wecomset) == 0 {
continue
}
content := "**【内部处理错误】当前标题: **" + title + "\n**【内部处理错误】当前异常: **" + e.Error() + "\n**【内部处理错误】发送时间: **" + triggerTime
content := "**Title: **" + title + "\n**Content: **" + e.Error() + "\n**Time: **" + triggerTime
sender.SendWecom(sender.WecomMessage{
Text: content,
Tokens: StringSetKeys(wecomset),
@ -145,7 +137,7 @@ func notifyToMaintainer(e error, title string) {
continue
}
content := "【内部处理错误】当前标题: " + title + "\n【内部处理错误】当前异常: " + e.Error() + "\n【内部处理错误】发送时间: " + triggerTime
content := "Title: " + title + "\nContent: " + e.Error() + "\nTime: " + triggerTime
sender.SendFeishu(sender.FeishuMessage{
Text: content,
AtMobiles: phones,

4
src/server/engine/worker.go
View File

@ -116,8 +116,7 @@ func (r RuleEval) Work() {
value, warnings, err = reader.Client.Query(context.Background(), promql, time.Now())
if err != nil {
logger.Errorf("rule_eval:%d promql:%s, error:%v", r.RuleID(), promql, err)
// 告警查询prometheus逻辑出错，发告警信息给管理员
notifyToMaintainer(err, "查询prometheus出错")
notifyToMaintainer(err, "failed to query prometheus")
return
}
@ -190,7 +189,6 @@ func (ws *WorkersType) Build(rids []int64) {
elst, err := models.AlertCurEventGetByRule(rules[hash].Id)
if err != nil {
logger.Errorf("worker_build: AlertCurEventGetByRule failed: %v", err)
notifyToMaintainer(err, "AlertCurEventGetByRule Error，ruleID="+fmt.Sprint(rules[hash].Id))
continue
}