code refactor notify plugin (#1065)

etc/script/notify/notify.go

@@ -41,13 +41,13 @@ func (n *N9EPlugin) Notify(bs []byte) {
 func (n *N9EPlugin) NotifyMaintainer(bs []byte) {
 	fmt.Println("do something... begin")
 	result := string(bs)
-	fmt.Println("%T",result)
+	fmt.Println(result)
 	fmt.Println("do something... end")
 }
 
 // will be loaded for alertingCall , The first letter must be capitalized to be exported
 var N9eCaller = N9EPlugin{
-	Name:        "n9e",
+	Name:        "N9EPlugin",
-	Description: "演示告警通过动态链接库方式通知",
+	Description: "Notification by lib",
 	BuildAt:     time.Now().Local().Format("2006/01/02 15:04:05"),
 }

src/notifier/notifier.go

@@ -0,0 +1,9 @@
+package notifier
+
+type Notifier interface {
+	Descript() string
+	Notify([]byte)
+	NotifyMaintainer([]byte)
+}
+
+var Instance Notifier

src/server/config/config.go

@@ -2,8 +2,11 @@ package config
 
 import (
 	"fmt"
+	"log"
 	"net"
 	"os"
+	"plugin"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
@@ -11,6 +14,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/koding/multiconfig"
 
+	"github.com/didi/nightingale/v5/src/notifier"
 	"github.com/didi/nightingale/v5/src/pkg/httpx"
 	"github.com/didi/nightingale/v5/src/pkg/logx"
 	"github.com/didi/nightingale/v5/src/pkg/ormx"
@@ -100,6 +104,33 @@ func MustLoad(fpaths ...string) {
 			}
 		}
 
+		if C.Alerting.CallPlugin.Enable {
+			if runtime.GOOS == "windows" {
+				fmt.Println("notify plugin on unsupported os:", runtime.GOOS)
+				os.Exit(1)
+			}
+
+			p, err := plugin.Open(C.Alerting.CallPlugin.PluginPath)
+			if err != nil {
+				fmt.Println("failed to load plugin:", err)
+				os.Exit(1)
+			}
+
+			caller, err := p.Lookup(C.Alerting.CallPlugin.Caller)
+			if err != nil {
+				fmt.Println("failed to lookup plugin Caller:", err)
+				os.Exit(1)
+			}
+
+			ins, ok := caller.(notifier.Notifier)
+			if !ok {
+				log.Println("notifier interface not implemented")
+				os.Exit(1)
+			}
+
+			notifier.Instance = ins
+		}
+
 		if C.WriterOpt.QueueMaxSize <= 0 {
 			C.WriterOpt.QueueMaxSize = 100000
 		}

src/server/engine/notify.go

@@ -9,8 +9,6 @@ import (
 	"net/http"
 	"os/exec"
 	"path"
-	"plugin"
-	"runtime"
 	"strings"
 	"time"
 
@@ -22,6 +20,7 @@ import (
 	"github.com/toolkits/pkg/slice"
 
 	"github.com/didi/nightingale/v5/src/models"
+	"github.com/didi/nightingale/v5/src/notifier"
 	"github.com/didi/nightingale/v5/src/pkg/sys"
 	"github.com/didi/nightingale/v5/src/pkg/tplx"
 	"github.com/didi/nightingale/v5/src/server/common/sender"
@@ -103,7 +102,6 @@ func alertingRedisPub(bs []byte) {
 
 func handleNotice(notice Notice, bs []byte) {
 	alertingCallScript(bs)
-
 	alertingCallPlugin(bs)
 
 	if len(config.C.Alerting.NotifyBuiltinChannels) == 0 {
@@ -398,12 +396,6 @@ func alertingCallScript(stdinBytes []byte) {
 	logger.Infof("event_notify: exec %s output: %s", fpath, buf.String())
 }
 
-type Notifier interface {
-	Descript() string
-	Notify([]byte)
-	NotifyMaintainer([]byte)
-}
-
 // call notify.so via golang plugin build
 // ig. etc/script/notify/notify.so
 func alertingCallPlugin(stdinBytes []byte) {
@@ -411,26 +403,8 @@ func alertingCallPlugin(stdinBytes []byte) {
 		return
 	}
 
-	if runtime.GOOS == "windows" {
+	logger.Debugf("alertingCallPlugin begin")
-		logger.Errorf("call notify plugin on unsupported os: %s", runtime.GOOS)
+	logger.Debugf("payload:", string(stdinBytes))
-		return
+	notifier.Instance.Notify(stdinBytes)
-	}
+	logger.Debugf("alertingCallPlugin done")
-
-	p, err := plugin.Open(config.C.Alerting.CallPlugin.PluginPath)
-	if err != nil {
-		logger.Errorf("failed to open notify plugin: %v", err)
-		return
-	}
-	caller, err := p.Lookup(config.C.Alerting.CallPlugin.Caller)
-	if err != nil {
-		logger.Errorf("failed to load caller: %v", err)
-		return
-	}
-	notifier, ok := caller.(Notifier)
-	if !ok {
-		logger.Errorf("notifier interface not implemented): %v", err)
-		return
-	}
-	notifier.Notify(stdinBytes)
-	logger.Debugf("alertingCallPlugin done. %s", notifier.Descript())
 }

src/server/engine/notify_maintainer.go

@@ -2,11 +2,11 @@ package engine
 
 import (
 	"encoding/json"
-	"plugin"
 	"runtime"
 	"time"
 
 	"github.com/didi/nightingale/v5/src/models"
+	"github.com/didi/nightingale/v5/src/notifier"
 	"github.com/didi/nightingale/v5/src/server/common/sender"
 	"github.com/didi/nightingale/v5/src/server/config"
 	"github.com/didi/nightingale/v5/src/server/memsto"
@@ -14,13 +14,13 @@ import (
 	"github.com/toolkits/pkg/logger"
 )
 
-type NoticeMaintainer struct {
+type MaintainMessage struct {
-	NotifyUsersObj []*models.User `json:"notify_user_obj" gorm:"-"`
+	Tos     []*models.User `json:"tos"`
-	Title          string         `json:"title"`
+	Title   string         `json:"title"`
-	Content        string         `json:"content"`
+	Content string         `json:"content"`
 }
 
-func noticeCallPlugin(stdinBytes []byte) {
+func notifyMaintainerWithPlugin(e error, title, triggerTime string, users []*models.User) {
 	if !config.C.Alerting.CallPlugin.Enable {
 		return
 	}
@@ -30,56 +30,48 @@ func noticeCallPlugin(stdinBytes []byte) {
 		return
 	}
 
-	p, err := plugin.Open(config.C.Alerting.CallPlugin.PluginPath)
+	stdinBytes, err := json.Marshal(MaintainMessage{
+		Tos:     users,
+		Title:   title,
+		Content: "Title: " + title + "\nContent: " + e.Error() + "\nTime: " + triggerTime,
+	})
+
 	if err != nil {
-		logger.Errorf("failed to open notify plugin: %v", err)
+		logger.Error("failed to marshal MaintainMessage:", err)
 		return
 	}
-	caller, err := p.Lookup(config.C.Alerting.CallPlugin.Caller)
+
-	if err != nil {
+	notifier.Instance.NotifyMaintainer(stdinBytes)
-		logger.Errorf("failed to load caller: %v", err)
+	logger.Debugf("notify maintainer with plugin done")
-		return
-	}
-	notifier, ok := caller.(Notifier)
-	if !ok {
-		logger.Errorf("notifier interface not implemented): %v", err)
-		return
-	}
-	notifier.NotifyMaintainer(stdinBytes)
-	logger.Debugf("noticeCallPlugin done. %s", notifier.Descript())
 }
 
 // notify to maintainer to handle the error
 func notifyToMaintainer(e error, title string) {
+	logger.Errorf("notifyToMaintainer, title:%s, error:%v", title, e)
 
-	logger.Errorf("notifyToMaintainer，title:%s, error:%v", title, e)
+	users := memsto.UserCache.GetMaintainerUsers()
-
+	if len(users) == 0 {
-	var noticeMaintainer NoticeMaintainer
-	maintainerUsers := memsto.UserCache.GetMaintainerUsers()
-	if len(maintainerUsers) == 0 {
 		return
 	}
-	triggerTime := time.Now().Format("2006/01/02 - 15:04:05")
-	noticeMaintainer.NotifyUsersObj = maintainerUsers
-	noticeMaintainer.Content = "【内部处理错误】当前标题: " + title + "\n【内部处理错误】当前异常: " + e.Error() + "\n【内部处理错误】发送时间: " + triggerTime
-	noticeMaintainer.Title = title
-	stdinBytes, err := json.Marshal(noticeMaintainer)
-	if err != nil {
-		logger.Errorf("notifyToMaintainer: failed to marshal noticeMaintainer: %v", err)
-	} else {
-		noticeCallPlugin(stdinBytes)
-	}
 
+	triggerTime := time.Now().Format("2006/01/02 - 15:04:05")
+
+	notifyMaintainerWithPlugin(e, title, triggerTime, users)
+	notifyMaintainerWithBuiltin(e, title, triggerTime, users)
+}
+
+func notifyMaintainerWithBuiltin(e error, title, triggerTime string, users []*models.User) {
 	if len(config.C.Alerting.NotifyBuiltinChannels) == 0 {
 		return
 	}
+
 	emailset := make(map[string]struct{})
 	phoneset := make(map[string]struct{})
 	wecomset := make(map[string]struct{})
 	dingtalkset := make(map[string]struct{})
 	feishuset := make(map[string]struct{})
 
-	for _, user := range maintainerUsers {
+	for _, user := range users {
 		if user.Email != "" {
 			emailset[user.Email] = struct{}{}
 		}
@@ -118,13 +110,13 @@ func notifyToMaintainer(e error, title string) {
 			if len(emailset) == 0 {
 				continue
 			}
-			content := "【内部处理错误】当前标题: " + title + "\n【内部处理错误】当前异常: " + e.Error() + "\n【内部处理错误】发送时间: " + triggerTime
+			content := "Title: " + title + "\nContent: " + e.Error() + "\nTime: " + triggerTime
 			sender.WriteEmail(title, content, StringSetKeys(emailset))
 		case "dingtalk":
 			if len(dingtalkset) == 0 {
 				continue
 			}
-			content := "**【内部处理错误】当前标题: **" + title + "\n**【内部处理错误】当前异常: **" + e.Error() + "\n**【内部处理错误】发送时间: **" + triggerTime
+			content := "**Title: **" + title + "\n**Content: **" + e.Error() + "\n**Time: **" + triggerTime
 			sender.SendDingtalk(sender.DingtalkMessage{
 				Title:     title,
 				Text:      content,
@@ -135,7 +127,7 @@ func notifyToMaintainer(e error, title string) {
 			if len(wecomset) == 0 {
 				continue
 			}
-			content := "**【内部处理错误】当前标题: **" + title + "\n**【内部处理错误】当前异常: **" + e.Error() + "\n**【内部处理错误】发送时间: **" + triggerTime
+			content := "**Title: **" + title + "\n**Content: **" + e.Error() + "\n**Time: **" + triggerTime
 			sender.SendWecom(sender.WecomMessage{
 				Text:   content,
 				Tokens: StringSetKeys(wecomset),
@@ -145,7 +137,7 @@ func notifyToMaintainer(e error, title string) {
 				continue
 			}
 
-			content := "【内部处理错误】当前标题: " + title + "\n【内部处理错误】当前异常: " + e.Error() + "\n【内部处理错误】发送时间: " + triggerTime
+			content := "Title: " + title + "\nContent: " + e.Error() + "\nTime: " + triggerTime
 			sender.SendFeishu(sender.FeishuMessage{
 				Text:      content,
 				AtMobiles: phones,

src/server/engine/worker.go

@@ -116,8 +116,7 @@ func (r RuleEval) Work() {
 		value, warnings, err = reader.Client.Query(context.Background(), promql, time.Now())
 		if err != nil {
 			logger.Errorf("rule_eval:%d promql:%s, error:%v", r.RuleID(), promql, err)
-			// 告警查询prometheus逻辑出错，发告警信息给管理员
+			notifyToMaintainer(err, "failed to query prometheus")
-			notifyToMaintainer(err, "查询prometheus出错")
 			return
 		}
 
@@ -190,7 +189,6 @@ func (ws *WorkersType) Build(rids []int64) {
 		elst, err := models.AlertCurEventGetByRule(rules[hash].Id)
 		if err != nil {
 			logger.Errorf("worker_build: AlertCurEventGetByRule failed: %v", err)
-			notifyToMaintainer(err, "AlertCurEventGetByRule Error，ruleID="+fmt.Sprint(rules[hash].Id))
 			continue
 		}