innov
/
nightingale1
1
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

not stable version

This commit is contained in:
Ulric Qin 2022-02-28 23:50:02 +08:00
parent b43f196d86
commit 403cb5a6ad
8 changed files with 286 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
etc/server.conf
View File

@ -54,17 +54,30 @@ IP = ""
# unit ms
Interval = 1000
[SMTP]
Host = "smtp.163.com"
Port = 994
User = "username"
Pass = "password"
From = "username@163.com"
InsecureSkipVerify = true
[Alerting]
NotifyScriptPath = "./etc/script/notify.py"
NotifyConcurrency = 100
TemplatesDir = "./etc/template"
NotifyConcurrency = 100
[Alerting.CallScript]
# built in sending capability in go code
# so, no need enable script sender
Enable = false
ScriptPath = "./etc/script/notify.py"
[Alerting.RedisPub]
Enable = false
# complete redis key: ${ChannelPrefix} + ${Cluster}
ChannelPrefix = "/alerts/"
[Alerting.GlobalCallback]
[Alerting.Webhook]
Enable = false
Url = "http://a.com/n9e/callback"
BasicAuthUser = ""

35
src/server/config/config.go
View File

@ -79,16 +79,16 @@ func MustLoad(fpaths ...string) {
C.Heartbeat.Endpoint = fmt.Sprintf("%s:%d", C.Heartbeat.IP, C.HTTP.Port)
C.Alerting.RedisPub.ChannelKey = C.Alerting.RedisPub.ChannelPrefix + C.ClusterName
if C.Alerting.GlobalCallback.Enable {
if C.Alerting.GlobalCallback.Timeout == "" {
C.Alerting.GlobalCallback.TimeoutDuration = time.Second * 5
if C.Alerting.Webhook.Enable {
if C.Alerting.Webhook.Timeout == "" {
C.Alerting.Webhook.TimeoutDuration = time.Second * 5
} else {
dur, err := time.ParseDuration(C.Alerting.GlobalCallback.Timeout)
dur, err := time.ParseDuration(C.Alerting.Webhook.Timeout)
if err != nil {
fmt.Println("failed to parse Alerting.GlobalCallback.Timeout")
fmt.Println("failed to parse Alerting.Webhook.Timeout")
os.Exit(1)
}
C.Alerting.GlobalCallback.TimeoutDuration = dur
C.Alerting.Webhook.TimeoutDuration = dur
}
}
@ -104,6 +104,7 @@ type Config struct {
Log logx.Config
HTTP httpx.Config
BasicAuth gin.Accounts
SMTP SMTPConfig
Heartbeat HeartbeatConfig
Alerting Alerting
NoData NoData
@ -123,12 +124,26 @@ type HeartbeatConfig struct {
Endpoint string
}
type SMTPConfig struct {
Host string
Port int
User string
Pass string
From string
InsecureSkipVerify bool
}
type Alerting struct {
NotifyScriptPath string
NotifyConcurrency int
TemplatesDir string
NotifyConcurrency int
CallScript CallScript
RedisPub RedisPub
GlobalCallback GlobalCallback
Webhook Webhook
}
type CallScript struct {
Enable bool
ScriptPath string
}
type RedisPub struct {
@ -137,7 +152,7 @@ type RedisPub struct {
ChannelKey string
}
type GlobalCallback struct {
type Webhook struct {
Enable bool
Url string
BasicAuthUser string

40
src/server/engine/callback.go
View File

@ -1,10 +1,6 @@
package engine
import (
"bytes"
"encoding/json"
"io/ioutil"
"net/http"
"strconv"
"strings"
"time"
@ -15,41 +11,9 @@ import (
"github.com/didi/nightingale/v5/src/pkg/ibex"
"github.com/didi/nightingale/v5/src/server/config"
"github.com/didi/nightingale/v5/src/server/memsto"
"github.com/didi/nightingale/v5/src/server/poster"
)
func PostJSON(url string, timeout time.Duration, v interface{}) (response []byte, code int, err error) {
var bs []byte
bs, err = json.Marshal(v)
if err != nil {
return
}
bf := bytes.NewBuffer(bs)
client := http.Client{
Timeout: timeout,
}
req, err := http.NewRequest("POST", url, bf)
req.Header.Set("Content-Type", "application/json")
var resp *http.Response
resp, err = client.Do(req)
if err != nil {
return
}
code = resp.StatusCode
if resp.Body != nil {
defer resp.Body.Close()
response, err = ioutil.ReadAll(resp.Body)
}
return
}
func callback(event *models.AlertCurEvent) {
urls := strings.Fields(event.Callbacks)
for _, url := range urls {
@ -68,7 +32,7 @@ func callback(event *models.AlertCurEvent) {
url = "http://" + url
}
resp, code, err := PostJSON(url, 5*time.Second, event)
resp, code, err := poster.PostJSON(url, 5*time.Second, event)
if err != nil {
logger.Errorf("event_callback(rule_id=%d url=%s) fail, resp: %s, err: %v, code: %d", event.RuleId, url, string(resp), err, code)
} else {

87
src/server/engine/notify.go
View File

@ -31,14 +31,14 @@ var fns = template.FuncMap{
"urlconvert": func(str string) interface{} { return template.URL(str) },
"timeformat": func(ts int64, pattern ...string) string {
defp := "2006-01-02 15:04:05"
if pattern != nil && len(pattern) > 0 {
if len(pattern) > 0 {
defp = pattern[0]
}
return time.Unix(ts, 0).Format(defp)
},
"timestamp": func(pattern ...string) string {
defp := "2006-01-02 15:04:05"
if pattern != nil && len(pattern) > 0 {
if len(pattern) > 0 {
defp = pattern[0]
}
return time.Now().Format(defp)
@ -89,7 +89,7 @@ type Notice struct {
Tpls map[string]string `json:"tpls"`
}
func buildStdin(event *models.AlertCurEvent) ([]byte, error) {
func genNotice(event *models.AlertCurEvent) Notice {
// build notice body with templates
ntpls := make(map[string]string)
for filename, tpl := range tpls {
@ -101,36 +101,40 @@ func buildStdin(event *models.AlertCurEvent) ([]byte, error) {
}
}
return json.Marshal(Notice{Event: event, Tpls: ntpls})
return Notice{Event: event, Tpls: ntpls}
}
func alertingRedisPub(bs []byte) {
// pub all alerts to redis
if config.C.Alerting.RedisPub.Enable {
err := storage.Redis.Publish(context.Background(), config.C.Alerting.RedisPub.ChannelKey, bs).Err()
if err != nil {
logger.Errorf("event_notify: redis publish %s err: %v", config.C.Alerting.RedisPub.ChannelKey, err)
}
}
}
func handleNotice(notice Notice, bs []byte) {
alertingCallScript(bs)
// TODO 弄个channel发邮件，学习daemon写法
// 收集tokens、phones，发呗
}
func notify(event *models.AlertCurEvent) {
logEvent(event, "notify")
stdin, err := buildStdin(event)
notice := genNotice(event)
stdinBytes, err := json.Marshal(notice)
if err != nil {
logger.Errorf("event_notify: build stdin failed: %v", err)
logger.Errorf("event_notify: failed to marshal notice: %v", err)
return
}
// pub all alerts to redis
if config.C.Alerting.RedisPub.Enable {
err = storage.Redis.Publish(context.Background(), config.C.Alerting.RedisPub.ChannelKey, stdin).Err()
if err != nil {
logger.Errorf("event_notify: redis publish %s err: %v", config.C.Alerting.RedisPub.ChannelKey, err)
}
}
alertingRedisPub(stdinBytes)
alertingWebhook(event)
if config.C.Alerting.GlobalCallback.Enable {
DoGlobalCallback(event)
}
// no notify.py? do nothing
if config.C.Alerting.NotifyScriptPath == "" {
return
}
callScript(stdin)
handleNotice(notice, stdinBytes)
// handle alert subscribes
subs, has := memsto.AlertSubscribeCache.Get(event.RuleId)
@ -144,8 +148,13 @@ func notify(event *models.AlertCurEvent) {
}
}
func DoGlobalCallback(event *models.AlertCurEvent) {
conf := config.C.Alerting.GlobalCallback
func alertingWebhook(event *models.AlertCurEvent) {
conf := config.C.Alerting.Webhook
if !conf.Enable {
return
}
if conf.Url == "" {
return
}
@ -159,7 +168,7 @@ func DoGlobalCallback(event *models.AlertCurEvent) {
req, err := http.NewRequest("POST", conf.Url, bf)
if err != nil {
logger.Warning("DoGlobalCallback failed to new request", err)
logger.Warning("alertingWebhook failed to new request", err)
return
}
@ -180,17 +189,17 @@ func DoGlobalCallback(event *models.AlertCurEvent) {
var resp *http.Response
resp, err = client.Do(req)
if err != nil {
logger.Warning("DoGlobalCallback failed to call url, error: ", err)
logger.Warning("alertingWebhook failed to call url, error: ", err)
return
}
var body []byte
if resp.Body != nil {
defer resp.Body.Close()
body, err = ioutil.ReadAll(resp.Body)
body, _ = ioutil.ReadAll(resp.Body)
}
logger.Debugf("DoGlobalCallback done, url: %s, response code: %d, body: %s", conf.Url, resp.StatusCode, string(body))
logger.Debugf("alertingWebhook done, url: %s, response code: %d, body: %s", conf.Url, resp.StatusCode, string(body))
}
func handleSubscribes(event models.AlertCurEvent, subs []*models.AlertSubscribe) {
@ -223,17 +232,27 @@ func handleSubscribe(event models.AlertCurEvent, sub *models.AlertSubscribe) {
fillUsers(&event)
stdin, err := buildStdin(&event)
notice := genNotice(&event)
stdinBytes, err := json.Marshal(notice)
if err != nil {
logger.Errorf("event_notify: build stdin failed when handle subscribe: %v", err)
logger.Errorf("event_notify: failed to marshal notice: %v", err)
return
}
callScript(stdin)
handleNotice(notice, stdinBytes)
}
func callScript(stdinBytes []byte) {
fpath := config.C.Alerting.NotifyScriptPath
func alertingCallScript(stdinBytes []byte) {
if !config.C.Alerting.CallScript.Enable {
return
}
// no notify.py? do nothing
if config.C.Alerting.CallScript.ScriptPath == "" {
return
}
fpath := config.C.Alerting.CallScript.ScriptPath
cmd := exec.Command(fpath)
cmd.Stdin = bytes.NewReader(stdinBytes)

42
src/server/poster/post.go Normal file
View File

@ -0,0 +1,42 @@
package poster
import (
"bytes"
"encoding/json"
"io/ioutil"
"net/http"
"time"
)
func PostJSON(url string, timeout time.Duration, v interface{}) (response []byte, code int, err error) {
var bs []byte
bs, err = json.Marshal(v)
if err != nil {
return
}
bf := bytes.NewBuffer(bs)
client := http.Client{
Timeout: timeout,
}
req, err := http.NewRequest("POST", url, bf)
req.Header.Set("Content-Type", "application/json")
var resp *http.Response
resp, err = client.Do(req)
if err != nil {
return
}
code = resp.StatusCode
if resp.Body != nil {
defer resp.Body.Close()
response, err = ioutil.ReadAll(resp.Body)
}
return
}

55
src/server/sender/dingtalk.go Normal file
View File

@ -0,0 +1,55 @@
package sender
import (
"time"
"github.com/didi/nightingale/v5/src/server/poster"
"github.com/toolkits/pkg/logger"
)
type DingtalkMessage struct {
Title string
Text string
AtMobiles []string
Tokens []string
}
type dingtalkMarkdown struct {
Title string `json:"title"`
Text string `json:"text"`
}
type dingtalkAt struct {
AtMobiles []string `json:"atMobiles"`
IsAtAll bool `json:"isAtAll"`
}
type dingtalk struct {
Msgtype string `json:"msgtype"`
Markdown dingtalkMarkdown `json:"markdown"`
At dingtalkAt `json:"at"`
}
func SendDingtalk(message DingtalkMessage) {
for i := 0; i < len(message.Tokens); i++ {
url := "https://oapi.dingtalk.com/robot/send?access_token=" + message.Tokens[i]
body := dingtalk{
Msgtype: "markdown",
Markdown: dingtalkMarkdown{
Title: message.Title,
Text: message.Text,
},
At: dingtalkAt{
AtMobiles: message.AtMobiles,
IsAtAll: false,
},
}
res, code, err := poster.PostJSON(url, time.Second*5, body)
if err != nil {
logger.Errorf("dingtalk_sender: result=fail url=%s code=%d error=%v response=%s", url, code, err, string(res))
} else {
logger.Infof("dingtalk_sender: result=succ url=%s code=%d response=%s", url, code, string(res))
}
}
}

52
src/server/sender/feishu.go Normal file
View File

@ -0,0 +1,52 @@
package sender
import (
"time"
"github.com/didi/nightingale/v5/src/server/poster"
"github.com/toolkits/pkg/logger"
)
type FeishuMessage struct {
Text string
AtMobiles []string
Tokens []string
}
type feishuContent struct {
Text string `json:"text"`
}
type feishuAt struct {
AtMobiles []string `json:"atMobiles"`
IsAtAll bool `json:"isAtAll"`
}
type feishu struct {
Msgtype string `json:"msg_type"`
Content feishuContent `json:"content"`
At feishuAt `json:"at"`
}
func SendFeishu(message FeishuMessage) {
for i := 0; i < len(message.Tokens); i++ {
url := "https://open.feishu.cn/open-apis/bot/v2/hook/" + message.Tokens[i]
body := feishu{
Msgtype: "text",
Content: feishuContent{
Text: message.Text,
},
At: feishuAt{
AtMobiles: message.AtMobiles,
IsAtAll: false,
},
}
res, code, err := poster.PostJSON(url, time.Second*5, body)
if err != nil {
logger.Errorf("feishu_sender: result=fail url=%s code=%d error=%v response=%s", url, code, err, string(res))
} else {
logger.Infof("feishu_sender: result=succ url=%s code=%d response=%s", url, code, string(res))
}
}
}

41
src/server/sender/wecom.go Normal file
View File

@ -0,0 +1,41 @@
package sender
import (
"time"
"github.com/didi/nightingale/v5/src/server/poster"
"github.com/toolkits/pkg/logger"
)
type WecomMessage struct {
Text string
Tokens []string
}
type wecomMarkdown struct {
Content string `json:"content"`
}
type wecom struct {
Msgtype string `json:"msgtype"`
Markdown wecomMarkdown `json:"markdown"`
}
func SendWecom(message WecomMessage) {
for i := 0; i < len(message.Tokens); i++ {
url := "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=" + message.Tokens[i]
body := wecom{
Msgtype: "markdown",
Markdown: wecomMarkdown{
Content: message.Text,
},
}
res, code, err := poster.PostJSON(url, time.Second*5, body)
if err != nil {
logger.Errorf("wecom_sender: result=fail url=%s code=%d error=%v response=%s", url, code, err, string(res))
} else {
logger.Infof("wecom_sender: result=succ url=%s code=%d response=%s", url, code, string(res))
}
}
}